Monitoring side of quantus/miner#9. The miner now exports build identity, per-device and per-job series (quantus/miner#14); this makes them readable. - asset/prometheus/quantus-rules.yml: recording rules joining the miner's series with the GPU series on a derived `host` and device index — hashrate by device and commit, hashrate per watt, batch time off the GPU, stale work, seal latency p95, idle waiting for the node. - script/infra-setup.sh (metrics role): prometheus.yml gains `rule_files:` for /etc/prometheus/rules.d, guarded like scrape_config_files; the dir; a sudoers grant for the rules file. - deploy.yaml deploy-metrics: pushes the rules (reload is already gated on content change); validate asserts the quantus-miner rule group loaded. - asset/grafana/quantus.json: "Performance by build" row, eight panels, and a deploy annotation from changes in miner_build_info. No credentials involved; Prometheus is the annotation source. - asset/gpu/nvidia-textfile.sh (gpu role, operator-shipped): five nvidia_gpu_event_reason_active{reason=...} gauges from nvidia-smi's clock event reasons, so a regression that coincides with thermal or power-brake capping can be told from a code regression. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5
100 lines
5.5 KiB
Bash
Executable File
100 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Write NVIDIA GPU metrics for node-exporter's textfile collector.
|
|
#
|
|
# Deliberately NOT a new exporter: node-exporter already runs on every host, is
|
|
# already scraped by the fleet Prometheus (job "node", targets from cichlid's
|
|
# inventory), and already has the textfile collector enabled. A textfile drop
|
|
# therefore needs no new port, no firewalld service, no scrape config and no
|
|
# entry in the port registry.
|
|
#
|
|
# DCGM was the alternative and is the wrong tool here: it targets datacenter
|
|
# cards and its GeForce support is partial. nvidia-smi reports everything we
|
|
# actually want.
|
|
set -euo pipefail
|
|
|
|
OUT_DIR="${OUT_DIR:-/var/lib/prometheus/node-exporter}"
|
|
OUT="${OUT_DIR}/nvidia.prom"
|
|
TMP="$(mktemp "${OUT}.XXXXXX")"
|
|
# The collector reads whole files, so a half-written one is a parse error and
|
|
# shows up as node_textfile_scrape_error. Build in a temp file, rename atomically.
|
|
trap 'rm -f "$TMP"' EXIT
|
|
|
|
QUERY=index,uuid,name,temperature.gpu,utilization.gpu,utilization.memory,\
|
|
memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed,\
|
|
clocks_event_reasons.sw_power_cap,clocks_event_reasons.hw_slowdown,clocks_event_reasons.sw_thermal_slowdown,\
|
|
clocks_event_reasons.hw_thermal_slowdown,clocks_event_reasons.hw_power_brake_slowdown
|
|
|
|
{
|
|
echo "# HELP nvidia_gpu_info Static GPU identity, always 1."
|
|
echo "# TYPE nvidia_gpu_info gauge"
|
|
echo "# HELP nvidia_gpu_temperature_celsius Core temperature."
|
|
echo "# TYPE nvidia_gpu_temperature_celsius gauge"
|
|
echo "# HELP nvidia_gpu_utilization_ratio Fraction of time the GPU was busy, 0-1."
|
|
echo "# TYPE nvidia_gpu_utilization_ratio gauge"
|
|
echo "# HELP nvidia_gpu_memory_utilization_ratio Fraction of time memory was accessed, 0-1."
|
|
echo "# TYPE nvidia_gpu_memory_utilization_ratio gauge"
|
|
echo "# HELP nvidia_gpu_memory_used_bytes Memory in use."
|
|
echo "# TYPE nvidia_gpu_memory_used_bytes gauge"
|
|
echo "# HELP nvidia_gpu_memory_total_bytes Memory installed."
|
|
echo "# TYPE nvidia_gpu_memory_total_bytes gauge"
|
|
echo "# HELP nvidia_gpu_power_watts Current board power draw."
|
|
echo "# TYPE nvidia_gpu_power_watts gauge"
|
|
echo "# HELP nvidia_gpu_power_limit_watts Enforced power limit."
|
|
echo "# TYPE nvidia_gpu_power_limit_watts gauge"
|
|
echo "# HELP nvidia_gpu_clock_sm_hertz Current SM clock."
|
|
echo "# TYPE nvidia_gpu_clock_sm_hertz gauge"
|
|
echo "# HELP nvidia_gpu_clock_memory_hertz Current memory clock."
|
|
echo "# TYPE nvidia_gpu_clock_memory_hertz gauge"
|
|
echo "# HELP nvidia_gpu_fan_speed_ratio Fan duty cycle, 0-1."
|
|
echo "# TYPE nvidia_gpu_fan_speed_ratio gauge"
|
|
echo "# HELP nvidia_gpu_event_reason_active 1 while the clock is being held down for this reason (power cap, thermal, brake), else 0."
|
|
echo "# TYPE nvidia_gpu_event_reason_active gauge"
|
|
echo "# HELP nvidia_gpu_scrape_ok 1 when nvidia-smi was queried successfully."
|
|
echo "# TYPE nvidia_gpu_scrape_ok gauge"
|
|
|
|
if ! smi=$(nvidia-smi --query-gpu="$QUERY" --format=csv,noheader,nounits 2>/dev/null); then
|
|
# Driver reloading, GPU fell off the bus, or nvidia-smi absent. Emit an
|
|
# explicit zero rather than a stale file or an empty one, so the dashboard
|
|
# shows "broken" instead of "flat".
|
|
echo "nvidia_gpu_scrape_ok 0"
|
|
exit 0
|
|
fi
|
|
echo "nvidia_gpu_scrape_ok 1"
|
|
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] || continue
|
|
IFS=',' read -r idx uuid name temp util umem mused mtotal pdraw plimit csm cmem fan \
|
|
r_pcap r_hwslow r_swtherm r_hwtherm r_brake \
|
|
<<< "$(printf '%s' "$line" | sed 's/, */,/g')"
|
|
# A label value may contain characters Prometheus treats specially.
|
|
esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; }
|
|
L="gpu=\"${idx}\",uuid=\"$(esc "$uuid")\",name=\"$(esc "$name")\""
|
|
|
|
num() { case "$1" in ''|*[!0-9.]*) return 1 ;; *) printf '%s' "$1" ;; esac; }
|
|
|
|
echo "nvidia_gpu_info{${L}} 1"
|
|
num "$temp" >/dev/null && echo "nvidia_gpu_temperature_celsius{${L}} ${temp}"
|
|
num "$util" >/dev/null && awk -v l="$L" -v v="$util" 'BEGIN{printf "nvidia_gpu_utilization_ratio{%s} %.4f\n", l, v/100}'
|
|
num "$umem" >/dev/null && awk -v l="$L" -v v="$umem" 'BEGIN{printf "nvidia_gpu_memory_utilization_ratio{%s} %.4f\n", l, v/100}'
|
|
num "$mused" >/dev/null && awk -v l="$L" -v v="$mused" 'BEGIN{printf "nvidia_gpu_memory_used_bytes{%s} %.0f\n", l, v*1048576}'
|
|
num "$mtotal" >/dev/null && awk -v l="$L" -v v="$mtotal" 'BEGIN{printf "nvidia_gpu_memory_total_bytes{%s} %.0f\n", l, v*1048576}'
|
|
num "$pdraw" >/dev/null && echo "nvidia_gpu_power_watts{${L}} ${pdraw}"
|
|
num "$plimit" >/dev/null && echo "nvidia_gpu_power_limit_watts{${L}} ${plimit}"
|
|
num "$csm" >/dev/null && awk -v l="$L" -v v="$csm" 'BEGIN{printf "nvidia_gpu_clock_sm_hertz{%s} %.0f\n", l, v*1000000}'
|
|
num "$cmem" >/dev/null && awk -v l="$L" -v v="$cmem" 'BEGIN{printf "nvidia_gpu_clock_memory_hertz{%s} %.0f\n", l, v*1000000}'
|
|
num "$fan" >/dev/null && awk -v l="$L" -v v="$fan" 'BEGIN{printf "nvidia_gpu_fan_speed_ratio{%s} %.4f\n", l, v/100}'
|
|
# Why a card is below its boost clock. A hashrate regression that
|
|
# coincides with a thermal or power-brake reason is not a code regression.
|
|
reason() { case "$2" in Active) echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 1" ;; "Not Active") echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 0" ;; esac; }
|
|
reason sw_power_cap "$r_pcap"
|
|
reason hw_slowdown "$r_hwslow"
|
|
reason sw_thermal_slowdown "$r_swtherm"
|
|
reason hw_thermal_slowdown "$r_hwtherm"
|
|
reason hw_power_brake_slowdown "$r_brake"
|
|
done <<< "$smi"
|
|
} > "$TMP"
|
|
|
|
chmod 0644 "$TMP"
|
|
mv -f "$TMP" "$OUT"
|
|
trap - EXIT
|