Files
quantus/asset/gpu/nvidia-textfile.sh
T
Rob ThijssenandClaude Fable 5.1 aa6bd855f4 metrics: performance by build — recording rules, dashboard row, throttle reasons
Monitoring side of quantus/miner#9. The miner now exports build identity,
per-device and per-job series (quantus/miner#14); this makes them readable.

- asset/prometheus/quantus-rules.yml: recording rules joining the miner's
  series with the GPU series on a derived `host` and device index —
  hashrate by device and commit, hashrate per watt, batch time off the GPU,
  stale work, seal latency p95, idle waiting for the node.
- script/infra-setup.sh (metrics role): prometheus.yml gains `rule_files:`
  for /etc/prometheus/rules.d, guarded like scrape_config_files; the dir;
  a sudoers grant for the rules file.
- deploy.yaml deploy-metrics: pushes the rules (reload is already gated on
  content change); validate asserts the quantus-miner rule group loaded.
- asset/grafana/quantus.json: "Performance by build" row, eight panels, and
  a deploy annotation from changes in miner_build_info. No credentials
  involved; Prometheus is the annotation source.
- asset/gpu/nvidia-textfile.sh (gpu role, operator-shipped): five
  nvidia_gpu_event_reason_active{reason=...} gauges from nvidia-smi's clock
  event reasons, so a regression that coincides with thermal or power-brake
  capping can be told from a code regression.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5
2026-09-03 14:13:15 +03:00

100 lines
5.5 KiB
Bash
Executable File

#!/usr/bin/env bash
# Write NVIDIA GPU metrics for node-exporter's textfile collector.
#
# Deliberately NOT a new exporter: node-exporter already runs on every host, is
# already scraped by the fleet Prometheus (job "node", targets from cichlid's
# inventory), and already has the textfile collector enabled. A textfile drop
# therefore needs no new port, no firewalld service, no scrape config and no
# entry in the port registry.
#
# DCGM was the alternative and is the wrong tool here: it targets datacenter
# cards and its GeForce support is partial. nvidia-smi reports everything we
# actually want.
set -euo pipefail
OUT_DIR="${OUT_DIR:-/var/lib/prometheus/node-exporter}"
OUT="${OUT_DIR}/nvidia.prom"
TMP="$(mktemp "${OUT}.XXXXXX")"
# The collector reads whole files, so a half-written one is a parse error and
# shows up as node_textfile_scrape_error. Build in a temp file, rename atomically.
trap 'rm -f "$TMP"' EXIT
QUERY=index,uuid,name,temperature.gpu,utilization.gpu,utilization.memory,\
memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed,\
clocks_event_reasons.sw_power_cap,clocks_event_reasons.hw_slowdown,clocks_event_reasons.sw_thermal_slowdown,\
clocks_event_reasons.hw_thermal_slowdown,clocks_event_reasons.hw_power_brake_slowdown
{
echo "# HELP nvidia_gpu_info Static GPU identity, always 1."
echo "# TYPE nvidia_gpu_info gauge"
echo "# HELP nvidia_gpu_temperature_celsius Core temperature."
echo "# TYPE nvidia_gpu_temperature_celsius gauge"
echo "# HELP nvidia_gpu_utilization_ratio Fraction of time the GPU was busy, 0-1."
echo "# TYPE nvidia_gpu_utilization_ratio gauge"
echo "# HELP nvidia_gpu_memory_utilization_ratio Fraction of time memory was accessed, 0-1."
echo "# TYPE nvidia_gpu_memory_utilization_ratio gauge"
echo "# HELP nvidia_gpu_memory_used_bytes Memory in use."
echo "# TYPE nvidia_gpu_memory_used_bytes gauge"
echo "# HELP nvidia_gpu_memory_total_bytes Memory installed."
echo "# TYPE nvidia_gpu_memory_total_bytes gauge"
echo "# HELP nvidia_gpu_power_watts Current board power draw."
echo "# TYPE nvidia_gpu_power_watts gauge"
echo "# HELP nvidia_gpu_power_limit_watts Enforced power limit."
echo "# TYPE nvidia_gpu_power_limit_watts gauge"
echo "# HELP nvidia_gpu_clock_sm_hertz Current SM clock."
echo "# TYPE nvidia_gpu_clock_sm_hertz gauge"
echo "# HELP nvidia_gpu_clock_memory_hertz Current memory clock."
echo "# TYPE nvidia_gpu_clock_memory_hertz gauge"
echo "# HELP nvidia_gpu_fan_speed_ratio Fan duty cycle, 0-1."
echo "# TYPE nvidia_gpu_fan_speed_ratio gauge"
echo "# HELP nvidia_gpu_event_reason_active 1 while the clock is being held down for this reason (power cap, thermal, brake), else 0."
echo "# TYPE nvidia_gpu_event_reason_active gauge"
echo "# HELP nvidia_gpu_scrape_ok 1 when nvidia-smi was queried successfully."
echo "# TYPE nvidia_gpu_scrape_ok gauge"
if ! smi=$(nvidia-smi --query-gpu="$QUERY" --format=csv,noheader,nounits 2>/dev/null); then
# Driver reloading, GPU fell off the bus, or nvidia-smi absent. Emit an
# explicit zero rather than a stale file or an empty one, so the dashboard
# shows "broken" instead of "flat".
echo "nvidia_gpu_scrape_ok 0"
exit 0
fi
echo "nvidia_gpu_scrape_ok 1"
while IFS= read -r line; do
[ -n "$line" ] || continue
IFS=',' read -r idx uuid name temp util umem mused mtotal pdraw plimit csm cmem fan \
r_pcap r_hwslow r_swtherm r_hwtherm r_brake \
<<< "$(printf '%s' "$line" | sed 's/, */,/g')"
# A label value may contain characters Prometheus treats specially.
esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; }
L="gpu=\"${idx}\",uuid=\"$(esc "$uuid")\",name=\"$(esc "$name")\""
num() { case "$1" in ''|*[!0-9.]*) return 1 ;; *) printf '%s' "$1" ;; esac; }
echo "nvidia_gpu_info{${L}} 1"
num "$temp" >/dev/null && echo "nvidia_gpu_temperature_celsius{${L}} ${temp}"
num "$util" >/dev/null && awk -v l="$L" -v v="$util" 'BEGIN{printf "nvidia_gpu_utilization_ratio{%s} %.4f\n", l, v/100}'
num "$umem" >/dev/null && awk -v l="$L" -v v="$umem" 'BEGIN{printf "nvidia_gpu_memory_utilization_ratio{%s} %.4f\n", l, v/100}'
num "$mused" >/dev/null && awk -v l="$L" -v v="$mused" 'BEGIN{printf "nvidia_gpu_memory_used_bytes{%s} %.0f\n", l, v*1048576}'
num "$mtotal" >/dev/null && awk -v l="$L" -v v="$mtotal" 'BEGIN{printf "nvidia_gpu_memory_total_bytes{%s} %.0f\n", l, v*1048576}'
num "$pdraw" >/dev/null && echo "nvidia_gpu_power_watts{${L}} ${pdraw}"
num "$plimit" >/dev/null && echo "nvidia_gpu_power_limit_watts{${L}} ${plimit}"
num "$csm" >/dev/null && awk -v l="$L" -v v="$csm" 'BEGIN{printf "nvidia_gpu_clock_sm_hertz{%s} %.0f\n", l, v*1000000}'
num "$cmem" >/dev/null && awk -v l="$L" -v v="$cmem" 'BEGIN{printf "nvidia_gpu_clock_memory_hertz{%s} %.0f\n", l, v*1000000}'
num "$fan" >/dev/null && awk -v l="$L" -v v="$fan" 'BEGIN{printf "nvidia_gpu_fan_speed_ratio{%s} %.4f\n", l, v/100}'
# Why a card is below its boost clock. A hashrate regression that
# coincides with a thermal or power-brake reason is not a code regression.
reason() { case "$2" in Active) echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 1" ;; "Not Active") echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 0" ;; esac; }
reason sw_power_cap "$r_pcap"
reason hw_slowdown "$r_hwslow"
reason sw_thermal_slowdown "$r_swtherm"
reason hw_thermal_slowdown "$r_hwtherm"
reason hw_power_brake_slowdown "$r_brake"
done <<< "$smi"
} > "$TMP"
chmod 0644 "$TMP"
mv -f "$TMP" "$OUT"
trap - EXIT