Rob Thijssen
598aea28f8
deploy / fetch (push) Successful in 16s
deploy / deploy-node (bob.hanzalova.internal, 0x134e73f06fa9bdb1dbfa909e149c563f5860ceb71a0e7307918f7033970edf59, benjy.hanzalova.internal quadbrat.hanzalova.internal, --public-addr /dns4/nh.thgttg.com/tcp/30333, --unsafe-rpc-external --rpc-methods safe --rpc-c… (push) Successful in 33s
deploy / deploy-miner (1, benjy.hanzalova.internal, bob.hanzalova.internal) (push) Successful in 41s
deploy / deploy-miner (1, quadbrat.hanzalova.internal, bob.hanzalova.internal) (push) Successful in 40s
deploy / deploy-metrics (push) Successful in 9s
node-exporter already runs on every host, is already scraped (job 'node', targets from cichlid's inventory, labelled by host), and already has the textfile collector enabled with a spool at /var/lib/prometheus/node-exporter. So a textfile drop needs no new exporter, no port allocation, no firewalld service and no scrape config -- which is why it is one-time provisioning rather than a deploy job. DCGM was the alternative and is the wrong tool: it targets datacenter cards and its GeForce support is partial. nvidia-smi reports everything wanted. The script writes to a temp file and renames atomically, because the collector reads whole files and a half-written one surfaces as node_textfile_scrape_error. When nvidia-smi fails it emits an explicit nvidia_gpu_scrape_ok 0 rather than leaving a stale file, so a broken driver reads as broken instead of the graphs quietly flatlining. Verified on the dual-GPU host: both cards reporting temperature, power, utilisation, memory, clocks and fan through node-exporter with node_textfile_scrape_error 0. Fleet-general rather than quantus-specific -- it lives here because there is no monitoring repo yet, and the dashboard says so.
87 lines
4.6 KiB
Bash
Executable File
87 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Write NVIDIA GPU metrics for node-exporter's textfile collector.
|
|
#
|
|
# Deliberately NOT a new exporter: node-exporter already runs on every host, is
|
|
# already scraped by the fleet Prometheus (job "node", targets from cichlid's
|
|
# inventory), and already has the textfile collector enabled. A textfile drop
|
|
# therefore needs no new port, no firewalld service, no scrape config and no
|
|
# entry in the port registry.
|
|
#
|
|
# DCGM was the alternative and is the wrong tool here: it targets datacenter
|
|
# cards and its GeForce support is partial. nvidia-smi reports everything we
|
|
# actually want.
|
|
set -euo pipefail
|
|
|
|
OUT_DIR="${OUT_DIR:-/var/lib/prometheus/node-exporter}"
|
|
OUT="${OUT_DIR}/nvidia.prom"
|
|
TMP="$(mktemp "${OUT}.XXXXXX")"
|
|
# The collector reads whole files, so a half-written one is a parse error and
|
|
# shows up as node_textfile_scrape_error. Build in a temp file, rename atomically.
|
|
trap 'rm -f "$TMP"' EXIT
|
|
|
|
QUERY=index,uuid,name,temperature.gpu,utilization.gpu,utilization.memory,\
|
|
memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed
|
|
|
|
{
|
|
echo "# HELP nvidia_gpu_info Static GPU identity, always 1."
|
|
echo "# TYPE nvidia_gpu_info gauge"
|
|
echo "# HELP nvidia_gpu_temperature_celsius Core temperature."
|
|
echo "# TYPE nvidia_gpu_temperature_celsius gauge"
|
|
echo "# HELP nvidia_gpu_utilization_ratio Fraction of time the GPU was busy, 0-1."
|
|
echo "# TYPE nvidia_gpu_utilization_ratio gauge"
|
|
echo "# HELP nvidia_gpu_memory_utilization_ratio Fraction of time memory was accessed, 0-1."
|
|
echo "# TYPE nvidia_gpu_memory_utilization_ratio gauge"
|
|
echo "# HELP nvidia_gpu_memory_used_bytes Memory in use."
|
|
echo "# TYPE nvidia_gpu_memory_used_bytes gauge"
|
|
echo "# HELP nvidia_gpu_memory_total_bytes Memory installed."
|
|
echo "# TYPE nvidia_gpu_memory_total_bytes gauge"
|
|
echo "# HELP nvidia_gpu_power_watts Current board power draw."
|
|
echo "# TYPE nvidia_gpu_power_watts gauge"
|
|
echo "# HELP nvidia_gpu_power_limit_watts Enforced power limit."
|
|
echo "# TYPE nvidia_gpu_power_limit_watts gauge"
|
|
echo "# HELP nvidia_gpu_clock_sm_hertz Current SM clock."
|
|
echo "# TYPE nvidia_gpu_clock_sm_hertz gauge"
|
|
echo "# HELP nvidia_gpu_clock_memory_hertz Current memory clock."
|
|
echo "# TYPE nvidia_gpu_clock_memory_hertz gauge"
|
|
echo "# HELP nvidia_gpu_fan_speed_ratio Fan duty cycle, 0-1."
|
|
echo "# TYPE nvidia_gpu_fan_speed_ratio gauge"
|
|
echo "# HELP nvidia_gpu_scrape_ok 1 when nvidia-smi was queried successfully."
|
|
echo "# TYPE nvidia_gpu_scrape_ok gauge"
|
|
|
|
if ! smi=$(nvidia-smi --query-gpu="$QUERY" --format=csv,noheader,nounits 2>/dev/null); then
|
|
# Driver reloading, GPU fell off the bus, or nvidia-smi absent. Emit an
|
|
# explicit zero rather than a stale file or an empty one, so the dashboard
|
|
# shows "broken" instead of "flat".
|
|
echo "nvidia_gpu_scrape_ok 0"
|
|
exit 0
|
|
fi
|
|
echo "nvidia_gpu_scrape_ok 1"
|
|
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] || continue
|
|
IFS=',' read -r idx uuid name temp util umem mused mtotal pdraw plimit csm cmem fan \
|
|
<<< "$(printf '%s' "$line" | sed 's/, */,/g')"
|
|
# A label value may contain characters Prometheus treats specially.
|
|
esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; }
|
|
L="gpu=\"${idx}\",uuid=\"$(esc "$uuid")\",name=\"$(esc "$name")\""
|
|
|
|
num() { case "$1" in ''|*[!0-9.]*) return 1 ;; *) printf '%s' "$1" ;; esac; }
|
|
|
|
echo "nvidia_gpu_info{${L}} 1"
|
|
num "$temp" >/dev/null && echo "nvidia_gpu_temperature_celsius{${L}} ${temp}"
|
|
num "$util" >/dev/null && awk -v l="$L" -v v="$util" 'BEGIN{printf "nvidia_gpu_utilization_ratio{%s} %.4f\n", l, v/100}'
|
|
num "$umem" >/dev/null && awk -v l="$L" -v v="$umem" 'BEGIN{printf "nvidia_gpu_memory_utilization_ratio{%s} %.4f\n", l, v/100}'
|
|
num "$mused" >/dev/null && awk -v l="$L" -v v="$mused" 'BEGIN{printf "nvidia_gpu_memory_used_bytes{%s} %.0f\n", l, v*1048576}'
|
|
num "$mtotal" >/dev/null && awk -v l="$L" -v v="$mtotal" 'BEGIN{printf "nvidia_gpu_memory_total_bytes{%s} %.0f\n", l, v*1048576}'
|
|
num "$pdraw" >/dev/null && echo "nvidia_gpu_power_watts{${L}} ${pdraw}"
|
|
num "$plimit" >/dev/null && echo "nvidia_gpu_power_limit_watts{${L}} ${plimit}"
|
|
num "$csm" >/dev/null && awk -v l="$L" -v v="$csm" 'BEGIN{printf "nvidia_gpu_clock_sm_hertz{%s} %.0f\n", l, v*1000000}'
|
|
num "$cmem" >/dev/null && awk -v l="$L" -v v="$cmem" 'BEGIN{printf "nvidia_gpu_clock_memory_hertz{%s} %.0f\n", l, v*1000000}'
|
|
num "$fan" >/dev/null && awk -v l="$L" -v v="$fan" 'BEGIN{printf "nvidia_gpu_fan_speed_ratio{%s} %.4f\n", l, v/100}'
|
|
done <<< "$smi"
|
|
} > "$TMP"
|
|
|
|
chmod 0644 "$TMP"
|
|
mv -f "$TMP" "$OUT"
|
|
trap - EXIT
|