Files
quantus/asset/prometheus/quantus-rules.yml
T
Rob ThijssenandClaude Fable 5.1 a0ce6d7467 rules: derive the short hostname so the GPU join matches node-exporter
node-exporter's host label is the short name (benjy), the first rule derived
the FQDN from the miner's instance, and quantus:device_hashrate_per_watt
produced no series. Verified against live labels on the metrics host.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5
2026-09-03 14:23:41 +03:00

56 lines
2.7 KiB
YAML

# Recording rules for the Quantus miners, deployed to
# /etc/prometheus/rules.d/quantus.yml on the metrics host (quantus/miner#9).
#
# The miner's series (job quantus-miner, instance host:9900) and the GPU
# series from node-exporter's textfile collector (job node, `host` label, the
# SHORT hostname as cichlid's inventory sets it) are joined on `host` derived
# as the first DNS label of the miner's instance, and on `device` (the miner's
# engine index) =
# `gpu` (nvidia-smi index). On single-card hosts both are 0; on a multi-card
# host the engine orders adapters discrete-first as Vulkan enumerates them,
# which is expected to match nvidia-smi's order but is not guaranteed.
groups:
- name: quantus-miner
interval: 30s
rules:
# Per-device throughput, keyed so it can be joined with the GPU series.
- record: quantus:device_hashrate:rate5m
expr: label_replace(rate(miner_device_hashes_total[5m]), "host", "$1", "instance", "([^.:]+).*")
# The same, carrying the build that produced it. The series set changes
# on every deploy, which is the annotation the dashboard draws.
- record: quantus:device_hashrate_by_commit:rate5m
expr: quantus:device_hashrate:rate5m * on(instance) group_left(commit, version) miner_build_info
# GPU draw keyed like the miner series.
- record: quantus:gpu_power_watts
expr: label_replace(nvidia_gpu_power_watts, "device", "$1", "gpu", "(.*)")
# Hashes per joule: the honest number when a power limit changes.
- record: quantus:device_hashrate_per_watt:rate5m
expr: quantus:device_hashrate:rate5m / on(host, device) group_left(name) quantus:gpu_power_watts
# Share of each batch spent off the GPU (submit gaps, readback). This is
# what pipelining and batch sizing reduce.
- record: quantus:gpu_batch_host_fraction:rate5m
expr: >
sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum{phase="host"}[5m]))
/
sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum[5m]))
# Work done for jobs that had already been superseded.
- record: quantus:stale_hash_fraction:rate1h
expr: >
sum by (instance) (rate(miner_stale_hashes_total[1h]))
/
sum by (instance) (rate(miner_hashes_total[1h]))
# Found to sent, 95th percentile.
- record: quantus:seal_latency_seconds:p95_5m
expr: histogram_quantile(0.95, sum by (instance, le) (rate(miner_seal_latency_seconds_bucket[5m])))
# Fraction of wall-clock spent waiting for the node to issue the next
# job after a result. Node-attributable, not miner-attributable.
- record: quantus:job_idle_fraction:rate5m
expr: rate(miner_job_idle_seconds_total[5m])