node-exporter's host label is the short name (benjy), the first rule derived the FQDN from the miner's instance, and quantus:device_hashrate_per_watt produced no series. Verified against live labels on the metrics host. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5
56 lines
2.7 KiB
YAML
56 lines
2.7 KiB
YAML
# Recording rules for the Quantus miners, deployed to
|
|
# /etc/prometheus/rules.d/quantus.yml on the metrics host (quantus/miner#9).
|
|
#
|
|
# The miner's series (job quantus-miner, instance host:9900) and the GPU
|
|
# series from node-exporter's textfile collector (job node, `host` label, the
|
|
# SHORT hostname as cichlid's inventory sets it) are joined on `host` derived
|
|
# as the first DNS label of the miner's instance, and on `device` (the miner's
|
|
# engine index) =
|
|
# `gpu` (nvidia-smi index). On single-card hosts both are 0; on a multi-card
|
|
# host the engine orders adapters discrete-first as Vulkan enumerates them,
|
|
# which is expected to match nvidia-smi's order but is not guaranteed.
|
|
groups:
|
|
- name: quantus-miner
|
|
interval: 30s
|
|
rules:
|
|
# Per-device throughput, keyed so it can be joined with the GPU series.
|
|
- record: quantus:device_hashrate:rate5m
|
|
expr: label_replace(rate(miner_device_hashes_total[5m]), "host", "$1", "instance", "([^.:]+).*")
|
|
|
|
# The same, carrying the build that produced it. The series set changes
|
|
# on every deploy, which is the annotation the dashboard draws.
|
|
- record: quantus:device_hashrate_by_commit:rate5m
|
|
expr: quantus:device_hashrate:rate5m * on(instance) group_left(commit, version) miner_build_info
|
|
|
|
# GPU draw keyed like the miner series.
|
|
- record: quantus:gpu_power_watts
|
|
expr: label_replace(nvidia_gpu_power_watts, "device", "$1", "gpu", "(.*)")
|
|
|
|
# Hashes per joule: the honest number when a power limit changes.
|
|
- record: quantus:device_hashrate_per_watt:rate5m
|
|
expr: quantus:device_hashrate:rate5m / on(host, device) group_left(name) quantus:gpu_power_watts
|
|
|
|
# Share of each batch spent off the GPU (submit gaps, readback). This is
|
|
# what pipelining and batch sizing reduce.
|
|
- record: quantus:gpu_batch_host_fraction:rate5m
|
|
expr: >
|
|
sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum{phase="host"}[5m]))
|
|
/
|
|
sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum[5m]))
|
|
|
|
# Work done for jobs that had already been superseded.
|
|
- record: quantus:stale_hash_fraction:rate1h
|
|
expr: >
|
|
sum by (instance) (rate(miner_stale_hashes_total[1h]))
|
|
/
|
|
sum by (instance) (rate(miner_hashes_total[1h]))
|
|
|
|
# Found to sent, 95th percentile.
|
|
- record: quantus:seal_latency_seconds:p95_5m
|
|
expr: histogram_quantile(0.95, sum by (instance, le) (rate(miner_seal_latency_seconds_bucket[5m])))
|
|
|
|
# Fraction of wall-clock spent waiting for the node to issue the next
|
|
# job after a result. Node-attributable, not miner-attributable.
|
|
- record: quantus:job_idle_fraction:rate5m
|
|
expr: rate(miner_job_idle_seconds_total[5m])
|