From a6cfa3bbf5db64c62e3bbf921db7adaf25770802 Mon Sep 17 00:00:00 2001 From: Rob Thijssen Date: Thu, 3 Sep 2026 09:52:00 +0300 Subject: [PATCH] fix: derive hashrate from the counter, not the gauge miner_hash_rate spikes to impossible values after a restart (1771 MH/s on a 4090 that peaks near 145) because it is computed per job over a tiny elapsed time. Hashrate panels now use rate(miner_hashes_total). Adds node and miner restart annotations so version changes are visible on every graph, a node version stat, a hash-rate-per-watt panel, and puts discovered peers on its own axis so connected is legible. --- asset/grafana/quantus.json | 160 +++++++++++++++++++++++++++++++------ 1 file changed, 135 insertions(+), 25 deletions(-) diff --git a/asset/grafana/quantus.json b/asset/grafana/quantus.json index d0be997..1c139a2 100644 --- a/asset/grafana/quantus.json +++ b/asset/grafana/quantus.json @@ -9,7 +9,7 @@ ], "timezone": "browser", "schemaVersion": 39, - "version": 6, + "version": 7, "refresh": "30s", "time": { "from": "now-6h", @@ -575,7 +575,34 @@ "mode": "palette-classic" } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "discovered" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 6, + 6 + ], + "fill": "dash" + } + } + ] + } + ] }, "options": { "legend": { @@ -587,7 +614,8 @@ "mode": "multi", "sort": "desc" } - } + }, + "description": "Discovered is an order of magnitude larger than connected, so it is on its own axis \u2014 otherwise the connected line reads as flat zero." }, { "type": "timeseries", @@ -948,7 +976,7 @@ }, { "type": "timeseries", - "title": "Hash rate by engine", + "title": "Hash rate (counter-derived)", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -966,8 +994,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "miner_hash_rate{job=\"quantus-miner\"}", - "legendFormat": "total", + "expr": "sum by (instance) (rate(miner_hashes_total{job=\"quantus-miner\"}[5m]))", + "legendFormat": "{{instance}}", "range": true, "refId": "A" }, @@ -977,21 +1005,10 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "miner_gpu_hash_rate{job=\"quantus-miner\"}", - "legendFormat": "gpu", + "expr": "sum(rate(miner_hashes_total{job=\"quantus-miner\"}[5m]))", + "legendFormat": "fleet", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "miner_cpu_hash_rate{job=\"quantus-miner\"}", - "legendFormat": "cpu", - "range": true, - "refId": "C" } ], "fieldConfig": { @@ -1019,11 +1036,11 @@ "sort": "desc" } }, - "description": "CPU is deliberately 0 (--cpu-workers 0): measured at ~1% of GPU throughput for a couple hundred watts." + "description": "Derived from miner_hashes_total, NOT the miner_hash_rate gauge. That gauge spikes to physically impossible values after a restart \u2014 1771 MH/s was observed on a 4090 that peaks near 145 \u2014 because it is computed per job over a tiny elapsed time. A counter derivative cannot do that." }, { "type": "timeseries", - "title": "Hashes computed (rate)", + "title": "Hash rate per watt", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -1041,7 +1058,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(miner_hashes_total{job=\"quantus-miner\"}[5m])", + "expr": "sum by (instance) (rate(miner_hashes_total{job=\"quantus-miner\"}[5m])) / on() group_left sum(nvidia_gpu_power_watts)", "legendFormat": "{{instance}}", "range": true, "refId": "A" @@ -1057,7 +1074,7 @@ "color": { "mode": "palette-classic" }, - "unit": "Hz" + "unit": "none" }, "overrides": [] }, @@ -1072,7 +1089,7 @@ "sort": "desc" } }, - "description": "Counter derivative \u2014 the honest check that work is being done, independent of the reported gauge." + "description": "Efficiency: hashes per second per watt of GPU draw. This is the number a power-limit change should move, and it is comparable across cards where raw hashrate is not." }, { "type": "row", @@ -2450,6 +2467,99 @@ "sort": "desc" } } + }, + { + "type": "stat", + "title": "Node version", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 112 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "substrate_build_info", + "legendFormat": "{{version}}", + "range": true, + "refId": "A" + } + ], + "description": "Read together with the yellow restart annotations: they mark where this changed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "textMode": "name", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } } - ] + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "hide": false, + "iconColor": "yellow", + "name": "node restart", + "expr": "changes(substrate_process_start_time_seconds[2m]) > 0", + "step": "1m", + "titleFormat": "node restart", + "textFormat": "a version or config change \u2014 treat data either side as separate populations" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "hide": false, + "iconColor": "orange", + "name": "miner restart", + "expr": "resets(miner_hashes_total[2m]) > 0", + "step": "1m", + "titleFormat": "miner restart", + "textFormat": "counter reset \u2014 hashrate readings near this line are unreliable" + } + ] + } }