diff --git a/asset/grafana/gpu.json b/asset/grafana/gpu.json index 5ccb15c..6377783 100644 --- a/asset/grafana/gpu.json +++ b/asset/grafana/gpu.json @@ -9,7 +9,7 @@ "description": "NVIDIA GPU temperature, power and utilisation across the fleet. Fed by node-exporter's textfile collector (asset/gpu/nvidia-textfile.sh in lair/quantus) \u2014 no separate exporter. Fleet-general rather than quantus-specific; move it if a monitoring repo appears.", "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", "time": { "from": "now-6h", @@ -20,7 +20,7 @@ "panels": [ { "type": "row", - "title": "Temperature", + "title": "Now", "collapsed": false, "gridPos": { "h": 1, @@ -30,6 +30,231 @@ }, "panels": [] }, + { + "type": "gauge", + "title": "GPU Busy", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_gpu_utilization_ratio", + "legendFormat": "{{host}} gpu{{gpu}}", + "range": true, + "refId": "A" + } + ], + "description": "One gauge per GPU. Mining pins this near 100% continuously; inference is spiky, so a flat 100% on a host that also serves inference is the contention showing up.", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "minVizWidth": 75, + "minVizHeight": 75, + "sizing": "auto", + "orientation": "auto" + } + }, + { + "type": "gauge", + "title": "GPU Temperature", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_gpu_temperature_celsius", + "legendFormat": "{{host}} gpu{{gpu}}", + "range": true, + "refId": "A" + } + ], + "description": "Amber at 80C, red at 87C \u2014 roughly where consumer NVIDIA cards begin thermal throttling. The 4090 was measured at 70C mining against ~40C idle.", + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "red", + "value": 87 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "minVizWidth": 75, + "minVizHeight": 75, + "sizing": "auto", + "orientation": "auto" + } + }, + { + "type": "gauge", + "title": "GPU Power", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_gpu_power_watts / nvidia_gpu_power_limit_watts", + "legendFormat": "{{host}} gpu{{gpu}}", + "range": true, + "refId": "A" + } + ], + "description": "Draw as a fraction of the card's enforced limit. Pinned at 100% means power-capped, which is the normal state while mining \u2014 and the reason lowering the cap raises efficiency: a 5090 does 0.60 MH/s per watt at 600W and 0.70 at its 400W floor.", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.9 + }, + { + "color": "red", + "value": 0.98 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "minVizWidth": 75, + "minVizHeight": 75, + "sizing": "auto", + "orientation": "auto" + } + }, + { + "type": "row", + "title": "Temperature", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, { "type": "timeseries", "title": "GPU temperature", @@ -41,7 +266,7 @@ "h": 9, "w": 24, "x": 0, - "y": 1 + "y": 9 }, "targets": [ { @@ -101,7 +326,7 @@ "sort": "desc" } }, - "description": "Thresholds mark where consumer NVIDIA cards begin thermal throttling (~83-87C depending on model). Sustained mining sits far higher than idle \u2014 the 4090 was measured at 70C under load against ~40C idle \u2014 so a rising baseline over days usually means dust or ambient, not a driver problem." + "description": "A rising baseline over days usually means dust or ambient, not a driver problem." }, { "type": "row", @@ -111,7 +336,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 18 }, "panels": [] }, @@ -126,7 +351,7 @@ "h": 9, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "targets": [ { @@ -177,7 +402,7 @@ "sort": "desc" } }, - "description": "Draw pinned at the limit means the card is power-capped, which is the normal state while mining. Measured efficiency: a 5090 does 0.60 MH/s per watt at 600W and 0.70 at its 400W floor, so capping lower is 18% more hash per watt." + "description": "Draw pinned at the limit means the card is power-capped \u2014 normal while mining." }, { "type": "timeseries", @@ -190,7 +415,7 @@ "h": 9, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "targets": [ { @@ -242,7 +467,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 28 }, "panels": [] }, @@ -257,7 +482,7 @@ "h": 8, "w": 8, "x": 0, - "y": 21 + "y": 29 }, "targets": [ { @@ -321,7 +546,7 @@ "h": 8, "w": 8, "x": 8, - "y": 21 + "y": 29 }, "targets": [ { @@ -361,7 +586,7 @@ "sort": "desc" } }, - "description": "Drops under a power or thermal cap; a good cross-check against the temperature panel." + "description": "Drops under a power or thermal cap; a cross-check against the temperature panel." }, { "type": "timeseries", @@ -374,7 +599,7 @@ "h": 8, "w": 8, "x": 16, - "y": 21 + "y": 29 }, "targets": [ { @@ -416,7 +641,7 @@ "sort": "desc" } }, - "description": "0 on cards with a passive idle mode. Fans at maximum with temperature still climbing is the failure worth alerting on." + "description": "0 on cards with a passive idle mode. Fans maxed with temperature still climbing is the failure worth alerting on." }, { "type": "stat", @@ -429,7 +654,7 @@ "h": 4, "w": 24, "x": 0, - "y": 29 + "y": 37 }, "targets": [ {