From 6d9e5ab89b1f2f9d2e7a3bb56fb60ab3729b23c6 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Sun, 14 Sep 2025 13:29:36 +0300 Subject: [PATCH] Add engine health dashboard for Grafana monitoring --- examples/grafana/engine-health-dashboard.json | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 examples/grafana/engine-health-dashboard.json diff --git a/examples/grafana/engine-health-dashboard.json b/examples/grafana/engine-health-dashboard.json new file mode 100644 index 0000000..e4681fe --- /dev/null +++ b/examples/grafana/engine-health-dashboard.json @@ -0,0 +1,293 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1, + "links": [], + "liveNow": false, + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["quantus", "miner", "engine", "health", "prometheus"], + "templating": { + "list": [ + { + "type": "datasource", + "name": "DS_PROMETHEUS", + "hide": 0, + "query": "prometheus", + "current": {}, + "options": [] + }, + { + "type": "query", + "name": "engine", + "label": "Engine(s)", + "hide": 0, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(miner_engine_backend, engine)", + "includeAll": true, + "multi": true, + "refresh": 2, + "sort": 1, + "query": "label_values(miner_engine_backend, engine)", + "regex": "", + "current": { "selected": false, "text": ["All"], "value": ["$__all"] } + }, + { + "type": "query", + "name": "backend", + "label": "Backend(s)", + "hide": 0, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(miner_engine_backend{engine=~\"$engine\"}, backend)", + "includeAll": true, + "multi": true, + "refresh": 2, + "sort": 1, + "query": "label_values(miner_engine_backend{engine=~\"$engine\"}, backend)", + "regex": "", + "current": { "selected": false, "text": ["All"], "value": ["$__all"] } + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": { "refresh_intervals": ["5s", "10s", "30s", "1m"] }, + "timezone": "", + "title": "Engine Estimated Rate & Health", + "uid": null, + "version": 1, + "panels": [ + { + "type": "stat", + "title": "Total Estimated Rate (All Engines/Backends)", + "description": "Sum of engine/backend estimated rates (nonces/sec)", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"})" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "value" + }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] } + }, + { + "type": "timeseries", + "title": "Estimated Rate by Engine/Backend (5m avg)", + "description": "avg_over_time(miner_job_estimated_rate[5m]) grouped by engine/backend", + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "avg_over_time(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"}[5m])", + "legendFormat": "{{engine}}/{{backend}}" + } + ], + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "bargauge", + "title": "Engine False Positives (1h increase)", + "description": "Increase in false positives over last hour by engine", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 5 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (engine) (increase(miner_candidates_false_positive_total{engine=~\"$engine\"}[1h]))", + "legendFormat": "{{engine}}" + } + ], + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showUnfilled": true + } + }, + { + "type": "bargauge", + "title": "Decision Mismatches (1h increase)", + "description": "Increase in host/device decision mismatches (parity sampler) by engine", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 5 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (engine) (increase(miner_sample_mismatch_total{engine=~\"$engine\"}[1h]))", + "legendFormat": "{{engine}}" + } + ], + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showUnfilled": true + } + }, + { + "type": "timeseries", + "title": "Jobs by Engine and Status (1m increase)", + "description": "Per-engine job status changes per minute", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 11 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (engine, status) (increase(miner_jobs_by_engine_total{engine=~\"$engine\"}[1m]))", + "legendFormat": "{{engine}}/{{status}}" + } + ], + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "stat", + "title": "Active Jobs", + "description": "Current number of running jobs", + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 11 }, + "datasource": "$DS_PROMETHEUS", + "targets": [{ "refId": "A", "expr": "miner_active_jobs" }], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "value" + }, + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + }, + { + "type": "timeseries", + "title": "Global Hash Rate (engine-reported)", + "description": "sum(miner_hash_rate) over time", + "gridPos": { "h": 7, "w": 12, "x": 18, "y": 11 }, + "datasource": "$DS_PROMETHEUS", + "targets": [{ "refId": "A", "expr": "sum(miner_hash_rate)" }], + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "timeseries", + "title": "Estimated Rate by Engine (5m avg, backend aggregated)", + "description": "avg_over_time(miner_job_estimated_rate[5m]) grouped by engine (backends summed)", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 18 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (engine) (avg_over_time(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"}[5m]))", + "legendFormat": "{{engine}}" + } + ], + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "table", + "title": "HTTP Requests by Endpoint/Code (1m increase)", + "description": "miner_http_requests_total grouped by endpoint/code over last minute", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 26 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (endpoint, code) (increase(miner_http_requests_total[1m]))", + "legendFormat": "{{endpoint}} {{code}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "auto" }, + "unit": "req/min", + "mappings": [] + }, + "overrides": [] + }, + "options": { "showHeader": true, "footer": { "show": false }, "frameIndex": 0 } + }, + { + "type": "bargauge", + "title": "Mine Requests by Result (5m increase)", + "description": "miner_mine_requests_total result breakdown", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 26 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (result) (increase(miner_mine_requests_total[5m]))", + "legendFormat": "{{result}}" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "decimals": 0 }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showUnfilled": true + } + }, + { + "type": "timeseries", + "title": "GPU (g2) Estimated Rate (5m avg, gpu-cuda only)", + "description": "avg_over_time(miner_job_estimated_rate{engine=\"gpu-cuda\", backend=\"g2\"}[5m])", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "avg_over_time(miner_job_estimated_rate{engine=\"gpu-cuda\", backend=\"g2\"}[5m])", + "legendFormat": "gpu-cuda/g2" + } + ], + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + } + ] +}