Add engine health dashboard for Grafana monitoring
This commit is contained in:
rob thijssen
@@ -0,0 +1,293 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"iteration": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["quantus", "miner", "engine", "health", "prometheus"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"type": "datasource",
|
||||
"name": "DS_PROMETHEUS",
|
||||
"hide": 0,
|
||||
"query": "prometheus",
|
||||
"current": {},
|
||||
"options": []
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"name": "engine",
|
||||
"label": "Engine(s)",
|
||||
"hide": 0,
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"definition": "label_values(miner_engine_backend, engine)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"query": "label_values(miner_engine_backend, engine)",
|
||||
"regex": "",
|
||||
"current": { "selected": false, "text": ["All"], "value": ["$__all"] }
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"name": "backend",
|
||||
"label": "Backend(s)",
|
||||
"hide": 0,
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"definition": "label_values(miner_engine_backend{engine=~\"$engine\"}, backend)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"query": "label_values(miner_engine_backend{engine=~\"$engine\"}, backend)",
|
||||
"regex": "",
|
||||
"current": { "selected": false, "text": ["All"], "value": ["$__all"] }
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"timepicker": { "refresh_intervals": ["5s", "10s", "30s", "1m"] },
|
||||
"timezone": "",
|
||||
"title": "Engine Estimated Rate & Health",
|
||||
"uid": null,
|
||||
"version": 1,
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Estimated Rate (All Engines/Backends)",
|
||||
"description": "Sum of engine/backend estimated rates (nonces/sec)",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"})"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"textMode": "value"
|
||||
},
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Estimated Rate by Engine/Backend (5m avg)",
|
||||
"description": "avg_over_time(miner_job_estimated_rate[5m]) grouped by engine/backend",
|
||||
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "avg_over_time(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"}[5m])",
|
||||
"legendFormat": "{{engine}}/{{backend}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Engine False Positives (1h increase)",
|
||||
"description": "Increase in false positives over last hour by engine",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 5 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (engine) (increase(miner_candidates_false_positive_total{engine=~\"$engine\"}[1h]))",
|
||||
"legendFormat": "{{engine}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] },
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showUnfilled": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Decision Mismatches (1h increase)",
|
||||
"description": "Increase in host/device decision mismatches (parity sampler) by engine",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 5 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (engine) (increase(miner_sample_mismatch_total{engine=~\"$engine\"}[1h]))",
|
||||
"legendFormat": "{{engine}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] },
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showUnfilled": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Jobs by Engine and Status (1m increase)",
|
||||
"description": "Per-engine job status changes per minute",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 11 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (engine, status) (increase(miner_jobs_by_engine_total{engine=~\"$engine\"}[1m]))",
|
||||
"legendFormat": "{{engine}}/{{status}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] },
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Active Jobs",
|
||||
"description": "Current number of running jobs",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 11 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [{ "refId": "A", "expr": "miner_active_jobs" }],
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"textMode": "value"
|
||||
},
|
||||
"fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Global Hash Rate (engine-reported)",
|
||||
"description": "sum(miner_hash_rate) over time",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 18, "y": 11 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [{ "refId": "A", "expr": "sum(miner_hash_rate)" }],
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"options": {
|
||||
"legend": { "displayMode": "list", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Estimated Rate by Engine (5m avg, backend aggregated)",
|
||||
"description": "avg_over_time(miner_job_estimated_rate[5m]) grouped by engine (backends summed)",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 18 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (engine) (avg_over_time(miner_job_estimated_rate{engine=~\"$engine\", backend=~\"$backend\"}[5m]))",
|
||||
"legendFormat": "{{engine}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"options": {
|
||||
"legend": { "displayMode": "list", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "HTTP Requests by Endpoint/Code (1m increase)",
|
||||
"description": "miner_http_requests_total grouped by endpoint/code over last minute",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 26 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (endpoint, code) (increase(miner_http_requests_total[1m]))",
|
||||
"legendFormat": "{{endpoint}} {{code}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "align": "auto" },
|
||||
"unit": "req/min",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": { "showHeader": true, "footer": { "show": false }, "frameIndex": 0 }
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Mine Requests by Result (5m increase)",
|
||||
"description": "miner_mine_requests_total result breakdown",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 26 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (result) (increase(miner_mine_requests_total[5m]))",
|
||||
"legendFormat": "{{result}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "none", "decimals": 0 },
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showUnfilled": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GPU (g2) Estimated Rate (5m avg, gpu-cuda only)",
|
||||
"description": "avg_over_time(miner_job_estimated_rate{engine=\"gpu-cuda\", backend=\"g2\"}[5m])",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 },
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "avg_over_time(miner_job_estimated_rate{engine=\"gpu-cuda\", backend=\"g2\"}[5m])",
|
||||
"legendFormat": "gpu-cuda/g2"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"options": {
|
||||
"legend": { "displayMode": "list", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user