diff --git a/asset/grafana/blackbeard-dashboards.yaml b/asset/grafana/blackbeard-dashboards.yaml new file mode 100644 index 0000000..26695fa --- /dev/null +++ b/asset/grafana/blackbeard-dashboards.yaml @@ -0,0 +1,17 @@ +# Grafana dashboard provider for blackbeard.observer, installed as +# /etc/grafana/provisioning/dashboards/blackbeard-dashboards.yaml on golgafrinchans, +# with the JSON under /etc/grafana/provisioning/dashboards/blackbeard/. Read at +# Grafana's start. The fleet's shape (lair/quantus, spine): file-provisioned and +# not editable in the UI, so a dashboard cannot drift from asset/grafana/. +apiVersion: 1 +providers: + - name: blackbeard + orgId: 1 + folder: blackbeard + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards/blackbeard + foldersFromFilesStructure: false diff --git a/asset/grafana/neard.json b/asset/grafana/neard.json new file mode 100644 index 0000000..314e941 --- /dev/null +++ b/asset/grafana/neard.json @@ -0,0 +1,761 @@ +{ + "uid": "blackbeard-neard", + "title": "NEAR nodes", + "description": "neard on frankie, gargravarr and quadbrat for blackbeard.observer (blackbeard/observer#39).", + "tags": [ + "near", + "blackbeard" + ], + "timezone": "browser", + "editable": false, + "refresh": "30s", + "schemaVersion": 39, + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(near_block_height_head{job=\"neard\"}, host)", + "refId": "host" + }, + "definition": "label_values(near_block_height_head{job=\"neard\"}, host)", + "multi": true, + "includeAll": true, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Sync", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "stat", + "title": "Head height", + "description": "Each node's chain head. A node still state-syncing reports the genesis-era height it started from.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_block_height_head{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none" + } + }, + { + "type": "stat", + "title": "Blocks behind its own header head", + "description": "Headers arrive first; this is how far block processing trails them. Near zero at the tip.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_header_head_height{job=\"neard\",host=~\"$host\"} - near_block_height_head{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none" + } + }, + { + "type": "stat", + "title": "Peers", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_peer_connections_total{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name", + "colorMode": "none", + "graphMode": "none" + } + }, + { + "type": "timeseries", + "title": "Blocks behind its own header head", + "description": "Falling means catching up; flat above zero means keeping pace without gaining; rising means falling behind the chain.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_header_head_height{job=\"neard\",host=~\"$host\"} - near_block_height_head{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Blocks processed per second", + "description": "The chain makes about 1.7 a second. Above that a node is gaining; below it, losing ground.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(near_block_processed_total{job=\"neard\",host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "State sync: parts downloaded", + "description": "Of each shard's state parts. Counters reset when neard restarts.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (host, shard_id) (near_state_sync_download_result{job=\"neard\",host=~\"$host\",type=\"part\",result=\"success\"}) / on (host, shard_id) near_state_sync_parts_per_shard{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}} shard {{shard_id}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "State sync: parts applied", + "description": "Applying is disk-bound; on gargravarr it is the slow half of a sync.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_state_sync_apply_part_delay_sec_count{job=\"neard\",host=~\"$host\"} / on (host, shard_id) near_state_sync_parts_per_shard{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}} shard {{shard_id}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Flat storage: deltas behind its head", + "description": "Every state read walks these. Past 100 neard warns 'too many hops'; a count that only grows means the head is held, e.g. by a state snapshot.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_flat_storage_cached_deltas{job=\"neard\",host=~\"$host\",shard_uid!=\"s0.v0\"}", + "legendFormat": "{{host}} {{shard_uid}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Chunk apply time per block", + "description": "Executing the tracked shards' chunks, per block. Grows with the shards a node carries.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(near_apply_all_chunks_time_sum{job=\"neard\",host=~\"$host\"}[$__rate_interval]) / rate(near_apply_all_chunks_time_count{job=\"neard\",host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "row", + "title": "Resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + } + }, + { + "type": "timeseries", + "title": "neard memory", + "description": "neard's own figure. gargravarr's unit caps it at 13 GiB high / 14 GiB max.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 30 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_memory_usage_bytes{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "neard CPU", + "description": "Percent of one core: 100 is one full core.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 30 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_cpu_usage_ratio{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "neard on-disk size", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 30 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (host) (near_rocksdb_live_sst_files_size{job=\"neard\",host=~\"$host\"})", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Disk busy (neard's disks)", + "description": "Share of time each disk was busy. 100% is the ceiling a sync hits on one SATA SSD.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\",host=~\"$host\",host=~\"frankie|gargravarr\",device=~\"sd[ab]\"}[$__rate_interval]) or rate(node_disk_io_time_seconds_total{job=\"node\",host=~\"$host\",host=\"quadbrat\",device=~\"sd[ef]\"}[$__rate_interval])", + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Waiting on disk (host)", + "description": "Share of time some task was stalled on IO, host-wide. Frankie's includes its Postgres standby.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_pressure_io_waiting_seconds_total{job=\"node\",host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Network", + "description": "Received above the line, sent below.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "near_received_bytes_per_second{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}} in", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "-near_sent_bytes_per_second{job=\"neard\",host=~\"$host\"}", + "legendFormat": "{{host}} out", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/asset/grafana/neard.py b/asset/grafana/neard.py new file mode 100644 index 0000000..60e870b --- /dev/null +++ b/asset/grafana/neard.py @@ -0,0 +1,124 @@ +"""Generate asset/grafana/neard.json, the NEAR nodes dashboard. + +Run `python3 asset/grafana/neard.py` after editing; the JSON is what Grafana +loads (file-provisioned, not editable in the UI), this is what to edit. +Series come from the `neard` scrape job (asset/prometheus/blackbeard-neard.yml) +joined to node_exporter by the `host` label both carry. +""" + +import json +from pathlib import Path + +DS = {"type": "prometheus", "uid": "prometheus"} +H = 'job="neard",host=~"$host"' +NODE = 'job="node",host=~"$host"' +# The disks neard's data is on, per host: frankie and gargravarr keep it on +# their two SATA SSDs, quadbrat on the two-SSD stripe (sde, sdf), not its root +# NVMe or the spinning array. +NEARD_DISKS = ( + 'rate(node_disk_io_time_seconds_total{job="node",host=~"$host",host=~"frankie|gargravarr",device=~"sd[ab]"}[$__rate_interval])' + ' or rate(node_disk_io_time_seconds_total{job="node",host=~"$host",host="quadbrat",device=~"sd[ef]"}[$__rate_interval])' +) + +panels = [] +y = 0 + + +def row(title): + global y + panels.append({"type": "row", "title": title, "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}}) + y += 1 + + +def panel(kind, title, targets, w, h, x, unit="short", desc="", extra=None): + p = { + "type": kind, "title": title, "description": desc, "datasource": DS, + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "targets": [{"datasource": DS, "expr": e, "legendFormat": l, "refId": chr(65 + i)} + for i, (e, l) in enumerate(targets)], + "fieldConfig": {"defaults": {"unit": unit}, "overrides": []}, + "options": {}, + } + if kind == "timeseries": + p["fieldConfig"]["defaults"]["custom"] = {"lineWidth": 2, "fillOpacity": 0, "showPoints": "never"} + p["options"] = {"legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"}} + if kind == "stat": + p["options"] = {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "textMode": "value_and_name", "colorMode": "none", "graphMode": "none"} + if extra: + for k, v in extra.items(): + p.setdefault(k, {}).update(v) if isinstance(v, dict) else p.__setitem__(k, v) + panels.append(p) + return p + + +row("Sync") +panel("stat", "Head height", [(f"near_block_height_head{{{H}}}", "{{host}}")], 8, 4, 0, + unit="none", desc="Each node's chain head. A node still state-syncing reports the genesis-era height it started from.") +panel("stat", "Blocks behind its own header head", [(f"near_header_head_height{{{H}}} - near_block_height_head{{{H}}}", "{{host}}")], 8, 4, 8, + unit="none", desc="Headers arrive first; this is how far block processing trails them. Near zero at the tip.") +panel("stat", "Peers", [(f"near_peer_connections_total{{{H}}}", "{{host}}")], 8, 4, 16, unit="none") +y += 4 +panel("timeseries", "Blocks behind its own header head", [(f"near_header_head_height{{{H}}} - near_block_height_head{{{H}}}", "{{host}}")], 12, 8, 0, + unit="none", desc="Falling means catching up; flat above zero means keeping pace without gaining; rising means falling behind the chain.") +panel("timeseries", "Blocks processed per second", [(f"rate(near_block_processed_total{{{H}}}[$__rate_interval])", "{{host}}")], 12, 8, 12, + unit="none", desc="The chain makes about 1.7 a second. Above that a node is gaining; below it, losing ground.") +y += 8 +panel("timeseries", "State sync: parts downloaded", [ + (f"sum by (host, shard_id) (near_state_sync_download_result{{{H},type=\"part\",result=\"success\"}}) / on (host, shard_id) near_state_sync_parts_per_shard{{{H}}}", "{{host}} shard {{shard_id}}")], + 12, 8, 0, unit="percentunit", desc="Of each shard's state parts. Counters reset when neard restarts.", + extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}}) +panel("timeseries", "State sync: parts applied", [ + (f"near_state_sync_apply_part_delay_sec_count{{{H}}} / on (host, shard_id) near_state_sync_parts_per_shard{{{H}}}", "{{host}} shard {{shard_id}}")], + 12, 8, 12, unit="percentunit", desc="Applying is disk-bound; on gargravarr it is the slow half of a sync.", + extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}}) +y += 8 +panel("timeseries", "Flat storage: deltas behind its head", [(f"near_flat_storage_cached_deltas{{{H},shard_uid!=\"s0.v0\"}}", "{{host}} {{shard_uid}}")], 12, 8, 0, + unit="none", desc="Every state read walks these. Past 100 neard warns 'too many hops'; a count that only grows means the head is held, e.g. by a state snapshot.") +panel("timeseries", "Chunk apply time per block", [ + (f"rate(near_apply_all_chunks_time_sum{{{H}}}[$__rate_interval]) / rate(near_apply_all_chunks_time_count{{{H}}}[$__rate_interval])", "{{host}}")], + 12, 8, 12, unit="s", desc="Executing the tracked shards' chunks, per block. Grows with the shards a node carries.") +y += 8 + +row("Resources") +panel("timeseries", "neard memory", [(f"near_memory_usage_bytes{{{H}}}", "{{host}}")], 8, 8, 0, + unit="bytes", desc="neard's own figure. gargravarr's unit caps it at 13 GiB high / 14 GiB max.") +panel("timeseries", "neard CPU", [(f"near_cpu_usage_ratio{{{H}}}", "{{host}}")], 8, 8, 8, + unit="percent", desc="Percent of one core: 100 is one full core.") +panel("timeseries", "neard on-disk size", [(f"sum by (host) (near_rocksdb_live_sst_files_size{{{H}}})", "{{host}}")], 8, 8, 16, unit="bytes") +y += 8 +panel("timeseries", "Disk busy (neard's disks)", [(NEARD_DISKS, "{{host}} {{device}}")], 8, 8, 0, + unit="percentunit", desc="Share of time each disk was busy. 100% is the ceiling a sync hits on one SATA SSD.", + extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}}) +panel("timeseries", "Waiting on disk (host)", [(f"rate(node_pressure_io_waiting_seconds_total{{{NODE}}}[$__rate_interval])", "{{host}}")], 8, 8, 8, + unit="percentunit", desc="Share of time some task was stalled on IO, host-wide. Frankie's includes its Postgres standby.", + extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}}) +panel("timeseries", "Network", [ + (f"near_received_bytes_per_second{{{H}}}", "{{host}} in"), + (f"-near_sent_bytes_per_second{{{H}}}", "{{host}} out")], 8, 8, 16, + unit="Bps", desc="Received above the line, sent below.") +y += 8 + +dashboard = { + "uid": "blackbeard-neard", + "title": "NEAR nodes", + "description": "neard on frankie, gargravarr and quadbrat for blackbeard.observer (blackbeard/observer#39).", + "tags": ["near", "blackbeard"], + "timezone": "browser", + "editable": False, + "refresh": "30s", + "schemaVersion": 39, + "time": {"from": "now-6h", "to": "now"}, + "templating": {"list": [{ + "name": "host", "label": "Host", "type": "query", "datasource": DS, + "query": {"query": 'label_values(near_block_height_head{job="neard"}, host)', "refId": "host"}, + "definition": 'label_values(near_block_height_head{job="neard"}, host)', + "multi": True, "includeAll": True, "current": {"selected": True, "text": "All", "value": "$__all"}, + "refresh": 2, "sort": 1, + }]}, + "panels": panels, +} +Path(__file__).with_name("neard.json").write_text(json.dumps(dashboard, indent=2) + "\n") +print(f"{len(panels)} panels")