`asset/grafana/neard.py` generates `neard.json` — sync (heads, blocks behind, state-sync parts downloaded and applied per shard, flat-storage deltas, chunk apply time) and resources (neard memory and CPU, on-disk size, the busy-ness of the disks neard is on, host IO pressure, network) — from the `neard` scrape job, joined to node_exporter by the `host` label both carry. File-provisioned like the fleet's other dashboards, under the `blackbeard` folder; installed on golgafrinchans by hand (2026-10-06). Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,17 @@
|
||||
# Grafana dashboard provider for blackbeard.observer, installed as
|
||||
# /etc/grafana/provisioning/dashboards/blackbeard-dashboards.yaml on golgafrinchans,
|
||||
# with the JSON under /etc/grafana/provisioning/dashboards/blackbeard/. Read at
|
||||
# Grafana's start. The fleet's shape (lair/quantus, spine): file-provisioned and
|
||||
# not editable in the UI, so a dashboard cannot drift from asset/grafana/.
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: blackbeard
|
||||
orgId: 1
|
||||
folder: blackbeard
|
||||
type: file
|
||||
disableDeletion: false
|
||||
allowUiUpdates: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards/blackbeard
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,761 @@
|
||||
{
|
||||
"uid": "blackbeard-neard",
|
||||
"title": "NEAR nodes",
|
||||
"description": "neard on frankie, gargravarr and quadbrat for blackbeard.observer (blackbeard/observer#39).",
|
||||
"tags": [
|
||||
"near",
|
||||
"blackbeard"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"editable": false,
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "host",
|
||||
"label": "Host",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"query": {
|
||||
"query": "label_values(near_block_height_head{job=\"neard\"}, host)",
|
||||
"refId": "host"
|
||||
},
|
||||
"definition": "label_values(near_block_height_head{job=\"neard\"}, host)",
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Sync",
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Head height",
|
||||
"description": "Each node's chain head. A node still state-syncing reports the genesis-era height it started from.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_block_height_head{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "none",
|
||||
"graphMode": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Blocks behind its own header head",
|
||||
"description": "Headers arrive first; this is how far block processing trails them. Near zero at the tip.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_header_head_height{job=\"neard\",host=~\"$host\"} - near_block_height_head{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "none",
|
||||
"graphMode": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Peers",
|
||||
"description": "",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_peer_connections_total{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "none",
|
||||
"graphMode": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Blocks behind its own header head",
|
||||
"description": "Falling means catching up; flat above zero means keeping pace without gaining; rising means falling behind the chain.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_header_head_height{job=\"neard\",host=~\"$host\"} - near_block_height_head{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Blocks processed per second",
|
||||
"description": "The chain makes about 1.7 a second. Above that a node is gaining; below it, losing ground.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 5
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "rate(near_block_processed_total{job=\"neard\",host=~\"$host\"}[$__rate_interval])",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "State sync: parts downloaded",
|
||||
"description": "Of each shard's state parts. Counters reset when neard restarts.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (host, shard_id) (near_state_sync_download_result{job=\"neard\",host=~\"$host\",type=\"part\",result=\"success\"}) / on (host, shard_id) near_state_sync_parts_per_shard{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}} shard {{shard_id}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "State sync: parts applied",
|
||||
"description": "Applying is disk-bound; on gargravarr it is the slow half of a sync.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 13
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_state_sync_apply_part_delay_sec_count{job=\"neard\",host=~\"$host\"} / on (host, shard_id) near_state_sync_parts_per_shard{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}} shard {{shard_id}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Flat storage: deltas behind its head",
|
||||
"description": "Every state read walks these. Past 100 neard warns 'too many hops'; a count that only grows means the head is held, e.g. by a state snapshot.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 21
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_flat_storage_cached_deltas{job=\"neard\",host=~\"$host\",shard_uid!=\"s0.v0\"}",
|
||||
"legendFormat": "{{host}} {{shard_uid}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Chunk apply time per block",
|
||||
"description": "Executing the tracked shards' chunks, per block. Grows with the shards a node carries.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 21
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "rate(near_apply_all_chunks_time_sum{job=\"neard\",host=~\"$host\"}[$__rate_interval]) / rate(near_apply_all_chunks_time_count{job=\"neard\",host=~\"$host\"}[$__rate_interval])",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Resources",
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 29
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "neard memory",
|
||||
"description": "neard's own figure. gargravarr's unit caps it at 13 GiB high / 14 GiB max.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 30
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_memory_usage_bytes{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "neard CPU",
|
||||
"description": "Percent of one core: 100 is one full core.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 30
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_cpu_usage_ratio{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "neard on-disk size",
|
||||
"description": "",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 30
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (host) (near_rocksdb_live_sst_files_size{job=\"neard\",host=~\"$host\"})",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk busy (neard's disks)",
|
||||
"description": "Share of time each disk was busy. 100% is the ceiling a sync hits on one SATA SSD.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 38
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "rate(node_disk_io_time_seconds_total{job=\"node\",host=~\"$host\",host=~\"frankie|gargravarr\",device=~\"sd[ab]\"}[$__rate_interval]) or rate(node_disk_io_time_seconds_total{job=\"node\",host=~\"$host\",host=\"quadbrat\",device=~\"sd[ef]\"}[$__rate_interval])",
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Waiting on disk (host)",
|
||||
"description": "Share of time some task was stalled on IO, host-wide. Frankie's includes its Postgres standby.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 38
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "rate(node_pressure_io_waiting_seconds_total{job=\"node\",host=~\"$host\"}[$__rate_interval])",
|
||||
"legendFormat": "{{host}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network",
|
||||
"description": "Received above the line, sent below.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 38
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "near_received_bytes_per_second{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}} in",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "-near_sent_bytes_per_second{job=\"neard\",host=~\"$host\"}",
|
||||
"legendFormat": "{{host}} out",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "Bps",
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Generate asset/grafana/neard.json, the NEAR nodes dashboard.
|
||||
|
||||
Run `python3 asset/grafana/neard.py` after editing; the JSON is what Grafana
|
||||
loads (file-provisioned, not editable in the UI), this is what to edit.
|
||||
Series come from the `neard` scrape job (asset/prometheus/blackbeard-neard.yml)
|
||||
joined to node_exporter by the `host` label both carry.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
DS = {"type": "prometheus", "uid": "prometheus"}
|
||||
H = 'job="neard",host=~"$host"'
|
||||
NODE = 'job="node",host=~"$host"'
|
||||
# The disks neard's data is on, per host: frankie and gargravarr keep it on
|
||||
# their two SATA SSDs, quadbrat on the two-SSD stripe (sde, sdf), not its root
|
||||
# NVMe or the spinning array.
|
||||
NEARD_DISKS = (
|
||||
'rate(node_disk_io_time_seconds_total{job="node",host=~"$host",host=~"frankie|gargravarr",device=~"sd[ab]"}[$__rate_interval])'
|
||||
' or rate(node_disk_io_time_seconds_total{job="node",host=~"$host",host="quadbrat",device=~"sd[ef]"}[$__rate_interval])'
|
||||
)
|
||||
|
||||
panels = []
|
||||
y = 0
|
||||
|
||||
|
||||
def row(title):
|
||||
global y
|
||||
panels.append({"type": "row", "title": title, "collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": y}})
|
||||
y += 1
|
||||
|
||||
|
||||
def panel(kind, title, targets, w, h, x, unit="short", desc="", extra=None):
|
||||
p = {
|
||||
"type": kind, "title": title, "description": desc, "datasource": DS,
|
||||
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
||||
"targets": [{"datasource": DS, "expr": e, "legendFormat": l, "refId": chr(65 + i)}
|
||||
for i, (e, l) in enumerate(targets)],
|
||||
"fieldConfig": {"defaults": {"unit": unit}, "overrides": []},
|
||||
"options": {},
|
||||
}
|
||||
if kind == "timeseries":
|
||||
p["fieldConfig"]["defaults"]["custom"] = {"lineWidth": 2, "fillOpacity": 0, "showPoints": "never"}
|
||||
p["options"] = {"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}}
|
||||
if kind == "stat":
|
||||
p["options"] = {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"textMode": "value_and_name", "colorMode": "none", "graphMode": "none"}
|
||||
if extra:
|
||||
for k, v in extra.items():
|
||||
p.setdefault(k, {}).update(v) if isinstance(v, dict) else p.__setitem__(k, v)
|
||||
panels.append(p)
|
||||
return p
|
||||
|
||||
|
||||
row("Sync")
|
||||
panel("stat", "Head height", [(f"near_block_height_head{{{H}}}", "{{host}}")], 8, 4, 0,
|
||||
unit="none", desc="Each node's chain head. A node still state-syncing reports the genesis-era height it started from.")
|
||||
panel("stat", "Blocks behind its own header head", [(f"near_header_head_height{{{H}}} - near_block_height_head{{{H}}}", "{{host}}")], 8, 4, 8,
|
||||
unit="none", desc="Headers arrive first; this is how far block processing trails them. Near zero at the tip.")
|
||||
panel("stat", "Peers", [(f"near_peer_connections_total{{{H}}}", "{{host}}")], 8, 4, 16, unit="none")
|
||||
y += 4
|
||||
panel("timeseries", "Blocks behind its own header head", [(f"near_header_head_height{{{H}}} - near_block_height_head{{{H}}}", "{{host}}")], 12, 8, 0,
|
||||
unit="none", desc="Falling means catching up; flat above zero means keeping pace without gaining; rising means falling behind the chain.")
|
||||
panel("timeseries", "Blocks processed per second", [(f"rate(near_block_processed_total{{{H}}}[$__rate_interval])", "{{host}}")], 12, 8, 12,
|
||||
unit="none", desc="The chain makes about 1.7 a second. Above that a node is gaining; below it, losing ground.")
|
||||
y += 8
|
||||
panel("timeseries", "State sync: parts downloaded", [
|
||||
(f"sum by (host, shard_id) (near_state_sync_download_result{{{H},type=\"part\",result=\"success\"}}) / on (host, shard_id) near_state_sync_parts_per_shard{{{H}}}", "{{host}} shard {{shard_id}}")],
|
||||
12, 8, 0, unit="percentunit", desc="Of each shard's state parts. Counters reset when neard restarts.",
|
||||
extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}})
|
||||
panel("timeseries", "State sync: parts applied", [
|
||||
(f"near_state_sync_apply_part_delay_sec_count{{{H}}} / on (host, shard_id) near_state_sync_parts_per_shard{{{H}}}", "{{host}} shard {{shard_id}}")],
|
||||
12, 8, 12, unit="percentunit", desc="Applying is disk-bound; on gargravarr it is the slow half of a sync.",
|
||||
extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}})
|
||||
y += 8
|
||||
panel("timeseries", "Flat storage: deltas behind its head", [(f"near_flat_storage_cached_deltas{{{H},shard_uid!=\"s0.v0\"}}", "{{host}} {{shard_uid}}")], 12, 8, 0,
|
||||
unit="none", desc="Every state read walks these. Past 100 neard warns 'too many hops'; a count that only grows means the head is held, e.g. by a state snapshot.")
|
||||
panel("timeseries", "Chunk apply time per block", [
|
||||
(f"rate(near_apply_all_chunks_time_sum{{{H}}}[$__rate_interval]) / rate(near_apply_all_chunks_time_count{{{H}}}[$__rate_interval])", "{{host}}")],
|
||||
12, 8, 12, unit="s", desc="Executing the tracked shards' chunks, per block. Grows with the shards a node carries.")
|
||||
y += 8
|
||||
|
||||
row("Resources")
|
||||
panel("timeseries", "neard memory", [(f"near_memory_usage_bytes{{{H}}}", "{{host}}")], 8, 8, 0,
|
||||
unit="bytes", desc="neard's own figure. gargravarr's unit caps it at 13 GiB high / 14 GiB max.")
|
||||
panel("timeseries", "neard CPU", [(f"near_cpu_usage_ratio{{{H}}}", "{{host}}")], 8, 8, 8,
|
||||
unit="percent", desc="Percent of one core: 100 is one full core.")
|
||||
panel("timeseries", "neard on-disk size", [(f"sum by (host) (near_rocksdb_live_sst_files_size{{{H}}})", "{{host}}")], 8, 8, 16, unit="bytes")
|
||||
y += 8
|
||||
panel("timeseries", "Disk busy (neard's disks)", [(NEARD_DISKS, "{{host}} {{device}}")], 8, 8, 0,
|
||||
unit="percentunit", desc="Share of time each disk was busy. 100% is the ceiling a sync hits on one SATA SSD.",
|
||||
extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}})
|
||||
panel("timeseries", "Waiting on disk (host)", [(f"rate(node_pressure_io_waiting_seconds_total{{{NODE}}}[$__rate_interval])", "{{host}}")], 8, 8, 8,
|
||||
unit="percentunit", desc="Share of time some task was stalled on IO, host-wide. Frankie's includes its Postgres standby.",
|
||||
extra={"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}})
|
||||
panel("timeseries", "Network", [
|
||||
(f"near_received_bytes_per_second{{{H}}}", "{{host}} in"),
|
||||
(f"-near_sent_bytes_per_second{{{H}}}", "{{host}} out")], 8, 8, 16,
|
||||
unit="Bps", desc="Received above the line, sent below.")
|
||||
y += 8
|
||||
|
||||
dashboard = {
|
||||
"uid": "blackbeard-neard",
|
||||
"title": "NEAR nodes",
|
||||
"description": "neard on frankie, gargravarr and quadbrat for blackbeard.observer (blackbeard/observer#39).",
|
||||
"tags": ["near", "blackbeard"],
|
||||
"timezone": "browser",
|
||||
"editable": False,
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"templating": {"list": [{
|
||||
"name": "host", "label": "Host", "type": "query", "datasource": DS,
|
||||
"query": {"query": 'label_values(near_block_height_head{job="neard"}, host)', "refId": "host"},
|
||||
"definition": 'label_values(near_block_height_head{job="neard"}, host)',
|
||||
"multi": True, "includeAll": True, "current": {"selected": True, "text": "All", "value": "$__all"},
|
||||
"refresh": 2, "sort": 1,
|
||||
}]},
|
||||
"panels": panels,
|
||||
}
|
||||
Path(__file__).with_name("neard.json").write_text(json.dumps(dashboard, indent=2) + "\n")
|
||||
print(f"{len(panels)} panels")
|
||||
Reference in New Issue
Block a user