From 867a71f514dae9fccc062985ad3530b40a44f6ae Mon Sep 17 00:00:00 2001 From: Rob Thijssen Date: Tue, 1 Sep 2026 05:34:50 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20telemetry=20census=20=E2=80=94=20node?= =?UTF-8?q?=20population=20vs=20distinct=20authors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain can only see miners that WIN blocks. A miner too small to take a block in any given window is invisible in the authorship leaderboard while still being connected and still burning electricity. Telemetry sees every node, so the two series together answer a question neither can alone: nodes flat while distinct authors falls means small miners are being out-competed, not quitting. Measured on the live feed: 32 nodes against 9-21 distinct authors, 229 aggregate CPU cores, 15 VMs, and five client versions in the wild -- half the network is still on 0.6.x/0.7.x/0.8.x, which is a fork risk at a runtime upgrade and worth knowing before 2026-09-09. python3-websockets is a SOFT dependency, packaged not pip: without it the exporter serves every chain metric and reports quantus_telemetry_up 0, so a missing package degrades rather than breaks. infra-setup.sh installs it and treats failure as non-fatal. The genesis hash is discovered from the node rather than configured -- one less value to get wrong when the chain changes for mainnet. --- .gitea/workflows/deploy.yaml | 5 + asset/arena/quantus-arena-exporter.py | 134 ++++++++++- asset/config/arena.env.tmpl | 6 + asset/grafana/quantus.json | 315 +++++++++++++++++++++++++- asset/systemd/quantus-arena.service | 12 +- script/infra-setup.sh | 11 + 6 files changed, 477 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index be9308f..1f87725 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -64,6 +64,9 @@ env: # Fleet Prometheus/Grafana host. Exporters are unauthenticated, so this is the # only host allowed to reach them. METRICS_HOST: golgafrinchans.kosherinata.internal + # substrate-telemetry feed for this chain. Found in /tmp/env-config.js on the + # telemetry site — NOT the /feed path on the main host, which 404s. + TELEMETRY_URL: wss://feed-telemetry.quantus.cat/feed # Arena exporter: network hashrate, difficulty, authorship leaderboard. # Derived from the service name per port-allocations.md §3. ARENA_PORT: "25033" @@ -233,6 +236,7 @@ jobs: QUANTUS_PUBLIC_ADDR: ${{ matrix.public_addr }} QUANTUS_CHAIN: ${{ env.CHAIN }} QUANTUS_RPC_EXPOSE: ${{ matrix.rpc_expose }} + QUANTUS_TELEMETRY_URL: ${{ env.TELEMETRY_URL }} run: | set -euo pipefail case "$QUANTUS_INNER_HASH" in @@ -260,6 +264,7 @@ jobs: pathlib.Path("node.env").write_text(out) a = pathlib.Path("asset/config/arena.env.tmpl").read_text() a = a.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"]) + a = a.replace("{{QUANTUS_TELEMETRY_URL}}", os.environ.get("QUANTUS_TELEMETRY_URL", "")) pathlib.Path("arena.env.partial").write_text(a) PY diff --git a/asset/arena/quantus-arena-exporter.py b/asset/arena/quantus-arena-exporter.py index a6304f8..2c8270c 100755 --- a/asset/arena/quantus-arena-exporter.py +++ b/asset/arena/quantus-arena-exporter.py @@ -124,6 +124,119 @@ def account_storage_key(account: bytes) -> str: + hashlib.blake2b(account, digest_size=16).digest() + account).hex() +# --- telemetry feed ------------------------------------------------------- +# +# substrate-telemetry's frontend feed. Gives the NODE population, which the +# chain cannot: a miner too small to win a block in any given window is +# invisible in the authorship leaderboard but still present here. Node count +# holding flat while distinct authors falls is the signature of small miners +# still running and simply being out-competed. +# +# Protocol: connect, send "subscribe:", then receive JSON arrays of +# alternating [action_code, payload, ...] pairs. +TELEMETRY_ACTIONS = {3: "added", 4: "removed", 11: "chain_stats"} + + +class Telemetry: + """Optional. Needs python3-websockets; without it the exporter still serves + every chain metric and simply reports quantus_telemetry_up 0.""" + + def __init__(self, url, genesis): + self.url = url + self.genesis = genesis + self.lock = threading.Lock() + self.nodes = {} # node_id -> dict(version, os, arch, cores, vm) + self.chain_node_count = None # authoritative count from action 11 + self.connected = False + self.errors = 0 + + def _ingest(self, arr): + for i in range(0, len(arr) - 1, 2): + code, payload = arr[i], arr[i + 1] + try: + if code == 3: # AddedNode + nid = payload[0] + d = payload[1] + hw = d[9] if len(d) > 9 and isinstance(d[9], dict) else {} + with self.lock: + self.nodes[nid] = { + "version": (d[2] or "unknown") if len(d) > 2 else "unknown", + "os": (d[5] or "unknown") if len(d) > 5 else "unknown", + "cores": hw.get("core_count") or 0, + "vm": bool(hw.get("is_virtual_machine")), + } + elif code == 4: # RemovedNode + with self.lock: + self.nodes.pop(payload, None) + elif code == 11: # AddedChain: [name, genesis, count] + if isinstance(payload, list) and len(payload) >= 3: + with self.lock: + self.chain_node_count = payload[2] + except (IndexError, TypeError, KeyError): + # One malformed payload must not kill the feed; the error + # counter makes it visible without taking the watcher down. + self.errors += 1 + + def run_forever(self): + try: + import websockets.sync.client as wsc + except ImportError: + LOG.warning("python3-websockets not installed — telemetry metrics disabled") + return + while True: + try: + with wsc.connect(self.url, open_timeout=20) as ws: + ws.send("subscribe:" + self.genesis) + self.connected = True + LOG.info("telemetry: subscribed to %s", self.genesis[:18]) + for msg in ws: + try: + arr = json.loads(msg) + except (ValueError, TypeError): + continue + if isinstance(arr, list): + self._ingest(arr) + except Exception as e: + self.errors += 1 + LOG.warning("telemetry: %s", e) + finally: + self.connected = False + with self.lock: + self.nodes.clear() + time.sleep(15) + + def render(self, g, out): + with self.lock: + nodes = dict(self.nodes) + count = self.chain_node_count + g("quantus_telemetry_up", "1 when the telemetry feed is connected.", + 1 if self.connected else 0) + g("quantus_telemetry_errors_total", "Telemetry feed errors since start.", self.errors) + if count is not None: + g("quantus_telemetry_chain_nodes", + "Nodes on this chain as reported by telemetry. Compare against " + "quantus_arena_distinct_miners: nodes flat while distinct authors " + "falls means small miners are still running and simply losing.", + count) + if not nodes: + return + g("quantus_telemetry_nodes_seen", "Nodes currently tracked from the feed.", len(nodes)) + g("quantus_telemetry_cores_total", + "Sum of reported CPU cores across nodes — a rough proxy for aggregate " + "CPU capacity attached to the network.", + sum(n["cores"] for n in nodes.values())) + g("quantus_telemetry_virtual_machines", + "Nodes reporting is_virtual_machine.", + sum(1 for n in nodes.values() if n["vm"])) + by_version = {} + for n in nodes.values(): + by_version[n["version"]] = by_version.get(n["version"], 0) + 1 + out.append("# HELP quantus_telemetry_nodes_by_version Nodes by client version.") + out.append("# TYPE quantus_telemetry_nodes_by_version gauge") + for v, c in sorted(by_version.items(), key=lambda kv: -kv[1])[:15]: + out.append('quantus_telemetry_nodes_by_version{version="%s"} %d' % (v, c)) + + class RpcError(Exception): pass @@ -196,8 +309,9 @@ def u512_le(hex_str: str) -> int: class Arena: def __init__(self, rpc, self_preimage, window_blocks, target_block_time, top_n, - reward_address=""): + reward_address="", telemetry=None): self.rpc = rpc + self.telemetry = telemetry self.reward_address = reward_address or None self.balance = None # (free, reserved, frozen, nonce) self.decimals = None @@ -396,6 +510,9 @@ class Arena: if self.decimals is not None: g("quantus_token_decimals", "Chain token decimals.", self.decimals) + if self.telemetry is not None: + self.telemetry.render(g, out) + if self.self_preimage: mine = counts.get(self.self_preimage, 0) g("quantus_blocks_authored_self", "Blocks we authored in the window.", mine) @@ -439,6 +556,8 @@ def main(): help="rolling leaderboard window, in blocks (~6h at a 6s target)") p.add_argument("--target-block-time", type=float, default=6.0) p.add_argument("--top-n", type=int, default=25) + p.add_argument("--telemetry-url", default="", + help="substrate-telemetry feed websocket; empty disables telemetry") p.add_argument("--reward-address", default="", help="our wormhole SS58 address, to track accrued rewards") p.add_argument("--self-preimage", default="", @@ -446,8 +565,17 @@ def main(): args = p.parse_args() logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - arena = Arena(Rpc(args.rpc_url), args.self_preimage, args.window_blocks, - args.target_block_time, args.top_n, args.reward_address) + rpc = Rpc(args.rpc_url) + telemetry = None + if args.telemetry_url: + # Genesis is discoverable, so it never needs configuring — one less + # value to get wrong when the chain changes for mainnet. + genesis = rpc.call("chain_getBlockHash", [0]) + telemetry = Telemetry(args.telemetry_url, genesis) + threading.Thread(target=telemetry.run_forever, daemon=True).start() + + arena = Arena(rpc, args.self_preimage, args.window_blocks, + args.target_block_time, args.top_n, args.reward_address, telemetry) Handler.arena = arena def loop(): diff --git a/asset/config/arena.env.tmpl b/asset/config/arena.env.tmpl index 17d00d6..89d315e 100644 --- a/asset/config/arena.env.tmpl +++ b/asset/config/arena.env.tmpl @@ -10,3 +10,9 @@ QUANTUS_INNER_HASH={{QUANTUS_INNER_HASH}} # reimplementing Poseidon2 over Goldilocks in the exporter for no gain. Empty is # tolerated — the exporter simply omits the balance metrics. QUANTUS_REWARD_ADDRESS={{QUANTUS_REWARD_ADDRESS}} + +# substrate-telemetry feed. Gives the NODE population, which the chain cannot: +# a miner too small to win a block in any window is invisible in the authorship +# leaderboard but still present here. Empty disables telemetry entirely. +# The genesis hash is discovered from the node, so it never needs configuring. +QUANTUS_TELEMETRY_URL={{QUANTUS_TELEMETRY_URL}} diff --git a/asset/grafana/quantus.json b/asset/grafana/quantus.json index 08bfc05..54c89df 100644 --- a/asset/grafana/quantus.json +++ b/asset/grafana/quantus.json @@ -9,7 +9,7 @@ ], "timezone": "browser", "schemaVersion": 39, - "version": 4, + "version": 5, "refresh": "30s", "time": { "from": "now-6h", @@ -1908,6 +1908,319 @@ "sort": "desc" } } + }, + { + "type": "row", + "title": "Network population (telemetry)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 84 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Nodes vs distinct block authors", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 14, + "x": 0, + "y": 85 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_telemetry_chain_nodes", + "legendFormat": "nodes on telemetry", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_arena_distinct_miners", + "legendFormat": "distinct authors (window)", + "range": true, + "refId": "B" + } + ], + "description": "THE comparison. The chain can only see miners that WIN blocks; telemetry sees every connected node. Nodes holding flat while distinct authors falls means small miners are still running and simply being out-competed as difficulty rises \u2014 not that they quit. Neither series alone can tell you that.", + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "piechart", + "title": "Client versions", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 85 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_telemetry_nodes_by_version", + "legendFormat": "{{version}}", + "range": true, + "refId": "A" + } + ], + "description": "Version spread across the network. A long tail of old clients is a fork risk at a runtime upgrade.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "type": "stat", + "title": "Aggregate CPU cores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 94 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_telemetry_cores_total", + "legendFormat": "cores", + "range": true, + "refId": "A" + } + ], + "description": "Sum of reported cores across nodes \u2014 a rough proxy for CPU capacity attached to the network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "type": "stat", + "title": "Virtual machines", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 94 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_telemetry_virtual_machines", + "legendFormat": "vms", + "range": true, + "refId": "A" + } + ], + "description": "Nodes self-reporting as VMs. Bare metal is the usual shape for a GPU miner.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "type": "stat", + "title": "Telemetry feed", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 94 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus_telemetry_up", + "legendFormat": "up", + "range": true, + "refId": "A" + } + ], + "description": "0 means the feed dropped or python3-websockets is missing; chain metrics are unaffected either way.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "down", + "color": "orange", + "index": 0 + }, + "1": { + "text": "connected", + "color": "green", + "index": 1 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } } ] } diff --git a/asset/systemd/quantus-arena.service b/asset/systemd/quantus-arena.service index 8104cdb..d9a2ae3 100644 --- a/asset/systemd/quantus-arena.service +++ b/asset/systemd/quantus-arena.service @@ -3,7 +3,14 @@ # from the PreRuntime digest in every block header. # # Runs on the node host and talks only to the node's loopback RPC, so it needs -# no chain data of its own and no credentials. Pure stdlib Python — no venv, no +# no chain data of its own and no credentials. +# +# Pure stdlib apart from ONE SOFT dependency: python3-websockets, used only for +# the telemetry feed. Without it the exporter still serves every chain metric and +# reports quantus_telemetry_up 0 — a missing package degrades rather than breaks. +# infra-setup.sh installs it from the Fedora repos; packaged, never pip. +# +# Otherwise pure stdlib Python — no venv, no # build step, nothing to keep patched beyond the interpreter. # # Separate service account from the node on purpose: this reads the chain over a @@ -29,7 +36,8 @@ ExecStart=/usr/bin/python3 /usr/local/bin/quantus-arena-exporter.py \ --window-blocks 3600 \ --target-block-time 6 \ --self-preimage ${QUANTUS_INNER_HASH} \ - --reward-address ${QUANTUS_REWARD_ADDRESS} + --reward-address ${QUANTUS_REWARD_ADDRESS} \ + --telemetry-url ${QUANTUS_TELEMETRY_URL} # The node may be down or restarting; reconnecting is normal operation. The # exporter already survives RPC errors internally, so a restart here means it diff --git a/script/infra-setup.sh b/script/infra-setup.sh index 55e0b44..b33de85 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -121,6 +121,17 @@ provision_node() { # journalctl -u for the deploy's log capture, without sudo. usermod -aG systemd-journal gitea_ci + # Soft dependency for the arena exporter's telemetry feed. Packaged, never + # pip. Its absence only disables telemetry metrics — the exporter still + # serves every chain metric — so this is deliberately not fatal. + if python3 -c "import websockets" >/dev/null 2>&1; then + echo " = python3-websockets already present" + elif dnf install -y python3-websockets >/dev/null 2>&1; then + echo " + installed python3-websockets" + else + echo " ! could not install python3-websockets — telemetry metrics will be disabled" >&2 + fi + tmp=/etc/sudoers.d/.quantus-node_gitea_ci.tmp cat > "$tmp" <<'SUDO' gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/quantus-node