diff --git a/asset/arena/quantus-arena-exporter.py b/asset/arena/quantus-arena-exporter.py index 68e2301..aab953d 100755 --- a/asset/arena/quantus-arena-exporter.py +++ b/asset/arena/quantus-arena-exporter.py @@ -415,7 +415,13 @@ class Arena: # Last tip block recorded: (height, author timestamp ms, preimage). Feeds # the per-author gap, drift and consecutive-block series below. self.last_tip = None - self.gap_hist = {} # preimage -> Hist(GAP_BUCKETS) + self.gap_hist = {} # preimage -> Hist(GAP_BUCKETS), by author timestamps + # Same, by the moment this exporter saw each block. Author timestamps + # are set when the node BUILDS the proposal (right after the previous + # block arrives), so they measure proposal timing; withheld blocks show + # in arrival timing. Resolution is the poll interval until telemetry + # first-seen replaces it (lair/quantus#11). + self.arrival_hist = {} # preimage -> Hist(GAP_BUCKETS) self.drift_hist = {} # preimage -> Hist(DRIFT_BUCKETS) self.consecutive = {} # preimage -> blocks authored directly after its own # Tip observations only — used for the block interval, which is real @@ -471,10 +477,11 @@ class Arena: last = self.last_tip if last is not None and height == last[0] + 1: self.gap_hist.setdefault(pre, Hist(GAP_BUCKETS)).observe((ts - last[1]) / 1000.0) + self.arrival_hist.setdefault(pre, Hist(GAP_BUCKETS)).observe(seen - last[3]) if last[2] == pre: self.consecutive[pre] = self.consecutive.get(pre, 0) + 1 self.drift_hist.setdefault(pre, Hist(DRIFT_BUCKETS)).observe(ts / 1000.0 - seen) - self.last_tip = (height, ts, pre) + self.last_tip = (height, ts, pre, seen) def poll_once(self): health = self.rpc.call("system_health", []) @@ -626,6 +633,7 @@ class Arena: # is merged into preimage="other" and the whole network into "all". with self.lock: gaps = {k: v for k, v in self.gap_hist.items()} + arrivals = {k: v for k, v in self.arrival_hist.items()} drifts = {k: v for k, v in self.drift_hist.items()} consecutive = dict(self.consecutive) @@ -647,10 +655,15 @@ class Arena: return shown if gaps: - out.append("# HELP quantus_block_gap_seconds Seconds between a block and the previous one, by the author of the later block (author timestamps). Bursts in the lowest buckets for one author right after others' blocks are withheld blocks.") + out.append("# HELP quantus_block_gap_seconds Seconds between a block's author timestamp and the previous block's, by the author of the later block. Author timestamps are set when the node builds the proposal, so this measures proposal timing; see quantus_block_arrival_gap_seconds for release timing.") out.append("# TYPE quantus_block_gap_seconds histogram") for labels, h in labelled(gaps): h.render(out, "quantus_block_gap_seconds", labels) + if arrivals: + out.append("# HELP quantus_block_arrival_gap_seconds Seconds between this exporter first seeing a block and first seeing the previous one, by the author of the later block. Resolution is the poll interval. Withheld blocks arrive right after others'.") + out.append("# TYPE quantus_block_arrival_gap_seconds histogram") + for labels, h in labelled(arrivals): + h.render(out, "quantus_block_arrival_gap_seconds", labels) if drifts: out.append("# HELP quantus_block_timestamp_drift_seconds Block timestamp minus the time this exporter first saw the block, by author. Negative is propagation and poll lag; positive is a future-dated block.") out.append("# TYPE quantus_block_timestamp_drift_seconds histogram") diff --git a/asset/grafana/quantus.json b/asset/grafana/quantus.json index 64eef6c..ed99d0d 100644 --- a/asset/grafana/quantus.json +++ b/asset/grafana/quantus.json @@ -9,7 +9,7 @@ ], "timezone": "browser", "schemaVersion": 39, - "version": 16, + "version": 18, "refresh": "30s", "time": { "from": "now-6h", @@ -1716,7 +1716,7 @@ "x": 0, "y": 62 }, - "description": "Blocks observed authored within the selected time range, by reward preimage (cumulative counter, exists from the exporter's restart onward). est. MH/s is the author's share of blocks times the network hashrate estimate over the range: a statistical figure, \u00b110% or so at a few hundred blocks. gap p10: the 10th percentile of seconds between the author's block and the previous one; near zero means blocks released right after others' (withheld). drift median: the author's block timestamp minus when we first saw it; a few seconds negative is normal, positive is future-dated. consecutive: blocks authored directly after the author's own previous block. Timing columns fill in from when the exporter started recording them. burst \u00d7: consecutive self-blocks divided by the count expected from the author's share (blocks \u00d7 share); about 1 is normal, sustained values above 2 mean blocks held and released together. Shown only where the expected count exceeds one. Cell colours: gap p10 red under 1 s and amber under 3 s; drift green in the normal negative band, amber once future-dated, red past 5 s, blue when more than 20 s in the past; burst amber above 2, red above 4.", + "description": "Blocks observed authored within the selected time range, by reward preimage (cumulative counter, exists from the exporter's restart onward). est. MH/s is the author's share of blocks times the network hashrate estimate over the range: a statistical figure, \u00b110% or so at a few hundred blocks. quick-follow \u00d7: the share of the author's blocks that arrived (as seen by the exporter, 1 s poll) within 2 s of the previous block, divided by the network's share; about 1 is normal, 2 or more means blocks released right after others' (withheld). Needs 10 samples. drift median: the author's block timestamp minus when we first saw it; a few seconds negative is normal, positive is future-dated. consecutive: blocks authored directly after the author's own previous block. Timing columns fill in from when the exporter started recording them. burst \u00d7: consecutive self-blocks divided by the count expected from the author's share (blocks \u00d7 share); about 1 is normal, sustained values above 2 mean blocks held and released together. Shown only where the expected count exceeds one. Cell colours: quick-follow amber above 2 and red above 3; drift green in the normal negative band, amber once future-dated, red past 5 s, blue when more than 20 s in the past; burst amber above 2, red above 4.", "targets": [ { "datasource": { @@ -1749,7 +1749,7 @@ }, "editorMode": "code", "refId": "C", - "expr": "(histogram_quantile(0.1, sum by (preimage, le) (increase(quantus_block_gap_seconds_bucket{preimage!~\"all|other\"}[$__range])))) == (histogram_quantile(0.1, sum by (preimage, le) (increase(quantus_block_gap_seconds_bucket{preimage!~\"all|other\"}[$__range]))))", + "expr": "((sum by (preimage) (increase(quantus_block_arrival_gap_seconds_bucket{le=\"2.0\",preimage!~\"all|other\"}[$__range])) / sum by (preimage) (increase(quantus_block_arrival_gap_seconds_count{preimage!~\"all|other\"}[$__range]))) / scalar(sum(increase(quantus_block_arrival_gap_seconds_bucket{le=\"2.0\",preimage=\"all\"}[$__range])) / sum(increase(quantus_block_arrival_gap_seconds_count{preimage=\"all\"}[$__range])))) and on(preimage) (sum by (preimage) (increase(quantus_block_arrival_gap_seconds_count{preimage!~\"all|other\"}[$__range])) >= 10)", "instant": true, "range": false, "format": "table" @@ -1845,7 +1845,7 @@ "self 1": "ours", "Value #A": "blocks", "Value #B": "est. MH/s", - "Value #C": "gap p10 (s)", + "Value #C": "quick-follow \u00d7", "Value #D": "drift median (s)", "Value #E": "consecutive", "Value #F": "burst \u00d7" @@ -2012,7 +2012,23 @@ { "matcher": { "id": "byName", - "options": "gap p10 (s)" + "options": "consecutive" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + }, + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "quick-follow \u00d7" }, "properties": [ { @@ -2042,15 +2058,19 @@ "mode": "absolute", "steps": [ { - "color": "red", + "color": "transparent", "value": null }, { - "color": "orange", - "value": 1 + "color": "green", + "value": 0 }, { - "color": "green", + "color": "orange", + "value": 2 + }, + { + "color": "red", "value": 3 } ] @@ -2091,9 +2111,13 @@ "mode": "absolute", "steps": [ { - "color": "blue", + "color": "transparent", "value": null }, + { + "color": "blue", + "value": -1000000 + }, { "color": "green", "value": -20 @@ -2111,22 +2135,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "consecutive" - }, - "properties": [ - { - "id": "decimals", - "value": 0 - }, - { - "id": "unit", - "value": "none" - } - ] - }, { "matcher": { "id": "byName", @@ -2160,9 +2168,13 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "transparent", "value": null }, + { + "color": "green", + "value": 0 + }, { "color": "orange", "value": 2 @@ -3614,8 +3626,8 @@ }, { "type": "timeseries", - "title": "Shortest gaps by author (p10, 6h)", - "description": "10th percentile of the time between a block and the previous one, by the author of the later block. The network's gaps are exponential around the target; an author whose p10 sits near zero is releasing blocks right after other people's, which is what withheld blocks look like.", + "title": "Quick follows by author (share of blocks arriving within 2 s, 6h)", + "description": "Share of each author's blocks that arrived within 2 s of the previous block, by the exporter's observation time (1 s poll). The network's own share is the baseline; an author well above it is releasing blocks right after other people's, which is what withheld blocks look like.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -3631,7 +3643,7 @@ "color": { "mode": "palette-classic" }, - "unit": "s", + "unit": "percentunit", "custom": { "lineWidth": 1, "fillOpacity": 0, @@ -3656,7 +3668,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.10, sum by (preimage, le) (rate(quantus_block_gap_seconds_bucket{preimage!=\"other\"}[6h])))", + "expr": "sum by (preimage) (increase(quantus_block_arrival_gap_seconds_bucket{le=\"2.0\",preimage!=\"other\"}[6h])) / sum by (preimage) (increase(quantus_block_arrival_gap_seconds_count{preimage!=\"other\"}[6h]))", "legendFormat": "{{preimage}}", "range": true, "refId": "A" diff --git a/asset/systemd/quantus-arena.service b/asset/systemd/quantus-arena.service index d9a2ae3..fcc26f4 100644 --- a/asset/systemd/quantus-arena.service +++ b/asset/systemd/quantus-arena.service @@ -31,6 +31,7 @@ EnvironmentFile=/etc/quantus-arena/arena.env WorkingDirectory=/var/lib/quantus-arena ExecStart=/usr/bin/python3 /usr/local/bin/quantus-arena-exporter.py \ + --poll-seconds 1 \ --rpc-url http://127.0.0.1:9944 \ --port 25033 \ --window-blocks 3600 \