diff --git a/.sqlx/query-18154cee1213de1b275c87babe63bd46b4191a20a9cf7b764ac9b197a594f7da.json b/.sqlx/query-18154cee1213de1b275c87babe63bd46b4191a20a9cf7b764ac9b197a594f7da.json new file mode 100644 index 0000000..4eb4d1f --- /dev/null +++ b/.sqlx/query-18154cee1213de1b275c87babe63bd46b4191a20a9cf7b764ac9b197a594f7da.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into block (\n chain, height, hash, miner, authored_at, observed_at, difficulty, at_tip\n )\n select * from unnest(\n $1::text[], $2::bigint[], $3::text[], $4::text[],\n $5::timestamptz[], $6::timestamptz[], $7::numeric[], $8::boolean[]\n )\n on conflict (chain, height) do update set\n hash = excluded.hash,\n miner = excluded.miner,\n authored_at = excluded.authored_at,\n observed_at = excluded.observed_at,\n -- coalesce: a reorg replacement re-observed from history has no\n -- difficulty to offer, and dropping the one already recorded for\n -- that height would put a hole in the series for no gain.\n difficulty = coalesce(excluded.difficulty, block.difficulty),\n at_tip = excluded.at_tip\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Int8Array", + "TextArray", + "TextArray", + "TimestamptzArray", + "TimestamptzArray", + "NumericArray", + "BoolArray" + ] + }, + "nullable": [] + }, + "hash": "18154cee1213de1b275c87babe63bd46b4191a20a9cf7b764ac9b197a594f7da" +} diff --git a/.sqlx/query-470022fab8a12aa88db0f1feb754f6fa9fca60a0bc2665ec862aa34a87f350a3.json b/.sqlx/query-470022fab8a12aa88db0f1feb754f6fa9fca60a0bc2665ec862aa34a87f350a3.json deleted file mode 100644 index 970af2a..0000000 --- a/.sqlx/query-470022fab8a12aa88db0f1feb754f6fa9fca60a0bc2665ec862aa34a87f350a3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n insert into block (chain, height, hash, miner, authored_at, observed_at, difficulty)\n select * from unnest(\n $1::text[], $2::bigint[], $3::text[], $4::text[],\n $5::timestamptz[], $6::timestamptz[], $7::numeric[]\n )\n on conflict (chain, height) do update set\n hash = excluded.hash,\n miner = excluded.miner,\n authored_at = excluded.authored_at,\n observed_at = excluded.observed_at,\n difficulty = excluded.difficulty\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "TextArray", - "Int8Array", - "TextArray", - "TextArray", - "TimestamptzArray", - "TimestamptzArray", - "NumericArray" - ] - }, - "nullable": [] - }, - "hash": "470022fab8a12aa88db0f1feb754f6fa9fca60a0bc2665ec862aa34a87f350a3" -} diff --git a/.sqlx/query-c0a3c4d0ac72b0a57096cd05af87bcb77d1e643224fb8c67c7f4569c27876c52.json b/.sqlx/query-c0a3c4d0ac72b0a57096cd05af87bcb77d1e643224fb8c67c7f4569c27876c52.json new file mode 100644 index 0000000..cf27779 --- /dev/null +++ b/.sqlx/query-c0a3c4d0ac72b0a57096cd05af87bcb77d1e643224fb8c67c7f4569c27876c52.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "\n with bounds as (\n select max(height) as hi from block where chain = $1\n )\n select (b.height / $2)::bigint as \"index!\",\n max(b.height) as \"last_height!\",\n count(*) as \"blocks!\",\n -- Rounded, because difficulty crosses every boundary as an\n -- integer decimal string and a mean is the one place it\n -- would otherwise acquire a fractional part.\n round(avg(b.difficulty))::text as difficulty,\n min(b.authored_at) as first_authored,\n max(b.authored_at) as last_authored,\n min(b.height) filter (where b.authored_at is not null)\n as timed_low,\n max(b.height) filter (where b.authored_at is not null)\n as timed_high,\n array_agg(distinct b.miner) as \"miners!\"\n from block b, bounds\n where b.chain = $1\n -- Buckets are anchored to absolute height, not to the newest\n -- block: the same reason `miner_series` bins from the epoch\n -- rather than from the range's start. Anchoring to the tip slides\n -- every boundary by one on every block, and the chart shifts\n -- under the reader between refreshes.\n and b.height >= ((bounds.hi / $2) - $3 + 1) * $2\n group by 1\n order by 1 asc\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "index!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_height!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "blocks!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "difficulty", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "first_authored", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "last_authored", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "timed_low", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "timed_high", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "miners!", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "c0a3c4d0ac72b0a57096cd05af87bcb77d1e643224fb8c67c7f4569c27876c52" +} diff --git a/CLAUDE.md b/CLAUDE.md index fa653de..8468571 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,22 @@ an explicit `at_tip` flag and backfilled blocks contribute no timing sample. The **Windows are block counts, never durations.** See `readme.md`. Anything that turns `Window` into a time range is wrong. +**The difficulty of block *h* lives in the state of block *h-1*.** +`pallet_qpow::on_finalize` retargets at the *end* of every block, so +`QPoWApi_get_difficulty` at the latest state returns what the *next* block must +meet, not what the tip met. Recording that against the block just seen is off by +one, which on a gradual retarget is a number that still looks entirely right. +`RpcClient::difficulty_for_child_of` asks at `header.parent_hash`, which is the +same call the miner made when it built the block. It needs state the node may +have pruned; an error there is `None`, never a fallback to the current value — +substituting it is how a gap fill stamps today's difficulty across a stretch of +old heights and draws a flat plateau in the history chart. + +**`observed_at` is only a measurement when `at_tip` is true.** The flag is now a +column (`0002`). A gap fill writes a whole batch of `observed_at` within the same +second, so any query treating it as timing data must filter on it. The chain +series sidesteps this entirely by using `authored_at`, the chain's own clock. + **`chain_subscribeNewHeads` skips blocks.** It reports the *best* head, so when several import at once the intermediates never arrive. It is a liveness signal, not a ledger — `ingest` fills gaps against `chain_getBlockHash`. Removing that @@ -118,6 +134,14 @@ site is therefore single-series — magnitude, one hue — with identity carried labels and row treatment. A second series in the accent colour is the one change that would break the palette, and it would not look broken. +This is why the stat-tile sparklines do not follow the usual "current period in +the accent" convention: the newest point is emphasised with `--data-bright`, a +lighter step of the same hue. They also do not start at zero — a sparkline is +read for shape — which is why the hover readout and the `aria-label` both carry +real numbers, and why the readout borrows the tile's note line instead of +floating a tooltip: five overlays in a row of 180px tiles is not a hover layer, +it is a pile. + `web/src/index.css` documents the validated values. Re-run the validator after touching them: diff --git a/crates/blackbeard-api/src/ingest.rs b/crates/blackbeard-api/src/ingest.rs index f72a258..4f2a576 100644 --- a/crates/blackbeard-api/src/ingest.rs +++ b/crates/blackbeard-api/src/ingest.rs @@ -167,6 +167,18 @@ async fn fill_gap(chain: &Arc, store: &Store, from: u64, to: u64) .ok() .flatten() .and_then(from_millis); + // The difficulty *this* block had to meet, not the one in force now. A + // pruning node has thrown away the state a gap this old needs, and the + // honest answer to that is `None`: stamping the current difficulty + // across a stretch of old heights is what produces a history chart with + // a flat plateau at today's value wherever the observer was away. + let difficulty = match chain.rpc.difficulty_for_child_of(&header.parent_hash).await { + Ok(d) => Some(blackbeard_core::hashrate::u512_to_dec(d).0), + Err(e) => { + tracing::debug!(chain = %chain.id(), height, error = %e, "no difficulty for a gap-filled block"); + None + } + }; let now = Utc::now(); batch.push(BlockRecord { chain: chain.id(), @@ -175,10 +187,8 @@ async fn fill_gap(chain: &Arc, store: &Store, from: u64, to: u64) miner: miner.clone(), authored_at, observed_at: now, - difficulty: chain - .read() - .difficulty - .map(|d| blackbeard_core::hashrate::u512_to_dec(d).0), + difficulty, + at_tip: false, }); observed.push(Observed { height, @@ -230,10 +240,19 @@ async fn record( .and_then(from_millis); let observed_at = Utc::now(); - let difficulty = chain - .read() - .difficulty - .map(|d| blackbeard_core::hashrate::u512_to_dec(d).0); + // Asked at the parent, which is the state the miner itself read when it + // built this block. The poller's figure is the difficulty for the block + // *after* this one — near enough to look right on a tile, and wrong in a + // history chart that is supposed to show when a retarget happened. + let difficulty = match chain.rpc.difficulty_for_child_of(&header.parent_hash).await { + Ok(d) => Some(blackbeard_core::hashrate::u512_to_dec(d).0), + Err(e) => { + // Unlike a gap fill, the tip's parent state is one block old and + // any node still has it. This failing means something else. + tracing::warn!(chain = %chain.id(), height, error = %e, "no difficulty for the block at the tip"); + None + } + }; if let Err(e) = store .record_blocks(&[BlockRecord { @@ -244,6 +263,7 @@ async fn record( authored_at, observed_at, difficulty, + at_tip: true, }]) .await { diff --git a/crates/blackbeard-api/src/routes.rs b/crates/blackbeard-api/src/routes.rs index 914b70a..67b66ae 100644 --- a/crates/blackbeard-api/src/routes.rs +++ b/crates/blackbeard-api/src/routes.rs @@ -16,8 +16,8 @@ use axum::response::{IntoResponse, Response}; use axum::routing::get; use axum::{Json, Router}; use blackbeard_entities::{ - ApiError, BigUintDec, ChainInfo, ChainSummary, LeaderboardRow, MinerDetail, MinerId, - MinerSeriesPoint, RecentBlock, Window, + ApiError, BigUintDec, ChainInfo, ChainSeries, ChainSummary, LeaderboardRow, MinerDetail, + MinerId, MinerSeriesPoint, RecentBlock, Window, }; use serde::{Deserialize, Serialize}; use tower_http::compression::CompressionLayer; @@ -47,6 +47,7 @@ pub fn router(state: AppState, allowed_origins: &[String]) -> Router { .route("/v1/chains/{chain}/summary", get(summary)) .route("/v1/chains/{chain}/leaderboard", get(leaderboard)) .route("/v1/chains/{chain}/blocks", get(blocks)) + .route("/v1/chains/{chain}/series", get(series)) .route("/v1/chains/{chain}/miners/{miner}", get(miner)) .route("/v1/ws", get(crate::ws::handler)) // Leaderboards are repetitive JSON and compress to a fraction of their @@ -187,6 +188,48 @@ async fn blocks( Ok(Json(runtime.ticker())) } +/// Points in a chain's history series. +/// +/// Every selectable window divides by this exactly (600, 3,600, 14,400 and +/// 100,800 blocks), which is what lets a bucket be a whole number of blocks and +/// a full window be exactly `SERIES_POINTS` buckets — the property the rolling +/// miner count relies on. The test below holds it. +/// +/// Fifty is sized for the mark, not for the data: these are sparklines a +/// hundred and forty pixels wide under a stat tile. More points would be +/// invisible detail on a line two pixels thick, and each one costs a bucket of +/// authors on the wire. +const SERIES_POINTS: u32 = 50; + +/// `GET /v1/chains/{chain}/series?window=day` +/// +/// The history behind the tiles at the top of the page. Two windows of buckets +/// are read rather than one: the oldest point still needs a whole window of +/// authors behind it to count distinct miners the way the tile does. +async fn series( + State(state): State, + Path(chain): Path, + Query(query): Query, +) -> Result, Failure> { + let runtime = state.chain(&chain).ok_or_else(|| unknown_chain(&chain))?; + let window = query.window()?; + let bucket_blocks = window.blocks() / SERIES_POINTS; + + let buckets = state + .store + .chain_series(&runtime.id(), bucket_blocks, SERIES_POINTS * 2) + .await + .map_err(database_unavailable)?; + + Ok(Json(ChainSeries { + chain: runtime.id(), + window, + window_blocks: window.blocks(), + bucket_blocks, + points: blackbeard_core::series::series(&buckets, SERIES_POINTS as usize, bucket_blocks), + })) +} + /// How far back a miner's history chart reaches, and how finely it is bucketed. /// /// Buckets are chosen so every range renders roughly the same number of points: @@ -280,13 +323,10 @@ fn parse_miner(raw: &str) -> Result { } fn database_unavailable(e: blackbeard_data::DataError) -> Failure { - tracing::warn!(error = %e, "a miner query could not reach the database"); + tracing::warn!(error = %e, "a history query could not reach the database"); Failure( StatusCode::SERVICE_UNAVAILABLE, - ApiError::new( - "database_unavailable", - "miner history is temporarily unavailable", - ), + ApiError::new("database_unavailable", "history is temporarily unavailable"), ) } @@ -335,6 +375,23 @@ mod tests { } } + #[test] + fn every_window_divides_into_a_whole_number_of_series_buckets() { + // The rolling miner count unions exactly SERIES_POINTS buckets to make + // one window. A window that did not divide evenly would make that union + // cover slightly more or less than the window the tile counts over, and + // the last point of the sparkline would not equal the number above it. + for w in Window::all() { + assert_eq!( + w.blocks() % SERIES_POINTS, + 0, + "{w:?} is {} blocks, which is not a multiple of {SERIES_POINTS}", + w.blocks() + ); + assert!(w.blocks() / SERIES_POINTS >= 1); + } + } + #[test] fn an_absent_window_parameter_means_the_default() { let q = WindowQuery { diff --git a/crates/blackbeard-cli/src/main.rs b/crates/blackbeard-cli/src/main.rs index e72c56a..c018e58 100644 --- a/crates/blackbeard-cli/src/main.rs +++ b/crates/blackbeard-cli/src/main.rs @@ -266,7 +266,6 @@ async fn backfill( }) .await?; - let difficulty = rpc.difficulty().await.ok().map(hashrate::u512_to_dec); println!("reading {from}..={to} from {rpc_url}"); // Written in batches rather than one statement: a week of blocks is a @@ -307,7 +306,19 @@ async fn backfill( // history look like it all arrived in the last minute, and every // per-miner chart would collapse into one bucket. observed_at: authored_at.unwrap_or_else(Utc::now), - difficulty: difficulty.as_ref().map(|d| d.0.clone()), + // Asked at the parent, per block: `pallet_qpow` retargets in + // `on_finalize`, so the difficulty stored against block h lives in + // the state of h-1. One current value copied across the whole range + // would draw a flat difficulty history over a period that in fact + // retargeted on every block. `None` where a pruned node cannot say. + difficulty: rpc + .difficulty_for_child_of(&header.parent_hash) + .await + .ok() + .map(|d| hashrate::u512_to_dec(d).0), + // Backfill is history by definition: no observation here timed + // anything, and `RollingWindow` must not treat it as if it had. + at_tip: false, }); if batch.len() >= BATCH { diff --git a/crates/blackbeard-core/src/lib.rs b/crates/blackbeard-core/src/lib.rs index 1739dfe..bd1bdc7 100644 --- a/crates/blackbeard-core/src/lib.rs +++ b/crates/blackbeard-core/src/lib.rs @@ -17,6 +17,7 @@ pub mod attribution; pub mod digest; pub mod hashrate; pub mod scale; +pub mod series; pub mod window; /// Failures decoding what a node handed us. diff --git a/crates/blackbeard-core/src/series.rs b/crates/blackbeard-core/src/series.rs new file mode 100644 index 0000000..626cc14 --- /dev/null +++ b/crates/blackbeard-core/src/series.rs @@ -0,0 +1,480 @@ +//! Turning buckets of blocks into the history behind the headline numbers. +//! +//! The database groups blocks into fixed ranges of height and hands back what +//! each range contained. This module turns those into the points a sparkline +//! draws, and it is where the three things that would quietly falsify that +//! chart are handled. +//! +//! **Block interval comes from author timestamps, never from observation +//! times.** `observed_at` is when this process happened to look, and a gap fill +//! writes a whole batch of it within the same second; deltas taken from it read +//! as a chain producing hundreds of blocks a second, and the hashrate derived +//! from that is wrong by orders of magnitude while still rendering as a number. +//! `authored_at` is the chain's own `Timestamp::set` inherent — the very value +//! `pallet_qpow` retargets against — so it says what the chain did regardless +//! of when anyone was watching. +//! +//! **A single bucket is too few samples to divide by.** Winning a block is a +//! Poisson process; measured on Planck, consecutive gaps ran 1.6 s, 4.4 s, +//! 13.5 s, 26 s, 27 s against a 6 s target. The mean of *n* such gaps still +//! carries a standard error of the mean over the square root of *n* — a +//! six-hour window's seventy-two-block bucket lands at ±1.8 s on a 15 s +//! interval, which across fifty points is a line that looks like a chain +//! changing size every few minutes and is nothing of the kind. Each point's +//! interval is therefore a moving average over [`SMOOTHING_BUCKETS`] buckets, +//! and never fewer than [`MIN_INTERVAL_SAMPLES`] blocks. +//! +//! **A rolling count cannot be summed from per-bucket counts.** Distinct miners +//! over a window is the tile's statistic, and a miner who won in two buckets +//! must not be counted twice — so buckets carry their author *sets* and this +//! module unions them, sliding a counter rather than rebuilding the set at +//! every point. + +use std::collections::HashMap; + +use blackbeard_entities::{BigUintDec, ChainSeriesPoint}; +use chrono::{DateTime, Utc}; + +/// Blocks a point's interval must be measured over before it is published. +/// +/// The same twenty as `RollingWindow::MIN_TIP_SAMPLES`, and for the same +/// reason: below it the mean swings by a factor of three between neighbouring +/// points and the line describes Poisson noise rather than the network. It is a +/// floor, not the usual case — only the shortest window has buckets small +/// enough to reach it. +pub const MIN_INTERVAL_SAMPLES: u32 = 20; + +/// Buckets each point's interval is averaged over. +/// +/// Three, so neighbouring points share two thirds of their samples and the line +/// moves when the chain does rather than when the sample happens to. It also +/// scales the smoothing with the window without a second rule: three buckets is +/// 36 blocks of an hour-long window and 6,048 of a week-long one, which is the +/// right answer both times — a short window has little to average and less to +/// hide, and a long one has plenty of both. +/// +/// The cost is that a point reaches back past its own bucket, so the line lags +/// a real step change by about a point. That is the ordinary price of a moving +/// average and cheap against drawing noise as signal. +pub const SMOOTHING_BUCKETS: u32 = 3; + +/// One fixed range of heights, aggregated. +/// +/// A bucket is a **block count**, never a duration — the same commitment the +/// leaderboard window makes. Time buckets would be meaningless across a stretch +/// where the observer was catching up, and every bucket here carries equal +/// statistical weight because every bucket is the same number of blocks. +#[derive(Debug, Clone, PartialEq)] +pub struct Bucket { + /// `height / bucket_blocks`, anchored to absolute height so a bucket's + /// boundaries are the same on every request forever. + pub index: i64, + /// Highest height in the bucket: where the point sits on the x-axis. + pub last_height: u64, + /// Blocks recorded. Below the bucket's full width for the newest bucket, + /// which is still filling, and anywhere the observer was not watching. + pub blocks: u32, + /// Mean difficulty over the blocks that have one, as a decimal string. + pub difficulty: Option, + /// Earliest author timestamp in the bucket. + pub first_authored: Option>, + /// Latest author timestamp in the bucket. + pub last_authored: Option>, + /// Lowest height in the bucket carrying an author timestamp. + pub timed_low: Option, + /// Highest height in the bucket carrying an author timestamp. + pub timed_high: Option, + /// Distinct authors in the bucket. The set, not a count: counts cannot be + /// unioned across buckets without double-counting. + pub miners: Vec, +} + +impl Bucket { + /// True when this bucket contributes a usable timing sample. + fn is_timed(&self) -> bool { + self.first_authored.is_some() && self.timed_low.is_some() + } +} + +/// Build the plotted points from `buckets`, oldest first. +/// +/// `points` is both the number of points to draw and — because a bucket is the +/// window divided by the point count — the number of buckets one full window +/// spans. `buckets` must therefore reach back `2 * points` so the oldest point +/// still has a whole window of authors behind it; anything shorter simply +/// yields `None` for that point's miner count rather than a partial one. +/// +/// Buckets with no blocks at all are emitted as points with every value `None`, +/// so an outage draws as a break in the line rather than being closed up into a +/// continuous one. Leading empty buckets are trimmed — a chain watched for +/// three hours under a week-long window should start where its history does, +/// not after forty blank points. +pub fn series(buckets: &[Bucket], points: usize, bucket_blocks: u32) -> Vec { + let (Some(first), Some(last)) = (buckets.first(), buckets.last()) else { + return Vec::new(); + }; + let by_index: HashMap = buckets.iter().map(|b| (b.index, b)).collect(); + + let newest = last.index; + let oldest_wanted = newest - points as i64 + 1; + // Never start before the first bucket that has anything in it, and never + // start later than the window's own beginning. + let start = oldest_wanted.max(first.index); + + // Three buckets, or more where a bucket is too narrow to reach the sample + // floor on its own. div_ceil, not plain division: rounding down would leave + // the shortest window measuring on half the samples it needs. + let smoothing = MIN_INTERVAL_SAMPLES + .div_ceil(bucket_blocks.max(1)) + .max(SMOOTHING_BUCKETS) as i64; + + let mut authors = AuthorWindow::default(); + // Prime the counter with everything strictly before the first point, so the + // first point already carries its full window rather than starting empty. + for index in (start - points as i64 + 1)..start { + if let Some(b) = by_index.get(&index) { + authors.add(&b.miners); + } + } + + let mut out = Vec::with_capacity((newest - start + 1).max(0) as usize); + for index in start..=newest { + if let Some(b) = by_index.get(&index) { + authors.add(&b.miners); + } + + let bucket = by_index.get(&index).copied(); + let interval = interval_at(&by_index, index, smoothing); + let difficulty = bucket.and_then(|b| b.difficulty.clone()).map(BigUintDec); + + out.push(ChainSeriesPoint { + // A bucket with no blocks still has a position: its last height is + // where it *would* have ended, which keeps the x-axis linear in + // height across the gap. + height: bucket.map_or_else( + || (index + 1).max(0) as u64 * u64::from(bucket_blocks) - 1, + |b| b.last_height, + ), + at: bucket.and_then(|b| b.last_authored), + blocks: bucket.map_or(0, |b| b.blocks), + network_hashrate: match (&difficulty, interval) { + (Some(d), Some(seconds)) => hashrate(d, seconds), + _ => None, + }, + difficulty, + block_interval_seconds: interval, + // The window is `points` buckets wide; a point that would need one + // older than anything recorded has an undercount, not a low count, + // and drawing it would show miners arriving who were always there. + miners: (index - points as i64 + 1 >= first.index).then(|| authors.distinct()), + }); + + if let Some(b) = by_index.get(&(index - points as i64 + 1)) { + authors.remove(&b.miners); + } + } + out +} + +/// Mean seconds per block over the `smoothing` buckets ending at `index`. +/// +/// Height difference, not sample count, is the divisor: a bucket the observer +/// only partly saw has fewer blocks in it than heights between its ends, and +/// dividing by the blocks it happened to record would read as the chain having +/// sped up over exactly the stretch where the record is worst. +fn interval_at(by_index: &HashMap, index: i64, smoothing: i64) -> Option { + let mut earliest: Option> = None; + let mut latest: Option> = None; + let mut low: Option = None; + let mut high: Option = None; + + for i in (index - smoothing + 1)..=index { + let Some(b) = by_index.get(&i).filter(|b| b.is_timed()) else { + continue; + }; + earliest = min_opt(earliest, b.first_authored); + latest = max_opt(latest, b.last_authored); + low = min_opt(low, b.timed_low); + high = max_opt(high, b.timed_high); + } + + let span = (latest? - earliest?).as_seconds_f64(); + let heights = high?.checked_sub(low?)?; + // A single timed block spans no heights, and author clocks are the authors' + // own — a run that reads as going backwards is skew, not a negative block + // time, and has nothing to say about how fast the chain is producing. + if span <= 0.0 || heights == 0 { + return None; + } + Some(span / heights as f64) +} + +/// Difficulty over interval, guarding the two ways it can produce a number that +/// is not one. +fn hashrate(difficulty: &BigUintDec, seconds: f64) -> Option { + let hashes: f64 = difficulty.as_str().parse().ok()?; + if seconds <= 0.0 || !hashes.is_finite() { + return None; + } + Some(hashes / seconds) +} + +fn min_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } +} + +fn max_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(a.max(b)), + (a, b) => a.or(b), + } +} + +/// A sliding count of distinct authors. +/// +/// Reference-counted rather than a set rebuilt per point: a week-long window is +/// fifty buckets of two thousand blocks, and rebuilding the union at every +/// point would union fifty sets fifty times to answer fifty questions that +/// differ by one bucket at each end. +#[derive(Debug, Default)] +struct AuthorWindow { + counts: HashMap, +} + +impl AuthorWindow { + fn add(&mut self, miners: &[String]) { + for m in miners { + *self.counts.entry(m.clone()).or_insert(0) += 1; + } + } + + fn remove(&mut self, miners: &[String]) { + for m in miners { + if let Some(count) = self.counts.get_mut(m) { + *count -= 1; + if *count == 0 { + self.counts.remove(m); + } + } + } + } + + fn distinct(&self) -> u32 { + self.counts.len() as u32 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn at(secs: i64) -> DateTime { + Utc.timestamp_opt(1_757_000_000 + secs, 0).unwrap() + } + + /// A bucket of `width` blocks ending at `index`, whose first block was + /// authored `first_at` seconds in and whose blocks are `interval` apart. + fn timed(index: i64, width: u32, first_at: i64, interval: i64, miners: &[&str]) -> Bucket { + let first_height = index as u64 * u64::from(width); + Bucket { + index, + last_height: first_height + u64::from(width) - 1, + blocks: width, + difficulty: Some("1200000000".into()), + first_authored: Some(at(first_at)), + last_authored: Some(at(first_at + i64::from(width - 1) * interval)), + timed_low: Some(first_height), + timed_high: Some(first_height + u64::from(width) - 1), + miners: miners.iter().map(|m| (*m).to_owned()).collect(), + } + } + + /// The uniform case: a chain producing every `interval` seconds since + /// height zero, so any run of buckets measures the same interval. + fn bucket(index: i64, width: u32, interval: i64, miners: &[&str]) -> Bucket { + timed( + index, + width, + index * i64::from(width) * interval, + interval, + miners, + ) + } + + fn empty(index: i64, width: u32) -> Bucket { + Bucket { + index, + last_height: (index + 1) as u64 * u64::from(width) - 1, + blocks: 0, + difficulty: None, + first_authored: None, + last_authored: None, + timed_low: None, + timed_high: None, + miners: Vec::new(), + } + } + + #[test] + fn no_buckets_is_an_empty_series_not_a_panic() { + assert!(series(&[], 4, 10).is_empty()); + } + + #[test] + fn interval_comes_from_author_timestamps_over_height_difference() { + // Ten blocks per bucket, authored 15 s apart. + let buckets: Vec = (0..8).map(|i| bucket(i, 10, 15, &["a"])).collect(); + let out = series(&buckets, 4, 10); + let last = out.last().unwrap(); + assert_eq!(last.block_interval_seconds, Some(15.0)); + // Difficulty over interval, and nothing else. + assert_eq!(last.network_hashrate, Some(1_200_000_000.0 / 15.0)); + } + + #[test] + fn a_bucket_narrower_than_the_sample_floor_is_smoothed_across_neighbours() { + // Four blocks per bucket is one sample short of nothing: the floor of + // twenty forces five buckets into every point's measurement, more than + // the three a wider bucket would get. + let buckets: Vec = (0..20).map(|i| bucket(i, 4, 10, &["a"])).collect(); + let out = series(&buckets, 8, 4); + assert!(out.iter().all(|p| p.block_interval_seconds == Some(10.0))); + + // And a single spike is diluted rather than drawn at full height: one + // bucket at four times the interval moves a five-bucket mean, but not + // to four times. It continues the timeline where bucket 18 left off — + // rescaling its absolute timestamps instead would be a chain that + // stopped for an hour, which is a different measurement. + let mut spiked = buckets.clone(); + spiked[19] = timed(19, 4, 75 * 10 + 40, 40, &["a"]); + let spike = series(&spiked, 8, 4).last().unwrap().block_interval_seconds; + let spike = spike.unwrap(); + assert!(spike > 10.0 && spike < 40.0, "{spike} was not smoothed"); + } + + #[test] + fn a_wide_bucket_is_still_averaged_across_its_neighbours() { + // Above the sample floor the smoothing does not switch off: a bucket of + // a hundred blocks already clears twenty, and a hundred samples of a + // Poisson process still carry ten percent of standard error — which + // across fifty points is a line that moves for no reason. + let mut buckets: Vec = (0..10).map(|i| bucket(i, 100, 10, &["a"])).collect(); + buckets[9] = timed(9, 100, 899 * 10 + 40, 40, &["a"]); + let out = series(&buckets, 4, 100); + let last = out.last().unwrap().block_interval_seconds.unwrap(); + assert!( + last > 10.0 && last < 40.0, + "{last} is the spike undiluted, not a three-bucket mean" + ); + } + + #[test] + fn one_timed_block_spans_no_heights_and_yields_no_interval() { + let mut b = bucket(0, 10, 15, &["a"]); + b.timed_high = b.timed_low; + b.last_authored = b.first_authored; + let out = series(&[b], 1, 10); + assert_eq!(out[0].block_interval_seconds, None); + assert_eq!(out[0].network_hashrate, None); + } + + #[test] + fn an_author_clock_running_backwards_is_skew_not_a_negative_block_time() { + let mut b = bucket(0, 10, 15, &["a"]); + b.last_authored = Some(at(0)); + b.first_authored = Some(at(600)); + assert_eq!(series(&[b], 1, 10)[0].block_interval_seconds, None); + } + + #[test] + fn miners_are_the_rolling_window_count_not_the_bucket_count() { + // Four buckets to a window; a different miner in each, plus one who + // wins in every bucket. Any point with a full window behind it sees + // five, even though no single bucket holds more than two. + let buckets = vec![ + bucket(0, 10, 15, &["always", "w"]), + bucket(1, 10, 15, &["always", "x"]), + bucket(2, 10, 15, &["always", "y"]), + bucket(3, 10, 15, &["always", "z"]), + bucket(4, 10, 15, &["always", "w"]), + bucket(5, 10, 15, &["always", "x"]), + bucket(6, 10, 15, &["always", "y"]), + bucket(7, 10, 15, &["always", "z"]), + ]; + let out = series(&buckets, 4, 10); + assert_eq!(out.len(), 4); + assert!( + out.iter().all(|p| p.miners == Some(5)), + "{:?}", + out.iter().map(|p| p.miners).collect::>() + ); + } + + #[test] + fn a_point_without_a_full_window_behind_it_reports_no_count_rather_than_a_low_one() { + // Five buckets of history, a four-bucket window: the series draws the + // newest four, and only the last two of those have a whole window + // behind them. The rest would undercount, and an undercount drawn as a + // line reads as miners arriving. + let buckets: Vec = (0..5).map(|i| bucket(i, 10, 15, &["a", "b"])).collect(); + let out = series(&buckets, 4, 10); + assert_eq!( + out.iter().map(|p| p.miners).collect::>(), + vec![None, None, Some(2), Some(2)] + ); + } + + #[test] + fn an_outage_is_a_break_in_the_line_not_a_line_across_it() { + // Buckets 2 and 3 were never observed. They must still occupy their + // place on the x-axis, with nothing plotted. + let buckets = vec![ + bucket(0, 10, 15, &["a"]), + bucket(1, 10, 15, &["a"]), + empty(2, 10), + empty(3, 10), + bucket(4, 10, 15, &["a"]), + ]; + let out = series(&buckets, 5, 10); + assert_eq!(out.len(), 5); + assert_eq!(out[2].blocks, 0); + assert_eq!(out[2].difficulty, None); + assert_eq!(out[2].network_hashrate, None); + // The gap keeps its position: heights stay monotonic across it. + assert!(out[1].height < out[2].height && out[2].height < out[4].height); + } + + #[test] + fn a_bucket_missing_from_the_rows_entirely_still_holds_its_place() { + // Postgres returns no row at all for a range with no blocks, which is + // not the same shape as a row of zeroes and must land in the same place. + let buckets = vec![bucket(0, 10, 15, &["a"]), bucket(3, 10, 15, &["a"])]; + let out = series(&buckets, 4, 10); + assert_eq!(out.len(), 4); + assert_eq!(out[1].blocks, 0); + assert_eq!(out[1].height, 19); + assert_eq!(out[3].height, 39); + } + + #[test] + fn leading_emptiness_is_trimmed_rather_than_drawn() { + // Two buckets of history under a window fifty wide: the chart starts + // where the history does. + let buckets = vec![bucket(48, 10, 15, &["a"]), bucket(49, 10, 15, &["a"])]; + let out = series(&buckets, 50, 10); + assert_eq!(out.len(), 2); + assert!(out.iter().all(|p| p.miners.is_none())); + } + + #[test] + fn the_newest_point_is_the_newest_bucket() { + let buckets: Vec = (0..10).map(|i| bucket(i, 10, 15, &["a"])).collect(); + let out = series(&buckets, 4, 10); + assert_eq!(out.last().unwrap().height, 99); + assert_eq!(out.len(), 4); + } +} diff --git a/crates/blackbeard-data/migrations/0002_block_at_tip.sql b/crates/blackbeard-data/migrations/0002_block_at_tip.sql new file mode 100644 index 0000000..1882b19 --- /dev/null +++ b/crates/blackbeard-data/migrations/0002_block_at_tip.sql @@ -0,0 +1,22 @@ +-- Record whether a block's `observed_at` is a tip observation. +-- +-- `RollingWindow::push` has always taken an `at_tip` flag, because the elapsed +-- time between importing two *historical* blocks measures the observer catching +-- up rather than the network producing — divide difficulty by that and the +-- hashrate is wrong by orders of magnitude while still rendering as a number. +-- That flag lived only in memory, so nothing in the database distinguished a +-- block seen as it was mined from one fetched minutes later by a gap fill, and +-- every row in a backfilled batch carries very nearly the same `observed_at`. +-- +-- Any query that treats `observed_at` as timing data therefore needs this +-- column. The chain series does not — it derives block interval from +-- `authored_at`, the chain's own clock, which is immune to when we looked — but +-- the propagation and withholding signal documented on `BlockObservation` is +-- precisely `authored_at` minus `observed_at`, and that difference means +-- nothing at all unless the observation was made at the tip. +-- +-- Existing rows default to false: most of them *were* tip observations, but +-- which ones is not recoverable, and false reads as "not known to be" — the +-- conservative direction for a flag whose whole job is to stop untrustworthy +-- timings being used as if they were measurements. +alter table block add column at_tip boolean not null default false; diff --git a/crates/blackbeard-data/src/rpc.rs b/crates/blackbeard-data/src/rpc.rs index 51abd74..5916ef0 100644 --- a/crates/blackbeard-data/src/rpc.rs +++ b/crates/blackbeard-data/src/rpc.rs @@ -181,17 +181,56 @@ impl RpcClient { } /// Current mining difficulty: expected hashes to win a block. + /// + /// This is the difficulty the **next** block must meet, not the one the tip + /// met — see [`Self::difficulty_for_child_of`] for why, and use that + /// instead whenever the difficulty of a *specific* block is wanted. pub async fn difficulty(&self) -> Result { - self.u512_runtime_call("QPoWApi_get_difficulty").await + self.u512_runtime_call("QPoWApi_get_difficulty", None).await + } + + /// The difficulty the child of `parent_hash` had to meet. + /// + /// `pallet_qpow::on_finalize` retargets at the *end* of every block, so the + /// `CurrentDifficulty` in the state of block *h* is the difficulty for + /// *h+1*. Asking at the latest state and calling the answer "the difficulty + /// of the block we just saw" is therefore off by one block — small, because + /// the retarget is gradual, and invisible, because the number still looks + /// entirely plausible. To record the difficulty of block *h*, ask at its + /// parent, which is the same call the miner made when it built it. + /// + /// Needs the state at `parent_hash`, so a pruning node answers only for the + /// recent past. The caller must treat an error as "not known" rather than + /// substituting the current value, which is how a gap fill would otherwise + /// stamp today's difficulty across a stretch of old heights. + pub async fn difficulty_for_child_of( + &self, + parent_hash: &str, + ) -> Result { + self.u512_runtime_call("QPoWApi_get_difficulty", Some(parent_hash)) + .await } /// The ceiling difficulty can reach. pub async fn max_difficulty(&self) -> Result { - self.u512_runtime_call("QPoWApi_get_max_difficulty").await + self.u512_runtime_call("QPoWApi_get_max_difficulty", None) + .await } - async fn u512_runtime_call(&self, api: &str) -> Result { - let v = self.call("state_call", json!([api, "0x"])).await?; + async fn u512_runtime_call( + &self, + api: &str, + at: Option<&str>, + ) -> Result { + // `state_call` takes the block hash third. Omitting it entirely and + // passing an explicit null are the same thing to the node, but building + // the shorter params list keeps the common call identical to what it + // has always sent. + let params = match at { + Some(hash) => json!([api, "0x", hash]), + None => json!([api, "0x"]), + }; + let v = self.call("state_call", params).await?; let hex = v.as_str().ok_or_else(|| DataError::Rpc { method: api.to_owned(), message: "runtime call did not return a hex string".into(), diff --git a/crates/blackbeard-data/src/store.rs b/crates/blackbeard-data/src/store.rs index 240b3b7..f4d1649 100644 --- a/crates/blackbeard-data/src/store.rs +++ b/crates/blackbeard-data/src/store.rs @@ -27,6 +27,7 @@ use std::time::Duration; use bigdecimal::BigDecimal; use blackbeard_core::attribution::NodeKey; +use blackbeard_core::series::Bucket; use blackbeard_core::window::Observed; use blackbeard_entities::{ChainId, MinerId}; use chrono::{DateTime, Utc}; @@ -80,8 +81,12 @@ pub struct BlockRecord { pub authored_at: Option>, /// When this observer first saw it. pub observed_at: DateTime, - /// Difficulty in force, as a decimal string. + /// Difficulty in force, as a decimal string. `None` where the node could + /// not answer for that height — better than a plausible wrong number. pub difficulty: Option, + /// True when `observed_at` is a tip observation rather than a gap fill. + /// Only then is it a measurement of anything; see `0002_block_at_tip.sql`. + pub at_tip: bool, } /// What the observer knows about a chain, for the `chain` table. @@ -229,20 +234,27 @@ impl Store { .iter() .map(|b| b.difficulty.as_deref().and_then(|d| d.parse().ok())) .collect(); + let at_tip: Vec = blocks.iter().map(|b| b.at_tip).collect(); let result = sqlx::query!( r#" - insert into block (chain, height, hash, miner, authored_at, observed_at, difficulty) + insert into block ( + chain, height, hash, miner, authored_at, observed_at, difficulty, at_tip + ) select * from unnest( $1::text[], $2::bigint[], $3::text[], $4::text[], - $5::timestamptz[], $6::timestamptz[], $7::numeric[] + $5::timestamptz[], $6::timestamptz[], $7::numeric[], $8::boolean[] ) on conflict (chain, height) do update set hash = excluded.hash, miner = excluded.miner, authored_at = excluded.authored_at, observed_at = excluded.observed_at, - difficulty = excluded.difficulty + -- coalesce: a reorg replacement re-observed from history has no + -- difficulty to offer, and dropping the one already recorded for + -- that height would put a hole in the series for no gain. + difficulty = coalesce(excluded.difficulty, block.difficulty), + at_tip = excluded.at_tip "#, &chains, &heights, @@ -251,6 +263,7 @@ impl Store { &authored as &[Option>], &observed, &difficulties as &[Option], + &at_tip, ) .execute(&self.pool) .await?; @@ -375,6 +388,71 @@ impl Store { .collect()) } + /// One bucket of a chain's history: every block in a fixed range of + /// heights, reduced to what the headline numbers are computed from. + /// + /// Deliberately raw. Turning these into hashrates and intervals is + /// arithmetic with two easy mistakes in it, so it happens in + /// `blackbeard-core`/the route where it can be unit-tested, not in SQL. + pub async fn chain_series( + &self, + chain: &ChainId, + bucket_blocks: u32, + buckets: u32, + ) -> Result, DataError> { + let rows = sqlx::query!( + r#" + with bounds as ( + select max(height) as hi from block where chain = $1 + ) + select (b.height / $2)::bigint as "index!", + max(b.height) as "last_height!", + count(*) as "blocks!", + -- Rounded, because difficulty crosses every boundary as an + -- integer decimal string and a mean is the one place it + -- would otherwise acquire a fractional part. + round(avg(b.difficulty))::text as difficulty, + min(b.authored_at) as first_authored, + max(b.authored_at) as last_authored, + min(b.height) filter (where b.authored_at is not null) + as timed_low, + max(b.height) filter (where b.authored_at is not null) + as timed_high, + array_agg(distinct b.miner) as "miners!" + from block b, bounds + where b.chain = $1 + -- Buckets are anchored to absolute height, not to the newest + -- block: the same reason `miner_series` bins from the epoch + -- rather than from the range's start. Anchoring to the tip slides + -- every boundary by one on every block, and the chart shifts + -- under the reader between refreshes. + and b.height >= ((bounds.hi / $2) - $3 + 1) * $2 + group by 1 + order by 1 asc + "#, + chain.as_str(), + i64::from(bucket_blocks), + i64::from(buckets), + ) + .fetch_all(&self.pool) + .await?; + + Ok(rows + .into_iter() + .map(|r| Bucket { + index: r.index, + last_height: r.last_height.max(0) as u64, + blocks: r.blocks.clamp(0, i64::from(u32::MAX)) as u32, + difficulty: r.difficulty, + first_authored: r.first_authored, + last_authored: r.last_authored, + timed_low: r.timed_low.map(|h| h.max(0) as u64), + timed_high: r.timed_high.map(|h| h.max(0) as u64), + miners: r.miners, + }) + .collect()) + } + /// Persist the attributions currently held. pub async fn save_attributions( &self, diff --git a/crates/blackbeard-data/tests/store.rs b/crates/blackbeard-data/tests/store.rs index a64ea93..f80827a 100644 --- a/crates/blackbeard-data/tests/store.rs +++ b/crates/blackbeard-data/tests/store.rs @@ -15,8 +15,9 @@ //! //! These exist because the queries they exercise are the ones a unit test //! cannot reach: an `unnest` batch upsert, the reorg semantics of the primary -//! key, and `date_bin` bucketing. Each has a failure mode that is silent rather -//! than loud — wrong rows, not an error. +//! key, `date_bin` bucketing, and the height bucketing behind the chain series. +//! Each has a failure mode that is silent rather than loud — wrong rows, not an +//! error. use std::time::Duration; @@ -111,6 +112,7 @@ fn block(height: u64, m: u8, secs: i64, difficulty: &str) -> BlockRecord { authored_at: Some(at(secs - 1)), observed_at: at(secs), difficulty: Some(difficulty.to_owned()), + at_tip: true, } } @@ -345,3 +347,130 @@ async fn a_failed_poll_does_not_blank_out_discovered_chain_properties() { f.cleanup().await; } + +#[tokio::test] +async fn the_chain_series_buckets_on_absolute_height() { + // Bucket boundaries are `height / bucket_blocks` on the absolute height, so + // they are the same on every request forever. Anchored to the newest block + // instead they would slide by one on every block, and the chart would shift + // under a reader watching it. + let f = fixture!("series_buckets"); + seed_chain(&f.store).await; + + // Heights 100..=139: four buckets of ten, indices 10 through 13. + let blocks: Vec<_> = (100..140) + .map(|h| block(h, (h % 4) as u8, h as i64 * 15, "1200000000")) + .collect(); + f.store.record_blocks(&blocks).await.unwrap(); + + let buckets = f.store.chain_series(&chain(), 10, 8).await.unwrap(); + assert_eq!( + buckets.iter().map(|b| b.index).collect::>(), + vec![10, 11, 12, 13] + ); + assert!(buckets.iter().all(|b| b.blocks == 10)); + assert_eq!(buckets[0].last_height, 109); + assert_eq!(buckets[3].last_height, 139); + + // Difficulty is a mean, carried as an integer decimal string — never a + // float, and never with the fractional part `avg` would otherwise give it. + assert_eq!(buckets[0].difficulty.as_deref(), Some("1200000000")); + + // Authors come back as sets so the caller can union adjacent buckets; four + // rotating miners over ten blocks is four distinct, not ten. + let mut miners = buckets[0].miners.clone(); + miners.sort(); + assert_eq!(miners.len(), 4); + + // Timing bounds are the author timestamps and the heights that carry them, + // which is what makes the interval a height difference rather than a + // sample count. + assert_eq!(buckets[0].timed_low, Some(100)); + assert_eq!(buckets[0].timed_high, Some(109)); + assert_eq!( + (buckets[0].last_authored.unwrap() - buckets[0].first_authored.unwrap()).num_seconds(), + 9 * 15 + ); + + f.cleanup().await; +} + +#[tokio::test] +async fn a_bucket_range_the_observer_missed_is_absent_rather_than_empty() { + // Postgres returns no row at all for a height range with no blocks. The + // series builder puts the point back; what matters here is that the query + // does not invent one, and that the window is counted back from the tip in + // buckets rather than in rows. + let f = fixture!("series_gap"); + seed_chain(&f.store).await; + + let mut blocks: Vec<_> = (0..10) + .map(|h| block(h, 1, h as i64 * 15, "1000")) + .collect(); + // Nothing in 10..29 — two whole buckets of outage. + blocks.extend((30..40).map(|h| block(h, 2, h as i64 * 15, "1000"))); + f.store.record_blocks(&blocks).await.unwrap(); + + let buckets = f.store.chain_series(&chain(), 10, 4).await.unwrap(); + assert_eq!( + buckets.iter().map(|b| b.index).collect::>(), + vec![0, 3], + "the missing buckets are absent, not zero-filled" + ); + + f.cleanup().await; +} + +#[tokio::test] +async fn a_block_with_no_difficulty_leaves_the_bucket_mean_to_the_rest() { + // A gap fill against a pruning node cannot learn the difficulty of an old + // block and records `None`. That must not poison the bucket: the mean is + // over the blocks that have one, and only a bucket with none at all is + // itself unknown. + let f = fixture!("series_null_difficulty"); + seed_chain(&f.store).await; + + let mut blocks: Vec<_> = (0..10) + .map(|h| block(h, 1, h as i64 * 15, "1000")) + .collect(); + for b in blocks.iter_mut().take(5) { + b.difficulty = None; + } + // A whole bucket with nothing known. + blocks.extend((10..20).map(|h| { + let mut b = block(h, 1, h as i64 * 15, "1000"); + b.difficulty = None; + b + })); + f.store.record_blocks(&blocks).await.unwrap(); + + let buckets = f.store.chain_series(&chain(), 10, 4).await.unwrap(); + assert_eq!(buckets[0].difficulty.as_deref(), Some("1000")); + assert_eq!(buckets[1].difficulty, None); + + f.cleanup().await; +} + +#[tokio::test] +async fn re_recording_a_block_without_a_difficulty_keeps_the_one_already_known() { + // A reorg replacement fetched from history has no difficulty to offer. The + // upsert coalesces rather than overwriting, because a hole in the series is + // a worse answer than the figure recorded when the height was at the tip. + let f = fixture!("series_difficulty_kept"); + seed_chain(&f.store).await; + + f.store + .record_blocks(&[block(7, 1, 100, "1234")]) + .await + .unwrap(); + let mut replacement = block(7, 2, 101, "1234"); + replacement.difficulty = None; + replacement.at_tip = false; + f.store.record_blocks(&[replacement]).await.unwrap(); + + let buckets = f.store.chain_series(&chain(), 10, 4).await.unwrap(); + assert_eq!(buckets[0].difficulty.as_deref(), Some("1234")); + assert_eq!(buckets[0].miners, vec![miner(2).0], "the reorg still won"); + + f.cleanup().await; +} diff --git a/crates/blackbeard-entities/src/lib.rs b/crates/blackbeard-entities/src/lib.rs index aa56885..8f5db70 100644 --- a/crates/blackbeard-entities/src/lib.rs +++ b/crates/blackbeard-entities/src/lib.rs @@ -19,12 +19,14 @@ mod block; mod chain; mod error; mod miner; +mod series; mod ws; pub use block::{BlockObservation, RecentBlock}; pub use chain::{ChainId, ChainInfo, ChainStatus, ChainSummary, ClientVersion, Tracking}; pub use error::{ApiError, EntityError}; pub use miner::{AttributionSource, LeaderboardRow, MinerDetail, MinerId, MinerSeriesPoint}; +pub use series::{ChainSeries, ChainSeriesPoint}; pub use ws::{ClientMessage, ServerMessage, Window}; use serde::{Deserialize, Serialize}; diff --git a/crates/blackbeard-entities/src/series.rs b/crates/blackbeard-entities/src/series.rs new file mode 100644 index 0000000..c706104 --- /dev/null +++ b/crates/blackbeard-entities/src/series.rs @@ -0,0 +1,74 @@ +//! The history behind the headline numbers. +//! +//! Every tile at the top of the page shows one current value. This is the same +//! four values over the recent past, at the resolution a sparkline can draw — +//! enough to answer "is this rising or falling", which a single number never +//! can. +//! +//! ## Every field is optional, and that is the point +//! +//! A point is not a promise that the observer knew the value at that height. A +//! stretch of blocks fetched from history has no trustworthy timing; a node +//! that has pruned its state cannot say what the difficulty was; a rolling +//! count needs a full window behind it before it means anything. Each of those +//! is a `None`, drawn as a break in the line, rather than a zero or a +//! carried-forward value — a chart that interpolates across what it does not +//! know is the same lie as a hashrate divided by a sync speed, and just as hard +//! to spot afterwards. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::{BigUintDec, ChainId, Window}; + +/// One bucket of blocks, reduced to the four plotted values. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] +#[ts(export, export_to = "ChainSeriesPoint.ts")] +pub struct ChainSeriesPoint { + /// Highest block height in the bucket. The x-axis is height, not time: + /// buckets are block counts, so their spacing in blocks is exact and their + /// spacing in time is whatever the chain actually did. + #[ts(type = "number")] + pub height: u64, + /// When the bucket's newest block was authored, for a human-readable + /// position. From the chain's own timestamp inherent, so it is the block's + /// time rather than the time this observer happened to look. + pub at: Option>, + /// Blocks recorded in the bucket, out of the bucket's full width. Below it + /// for the newest bucket, which is still filling, and anywhere the observer + /// was not watching. + pub blocks: u32, + /// Mean difficulty across the bucket: expected hashes per block. + pub difficulty: Option, + /// Mean seconds per block across the bucket, from author timestamps. + pub block_interval_seconds: Option, + /// Mean difficulty over mean interval. Never falls back to the configured + /// target the way the live tile may: a nominal figure can be labelled + /// nominal beside a number, but not at one point of sixty in a sparkline. + pub network_hashrate: Option, + /// Distinct authors over the **full window ending here** — the same + /// statistic the tile shows, not a count within the bucket, so the last + /// point of the line is the number printed above it. `None` until a full + /// window of history precedes the point, rather than a count that ramps up + /// from nothing and reads as miners arriving. + pub miners: Option, +} + +/// A chain's recent history, one series per headline tile. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] +#[ts(export, export_to = "ChainSeries.ts")] +pub struct ChainSeries { + /// Which chain. + pub chain: ChainId, + /// The window these points span, matching the leaderboard's. + pub window: Window, + /// The window's length in blocks. + pub window_blocks: u32, + /// Blocks per point. The window divided by the number of points, exactly — + /// every selectable window is a multiple of the point count. + pub bucket_blocks: u32, + /// Oldest first. Fewer than the full count before the observer has watched + /// a whole window. + pub points: Vec, +} diff --git a/readme.md b/readme.md index 1902886..1f1c27a 100644 --- a/readme.md +++ b/readme.md @@ -115,6 +115,23 @@ million blocks. The same reasoning is why a *measured* block interval is only used when the node is at the tip; while syncing, the configured target is used and every hashrate on the page is labelled **nominal**. +The sparklines under the headline tiles hold the same line. They are bucketed +by **height**, not by time: every point is the same number of blocks, so every +point carries the same statistical weight and a stretch where nothing was +recorded stays the same width on the axis as one where everything was. A bucket +is the window divided by fifty, which is why every window is a multiple of it — +a full window is then exactly fifty buckets, and the rolling miner count at the +last point is the same number printed in the tile above it. + +Their block interval comes from the author's own timestamp inherent rather than +from when this observer saw the block, for the same reason the *measured* +interval only counts tip observations: a batch of blocks fetched from history +all carry very nearly the same observation time, and dividing by that reads as +a chain producing hundreds of blocks a second. Each point is a moving average +over three buckets, because the mean of one bucket's gaps still carries enough +Poisson noise to draw a chain that looks like it is changing size every few +minutes. + ## Data Postgres on `magrathea.kosherinata.internal`, mTLS and passwordless — the host's @@ -234,6 +251,10 @@ a `probe` run. of blocks my hashrate predicts?" honestly needs a hashrate the observer cannot see. `lair/quantus` issue #1 tracks the same gap from the exporter side. - **A telemetry name is an inference.** See *Names* above. +- **History has holes, and shows them.** Difficulty is a state read, so a + pruning node cannot say what it was at a height fetched long after the fact, + and a stretch the observer was not running for was never recorded at all. + Both are gaps in the sparkline rather than a line drawn across them. ## Related diff --git a/web/src/api/generated/ChainSeries.ts b/web/src/api/generated/ChainSeries.ts new file mode 100644 index 0000000..57c447d --- /dev/null +++ b/web/src/api/generated/ChainSeries.ts @@ -0,0 +1,31 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ChainId } from "./ChainId"; +import type { ChainSeriesPoint } from "./ChainSeriesPoint"; +import type { Window } from "./Window"; + +/** + * A chain's recent history, one series per headline tile. + */ +export type ChainSeries = { +/** + * Which chain. + */ +chain: ChainId, +/** + * The window these points span, matching the leaderboard's. + */ +window: Window, +/** + * The window's length in blocks. + */ +window_blocks: number, +/** + * Blocks per point. The window divided by the number of points, exactly — + * every selectable window is a multiple of the point count. + */ +bucket_blocks: number, +/** + * Oldest first. Fewer than the full count before the observer has watched + * a whole window. + */ +points: Array, }; diff --git a/web/src/api/generated/ChainSeriesPoint.ts b/web/src/api/generated/ChainSeriesPoint.ts new file mode 100644 index 0000000..4ec80b6 --- /dev/null +++ b/web/src/api/generated/ChainSeriesPoint.ts @@ -0,0 +1,47 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BigUintDec } from "./BigUintDec"; + +/** + * One bucket of blocks, reduced to the four plotted values. + */ +export type ChainSeriesPoint = { +/** + * Highest block height in the bucket. The x-axis is height, not time: + * buckets are block counts, so their spacing in blocks is exact and their + * spacing in time is whatever the chain actually did. + */ +height: number, +/** + * When the bucket's newest block was authored, for a human-readable + * position. From the chain's own timestamp inherent, so it is the block's + * time rather than the time this observer happened to look. + */ +at: string | null, +/** + * Blocks recorded in the bucket, out of the bucket's full width. Below it + * for the newest bucket, which is still filling, and anywhere the observer + * was not watching. + */ +blocks: number, +/** + * Mean difficulty across the bucket: expected hashes per block. + */ +difficulty: BigUintDec | null, +/** + * Mean seconds per block across the bucket, from author timestamps. + */ +block_interval_seconds: number | null, +/** + * Mean difficulty over mean interval. Never falls back to the configured + * target the way the live tile may: a nominal figure can be labelled + * nominal beside a number, but not at one point of sixty in a sparkline. + */ +network_hashrate: number | null, +/** + * Distinct authors over the **full window ending here** — the same + * statistic the tile shows, not a count within the bucket, so the last + * point of the line is the number printed above it. `None` until a full + * window of history precedes the point, rather than a count that ramps up + * from nothing and reads as miners arriving. + */ +miners: number | null, }; diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index 7ec9b00..910cb89 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -7,6 +7,7 @@ */ import type { ApiError } from './generated/ApiError' +import type { ChainSeries } from './generated/ChainSeries' import type { MinerDetail } from './generated/MinerDetail' import type { Window as WindowName } from './generated/Window' @@ -39,6 +40,25 @@ async function get(path: string, signal?: AbortSignal): Promise { return (await response.json()) as T } +/** + * The history behind the headline tiles. + * + * A query rather than a stream, like a miner's page: pushing fifty buckets to + * every connected tab on every block would be the whole page's bandwidth spent + * on a line whose newest point moves once a bucket. The caller refetches when + * the chain crosses a bucket boundary, which is the only moment it changes. + */ +export function fetchSeries( + chain: string, + windowName: WindowName, + signal?: AbortSignal, +): Promise { + return get( + `/chains/${encodeURIComponent(chain)}/series?window=${windowName}`, + signal, + ) +} + /** One miner's standing and history. */ export function fetchMiner( chain: string, diff --git a/web/src/components/Sparkline.tsx b/web/src/components/Sparkline.tsx new file mode 100644 index 0000000..fe28532 --- /dev/null +++ b/web/src/components/Sparkline.tsx @@ -0,0 +1,179 @@ +/** + * The trend under a stat tile. + * + * A stat tile answers "what is it now" and cannot answer "and is that high". + * The sparkline is the smallest mark that can: one series, one hue, no axes, no + * legend — the tile's label already names what is plotted and the tile's value + * already prints where the line ends. + * + * Three things it deliberately does not do: + * + * **It does not start at zero.** A sparkline is read for shape, not magnitude, + * and a hashrate that has moved 3% would be a flat line on a zero baseline. The + * band is the window's own min to max, which is why the accessible label and + * the hover readout both carry real numbers — the height of the line is not a + * quantity a reader should be estimating. + * + * **It does not join across a gap.** A `null` is a value this observer does not + * know — an outage, a pruned difficulty, a window not yet full — and a line + * drawn across it would assert history that was never recorded. Each run of + * known values is its own path. + * + * **It does not introduce a second colour.** Bronze and crimson are adjacent + * hues that fail CVD separation as a categorical pair (see `index.css`), so the + * "current period in the accent" convention would be exactly the change that + * breaks this palette. Emphasis on the newest point is a lighter step of the + * same hue instead. + * + * The SVG is measured rather than scaled to fit: one unit stays one CSS pixel, + * so the 2px stroke is 2px on every screen and the crosshair lands under the + * pointer instead of near it. + */ + +import { useCallback, useEffect, useRef, useState } from 'react' + +const HEIGHT = 30 +/** + * Room for the terminal marker and the stroke's round cap at all four edges. + * Without the horizontal half the newest point — the one the eye goes to — is + * sliced in two by the viewport boundary. + */ +const PAD = 4 +/** Used until the container is measured, and if ResizeObserver is absent. */ +const FALLBACK_WIDTH = 140 + +/** One plotted value and how to say it out loud. */ +export interface SparkPoint { + /** `null` where the observer has no figure for this bucket. */ + value: number | null + /** The readout for this point: a formatted value, and when it was. */ + label: string +} + +function useMeasuredWidth(): [React.RefObject, number] { + const ref = useRef(null) + const [width, setWidth] = useState(FALLBACK_WIDTH) + useEffect(() => { + const node = ref.current + if (!node) return + const observer = new ResizeObserver(([entry]) => { + if (entry) setWidth(Math.max(60, entry.contentRect.width)) + }) + observer.observe(node) + return () => observer.disconnect() + }, []) + return [ref, width] +} + +export function Sparkline({ + points, + label, + onProbe, +}: { + points: SparkPoint[] + /** What the series is, for the accessible name. The tile's label. */ + label: string + /** Called with the hovered or focused index, and `null` on leaving. */ + onProbe: (index: number | null) => void +}) { + const [box, width] = useMeasuredWidth() + const [active, setActive] = useState(null) + + const probe = useCallback( + (index: number | null) => { + setActive(index) + onProbe(index) + }, + [onProbe], + ) + + const known = points.filter((p) => p.value !== null) + const values = known.map((p) => p.value as number) + const low = Math.min(...values) + const high = Math.max(...values) + const span = high - low + const plot = Math.max(1, width - PAD * 2) + const x = (i: number) => + points.length > 1 ? PAD + (i / (points.length - 1)) * plot : PAD + plot / 2 + // A series that never moved sits on the centre line rather than on the floor, + // where a flat line reads as "zero" instead of "unchanged". + const y = (v: number) => + span === 0 ? HEIGHT / 2 : HEIGHT - PAD - ((v - low) / span) * (HEIGHT - PAD * 2) + + // Each run of known values is its own path, so a gap stays a gap. + const runs: string[] = [] + let run: string[] = [] + points.forEach((p, i) => { + if (p.value === null) { + if (run.length > 1) runs.push(run.join(' ')) + run = [] + return + } + run.push(`${run.length === 0 ? 'M' : 'L'}${x(i).toFixed(1)} ${y(p.value).toFixed(1)}`) + }) + if (run.length > 1) runs.push(run.join(' ')) + + const lastKnown = points.reduce((found, p, i) => (p.value === null ? found : i), -1) + const marker = active !== null && points[active]?.value != null ? active : lastKnown + + const onPointer = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect() + if (rect.width === 0) return + // Inverts the same inset the plot uses, so the crosshair lands on the point + // under the pointer rather than one over near the ends. + const fraction = (event.clientX - rect.left - PAD) / Math.max(1, rect.width - PAD * 2) + const index = Math.round(fraction * (points.length - 1)) + probe(Math.min(points.length - 1, Math.max(0, index))) + } + + const onKey = (event: React.KeyboardEvent) => { + if (event.key !== 'ArrowLeft' && event.key !== 'ArrowRight') return + event.preventDefault() + const from = active ?? points.length - 1 + probe(Math.min(points.length - 1, Math.max(0, from + (event.key === 'ArrowLeft' ? -1 : 1)))) + } + + // One point is a dot, not a trend, and no points is nothing at all. Both + // still occupy the slot, so the tiles in a row keep their baselines aligned — + // and the slot keeps the ref, so a later refetch that does have a series is + // measured rather than stuck on the fallback width. + return ( +
+ {known.length >= 2 && ( + probe(null)} + onFocus={() => probe(points.length - 1)} + onBlur={() => probe(null)} + onKeyDown={onKey} + > + {active !== null && ( + + )} + {runs.map((d) => ( + + ))} + {marker >= 0 && points[marker]?.value != null && ( + + )} + + )} +
+ ) +} diff --git a/web/src/components/StatBar.tsx b/web/src/components/StatBar.tsx index f5f8747..f0ad803 100644 --- a/web/src/components/StatBar.tsx +++ b/web/src/components/StatBar.tsx @@ -5,11 +5,34 @@ * and a bar chart of five unrelated quantities would be a chart of nothing. * Network hashrate leads as the hero figure because it is the one number that * answers "how hard is this to win right now". + * + * Under four of them sits the same value's recent history, because a number on + * its own cannot say whether it is high. Height gets none: it only ever goes + * up, and a straight diagonal is a line with nothing to report. + * + * The series is fetched rather than pushed, and refetched only when the chain + * crosses a bucket boundary — the one moment a point can change. Everything + * else on this page arrives on the socket; fifty buckets per block to every + * open tab would not. */ +import { useCallback, useEffect, useMemo, useState } from 'react' + +import type { ChainSeries } from '../api/generated/ChainSeries' +import type { ChainSeriesPoint } from '../api/generated/ChainSeriesPoint' import type { ChainSummary } from '../api/generated/ChainSummary' import type { Window as WindowName } from '../api/generated/Window' +import { fetchSeries } from '../api/rest' import { bigNumber, hashrate, height, seconds, windowSpan } from '../lib/format' +import { Sparkline, type SparkPoint } from './Sparkline' + +/** Kept in step with `Window::blocks()` in the entities crate. */ +const WINDOW_BLOCKS: Record = { + hour: 600, + six_hours: 3600, + day: 14400, + week: 100800, +} function Stat({ label, @@ -18,6 +41,7 @@ function Stat({ hero, noteClass, title, + trend, }: { label: string value: string @@ -25,6 +49,7 @@ function Stat({ hero?: boolean noteClass?: string title?: string + trend?: React.ReactNode }) { return (
@@ -33,10 +58,65 @@ function Stat({ {value}
{note ?? ''}
+ {trend ??
}
) } +/** + * A tile whose note doubles as the sparkline's readout. + * + * A floating tooltip per tile would put five overlays in a row a hundred and + * forty pixels wide. Borrowing the caption line instead keeps the number in the + * place the reader is already looking, costs no layout, and falls back to the + * caption the moment the pointer leaves. + */ +function TrendStat({ + label, + value, + note, + hero, + noteClass, + title, + points, +}: { + label: string + value: string + note: string + hero?: boolean + noteClass?: string + title?: string + points: SparkPoint[] | null +}) { + const [probed, setProbed] = useState(null) + const onProbe = useCallback((index: number | null) => setProbed(index), []) + const reading = probed !== null ? points?.[probed]?.label : undefined + + return ( + } + : {})} + /> + ) +} + +/** When a point was, in the shortest form that still places it. */ +function when(at: string | null, spanSeconds: number): string { + if (!at) return '' + return new Date(at).toLocaleString(undefined, { + ...(spanSeconds > 86_400 ? { month: 'short', day: 'numeric' } : {}), + hour: '2-digit', + minute: '2-digit', + }) +} + export function StatBar({ summary, windowName, @@ -44,6 +124,77 @@ export function StatBar({ summary: ChainSummary | null windowName: WindowName }) { + const chain = summary?.chain ?? null + const windowBlocks = WINDOW_BLOCKS[windowName] + const bucketBlocks = windowBlocks / 50 + // The identity of the newest bucket. A new block only changes the chart when + // it opens a new one, so this is the whole refetch condition. + const bucket = Math.floor((summary?.height ?? 0) / bucketBlocks) + const [series, setSeries] = useState(null) + + useEffect(() => { + if (!chain) return + const controller = new AbortController() + fetchSeries(chain, windowName, controller.signal) + .then(setSeries) + .catch(() => { + // The tiles are the point and they come from the socket; history that + // will not load is a missing sparkline, never a broken page. + if (!controller.signal.aborted) setSeries(null) + }) + return () => controller.abort() + }, [chain, windowName, bucket]) + + const interval = summary + ? (summary.block_interval_seconds ?? summary.target_block_time_seconds) + : 6 + const spanSeconds = windowBlocks * interval + + // One pass over the points per tile, memoised: these rebuild only when the + // series does, not on every hover. + const trends = useMemo(() => { + const points = series?.points ?? null + if (!points || points.length === 0) return null + const build = ( + pick: (p: ChainSeriesPoint) => number | null, + say: (v: number) => string, + ): SparkPoint[] => + points.map((p) => { + const value = pick(p) + const at = when(p.at, spanSeconds) + return { + value, + label: value === null ? 'no data' : at ? `${say(value)} · ${at}` : say(value), + } + }) + + return { + hashrate: build( + (p) => p.network_hashrate, + (v) => hashrate(v), + ), + difficulty: build( + (p) => { + if (!p.difficulty) return null + // Difficulty crosses the wire as a decimal string so it is never + // rounded in transit. Plotting is the one place rounding it is + // right: a sparkline is a shape, and a float carries the magnitude. + const n = Number(p.difficulty) + return Number.isFinite(n) ? n : null + }, + (v) => bigNumber(BigInt(Math.round(v)).toString()), + ), + interval: build( + (p) => p.block_interval_seconds, + (v) => seconds(v), + ), + miners: build( + (p) => p.miners, + (v) => `${v} miner${v === 1 ? '' : 's'}`, + ), + } + }, [series, spanSeconds]) + if (!summary) { return (
@@ -54,12 +205,9 @@ export function StatBar({ ) } - const interval = summary.block_interval_seconds ?? summary.target_block_time_seconds - const windowBlocks = { hour: 600, six_hours: 3600, day: 14400, week: 100800 }[windowName] - return (
- - - -
) diff --git a/web/src/index.css b/web/src/index.css index 3539ed0..b60cfee 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -415,6 +415,60 @@ a { color: var(--warn); } +/* The note line doubles as the sparkline's readout, so a hover changes its + text. Brightening it says the line is being read rather than captioned — + without moving anything, which at this size would be a flicker. */ +.stat-note.reading { + color: var(--text-secondary); + font-variant-numeric: tabular-nums; +} + +/* ---- sparklines ---------------------------------------------------------- */ + +/* Reserved on every tile, drawn on four: the fifth (height) has nothing worth + plotting, and an absent slot there would leave one tile's baselines out of + step with the rest of the row. */ +.spark { + height: 30px; + margin-top: 9px; +} + +.spark-svg { + display: block; + touch-action: none; +} + +.spark-svg:focus-visible { + outline: 2px solid var(--data-bright); + outline-offset: 2px; +} + +/* Single series, single hue — the same bronze every other mark on the site + uses. The palette's second colour is reserved for identity and alarm, and as + an adjacent hue it would not survive CVD as a second series here. */ +.spark-line { + fill: none; + stroke: var(--data); + stroke-width: 2; + stroke-linejoin: round; + stroke-linecap: round; +} + +/* Emphasis on the newest point, as a lighter step rather than a second hue. + `--data-bright` sits above the validator's categorical lightness band (L + 0.762 against a 0.48–0.67 ceiling) — that check governs slots a reader has to + tell apart from each other, which a single-hue emphasis step is not. It is + the same token the hero figure already wears, and it clears contrast against + the tile surface. Do not "fix" it by giving the marker the accent colour. */ +.spark-marker { + fill: var(--data-bright); +} + +.spark-crosshair { + stroke: var(--border-strong); + stroke-width: 1; +} + /* ---- leaderboard --------------------------------------------------------- */ .board {