diff --git a/CLAUDE.md b/CLAUDE.md index ade5c1f..2397cdc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,6 +179,25 @@ height. `lower_first_seen` uses `least`, so a zero left in place would win forever and claim every runtime began at block 0 — it special-cases zero for that reason. Do not simplify it back to a plain `least`. +**A wrong storage hasher reads as absent, not as an error.** A key is +`twox128(pallet prefix) ++ twox128(item) ++ hashed keys`, and each key's hash is +declared per entry — `System::Account` is `Blake2_128Concat`, other maps are +`Twox64Concat`. Get it wrong and `state_getStorage` returns null, which is +indistinguishable from an empty entry. `Runtime::storage_key` therefore takes +every part from the metadata, including `PalletStorageMetadata.prefix`, which is +usually the pallet name and is **not required to be**. Its test pins two keys +against ones read off the live chain by hand; keep it that way, because nothing +else would catch a mistake here. + +**Absent is not zero, and only the modifier knows which.** An `optional` entry +holding nothing means nothing. A `default` entry holding nothing means the +runtime's default, which is a value the chain never wrote. `StorageTarget` +carries the default so a caller can tell them apart, and `read_balance` +deliberately does *not* apply it: `AccountInfo` defaults to a zeroed struct, and +Substrate reaps empty accounts, so falling back would turn "this account does +not exist" into "this account holds nothing". The treasury address on mainnet is +exactly that case. + ## Facts established by measurement Taken from the live **Planck** chain, 2026-09-04. Don't re-derive or contradict diff --git a/Cargo.lock b/Cargo.lock index 767a500..49cc10a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,6 +252,7 @@ dependencies = [ "futures-util", "hex", "primitive-types", + "scale-value", "serde", "serde_json", "sqlx", @@ -295,6 +296,7 @@ dependencies = [ "serde_json", "thiserror 2.0.20", "tracing", + "twox-hash", ] [[package]] @@ -3039,6 +3041,12 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + [[package]] name = "typenum" version = "1.20.1" diff --git a/Cargo.toml b/Cargo.toml index ed3c5a8..5c8dd65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ scale-value = { version = "0.18", default-features = false } qp-poseidon-core = { version = "3.1.0", default-features = false } # SS58 checksums only. Base58 is hand-rolled; this is not. blake2 = { version = "0.10", default-features = false } +twox-hash = { version = "2", default-features = false, features = ["xxhash64"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } serde = { version = "1", features = ["derive"] } diff --git a/crates/blackbeard-api/Cargo.toml b/crates/blackbeard-api/Cargo.toml index 299ff34..7efae0f 100644 --- a/crates/blackbeard-api/Cargo.toml +++ b/crates/blackbeard-api/Cargo.toml @@ -25,6 +25,7 @@ figment.workspace = true futures-util.workspace = true hex.workspace = true primitive-types.workspace = true +scale-value.workspace = true serde.workspace = true sqlx.workspace = true serde_json.workspace = true diff --git a/crates/blackbeard-api/src/routes.rs b/crates/blackbeard-api/src/routes.rs index 6dee98c..365d4e8 100644 --- a/crates/blackbeard-api/src/routes.rs +++ b/crates/blackbeard-api/src/routes.rs @@ -17,10 +17,10 @@ use axum::routing::get; use axum::{Json, Router}; use blackbeard_entities::{ AccountDetail, AccountEvent, AccountRow, ActivitySource, ApiError, BigUintDec, BlockDetail, - CallIndex, CallSummary, ChainInfo, ChainSeries, ChainSummary, EventSummary, LeaderboardRow, - MinerDetail, MinerId, MinerSeriesPoint, RecentBlock, RewardSummary, RuntimeConstant, - RuntimeDetail, RuntimeField, RuntimePallet, RuntimeSignedExtension, RuntimeStorage, - RuntimeSummary, RuntimeVariant, Window, + CallIndex, CallSummary, ChainInfo, ChainSeries, ChainState, ChainSummary, EventSummary, + LeaderboardRow, MinerDetail, MinerId, MinerSeriesPoint, RecentBlock, RewardSummary, + RuntimeConstant, RuntimeDetail, RuntimeField, RuntimePallet, RuntimeSignedExtension, + RuntimeStorage, RuntimeSummary, RuntimeVariant, StateEntry, Window, }; use serde::{Deserialize, Serialize}; use tower_http::compression::CompressionLayer; @@ -61,6 +61,7 @@ pub fn router(state: AppState, allowed_origins: &[String]) -> Router { "/v1/chains/{chain}/events/{pallet}/{variant}", get(event_feed), ) + .route("/v1/chains/{chain}/state", get(chain_state)) .route("/v1/chains/{chain}/runtimes", get(runtimes)) .route("/v1/chains/{chain}/runtimes/{spec}", get(runtime)) .route("/v1/ws", get(crate::ws::handler)) @@ -736,6 +737,104 @@ async fn accounts( )) } +/// Every storage entry readable without a key, decoded. +/// +/// The half of a chain that history cannot reach. This chain's treasury address +/// was set at genesis, so no extrinsic ever carried it and no event ever +/// announced it — it exists only here, and until the observer could read state +/// the honest answer to "which address is the treasury" was that we did not +/// know. +/// +/// Read live rather than indexed. State is what is true *now*, and a cached +/// copy of now is a copy of some earlier now that looks identical. +async fn chain_state( + State(state): State, + Path(chain): Path, +) -> Result, Failure> { + let runtime = state.chain(&chain).ok_or_else(|| unknown_chain(&chain))?; + let (parsed, spec_version, height) = { + let inner = runtime.read(); + (inner.current_runtime(), inner.spec_version, inner.height) + }; + let Some(parsed) = parsed else { + return Err(Failure( + StatusCode::SERVICE_UNAVAILABLE, + ApiError::new( + "runtime_unknown", + "no runtime metadata cached for this chain yet", + ), + )); + }; + + let described = parsed.describe(); + let mut entries = Vec::new(); + for (pallet, name) in parsed.plain_storage() { + // Shape and documentation from the description already computed, so the + // page reads the same as the runtime page it mirrors. + let declared = described + .pallets + .iter() + .find(|p| p.name == pallet) + .and_then(|p| p.storage.iter().find(|s| s.name == name)); + let (type_name, modifier, docs) = match declared { + Some(d) => ( + d.shape.clone(), + d.modifier.clone(), + d.docs + .iter() + .find(|x| !x.trim().is_empty()) + .map(|x| x.trim().to_owned()), + ), + None => (String::new(), String::new(), None), + }; + + let mut entry = StateEntry { + pallet: pallet.clone(), + name: name.clone(), + type_name, + modifier, + docs, + value: None, + is_default: false, + error: None, + }; + + match parsed.storage_key(&pallet, &name, &[]) { + Ok(target) => { + let key = format!("0x{}", hex::encode(&target.key)); + match runtime.rpc.storage(&key, None).await { + Ok(Some(raw)) => match parsed.decode_storage(&target, &raw) { + Ok(v) => entry.value = Some(v), + Err(e) => entry.error = Some(e.to_string()), + }, + Ok(None) => { + // Nothing stored. For a `default` entry the runtime + // supplies a value; for an `optional` one, absent is + // the answer and must not be dressed as a zero. + entry.is_default = true; + if let Some(default) = &target.default { + match parsed.decode_storage(&target, default) { + Ok(v) => entry.value = Some(v), + Err(e) => entry.error = Some(e.to_string()), + } + } + } + Err(e) => entry.error = Some(e.to_string()), + } + } + Err(e) => entry.error = Some(e.to_string()), + } + entries.push(entry); + } + + Ok(Json(ChainState { + chain: runtime.id(), + spec_version, + height, + entries, + })) +} + /// Every dispatchable the runtime declares, with what has been done with it. /// /// The declared surface comes from the metadata already parsed for decoding; @@ -1211,6 +1310,8 @@ async fn account( .await .map_err(database_unavailable)?; + let (balance, account_state) = read_balance(&runtime, &account).await; + // The name, if telemetry has resolved one, through exactly the path the // leaderboard uses — so an account page and a standings row cannot // disagree about who someone is. Only a real name: `attribute` falls back @@ -1233,6 +1334,8 @@ async fn account( account, miner, display, + balance, + state: account_state, rewards: RewardSummary { count: totals.count, total: BigUintDec(totals.total), @@ -1274,6 +1377,62 @@ async fn account( })) } +/// An account's `System::Account` entry, and its free balance out of it. +/// +/// Live from state, not summed from events: those are different numbers and +/// only one of them is a balance. An account's rewards say what it was paid; a +/// balance says what it has, and they differ by every transfer out. +/// +/// **Absent is not zero.** Substrate reaps an account with nothing in it, so a +/// missing entry means the account does not exist rather than that it holds +/// nothing — and the two read very differently for an address like this chain's +/// treasury, which has never been funded. `None` rather than `0`. +/// +/// The one named field lookup in the whole decoding path is `data.free`, and it +/// is here rather than in `blackbeard-core` for exactly that reason: the core +/// knows nothing about what any field means, and this is a presentation +/// convenience for the number a reader came for. The full entry rides alongside +/// it so the convenience is checkable. +async fn read_balance( + runtime: &crate::state::ChainRuntime, + account_hex: &str, +) -> (Option, Option) { + let Some(parsed) = runtime.read().current_runtime() else { + return (None, None); + }; + let Ok(bytes) = hex::decode(account_hex.trim_start_matches("0x")) else { + return (None, None); + }; + let Ok(target) = parsed.storage_key( + "System", + "Account", + &[scale_value::Value::from_bytes(&bytes)], + ) else { + return (None, None); + }; + let key = format!("0x{}", hex::encode(&target.key)); + // Deliberately not falling back to the entry's default. `AccountInfo` + // defaults to a zeroed struct, and rendering that would turn "this account + // does not exist" into "this account holds nothing", which is a claim about + // a different thing. + let Ok(Some(raw)) = runtime.rpc.storage(&key, None).await else { + return (None, None); + }; + let Ok(value) = parsed.decode_storage(&target, &raw) else { + return (None, None); + }; + let free = value + .get("data") + .and_then(|d| d.get("free")) + .and_then(|f| match f { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Number(n) => Some(n.to_string()), + _ => None, + }) + .map(BigUintDec); + (free, Some(value)) +} + /// An SS58 address for this network, as the raw account id. /// /// Validated here rather than passed through, for the same reason a preimage diff --git a/crates/blackbeard-core/Cargo.toml b/crates/blackbeard-core/Cargo.toml index 9af642f..7b95069 100644 --- a/crates/blackbeard-core/Cargo.toml +++ b/crates/blackbeard-core/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true blackbeard-entities.workspace = true chrono.workspace = true blake2.workspace = true +twox-hash.workspace = true frame-metadata.workspace = true hex.workspace = true parity-scale-codec.workspace = true diff --git a/crates/blackbeard-core/src/runtime.rs b/crates/blackbeard-core/src/runtime.rs index 688181c..0399079 100644 --- a/crates/blackbeard-core/src/runtime.rs +++ b/crates/blackbeard-core/src/runtime.rs @@ -53,6 +53,9 @@ pub enum RuntimeError { /// A value did not match the type the registry said it would. #[error("decoding against the registry failed: {0}")] Decode(String), + /// The runtime declares no such storage entry, or not in that shape. + #[error("{0}")] + NoStorageEntry(String), } /// One decoded event, named as the runtime names it. @@ -228,6 +231,66 @@ struct ExtrinsicTypes { call: u32, } +/// Where a storage value lives and what it decodes as. +#[derive(Debug, Clone, PartialEq)] +pub struct StorageTarget { + /// The full storage key, ready for `state_getStorage`. + pub key: Vec, + /// The registry type its value decodes as. + pub value_ty: u32, + /// The SCALE-encoded default for a `Default` entry, which is what the chain + /// means when it returns nothing. `None` for an `Optional` entry, where + /// nothing means nothing. + pub default: Option>, +} + +/// `twox128`, as Substrate builds storage prefixes: two little-endian xxhash64 +/// digests, seeded 0 and 1, concatenated. +fn twox_128(input: &[u8]) -> [u8; 16] { + use twox_hash::XxHash64; + let mut out = [0u8; 16]; + out[..8].copy_from_slice(&XxHash64::oneshot(0, input).to_le_bytes()); + out[8..].copy_from_slice(&XxHash64::oneshot(1, input).to_le_bytes()); + out +} + +/// Hash one map key the way its entry declares. +/// +/// The `Concat` variants keep the key after its hash, which is what makes a map +/// enumerable — `state_getKeys` can hand back the keys and a reader can recover +/// them. The plain variants do not, and a map using one is write-only from +/// outside the runtime. +fn hash_key(hasher: &frame_metadata::v14::StorageHasher, encoded: &[u8]) -> Vec { + use blake2::digest::consts::{U16, U32}; + use blake2::{Blake2b, Digest}; + use frame_metadata::v14::StorageHasher as H; + use twox_hash::XxHash64; + + match hasher { + H::Blake2_128 => Blake2b::::digest(encoded).to_vec(), + H::Blake2_256 => Blake2b::::digest(encoded).to_vec(), + H::Blake2_128Concat => { + let mut v = Blake2b::::digest(encoded).to_vec(); + v.extend_from_slice(encoded); + v + } + H::Twox128 => twox_128(encoded).to_vec(), + H::Twox256 => { + let mut v = Vec::with_capacity(32); + for seed in 0..4u64 { + v.extend_from_slice(&XxHash64::oneshot(seed, encoded).to_le_bytes()); + } + v + } + H::Twox64Concat => { + let mut v = XxHash64::oneshot(0, encoded).to_le_bytes().to_vec(); + v.extend_from_slice(encoded); + v + } + H::Identity => encoded.to_vec(), + } +} + /// A runtime's description of itself, ready to decode with. pub struct Runtime { metadata: RuntimeMetadataV14, @@ -527,6 +590,155 @@ impl Runtime { )) } + /// The storage key for one entry, and the type its value decodes as. + /// + /// ## Why this is computed rather than looked up + /// + /// A storage key is `twox128(pallet prefix) ++ twox128(item) ++ hashed + /// keys`, and *which* hash each key uses is declared per entry in the + /// metadata — `System::Account` is `Blake2_128Concat`, another map is + /// `Twox64Concat`, and picking wrong yields a key that reads as absent + /// rather than as an error. Every part of that comes from the runtime's own + /// description, including the pallet's storage prefix, which is usually the + /// pallet name and is not required to be. + /// + /// `keys` are SCALE-encoded against the key type the entry declares, so a + /// map keyed on an account, a `u32` or a tuple all work without this + /// knowing which it is. + pub fn storage_key( + &self, + pallet: &str, + item: &str, + keys: &[scale_value::Value], + ) -> Result { + let pallet_meta = self + .metadata + .pallets + .iter() + .find(|p| p.name == pallet) + .ok_or_else(|| RuntimeError::NoStorageEntry(format!("no pallet {pallet}")))?; + let storage = pallet_meta + .storage + .as_ref() + .ok_or_else(|| RuntimeError::NoStorageEntry(format!("{pallet} declares no storage")))?; + let entry = storage + .entries + .iter() + .find(|e| e.name == item) + .ok_or_else(|| RuntimeError::NoStorageEntry(format!("no {pallet}::{item}")))?; + + let mut key = Vec::with_capacity(64); + key.extend_from_slice(&twox_128(storage.prefix.as_bytes())); + key.extend_from_slice(&twox_128(item.as_bytes())); + + let value_ty = match &entry.ty { + frame_metadata::v14::StorageEntryType::Plain(t) => { + if !keys.is_empty() { + return Err(RuntimeError::NoStorageEntry(format!( + "{pallet}::{item} takes no key" + ))); + } + t.id + } + frame_metadata::v14::StorageEntryType::Map { + hashers, + key: key_ty, + value, + } => { + if keys.len() != hashers.len() { + return Err(RuntimeError::NoStorageEntry(format!( + "{pallet}::{item} takes {} key(s), given {}", + hashers.len(), + keys.len() + ))); + } + // One hasher means one key of the declared type. Several means + // the declared type is a tuple and each element takes its own + // hasher — the shape `StorageDoubleMap` and `StorageNMap` + // produce. + let key_tys: Vec = if hashers.len() == 1 { + vec![key_ty.id] + } else { + match self.metadata.types.resolve(key_ty.id).map(|t| &t.type_def) { + Some(scale_info::TypeDef::Tuple(t)) => { + t.fields.iter().map(|f| f.id).collect() + } + _ => { + return Err(RuntimeError::NoStorageEntry(format!( + "{pallet}::{item} has {} hashers and a non-tuple key", + hashers.len() + ))); + } + } + }; + for ((value, ty), hasher) in keys.iter().zip(key_tys).zip(hashers.iter()) { + let mut encoded = Vec::new(); + scale_value::scale::encode_as_type( + value, + ty, + &self.metadata.types, + &mut encoded, + ) + .map_err(|e| RuntimeError::Decode(format!("key does not fit its type: {e}")))?; + key.extend_from_slice(&hash_key(hasher, &encoded)); + } + value.id + } + }; + + Ok(StorageTarget { + key, + value_ty, + // What the chain returns when the entry is empty. `Optional` means + // absent is genuinely absent; `Default` means absent is this value, + // which is not the same thing and is why the modifier is carried. + default: match entry.modifier { + frame_metadata::v14::StorageEntryModifier::Optional => None, + frame_metadata::v14::StorageEntryModifier::Default => Some(entry.default.clone()), + }, + }) + } + + /// Decode a storage value against the type its entry declares. + /// + /// Normalised the same way event fields are — accounts as `0x` hex, big + /// integers as decimal strings — so one renderer serves state, events and + /// call arguments alike. + pub fn decode_storage( + &self, + target: &StorageTarget, + raw: &[u8], + ) -> Result { + let mut cursor = raw; + let value = + scale_value::scale::decode_as_type(&mut cursor, target.value_ty, &self.metadata.types) + .map_err(|e| RuntimeError::Decode(e.to_string()))?; + if !cursor.is_empty() { + return Err(RuntimeError::Decode(format!( + "{} trailing bytes in storage value", + cursor.len() + ))); + } + let mut accounts = BTreeSet::new(); + Ok(self.normalise(&value, &mut accounts)) + } + + /// Every plain storage entry, which is everything readable without a key. + pub fn plain_storage(&self) -> Vec<(String, String)> { + let mut out = Vec::new(); + for pallet in &self.metadata.pallets { + let Some(storage) = &pallet.storage else { + continue; + }; + for e in &storage.entries { + if matches!(e.ty, frame_metadata::v14::StorageEntryType::Plain(_)) { + out.push((pallet.name.clone(), e.name.clone())); + } + } + } + out + } + /// Everything this runtime says about itself. /// /// The metadata is already in memory for decoding; this is the same @@ -1343,6 +1555,80 @@ mod tests { assert!(matches!(err, RuntimeError::Decode(_)), "{err:?}"); } + /// Pinned against keys read off the live chain by hand. A wrong hasher + /// yields a key that reads as *absent* rather than as an error, so a + /// mistake here is invisible without a fixed answer to check against. + #[test] + fn a_storage_key_matches_what_the_chain_answers_to() { + let rt = Runtime::from_metadata(&metadata()).expect("parses"); + + // Plain entry, no key: twox128(prefix) ++ twox128(item). + let t = rt + .storage_key("TreasuryPallet", "TreasuryAccount", &[]) + .expect("declared"); + assert_eq!( + hex::encode(&t.key), + "738724b52124086a5b7d669ee9241cec6018b2b279b2e61a028ee62879c46608" + ); + // `optional`, so nothing means nothing rather than a default. + assert_eq!(t.default, None); + + // A map keyed on an account, hashed `Blake2_128Concat` — the key keeps + // the account after its hash, which is why it ends in it. + let account = + hex::decode("c6801725b054b06a3d5030f25caa3f2e9094d7722efe4f9eddbce1227798ec7c") + .unwrap(); + let key = scale_value::Value::from_bytes(&account); + let a = rt + .storage_key("System", "Account", &[key]) + .expect("declared"); + let rendered = hex::encode(&a.key); + assert!( + rendered.starts_with("26aa394eea5630e07c48ae0c9558cef7"), + "{rendered}" + ); + assert!(rendered.ends_with(&hex::encode(&account)), "{rendered}"); + assert_eq!(a.key.len(), 16 + 16 + 16 + 32); + // `AccountInfo` has a default, so an absent entry means zero rather + // than "no such account" — the distinction the modifier carries. + assert!(a.default.is_some()); + } + + #[test] + fn a_key_of_the_wrong_shape_is_refused() { + let rt = Runtime::from_metadata(&metadata()).expect("parses"); + // A plain entry takes none. + assert!(matches!( + rt.storage_key( + "TreasuryPallet", + "TreasuryAccount", + &[scale_value::Value::u128(1)] + ), + Err(RuntimeError::NoStorageEntry(_)) + )); + // A map takes exactly as many as it declares hashers. + assert!(matches!( + rt.storage_key("System", "Account", &[]), + Err(RuntimeError::NoStorageEntry(_)) + )); + assert!(matches!( + rt.storage_key("System", "NoSuchThing", &[]), + Err(RuntimeError::NoStorageEntry(_)) + )); + } + + /// Everything readable without a key, which is what a state page can show + /// without being told anything. + #[test] + fn plain_entries_are_discoverable() { + let rt = Runtime::from_metadata(&metadata()).expect("parses"); + let plain = rt.plain_storage(); + assert!(plain.contains(&("TreasuryPallet".into(), "TreasuryAccount".into()))); + assert!(plain.contains(&("QPoW".into(), "CurrentDifficulty".into()))); + // A map is not plain, and must not appear. + assert!(!plain.contains(&("System".into(), "Account".into()))); + } + #[test] fn a_block_decoded_against_the_wrong_runtime_fails_loudly() { // Trailing bytes mean the registry and the data disagree. Storing a diff --git a/crates/blackbeard-data/src/rpc.rs b/crates/blackbeard-data/src/rpc.rs index 776a085..fbbbba9 100644 --- a/crates/blackbeard-data/src/rpc.rs +++ b/crates/blackbeard-data/src/rpc.rs @@ -245,9 +245,28 @@ impl RpcClient { /// `Ok(None)` where the node has pruned the state it lives in — an answer, /// not a failure, and the reason metadata is cached rather than re-fetched. pub async fn events_at(&self, hash: &str) -> Result>, DataError> { - let v = self - .call("state_getStorage", json!([SYSTEM_EVENTS_KEY, hash])) - .await?; + self.storage(SYSTEM_EVENTS_KEY, Some(hash)).await + } + + /// Read one storage value, by key. + /// + /// `None` means the chain holds nothing there, which for a `Default` entry + /// means its default and for an `Optional` entry means nothing — a + /// distinction only the metadata knows, so it is the caller's to make. + /// + /// At the tip when `hash` is `None`. Against an old block it needs state + /// the node may have pruned, which is an absent value rather than an error + /// and is why nothing here treats absence as authoritative on its own. + pub async fn storage( + &self, + key: &str, + hash: Option<&str>, + ) -> Result>, DataError> { + let params = match hash { + Some(h) => json!([key, h]), + None => json!([key]), + }; + let v = self.call("state_getStorage", params).await?; let Some(hex) = v.as_str() else { return Ok(None); }; diff --git a/crates/blackbeard-entities/src/account.rs b/crates/blackbeard-entities/src/account.rs index 9a3dd0c..659a2d2 100644 --- a/crates/blackbeard-entities/src/account.rs +++ b/crates/blackbeard-entities/src/account.rs @@ -36,6 +36,21 @@ pub struct AccountDetail { pub miner: Option, /// The telemetry name held for that miner, if one is. pub display: Option, + /// The account's free balance right now, read from chain state rather than + /// inferred from the flows below. + /// + /// `null` when the node would not answer, and — importantly — also when the + /// account does not exist. On Substrate an account with nothing in it is + /// not an account with zero, it is absent, and those read differently: the + /// treasury address this chain has configured has never been funded, and + /// showing it a zero would imply it had been and spent. + pub balance: Option, + /// The whole `System::Account` entry as the runtime describes it — nonce, + /// consumers, providers, and the balance breakdown. Carried alongside the + /// single number above because the number is what a reader came for and the + /// rest is what makes it checkable. + #[ts(type = "Record | null")] + pub state: Option, /// Mining rewards, over the whole indexed record. pub rewards: RewardSummary, /// Everything involving this account, newest first. diff --git a/crates/blackbeard-entities/src/lib.rs b/crates/blackbeard-entities/src/lib.rs index 92e9cf4..28c5649 100644 --- a/crates/blackbeard-entities/src/lib.rs +++ b/crates/blackbeard-entities/src/lib.rs @@ -23,6 +23,7 @@ mod error; mod miner; mod runtime; mod series; +mod state; mod ws; pub use account::{AccountDetail, AccountEvent, AccountRow, ActivitySource, RewardSummary}; @@ -36,6 +37,7 @@ pub use runtime::{ RuntimeStorage, RuntimeSummary, RuntimeVariant, }; pub use series::{ChainSeries, ChainSeriesPoint}; +pub use state::{ChainState, StateEntry}; pub use ws::{ClientMessage, ServerMessage, Window}; use serde::{Deserialize, Serialize}; diff --git a/crates/blackbeard-entities/src/state.rs b/crates/blackbeard-entities/src/state.rs new file mode 100644 index 0000000..48951de --- /dev/null +++ b/crates/blackbeard-entities/src/state.rs @@ -0,0 +1,60 @@ +//! Chain state, decoded against the runtime that describes it. +//! +//! Everything else on this site is *history* — what happened, from headers, +//! events and extrinsics. This is the other half: what is true right now. +//! +//! It matters for the questions history cannot answer. "What has this account +//! been paid" is a sum over events; "what does it hold" is a storage read, and +//! the two differ by every transfer out. "Which address is the treasury" is not +//! an event at all on this chain — it was set at genesis, so no extrinsic ever +//! carried it and no event ever announced it. It exists only in state. + +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::ChainId; + +/// One storage entry, read and decoded. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "StateEntry.ts")] +pub struct StateEntry { + /// Owning pallet, as the runtime names it. + pub pallet: String, + /// The entry, as the runtime names it. + pub name: String, + /// The value's type, rendered from the registry. + pub type_name: String, + /// `optional` or `default`. + pub modifier: String, + /// The first line of the runtime's documentation for it. + pub docs: Option, + /// The decoded value. Accounts as `0x` hex, big integers as decimal + /// strings — the same normalisation events and call arguments get, so one + /// renderer serves all three. + #[ts(type = "unknown")] + pub value: Option, + /// Whether the chain held nothing there. + /// + /// Not the same as `value` being null: for a `default` entry the runtime + /// supplies a value when storage is empty, and this says the value shown is + /// that default rather than something the chain wrote. + pub is_default: bool, + /// Why it could not be read, when it could not. + pub error: Option, +} + +/// Every storage entry readable without a key, decoded. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "ChainState.ts")] +pub struct ChainState { + /// Which chain. + pub chain: ChainId, + /// The runtime the entries were described by. + #[ts(type = "number")] + pub spec_version: Option, + /// The block this state was read at. + #[ts(type = "number")] + pub height: Option, + /// The entries, in pallet declaration order. + pub entries: Vec, +} diff --git a/readme.md b/readme.md index b8fefff..fce4575 100644 --- a/readme.md +++ b/readme.md @@ -190,6 +190,40 @@ become one decimal rather than four or eight little-endian limbs. Difficulty as `[1189189, 0, 0, 0, 0, 0, 0, 0]` is a correct description of the bytes and tells a reader nothing. +## State: what is true now + +Everything else here is history — headers, events, extrinsics, a record of what +happened. `/quantus/state` is the other half, and it exists because of a +question history could not answer. + +Which address is this chain's treasury? `TreasuryPallet::set_treasury_account` +reads *never* on the call index and `TreasuryAccountUpdated` reads *never* on +the event index. Both are true: the address was set at genesis, so no extrinsic +ever carried it and no event ever announced it. It exists only in state, and +until the observer could read state the honest answer was that we did not know. +It is `qzjsuLN7Nhu4bjvmUbjSTr2ZTeZ7oRxXpQP9fdv6PcHUCRrVR`, and it **has never +been funded** — no `System::Account` entry at all. + +A storage key is `twox128(pallet prefix) ++ twox128(item) ++ hashed keys`, and +*which* hash each key uses is declared per entry: `System::Account` is +`Blake2_128Concat`, another map is `Twox64Concat`. Picking wrong yields a key +that reads as **absent rather than as an error**, which is why +`Runtime::storage_key` computes every part of it from the runtime's own +description — including the pallet's storage prefix, which is usually the pallet +name and is not required to be — and why its test pins two keys against ones +read off the live chain by hand. + +Absent is not zero, and the metadata is what knows the difference. An `optional` +entry holding nothing means nothing; a `default` entry holding nothing means the +value the runtime supplies, which the page marks as *default* rather than +passing off as something the chain wrote. The same distinction is why an +unfunded account shows **—** and "no account on chain" rather than a balance of +zero: Substrate reaps an empty account, so a missing entry means it does not +exist, and a zero would say it had been funded and spent. + +Read live on every request. State is what is true now, and a cached copy of now +is a copy of some earlier now wearing the same face. + ## What the chain can do, and what it has done `/quantus/event` is its other half, and the one a chain analyst reads for diff --git a/web/src/App.tsx b/web/src/App.tsx index 4b814d4..4445af2 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -25,6 +25,7 @@ import { MinerPanel } from './components/MinerPanel' import { RuntimePanel } from './components/RuntimePanel' import { RuntimesIndex } from './components/RuntimesIndex' import { SectionNav } from './components/SectionNav' +import { StateIndex } from './components/StateIndex' import { StatBar } from './components/StatBar' import { seconds, windowSpan } from './lib/format' import { href, parse, WINDOWS } from './lib/routes' @@ -243,6 +244,13 @@ export default function App() { symbol={info?.token_symbol ?? ''} /> )} + {route.index === 'state' && chain && ( + + )} {route.index === 'runtime' && chain && ( )} diff --git a/web/src/api/generated/AccountDetail.ts b/web/src/api/generated/AccountDetail.ts index f08550e..afba6df 100644 --- a/web/src/api/generated/AccountDetail.ts +++ b/web/src/api/generated/AccountDetail.ts @@ -1,5 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AccountEvent } from "./AccountEvent"; +import type { BigUintDec } from "./BigUintDec"; import type { ChainId } from "./ChainId"; import type { MinerId } from "./MinerId"; import type { RewardSummary } from "./RewardSummary"; @@ -35,6 +36,24 @@ miner: MinerId | null, * The telemetry name held for that miner, if one is. */ display: string | null, +/** + * The account's free balance right now, read from chain state rather than + * inferred from the flows below. + * + * `null` when the node would not answer, and — importantly — also when the + * account does not exist. On Substrate an account with nothing in it is + * not an account with zero, it is absent, and those read differently: the + * treasury address this chain has configured has never been funded, and + * showing it a zero would imply it had been and spent. + */ +balance: BigUintDec | null, +/** + * The whole `System::Account` entry as the runtime describes it — nonce, + * consumers, providers, and the balance breakdown. Carried alongside the + * single number above because the number is what a reader came for and the + * rest is what makes it checkable. + */ +state: Record | null, /** * Mining rewards, over the whole indexed record. */ diff --git a/web/src/api/generated/ChainState.ts b/web/src/api/generated/ChainState.ts new file mode 100644 index 0000000..a164248 --- /dev/null +++ b/web/src/api/generated/ChainState.ts @@ -0,0 +1,24 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ChainId } from "./ChainId"; +import type { StateEntry } from "./StateEntry"; + +/** + * Every storage entry readable without a key, decoded. + */ +export type ChainState = { +/** + * Which chain. + */ +chain: ChainId, +/** + * The runtime the entries were described by. + */ +spec_version: number, +/** + * The block this state was read at. + */ +height: number, +/** + * The entries, in pallet declaration order. + */ +entries: Array, }; diff --git a/web/src/api/generated/StateEntry.ts b/web/src/api/generated/StateEntry.ts new file mode 100644 index 0000000..f695ae3 --- /dev/null +++ b/web/src/api/generated/StateEntry.ts @@ -0,0 +1,44 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * One storage entry, read and decoded. + */ +export type StateEntry = { +/** + * Owning pallet, as the runtime names it. + */ +pallet: string, +/** + * The entry, as the runtime names it. + */ +name: string, +/** + * The value's type, rendered from the registry. + */ +type_name: string, +/** + * `optional` or `default`. + */ +modifier: string, +/** + * The first line of the runtime's documentation for it. + */ +docs: string | null, +/** + * The decoded value. Accounts as `0x` hex, big integers as decimal + * strings — the same normalisation events and call arguments get, so one + * renderer serves all three. + */ +value: unknown, +/** + * Whether the chain held nothing there. + * + * Not the same as `value` being null: for a `default` entry the runtime + * supplies a value when storage is empty, and this says the value shown is + * that default rather than something the chain wrote. + */ +is_default: boolean, +/** + * Why it could not be read, when it could not. + */ +error: string | null, }; diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index 96afbd8..146ff6f 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -14,6 +14,7 @@ import type { BlockDetail } from './generated/BlockDetail' import type { BlockExtrinsic } from './generated/BlockExtrinsic' import type { CallIndex } from './generated/CallIndex' import type { ChainSeries } from './generated/ChainSeries' +import type { ChainState } from './generated/ChainState' import type { RecentBlock } from './generated/RecentBlock' import type { MinerDetail } from './generated/MinerDetail' import type { RuntimeDetail } from './generated/RuntimeDetail' @@ -147,6 +148,16 @@ export function fetchCallFeed( ) } +/** + * Every storage entry readable without a key, decoded. + * + * Live on every call. State is what is true now, and caching now produces a + * copy of some earlier now that is indistinguishable from the real thing. + */ +export function fetchState(chain: string, signal?: AbortSignal): Promise { + return get(`/chains/${encodeURIComponent(chain)}/state`, signal) +} + /** Accounts ranked by what they have been paid, most first. */ export function fetchAccounts(chain: string, signal?: AbortSignal): Promise { return get(`/chains/${encodeURIComponent(chain)}/accounts`, signal) diff --git a/web/src/components/AccountPanel.tsx b/web/src/components/AccountPanel.tsx index 7496954..59e9694 100644 --- a/web/src/components/AccountPanel.tsx +++ b/web/src/components/AccountPanel.tsx @@ -55,6 +55,14 @@ export function AccountPanel({ const [detail, setDetail] = useState(null) const [error, setError] = useState(null) + // Pulled out of the whole `System::Account` entry rather than sent as their + // own fields: they are the runtime's names, and reading them here keeps the + // wire format the chain's own shape. + const st = detail?.state as Record | null | undefined + const data = (st?.data ?? null) as Record | null + const reserved = data?.reserved !== undefined ? String(data.reserved) : null + const nonce = st?.nonce !== undefined ? String(st.nonce) : null + useEffect(() => { const controller = new AbortController() setDetail(null) @@ -115,6 +123,40 @@ export function AccountPanel({ reading 0, 0, — and — is not a summary of anything; it is the mining shape of this page imposed on an account that has never mined, and it pushes the history that does exist below the fold. */} + {/* Held now, from state — a different number from anything summed + below, and the one most people came for. Absent is not zero: + Substrate reaps an empty account, so a missing entry means the + account does not exist, and a 0 would say it had been funded and + spent. */} +
+
+
Balance
+
+ {detail.balance ? tokens(detail.balance, decimals, 4) : '—'} +
+
+ {detail.balance ? `${symbol} held now` : 'no account on chain'} +
+
+ {reserved !== null && reserved !== '0' && ( +
+
Reserved
+
{tokens(reserved, decimals, 4)}
+
{symbol} in deposits
+
+ )} +
+
Nonce
+
{nonce === null ? '—' : fmtHeight(Number(nonce))}
+
transactions sent
+
+
+ {detail.rewards.count > 0 && (
diff --git a/web/src/components/StateIndex.tsx b/web/src/components/StateIndex.tsx new file mode 100644 index 0000000..36b70fb --- /dev/null +++ b/web/src/components/StateIndex.tsx @@ -0,0 +1,144 @@ +/** + * What is true right now. + * + * Everything else on this site is history — headers, events, extrinsics, all of + * it a record of what happened. This is the other half, and it answers the + * questions history cannot. + * + * The treasury is the case that made it necessary. This chain's treasury + * address was set at genesis, so no extrinsic ever carried it and no event ever + * announced it: `TreasuryPallet::set_treasury_account` shows *never* on the + * call index and `TreasuryAccountUpdated` shows *never* on the event index, and + * both are true. The address exists only in state, and until the observer could + * read state the honest answer to "which address is the treasury" was that we + * did not know. + * + * Read live on every request rather than indexed. State is what is true now, + * and a cached copy of now is a copy of some earlier now wearing the same face. + */ + +import { useEffect, useState } from 'react' + +import type { ChainState } from '../api/generated/ChainState' +import { RequestFailed, fetchState } from '../api/rest' +import { height as fmtHeight } from '../lib/format' +import { Payload } from './Payload' + +export function StateIndex({ + chain, + decimals, + symbol, +}: { + chain: string + decimals: number + symbol: string +}) { + const [state, setState] = useState(null) + const [error, setError] = useState(null) + + useEffect(() => { + const controller = new AbortController() + setState(null) + setError(null) + fetchState(chain, controller.signal) + .then(setState) + .catch((e: unknown) => { + if (controller.signal.aborted) return + setError(e instanceof RequestFailed ? e.message : 'Could not reach the observer.') + }) + return () => controller.abort() + }, [chain]) + + return ( +
+
+

State

+ {state && ( + + {state.entries.length} entries at #{fmtHeight(state.height)} + {state.spec_version !== null && ` · runtime v${state.spec_version}`} + + )} +
+ + {error &&

{error}

} + {!error && state === null &&

Reading the chain…

} + + {state !== null && ( + <> +
+ + + + + + + + + + {state.entries.map((e) => ( + + + + + ))} + +
+ Every storage entry readable without a key, decoded +
+ Entry + + Value +
+
+ + {e.pallet}::{e.name} + + {e.type_name} + {e.docs && {e.docs}} +
+ 		+ {e.error ? ( + + unreadable + + ) : e.value === null || e.value === undefined ? ( + /* An `optional` entry with nothing in it. Not zero, not + empty — the chain holds nothing there, and for the + treasury address that difference is the answer. */ + unset + ) : ( + + {typeof e.value === 'object' ? ( + + ) : ( + {String(e.value)} + )} + {e.is_default && ( + + default + + )} + + )} +
+
+ +

+ Read live from the chain at the block above, and decoded against the runtime's own + declaration of each entry's type — including which hash its key uses, which is why a + value appears here at all rather than as a key nobody can compute. Only entries needing + no key are listed; a map like System::Account is read per account, on its + own page. +

+ + )} +
+ ) +} diff --git a/web/src/index.css b/web/src/index.css index 8640b49..dc9a0d9 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -1326,3 +1326,12 @@ tr.mine .share-bar > i { .call-unused code { color: var(--text-secondary); } + +/* A decoded storage value and, when the chain held nothing, the note that this + is the runtime's default rather than something it wrote. */ +.state-value { + display: inline-flex; + flex-wrap: wrap; + align-items: baseline; + gap: 4px 10px; +} diff --git a/web/src/lib/routes.ts b/web/src/lib/routes.ts index 9a85f44..45ebc65 100644 --- a/web/src/lib/routes.ts +++ b/web/src/lib/routes.ts @@ -56,7 +56,7 @@ export interface Route { } /** The kinds that have an index. */ -export type SectionName = 'block' | 'call' | 'event' | 'account' | 'runtime' +export type SectionName = 'block' | 'call' | 'event' | 'account' | 'state' | 'runtime' /** * The sections, in the order the nav shows them. @@ -70,6 +70,7 @@ export const SECTIONS: { id: SectionName; label: string }[] = [ { id: 'call', label: 'Calls' }, { id: 'event', label: 'Events' }, { id: 'account', label: 'Accounts' }, + { id: 'state', label: 'State' }, { id: 'runtime', label: 'Runtimes' }, ]