diff --git a/.sqlx/query-758ab5f43dc25b00170f77d955052aebe0649de9000928159d0f6c737dc0589a.json b/.sqlx/query-758ab5f43dc25b00170f77d955052aebe0649de9000928159d0f6c737dc0589a.json new file mode 100644 index 0000000..022a540 --- /dev/null +++ b/.sqlx/query-758ab5f43dc25b00170f77d955052aebe0649de9000928159d0f6c737dc0589a.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select height, event_index, pallet, variant, phase, extrinsic_index,\n fields as \"fields!\", at, accounts as \"accounts!\"\n from chain_event\n where chain = $1\n and pallet = 'Wormhole' and variant = 'NativeTransferred'\n and accounts && $2::text[]\n and fields ->> 'to' = any($2::text[])\n and (height, event_index) > ($3, $4)\n order by height, event_index\n limit $5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "event_index", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "pallet", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "variant", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "phase", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "extrinsic_index", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "fields!", + "type_info": "Jsonb" + }, + { + "ordinal": 7, + "name": "at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "accounts!", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Int4", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "758ab5f43dc25b00170f77d955052aebe0649de9000928159d0f6c737dc0589a" +} diff --git a/crates/blackbeard-api/src/routes.rs b/crates/blackbeard-api/src/routes.rs index cf6bdaf..4f4e028 100644 --- a/crates/blackbeard-api/src/routes.rs +++ b/crates/blackbeard-api/src/routes.rs @@ -79,11 +79,29 @@ pub fn router(state: AppState, allowed_origins: &[String]) -> Router { .route("/v1/chains/{chain}/runtimes", get(runtimes)) .route("/v1/chains/{chain}/runtimes/{spec}", get(runtime)) .route("/v1/ws", get(crate::ws::handler)) + .layer(cors) + // Outside the site's allowlist, with its own. Wallets call this from + // browser extensions, whose origins cannot be listed: Chrome's id + // depends on the install, Firefox's is a random uuid per install. It + // serves public chain data, read-only, so any origin may read it. Merged + // after `.layer(cors)` rather than routed before it, so the allowlist + // never sees these requests and cannot answer a preflight for them. + .merge( + Router::new() + .route( + "/v1/chains/{chain}/wormhole/deposits", + get(wormhole_deposits), + ) + .layer( + CorsLayer::new() + .allow_methods([Method::GET]) + .allow_origin(tower_http::cors::Any), + ), + ) // Leaderboards are repetitive JSON and compress to a fraction of their // size; the snapshot a browser fetches on first paint is the largest // single response this API serves. .layer(CompressionLayer::new()) - .layer(cors) .with_state(state) } @@ -1535,6 +1553,145 @@ fn node_info( /// How many rows an index page carries. const INDEX_PAGE: i64 = 50; +/// Most addresses one deposits request may name. A wallet's gap-limit window is +/// forty (twenty receive, twenty change); this leaves room without letting one +/// request become a scan of the chain. +const DEPOSIT_ADDRESSES_MAX: usize = 100; + +/// Deposits per page, by default and at most. +const DEPOSIT_PAGE: i64 = 500; +const DEPOSIT_PAGE_MAX: i64 = 1000; + +#[derive(Debug, Deserialize)] +struct DepositsQuery { + /// Comma-separated addresses, SS58 or `0x` hex. + to: Option, + /// `height:event_index`, exclusive. + after: Option, + limit: Option, +} + +/// An address as `0x` hex: SS58 at this network's prefix, or 32 bytes of hex. +fn parse_deposit_address(raw: &str) -> Result { + let hex = raw.strip_prefix("0x").unwrap_or(raw); + if raw.starts_with("0x") && hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit()) { + return Ok(format!("0x{}", hex.to_ascii_lowercase())); + } + parse_address(raw) +} + +fn parse_cursor(raw: &str) -> Result<(u64, u32), Failure> { + raw.split_once(':') + .and_then(|(h, i)| Some((h.parse().ok()?, i.parse().ok()?))) + .ok_or_else(|| { + Failure( + StatusCode::BAD_REQUEST, + ApiError::new( + "malformed_cursor", + format!("`{raw}` is not a `height:event_index` cursor"), + ), + ) + }) +} + +/// An event field that is an integer. The decoder writes one past 2^53 as a +/// string, so both spellings are accepted. +fn field_integer(fields: &serde_json::Value, name: &str) -> Option { + match fields.get(name)? { + serde_json::Value::Number(n) => n.as_u64().map(|v| v.to_string()), + serde_json::Value::String(s) if !s.is_empty() && s.chars().all(|c| c.is_ascii_digit()) => { + Some(s.clone()) + } + _ => None, + } +} + +/// Wormhole deposits to a set of addresses, oldest first. +/// +/// For a wallet working out what it can still spend, without telling Quantus's +/// indexer which addresses are one person's. See blackbeard/observer#22. +async fn wormhole_deposits( + State(state): State, + Path(chain): Path, + Query(query): Query, +) -> Result, Failure> { + let runtime = state.chain(&chain).ok_or_else(|| unknown_chain(&chain))?; + let id = runtime.id(); + + let mut tos: Vec = Vec::new(); + for raw in query + .to + .as_deref() + .unwrap_or("") + .split(',') + .map(str::trim) + .filter(|a| !a.is_empty()) + { + let hex = parse_deposit_address(raw)?; + if !tos.contains(&hex) { + tos.push(hex); + } + } + if tos.is_empty() || tos.len() > DEPOSIT_ADDRESSES_MAX { + return Err(Failure( + StatusCode::BAD_REQUEST, + ApiError::new( + "bad_addresses", + format!("`to` must name between 1 and {DEPOSIT_ADDRESSES_MAX} addresses"), + ), + )); + } + let after = match query.after.as_deref() { + Some(raw) => parse_cursor(raw)?, + None => (0, 0), + }; + let limit = query + .limit + .unwrap_or(DEPOSIT_PAGE) + .clamp(1, DEPOSIT_PAGE_MAX); + + let rows = state + .store + .wormhole_deposits(&id, &tos, after, limit) + .await + .map_err(database_unavailable)?; + let scan = state + .store + .event_scan(&id) + .await + .map_err(database_unavailable)?; + + let full = rows.len() as i64 == limit; + let next = rows + .last() + .filter(|_| full) + .map(|r| format!("{}:{}", r.height, r.event_index)); + let deposits = rows + .into_iter() + .filter_map(|r| { + let to = r.fields.get("to")?.as_str()?.to_string(); + Some(blackbeard_entities::WormholeDeposit { + address: blackbeard_core::wormhole::ss58_of(&to)?, + amount: BigUintDec(field_integer(&r.fields, "amount")?), + transfer_count: field_integer(&r.fields, "transfer_count")?.parse().ok()?, + leaf_index: field_integer(&r.fields, "leaf_index")?.parse().ok()?, + height: r.height, + event_index: r.event_index, + at: r.at, + to, + }) + }) + .collect(); + + Ok(Json(blackbeard_entities::WormholeDeposits { + chain: id, + deposits, + next, + indexed_from: scan.map(|(low, _)| low), + indexed_to: scan.map(|(_, high)| high), + })) +} + /// Paging cursor for the block index. #[derive(Debug, Deserialize)] struct BeforeQuery { @@ -3311,6 +3468,53 @@ fn database_unavailable(e: blackbeard_data::DataError) -> Failure { mod tests { use super::*; + /// Deposits take either spelling of an address and give back one: SS58 at + /// prefix 189 (mainnet block 13160's miner) and its hex, in any case. + #[test] + fn deposit_addresses_accept_ss58_and_hex() { + let hex = "0xc6801725b054b06a3d5030f25caa3f2e9094d7722efe4f9eddbce1227798ec7c"; + let from_ss58 = + parse_deposit_address("qzowWAgbzjc2XfHY4vyEo2eVLKbknTESUFoXnisQuUh1x1koo").ok(); + let from_hex = + parse_deposit_address(&hex.to_ascii_uppercase().replacen("0X", "0x", 1)).ok(); + + assert_eq!(from_ss58.as_deref(), Some(hex)); + assert_eq!(from_hex.as_deref(), Some(hex)); + assert!(parse_deposit_address("0x1234").is_err()); + assert!(parse_deposit_address("not an address").is_err()); + } + + #[test] + fn deposit_cursor_is_height_and_event_index() { + assert_eq!(parse_cursor("55828:4").ok(), Some((55828, 4))); + assert!(parse_cursor("55828").is_err()); + assert!(parse_cursor("a:b").is_err()); + } + + /// The decoder writes an integer past 2^53 as a string; a u128 amount must + /// come through either way, and never as a float. + #[test] + fn deposit_integers_read_both_spellings() { + let fields = serde_json::json!({ + "small": 300_000_000_000u64, + "large": "340282366920938463463374607431768211455", + "negative": -1, + "float": 1.5, + }); + + assert_eq!( + field_integer(&fields, "small").as_deref(), + Some("300000000000") + ); + assert_eq!( + field_integer(&fields, "large").as_deref(), + Some("340282366920938463463374607431768211455") + ); + assert_eq!(field_integer(&fields, "negative"), None); + assert_eq!(field_integer(&fields, "float"), None); + assert_eq!(field_integer(&fields, "missing"), None); + } + /// The runtime's own spelling is unambiguous and unreadable. Only the /// associated-type qualifier goes; a generic is information. /// The chart reaches back as far as the window it sits under. diff --git a/crates/blackbeard-data/src/store.rs b/crates/blackbeard-data/src/store.rs index b8603e3..e28712b 100644 --- a/crates/blackbeard-data/src/store.rs +++ b/crates/blackbeard-data/src/store.rs @@ -1632,6 +1632,57 @@ impl Store { .collect()) } + /// Wormhole deposits to any of `tos`, oldest first, strictly after a + /// `(height, event_index)` cursor. + /// + /// Oldest first so a wallet can keep what it has and resume. `accounts &&` + /// is what the GIN index answers; the `fields ->> 'to'` test is what makes a + /// row a deposit *to* one of these addresses rather than one that merely + /// mentions it, since `accounts` holds the sender too. + pub async fn wormhole_deposits( + &self, + chain: &ChainId, + tos: &[String], + after: (u64, u32), + limit: i64, + ) -> Result, DataError> { + let rows = sqlx::query!( + r#" + select height, event_index, pallet, variant, phase, extrinsic_index, + fields as "fields!", at, accounts as "accounts!" + from chain_event + where chain = $1 + and pallet = 'Wormhole' and variant = 'NativeTransferred' + and accounts && $2::text[] + and fields ->> 'to' = any($2::text[]) + and (height, event_index) > ($3, $4) + order by height, event_index + limit $5 + "#, + chain.as_str(), + tos, + after.0 as i64, + after.1 as i32, + limit, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| StoredEvent { + height: r.height.max(0) as u64, + event_index: r.event_index.max(0) as u32, + pallet: r.pallet, + variant: r.variant, + phase: r.phase, + extrinsic_index: r.extrinsic_index.map(|i| i.max(0) as u32), + fields: r.fields, + at: r.at, + accounts: r.accounts, + }) + .collect()) + } + /// Every dispatch of one kind of call, newest first. pub async fn extrinsics_of_kind( &self, diff --git a/crates/blackbeard-data/tests/store.rs b/crates/blackbeard-data/tests/store.rs index a3e1806..42c027d 100644 --- a/crates/blackbeard-data/tests/store.rs +++ b/crates/blackbeard-data/tests/store.rs @@ -711,3 +711,104 @@ async fn node_telemetry_overwrites_but_a_null_does_not_erase() { f.cleanup().await; } + +/// Deposits by recipient: the query behind a wallet's wormhole balance +/// (blackbeard/observer#22). +/// +/// The ways it can be wrong are all silent: a deposit *from* one of the +/// addresses counted as one *to* it, another event kind leaking in, a cursor +/// that skips or repeats a same-block sibling, or another chain's rows. +#[tokio::test] +async fn wormhole_deposits_are_to_the_addresses_oldest_first_and_resumable() { + use blackbeard_data::store::EventRecord; + + let Some(fx) = Fixture::new("wormhole_deposits").await else { + return; + }; + seed_chain(&fx.store).await; + fx.store + .upsert_chain(&ChainRecord { + chain: ChainId("quantus".into()), + display_name: "Quantus".into(), + mainnet: true, + genesis: Some("0xfb54".into()), + token_symbol: Some("QTC".into()), + token_decimals: Some(12), + target_block_time: 12.0, + }) + .await + .unwrap(); + + let mine = format!("0x{:064x}", 0xa1); + let also_mine = format!("0x{:064x}", 0xa2); + let theirs = format!("0x{:064x}", 0xb1); + let minter = format!("0x{}", "01".repeat(32)); + let event = + |chain: &str, height: u64, index: u32, variant: &str, from: &str, to: &str, count: u64| { + EventRecord { + chain: ChainId(chain.into()), + height, + event_index: index, + pallet: "Wormhole".into(), + variant: variant.into(), + phase: "ApplyExtrinsic".into(), + extrinsic_index: Some(0), + fields: serde_json::json!({ + "from": from, "to": to, "amount": 300_000_000_000u64, + "transfer_count": count, "leaf_index": height * 10 + index as u64, + }), + accounts: vec![from.to_owned(), to.to_owned()], + at: Some(at(height as i64)), + } + }; + + fx.store + .record_events(&[ + event("planck", 10, 4, "NativeTransferred", &minter, &mine, 0), + // same block, lower index: must come first + event("planck", 10, 2, "NativeTransferred", &minter, &also_mine, 0), + // from mine to theirs: mentions mine, is not a deposit to it + event("planck", 11, 1, "NativeTransferred", &mine, &theirs, 0), + event("planck", 12, 3, "AssetTransferred", &minter, &mine, 1), + event("planck", 13, 0, "NativeTransferred", &minter, &mine, 1), + // the same recipient on another chain + event("quantus", 10, 1, "NativeTransferred", &minter, &mine, 0), + ]) + .await + .unwrap(); + + let tos = vec![mine.clone(), also_mine.clone()]; + let all = fx + .store + .wormhole_deposits(&chain(), &tos, (0, 0), 100) + .await + .unwrap(); + let keys: Vec<(u64, u32)> = all.iter().map(|e| (e.height, e.event_index)).collect(); + assert_eq!(keys, vec![(10, 2), (10, 4), (13, 0)]); + + // A page of one, resumed from its last row, walks the same three without + // skipping the same-block sibling or repeating a row. + let mut cursor = (0, 0); + let mut walked = Vec::new(); + loop { + let page = fx + .store + .wormhole_deposits(&chain(), &tos, cursor, 1) + .await + .unwrap(); + let Some(last) = page.last() else { break }; + cursor = (last.height, last.event_index); + walked.push(cursor); + } + assert_eq!(walked, keys); + + let theirs_only = fx + .store + .wormhole_deposits(&chain(), std::slice::from_ref(&theirs), (0, 0), 100) + .await + .unwrap(); + assert_eq!(theirs_only.len(), 1); + assert_eq!(theirs_only[0].height, 11); + + fx.cleanup().await; +} diff --git a/crates/blackbeard-entities/src/chain.rs b/crates/blackbeard-entities/src/chain.rs index dab3c69..c0efc79 100644 --- a/crates/blackbeard-entities/src/chain.rs +++ b/crates/blackbeard-entities/src/chain.rs @@ -678,6 +678,63 @@ pub struct WormholeSummary { pub indexed_to: Option, } +/// One deposit into a wormhole address: a `Wormhole::NativeTransferred` to it. +/// +/// Everything a wallet needs to tell what is still spendable. The address's +/// `System::Account` balance cannot: an exit credits the exit account and +/// records a nullifier, and nothing ever debits the wormhole address, so its +/// balance is a receipt total. A deposit is unspent while the nullifier derived +/// from the address's secret and this `transfer_count` is unused, and the +/// wallet works that out itself: the nullifier needs the secret, and naming it +/// to anyone would pair this deposit with its exit. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "WormholeDeposit.ts")] +pub struct WormholeDeposit { + /// Recipient, as `0x` hex. + pub to: String, + /// The same recipient in SS58, prefix 189. + pub address: String, + /// Planck, as a decimal string: a u128 does not survive a JSON number. + pub amount: BigUintDec, + /// This recipient's transfer counter when the deposit landed, from 0. With + /// the secret, it is what the nullifier is derived from. + #[ts(type = "number")] + pub transfer_count: u64, + /// Where the deposit sits in the zk tree, for the Merkle proof an exit needs. + #[ts(type = "number")] + pub leaf_index: u64, + /// Block it landed in. + #[ts(type = "number")] + pub height: u64, + /// Position in that block's events. With the height, the paging cursor. + #[ts(type = "number")] + pub event_index: u32, + /// The block's own timestamp, when the node answered for it. + pub at: Option>, +} + +/// A page of wormhole deposits to a set of addresses, oldest first. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "WormholeDeposits.ts")] +pub struct WormholeDeposits { + /// Which chain. + pub chain: ChainId, + /// The deposits. + pub deposits: Vec, + /// Cursor for the next page, as `height:event_index`, when this one was + /// full. Absent when there is nothing more *in the indexed range*. + pub next: Option, + /// Lowest block whose events this observer has read. A deposit below it is + /// not missing from the chain, only from this answer: a client must not add + /// these up and call the result a balance unless the range covers genesis, + /// or it has checked the count against `Wormhole::TransferCount`. + #[ts(type = "number")] + pub indexed_from: Option, + /// Highest. + #[ts(type = "number")] + pub indexed_to: Option, +} + /// Credits out of the wormhole, grouped by what the address appears to be. /// /// This exists because the obvious reading of exit batches is wrong. Grouping diff --git a/crates/blackbeard-entities/src/lib.rs b/crates/blackbeard-entities/src/lib.rs index 0be8eef..dba9e8f 100644 --- a/crates/blackbeard-entities/src/lib.rs +++ b/crates/blackbeard-entities/src/lib.rs @@ -35,7 +35,7 @@ pub use chain::{ AccountFlows, ChainId, ChainInfo, ChainStatus, ChainSummary, ClientVersion, Concentration, DailyActivity, DailyWormhole, Distribution, ExitCohort, LockCounts, MinerFlow, NetworkSummary, PalletPot, Tracking, TransferRoute, VestingPoint, VestingSchedule, VestingSummary, - WormholeSummary, + WormholeDeposit, WormholeDeposits, WormholeSummary, }; pub use error::{ApiError, EntityError}; pub use miner::{ diff --git a/web/src/api/generated/WormholeDeposit.ts b/web/src/api/generated/WormholeDeposit.ts new file mode 100644 index 0000000..b812097 --- /dev/null +++ b/web/src/api/generated/WormholeDeposit.ts @@ -0,0 +1,48 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BigUintDec } from "./BigUintDec"; + +/** + * One deposit into a wormhole address: a `Wormhole::NativeTransferred` to it. + * + * Everything a wallet needs to tell what is still spendable. The address's + * `System::Account` balance cannot: an exit credits the exit account and + * records a nullifier, and nothing ever debits the wormhole address, so its + * balance is a receipt total. A deposit is unspent while the nullifier derived + * from the address's secret and this `transfer_count` is unused, and the + * wallet works that out itself: the nullifier needs the secret, and naming it + * to anyone would pair this deposit with its exit. + */ +export type WormholeDeposit = { +/** + * Recipient, as `0x` hex. + */ +to: string, +/** + * The same recipient in SS58, prefix 189. + */ +address: string, +/** + * Planck, as a decimal string: a u128 does not survive a JSON number. + */ +amount: BigUintDec, +/** + * This recipient's transfer counter when the deposit landed, from 0. With + * the secret, it is what the nullifier is derived from. + */ +transfer_count: number, +/** + * Where the deposit sits in the zk tree, for the Merkle proof an exit needs. + */ +leaf_index: number, +/** + * Block it landed in. + */ +height: number, +/** + * Position in that block's events. With the height, the paging cursor. + */ +event_index: number, +/** + * The block's own timestamp, when the node answered for it. + */ +at: string | null, }; diff --git a/web/src/api/generated/WormholeDeposits.ts b/web/src/api/generated/WormholeDeposits.ts new file mode 100644 index 0000000..76b21cc --- /dev/null +++ b/web/src/api/generated/WormholeDeposits.ts @@ -0,0 +1,32 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ChainId } from "./ChainId"; +import type { WormholeDeposit } from "./WormholeDeposit"; + +/** + * A page of wormhole deposits to a set of addresses, oldest first. + */ +export type WormholeDeposits = { +/** + * Which chain. + */ +chain: ChainId, +/** + * The deposits. + */ +deposits: Array, +/** + * Cursor for the next page, as `height:event_index`, when this one was + * full. Absent when there is nothing more *in the indexed range*. + */ +next: string | null, +/** + * Lowest block whose events this observer has read. A deposit below it is + * not missing from the chain, only from this answer: a client must not add + * these up and call the result a balance unless the range covers genesis, + * or it has checked the count against `Wormhole::TransferCount`. + */ +indexed_from: number, +/** + * Highest. + */ +indexed_to: number, };