diff --git a/CLAUDE.md b/CLAUDE.md index 070cfd9..3a9510b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -507,6 +507,25 @@ trades that never happened. as `intents.near`. Intents arriving as cross-contract receipts are not read; the price response says so in `reads` rather than leaving it to be found. +**Three nodes, split by shard, because a new shard cannot join a running one +in time.** neard starts syncing a newly tracked shard only at an epoch +boundary, so adding one to a live node makes it usable about an epoch and a +half later. A node started empty syncs every tracked shard at once. So, as of +2026-10-06 (blackbeard/observer#39): + + frankie shard 7 intents.near rpc_urls (via near.internal) + gargravarr shards 4, 8 qtc.omft.near, pool pool_rpc_urls + quadbrat shards 10, 12 USDC, Ref v1 not read yet + +State snapshots are **off** on gargravarr and quadbrat. A snapshot holds flat +storage's head until it finishes, and on gargravarr's SATA disks after a fresh +sync it did not finish: every state read walked hundreds of deltas and the +catch-up ran slower than the chain. We consume state and serve none, so the +snapshot bought nothing. quadbrat's P2P port is **24568**, not 24567 — the +hanzalova WAN forwards 24567 to gargravarr, and neard tells peers its own listen +port, so the forwarded port and the node's must match. neard's metrics are on +the RPC port at `/metrics`, scraped by `asset/prometheus/blackbeard-neard.yml`. + **On a one-shard node, `tx_status` must not wait for execution.** Our node tracks only intents.near's shard. `EXPERIMENTAL_tx_status` with `EXECUTED` waits for every receipt in the transaction, including the withdrawals on diff --git a/asset/config/config.toml.tmpl b/asset/config/config.toml.tmpl index 68ade62..ec9af23 100644 --- a/asset/config/config.toml.tmpl +++ b/asset/config/config.toml.tmpl @@ -147,6 +147,11 @@ max_gap_fill_blocks = 5000 asset = "nep141:qtc.omft.near" start_height = 218346736 rpc_urls = ["https://near.internal"] +# gargravarr tracks the shards of dclv2.ref-labs.near and qtc.omft.near (8 and +# 4), where the pool's whitelisted accounts sign their direct calls +# (blackbeard/observer#39). Plain HTTP on the hanzalova mesh, admitted to bob +# alone by a firewalld rich rule, as frankie's RPC is to the proxy. +pool_rpc_urls = ["http://gargravarr.hanzalova.internal:24403"] [[chains]] id = "heisenberg" diff --git a/asset/prometheus/blackbeard-neard.yml b/asset/prometheus/blackbeard-neard.yml new file mode 100644 index 0000000..dc3d6eb --- /dev/null +++ b/asset/prometheus/blackbeard-neard.yml @@ -0,0 +1,28 @@ +# neard (NEAR mainnet RPC nodes) for blackbeard.observer. Installed by hand at +# /etc/prometheus/scrape_configs.d/blackbeard-neard.yml on golgafrinchans +# (2026-10-06); this copy is the record. Reload with `curl -X POST +# http://127.0.0.1:26559/-/reload` there after `promtool check config`. +# neard serves Prometheus metrics on its JSON-RPC port at /metrics; each host +# admits this one source (10.3.101.4) to that port with a firewalld rich rule. +# +# Targets are addresses, not names: this host resolves through kosherinata +# Unbound, which carries no record for gargravarr or frankie (hanzalova only). +# The "shards" label is what each node tracks (blackbeard/observer#39). +scrape_configs: + - job_name: neard + metrics_path: /metrics + scrape_interval: 30s + scrape_timeout: 20s + static_configs: + - targets: ["10.6.0.70:24403"] + labels: + host: frankie + shards: "7" + - targets: ["10.6.0.107:24403"] + labels: + host: gargravarr + shards: "4,8" + - targets: ["10.6.0.76:24403"] + labels: + host: quadbrat + shards: "10,12" diff --git a/crates/blackbeard-api/src/config.rs b/crates/blackbeard-api/src/config.rs index fa811b2..405f8f2 100644 --- a/crates/blackbeard-api/src/config.rs +++ b/crates/blackbeard-api/src/config.rs @@ -232,6 +232,13 @@ pub struct IntentsConfig { /// pool for the asset needs no config change. #[serde(default = "default_dcl_contracts")] pub dcl_contracts: Vec, + /// NEAR JSON-RPC endpoints of a node tracking the shards the pool contracts + /// and the asset's token live on (blackbeard/observer#39). Transactions + /// signed on those shards that call a pool directly — the whitelisted + /// accounts trading or moving liquidity without intents.near — are found + /// there and queued for `refdcl` like the relay's. Empty turns the scan off. + #[serde(default)] + pub pool_rpc_urls: Vec, } impl IntentsConfig { diff --git a/crates/blackbeard-api/src/intents.rs b/crates/blackbeard-api/src/intents.rs index f4d69af..8066b0e 100644 --- a/crates/blackbeard-api/src/intents.rs +++ b/crates/blackbeard-api/src/intents.rs @@ -119,6 +119,7 @@ pub fn spawn(chain: Arc, store: Store) { near.clone(), config.clone(), ); + crate::poolscan::spawn(Arc::clone(&chain), store.clone(), config.clone()); tokio::spawn(scan(chain, store, near, config)); } diff --git a/crates/blackbeard-api/src/main.rs b/crates/blackbeard-api/src/main.rs index 52317c3..6a8a0d2 100644 --- a/crates/blackbeard-api/src/main.rs +++ b/crates/blackbeard-api/src/main.rs @@ -11,6 +11,7 @@ mod config; mod ingest; mod intents; +mod poolscan; mod refdcl; mod registry; mod routes; diff --git a/crates/blackbeard-api/src/poolscan.rs b/crates/blackbeard-api/src/poolscan.rs new file mode 100644 index 0000000..5a4cfe8 --- /dev/null +++ b/crates/blackbeard-api/src/poolscan.rs @@ -0,0 +1,243 @@ +//! Direct calls to the asset's pools, read on the shards the calls are signed on. +//! +//! `intents::scan` finds a pool swap only when the 1Click relay made it, because +//! it reads intents.near's shard and a transaction lands in its *signer's* +//! shard. The pool's whitelisted accounts also call it themselves — to trade, to +//! add or pull liquidity — and those transactions are signed on other shards: +//! `liq.selfevident.near` and `solver-ref-2.near` are on the shard the token +//! contract is on, the pool on another. Our coverage figure measured the gap +//! (#37: 99.75% of buys); this closes it for every signer on a shard a tracked +//! node holds (blackbeard/observer#39). +//! +//! It only finds candidates. Each goes into `ref_pending` and is read whole by +//! `refdcl::fetch`, the same path as the relay's, so a swap is parsed one way +//! whichever road it arrived by and recorded once on its `(tx, receipt, index)` +//! key. A candidate that turns out not to be a swap — a liquidity change, say — +//! yields no swap event and records nothing. +//! +//! Only what the node holds is read: forward from wherever the cursor stands, +//! and back no further than the oldest block the node has. History before the +//! node synced is the coverage figure's to account for, not a public endpoint's +//! to be paced through for days. + +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use blackbeard_core::intents as core; +use blackbeard_data::near::NearClient; +use blackbeard_data::store::Store; + +use crate::config::IntentsConfig; +use crate::state::ChainRuntime; + +/// NEAR heights read per pass, as for the intents scan. +const BATCH: u64 = 64; + +/// Wait at the tip before asking again. +const TIP_WAIT: Duration = Duration::from_secs(1); + +/// Wait after a failure — including the node not having synced the shards yet. +const RETRY: Duration = Duration::from_secs(15); + +/// How long a NEAR call may take. +const NEAR_TIMEOUT: Duration = Duration::from_secs(20); + +/// The cursor's key in `intents_scan`, distinct from the intents scan's own so +/// the two never share a position. +fn cursor_key(asset: &str) -> String { + format!("pool-calls:{asset}") +} + +/// Start the scan, if a node is configured for it. +pub fn spawn(chain: Arc, store: Store, config: IntentsConfig) { + if config.pool_rpc_urls.is_empty() + || config.dcl_contracts.is_empty() + || config.token_contract().is_none() + { + return; + } + // The node's own endpoints for everything it can answer; the archive is + // never consulted, because nothing below the node's oldest block is read. + let near = match NearClient::new( + config.pool_rpc_urls.clone(), + config.pool_rpc_urls.clone(), + config.view_urls.clone(), + NEAR_TIMEOUT, + ) { + Ok(n) => n, + Err(e) => { + tracing::error!(chain = %chain.id(), error = %e, "pool call scan not started"); + return; + } + }; + tracing::info!(chain = %chain.id(), endpoints = ?config.pool_rpc_urls, + "following direct calls to the asset's pools"); + tokio::spawn(scan(chain, store, near, config)); +} + +/// Whether a transaction calls one of `pools`: directly, or by sending a token +/// to it with `ft_transfer_call`, which is how a DCL swap is paid for. +fn calls_pool(tx: &blackbeard_data::near::ChunkTransaction, pools: &BTreeSet) -> bool { + pools.contains(&tx.receiver_id) + || tx + .ft_transfer_call_receivers() + .iter() + .any(|r| pools.contains(r)) +} + +async fn scan(chain: Arc, store: Store, near: NearClient, config: IntentsConfig) { + let id = chain.id(); + let key = cursor_key(&config.asset); + let pools = crate::refdcl::contracts(&config); + let asset_contract = config.token_contract().unwrap_or_default().to_owned(); + loop { + if chain.read().current_runtime().is_none() { + tokio::time::sleep(RETRY).await; + continue; + } + let (tip, oldest) = match near.heights().await { + Ok(h) => h, + Err(e) => { + tracing::debug!(chain = %id, error = %e, "pool call node not answering"); + tokio::time::sleep(RETRY).await; + continue; + } + }; + let cursor = match store.intents_scan(&id, &key).await { + Ok(c) => c, + Err(e) => { + tracing::warn!(chain = %id, error = %e, "pool call cursor unreadable"); + tokio::time::sleep(RETRY).await; + continue; + } + }; + let (low, high) = cursor.unwrap_or((tip, tip.saturating_sub(1))); + let floor = oldest.max(config.start_height); + let (heights, descending): (Vec, bool) = if high < tip { + ((high + 1..=tip.min(high + BATCH)).collect(), false) + } else if low > floor { + let from = low.saturating_sub(BATCH).max(floor); + ((from..low).rev().collect(), true) + } else { + tokio::time::sleep(TIP_WAIT).await; + continue; + }; + + let mut layout = None; + for &height in &heights { + match near.shard_layout(height).await { + Ok(l) => { + layout = Some(l); + break; + } + Err(e) if blackbeard_data::near::unknown(&e) => continue, + Err(e) => { + tracing::warn!(chain = %id, error = %e, "pool call shard layout unreadable"); + break; + } + } + } + let Some(layout) = layout else { + tokio::time::sleep(RETRY).await; + continue; + }; + // The shards a direct call can be signed on that this node holds: the + // pools' and the token's. intents.near's is left to the intents scan. + let shard_of = + |account: &str| core::shard_for(account, &layout.boundary_accounts, &layout.shard_ids); + let relay_shard = shard_of(&config.contract); + let shards: BTreeSet = pools + .iter() + .map(String::as_str) + .chain(std::iter::once(asset_contract.as_str())) + .filter_map(shard_of) + .filter(|s| Some(*s) != relay_shard) + .collect(); + + let mut reached = None; + 'heights: for &height in &heights { + for &shard in &shards { + let chunk = match near.chunk(height, shard, oldest).await { + Ok(c) => c, + Err(e) => { + tracing::warn!(chain = %id, near_height = height, shard, error = %e, + "pool call chunk unreadable; will retry"); + break 'heights; + } + }; + let Some(chunk) = chunk else { continue }; + for tx in chunk.transactions.iter().filter(|t| calls_pool(t, &pools)) { + if let Err(e) = store + .queue_ref_pending(&id, &tx.hash, &tx.signer_id, height, &tx.signer_id) + .await + { + tracing::warn!(chain = %id, tx = %tx.hash, error = %e, + "direct pool call not queued"); + break 'heights; + } + tracing::info!(chain = %id, near_height = height, shard, tx = %tx.hash, + signer = %tx.signer_id, "direct pool call queued"); + } + } + reached = Some(height); + } + + if let Some(reached) = reached { + let (new_low, new_high) = if descending { + (reached, high) + } else { + (low.min(heights[0]), reached) + }; + if let Err(e) = store.set_intents_scan(&id, &key, new_low, new_high).await { + tracing::warn!(chain = %id, error = %e, "pool call cursor not saved"); + } + } else { + tokio::time::sleep(RETRY).await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tx(receiver: &str, actions: serde_json::Value) -> blackbeard_data::near::ChunkTransaction { + serde_json::from_value(serde_json::json!({ + "hash": "h", "signer_id": "liq.selfevident.near", + "receiver_id": receiver, "actions": actions, + })) + .unwrap() + } + + #[test] + fn a_pool_call_is_a_call_to_the_pool_or_a_transfer_into_it() { + let pools: BTreeSet = ["dclv2.ref-labs.near".to_owned()].into(); + let direct = tx( + "dclv2.ref-labs.near", + serde_json::json!([{"FunctionCall": {"method_name": "remove_liquidity", "args": "e30=", "gas": 1, "deposit": "0"}}]), + ); + // {"receiver_id":"dclv2.ref-labs.near","amount":"1","msg":""} + let paid = tx( + "qtc.omft.near", + serde_json::json!([{"FunctionCall": {"method_name": "ft_transfer_call", + "args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIiwibXNnIjoiIn0=", + "gas": 1, "deposit": "1"}}]), + ); + // A plain transfer of the token to somebody else is not a pool call. + let elsewhere = tx( + "qtc.omft.near", + serde_json::json!([{"FunctionCall": {"method_name": "ft_transfer", + "args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIn0=", + "gas": 1, "deposit": "1"}}]), + ); + assert!(calls_pool(&direct, &pools)); + assert!(calls_pool(&paid, &pools)); + assert!(!calls_pool(&elsewhere, &pools)); + } + + #[test] + fn the_cursor_never_shares_the_intents_scan_key() { + assert_ne!(cursor_key("nep141:qtc.omft.near"), "nep141:qtc.omft.near"); + } +} diff --git a/crates/blackbeard-api/src/refdcl.rs b/crates/blackbeard-api/src/refdcl.rs index b74ff01..53dbf3e 100644 --- a/crates/blackbeard-api/src/refdcl.rs +++ b/crates/blackbeard-api/src/refdcl.rs @@ -32,9 +32,11 @@ use crate::state::ChainRuntime; /// What this reads, for the pool response to carry. pub const READS: &str = "Ref DCL pools holding the asset, read from the pool contract once a \ - minute. Swaps are those a solver made through NEAR Intents — found on intents.near's own \ - shard and fetched whole from a public endpoint. A swap made by calling the pool directly is \ - not read; the indexed totals beside the pool's own counters show how much that leaves out."; + minute. Swaps are those a solver made through NEAR Intents, found on intents.near's own \ + shard, and those made by calling the pool directly from an account on the pool's or the \ + token's shard, found on a node of ours tracking both since it synced; each is fetched whole \ + from a public endpoint. A direct call signed on any other shard, or before that node synced, \ + is not read; the indexed totals beside the pool's own counters show how much that leaves out."; /// How long to wait before fetching a queued transaction, in NEAR blocks — long /// enough for the pool's receipt, which runs a block or two after the relay's. diff --git a/crates/blackbeard-data/src/near.rs b/crates/blackbeard-data/src/near.rs index ca2e09a..538c6f5 100644 --- a/crates/blackbeard-data/src/near.rs +++ b/crates/blackbeard-data/src/near.rs @@ -47,6 +47,27 @@ impl ChunkTransaction { == Some(method) }) } + + /// Where each `ft_transfer_call` in it sends its tokens: the `receiver_id` + /// inside the call's arguments, not the transaction's own receiver, which + /// is the token contract. A transfer whose arguments do not decode is + /// skipped rather than guessed at. + pub fn ft_transfer_call_receivers(&self) -> Vec { + use base64::Engine as _; + self.actions + .iter() + .filter_map(|a| a.get("FunctionCall")) + .filter(|f| f.get("method_name").and_then(Value::as_str) == Some("ft_transfer_call")) + .filter_map(|f| f.get("args").and_then(Value::as_str)) + .filter_map(|b| base64::engine::general_purpose::STANDARD.decode(b).ok()) + .filter_map(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter_map(|v| { + v.get("receiver_id") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .collect() + } } /// One shard's chunk at one height. @@ -516,6 +537,26 @@ pub async fn oneclick_lists( #[cfg(test)] mod tests { + #[test] + fn a_transfer_call_is_read_for_where_it_sends_the_tokens() { + // `{"receiver_id":"dclv2.ref-labs.near","amount":"1","msg":""}`, as the + // RPC renders a FunctionCall's args: base64. + let tx: super::ChunkTransaction = serde_json::from_value(serde_json::json!({ + "hash": "h", "signer_id": "liq.selfevident.near", "receiver_id": "qtc.omft.near", + "actions": [ + {"FunctionCall": {"method_name": "ft_transfer_call", + "args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIiwibXNnIjoiIn0=", + "gas": 1, "deposit": "1"}}, + {"FunctionCall": {"method_name": "ft_transfer", "args": "e30=", "gas": 1, "deposit": "1"}}, + {"FunctionCall": {"method_name": "ft_transfer_call", "args": "not base64!", "gas": 1, "deposit": "1"}} + ] + })).unwrap(); + assert_eq!( + tx.ft_transfer_call_receivers(), + vec!["dclv2.ref-labs.near".to_owned()] + ); + } + use super::*; #[test]