feat(api): read direct calls to the asset's pools on the shards they are signed on
The relay's pool swaps are found on intents.near's shard; a whitelisted account calling the pool itself signs on its own shard, which nothing read — #37's coverage figure measured the gap at 99.75% of buys. `poolscan` follows the pools' and the token's shards on gargravarr (`pool_rpc_urls`), which tracks shards 4 and 8, and queues every transaction that calls a pool or `ft_transfer_call`s tokens into one. `refdcl::fetch` reads each whole, as it does the relay's, so a swap is parsed one way and recorded once whichever road it came by. Reads only what the node holds; history before it synced stays the coverage figure's to account for. Records the three-node shard split and the Prometheus neard scrape config (installed on golgafrinchans) in the repo. Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj
This commit is contained in:
@@ -507,6 +507,25 @@ trades that never happened.
|
||||
as `intents.near`. Intents arriving as cross-contract receipts are not read; the
|
||||
price response says so in `reads` rather than leaving it to be found.
|
||||
|
||||
**Three nodes, split by shard, because a new shard cannot join a running one
|
||||
in time.** neard starts syncing a newly tracked shard only at an epoch
|
||||
boundary, so adding one to a live node makes it usable about an epoch and a
|
||||
half later. A node started empty syncs every tracked shard at once. So, as of
|
||||
2026-10-06 (blackbeard/observer#39):
|
||||
|
||||
frankie shard 7 intents.near rpc_urls (via near.internal)
|
||||
gargravarr shards 4, 8 qtc.omft.near, pool pool_rpc_urls
|
||||
quadbrat shards 10, 12 USDC, Ref v1 not read yet
|
||||
|
||||
State snapshots are **off** on gargravarr and quadbrat. A snapshot holds flat
|
||||
storage's head until it finishes, and on gargravarr's SATA disks after a fresh
|
||||
sync it did not finish: every state read walked hundreds of deltas and the
|
||||
catch-up ran slower than the chain. We consume state and serve none, so the
|
||||
snapshot bought nothing. quadbrat's P2P port is **24568**, not 24567 — the
|
||||
hanzalova WAN forwards 24567 to gargravarr, and neard tells peers its own listen
|
||||
port, so the forwarded port and the node's must match. neard's metrics are on
|
||||
the RPC port at `/metrics`, scraped by `asset/prometheus/blackbeard-neard.yml`.
|
||||
|
||||
**On a one-shard node, `tx_status` must not wait for execution.** Our node
|
||||
tracks only intents.near's shard. `EXPERIMENTAL_tx_status` with `EXECUTED`
|
||||
waits for every receipt in the transaction, including the withdrawals on
|
||||
|
||||
@@ -147,6 +147,11 @@ max_gap_fill_blocks = 5000
|
||||
asset = "nep141:qtc.omft.near"
|
||||
start_height = 218346736
|
||||
rpc_urls = ["https://near.internal"]
|
||||
# gargravarr tracks the shards of dclv2.ref-labs.near and qtc.omft.near (8 and
|
||||
# 4), where the pool's whitelisted accounts sign their direct calls
|
||||
# (blackbeard/observer#39). Plain HTTP on the hanzalova mesh, admitted to bob
|
||||
# alone by a firewalld rich rule, as frankie's RPC is to the proxy.
|
||||
pool_rpc_urls = ["http://gargravarr.hanzalova.internal:24403"]
|
||||
|
||||
[[chains]]
|
||||
id = "heisenberg"
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# neard (NEAR mainnet RPC nodes) for blackbeard.observer. Installed by hand at
|
||||
# /etc/prometheus/scrape_configs.d/blackbeard-neard.yml on golgafrinchans
|
||||
# (2026-10-06); this copy is the record. Reload with `curl -X POST
|
||||
# http://127.0.0.1:26559/-/reload` there after `promtool check config`.
|
||||
# neard serves Prometheus metrics on its JSON-RPC port at /metrics; each host
|
||||
# admits this one source (10.3.101.4) to that port with a firewalld rich rule.
|
||||
#
|
||||
# Targets are addresses, not names: this host resolves through kosherinata
|
||||
# Unbound, which carries no record for gargravarr or frankie (hanzalova only).
|
||||
# The "shards" label is what each node tracks (blackbeard/observer#39).
|
||||
scrape_configs:
|
||||
- job_name: neard
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 20s
|
||||
static_configs:
|
||||
- targets: ["10.6.0.70:24403"]
|
||||
labels:
|
||||
host: frankie
|
||||
shards: "7"
|
||||
- targets: ["10.6.0.107:24403"]
|
||||
labels:
|
||||
host: gargravarr
|
||||
shards: "4,8"
|
||||
- targets: ["10.6.0.76:24403"]
|
||||
labels:
|
||||
host: quadbrat
|
||||
shards: "10,12"
|
||||
@@ -232,6 +232,13 @@ pub struct IntentsConfig {
|
||||
/// pool for the asset needs no config change.
|
||||
#[serde(default = "default_dcl_contracts")]
|
||||
pub dcl_contracts: Vec<String>,
|
||||
/// NEAR JSON-RPC endpoints of a node tracking the shards the pool contracts
|
||||
/// and the asset's token live on (blackbeard/observer#39). Transactions
|
||||
/// signed on those shards that call a pool directly — the whitelisted
|
||||
/// accounts trading or moving liquidity without intents.near — are found
|
||||
/// there and queued for `refdcl` like the relay's. Empty turns the scan off.
|
||||
#[serde(default)]
|
||||
pub pool_rpc_urls: Vec<String>,
|
||||
}
|
||||
|
||||
impl IntentsConfig {
|
||||
|
||||
@@ -119,6 +119,7 @@ pub fn spawn(chain: Arc<ChainRuntime>, store: Store) {
|
||||
near.clone(),
|
||||
config.clone(),
|
||||
);
|
||||
crate::poolscan::spawn(Arc::clone(&chain), store.clone(), config.clone());
|
||||
tokio::spawn(scan(chain, store, near, config));
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
mod config;
|
||||
mod ingest;
|
||||
mod intents;
|
||||
mod poolscan;
|
||||
mod refdcl;
|
||||
mod registry;
|
||||
mod routes;
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
//! Direct calls to the asset's pools, read on the shards the calls are signed on.
|
||||
//!
|
||||
//! `intents::scan` finds a pool swap only when the 1Click relay made it, because
|
||||
//! it reads intents.near's shard and a transaction lands in its *signer's*
|
||||
//! shard. The pool's whitelisted accounts also call it themselves — to trade, to
|
||||
//! add or pull liquidity — and those transactions are signed on other shards:
|
||||
//! `liq.selfevident.near` and `solver-ref-2.near` are on the shard the token
|
||||
//! contract is on, the pool on another. Our coverage figure measured the gap
|
||||
//! (#37: 99.75% of buys); this closes it for every signer on a shard a tracked
|
||||
//! node holds (blackbeard/observer#39).
|
||||
//!
|
||||
//! It only finds candidates. Each goes into `ref_pending` and is read whole by
|
||||
//! `refdcl::fetch`, the same path as the relay's, so a swap is parsed one way
|
||||
//! whichever road it arrived by and recorded once on its `(tx, receipt, index)`
|
||||
//! key. A candidate that turns out not to be a swap — a liquidity change, say —
|
||||
//! yields no swap event and records nothing.
|
||||
//!
|
||||
//! Only what the node holds is read: forward from wherever the cursor stands,
|
||||
//! and back no further than the oldest block the node has. History before the
|
||||
//! node synced is the coverage figure's to account for, not a public endpoint's
|
||||
//! to be paced through for days.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use blackbeard_core::intents as core;
|
||||
use blackbeard_data::near::NearClient;
|
||||
use blackbeard_data::store::Store;
|
||||
|
||||
use crate::config::IntentsConfig;
|
||||
use crate::state::ChainRuntime;
|
||||
|
||||
/// NEAR heights read per pass, as for the intents scan.
|
||||
const BATCH: u64 = 64;
|
||||
|
||||
/// Wait at the tip before asking again.
|
||||
const TIP_WAIT: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Wait after a failure — including the node not having synced the shards yet.
|
||||
const RETRY: Duration = Duration::from_secs(15);
|
||||
|
||||
/// How long a NEAR call may take.
|
||||
const NEAR_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// The cursor's key in `intents_scan`, distinct from the intents scan's own so
|
||||
/// the two never share a position.
|
||||
fn cursor_key(asset: &str) -> String {
|
||||
format!("pool-calls:{asset}")
|
||||
}
|
||||
|
||||
/// Start the scan, if a node is configured for it.
|
||||
pub fn spawn(chain: Arc<ChainRuntime>, store: Store, config: IntentsConfig) {
|
||||
if config.pool_rpc_urls.is_empty()
|
||||
|| config.dcl_contracts.is_empty()
|
||||
|| config.token_contract().is_none()
|
||||
{
|
||||
return;
|
||||
}
|
||||
// The node's own endpoints for everything it can answer; the archive is
|
||||
// never consulted, because nothing below the node's oldest block is read.
|
||||
let near = match NearClient::new(
|
||||
config.pool_rpc_urls.clone(),
|
||||
config.pool_rpc_urls.clone(),
|
||||
config.view_urls.clone(),
|
||||
NEAR_TIMEOUT,
|
||||
) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::error!(chain = %chain.id(), error = %e, "pool call scan not started");
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::info!(chain = %chain.id(), endpoints = ?config.pool_rpc_urls,
|
||||
"following direct calls to the asset's pools");
|
||||
tokio::spawn(scan(chain, store, near, config));
|
||||
}
|
||||
|
||||
/// Whether a transaction calls one of `pools`: directly, or by sending a token
|
||||
/// to it with `ft_transfer_call`, which is how a DCL swap is paid for.
|
||||
fn calls_pool(tx: &blackbeard_data::near::ChunkTransaction, pools: &BTreeSet<String>) -> bool {
|
||||
pools.contains(&tx.receiver_id)
|
||||
|| tx
|
||||
.ft_transfer_call_receivers()
|
||||
.iter()
|
||||
.any(|r| pools.contains(r))
|
||||
}
|
||||
|
||||
async fn scan(chain: Arc<ChainRuntime>, store: Store, near: NearClient, config: IntentsConfig) {
|
||||
let id = chain.id();
|
||||
let key = cursor_key(&config.asset);
|
||||
let pools = crate::refdcl::contracts(&config);
|
||||
let asset_contract = config.token_contract().unwrap_or_default().to_owned();
|
||||
loop {
|
||||
if chain.read().current_runtime().is_none() {
|
||||
tokio::time::sleep(RETRY).await;
|
||||
continue;
|
||||
}
|
||||
let (tip, oldest) = match near.heights().await {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::debug!(chain = %id, error = %e, "pool call node not answering");
|
||||
tokio::time::sleep(RETRY).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let cursor = match store.intents_scan(&id, &key).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(chain = %id, error = %e, "pool call cursor unreadable");
|
||||
tokio::time::sleep(RETRY).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let (low, high) = cursor.unwrap_or((tip, tip.saturating_sub(1)));
|
||||
let floor = oldest.max(config.start_height);
|
||||
let (heights, descending): (Vec<u64>, bool) = if high < tip {
|
||||
((high + 1..=tip.min(high + BATCH)).collect(), false)
|
||||
} else if low > floor {
|
||||
let from = low.saturating_sub(BATCH).max(floor);
|
||||
((from..low).rev().collect(), true)
|
||||
} else {
|
||||
tokio::time::sleep(TIP_WAIT).await;
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut layout = None;
|
||||
for &height in &heights {
|
||||
match near.shard_layout(height).await {
|
||||
Ok(l) => {
|
||||
layout = Some(l);
|
||||
break;
|
||||
}
|
||||
Err(e) if blackbeard_data::near::unknown(&e) => continue,
|
||||
Err(e) => {
|
||||
tracing::warn!(chain = %id, error = %e, "pool call shard layout unreadable");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(layout) = layout else {
|
||||
tokio::time::sleep(RETRY).await;
|
||||
continue;
|
||||
};
|
||||
// The shards a direct call can be signed on that this node holds: the
|
||||
// pools' and the token's. intents.near's is left to the intents scan.
|
||||
let shard_of =
|
||||
|account: &str| core::shard_for(account, &layout.boundary_accounts, &layout.shard_ids);
|
||||
let relay_shard = shard_of(&config.contract);
|
||||
let shards: BTreeSet<u64> = pools
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.chain(std::iter::once(asset_contract.as_str()))
|
||||
.filter_map(shard_of)
|
||||
.filter(|s| Some(*s) != relay_shard)
|
||||
.collect();
|
||||
|
||||
let mut reached = None;
|
||||
'heights: for &height in &heights {
|
||||
for &shard in &shards {
|
||||
let chunk = match near.chunk(height, shard, oldest).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(chain = %id, near_height = height, shard, error = %e,
|
||||
"pool call chunk unreadable; will retry");
|
||||
break 'heights;
|
||||
}
|
||||
};
|
||||
let Some(chunk) = chunk else { continue };
|
||||
for tx in chunk.transactions.iter().filter(|t| calls_pool(t, &pools)) {
|
||||
if let Err(e) = store
|
||||
.queue_ref_pending(&id, &tx.hash, &tx.signer_id, height, &tx.signer_id)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(chain = %id, tx = %tx.hash, error = %e,
|
||||
"direct pool call not queued");
|
||||
break 'heights;
|
||||
}
|
||||
tracing::info!(chain = %id, near_height = height, shard, tx = %tx.hash,
|
||||
signer = %tx.signer_id, "direct pool call queued");
|
||||
}
|
||||
}
|
||||
reached = Some(height);
|
||||
}
|
||||
|
||||
if let Some(reached) = reached {
|
||||
let (new_low, new_high) = if descending {
|
||||
(reached, high)
|
||||
} else {
|
||||
(low.min(heights[0]), reached)
|
||||
};
|
||||
if let Err(e) = store.set_intents_scan(&id, &key, new_low, new_high).await {
|
||||
tracing::warn!(chain = %id, error = %e, "pool call cursor not saved");
|
||||
}
|
||||
} else {
|
||||
tokio::time::sleep(RETRY).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tx(receiver: &str, actions: serde_json::Value) -> blackbeard_data::near::ChunkTransaction {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"hash": "h", "signer_id": "liq.selfevident.near",
|
||||
"receiver_id": receiver, "actions": actions,
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pool_call_is_a_call_to_the_pool_or_a_transfer_into_it() {
|
||||
let pools: BTreeSet<String> = ["dclv2.ref-labs.near".to_owned()].into();
|
||||
let direct = tx(
|
||||
"dclv2.ref-labs.near",
|
||||
serde_json::json!([{"FunctionCall": {"method_name": "remove_liquidity", "args": "e30=", "gas": 1, "deposit": "0"}}]),
|
||||
);
|
||||
// {"receiver_id":"dclv2.ref-labs.near","amount":"1","msg":""}
|
||||
let paid = tx(
|
||||
"qtc.omft.near",
|
||||
serde_json::json!([{"FunctionCall": {"method_name": "ft_transfer_call",
|
||||
"args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIiwibXNnIjoiIn0=",
|
||||
"gas": 1, "deposit": "1"}}]),
|
||||
);
|
||||
// A plain transfer of the token to somebody else is not a pool call.
|
||||
let elsewhere = tx(
|
||||
"qtc.omft.near",
|
||||
serde_json::json!([{"FunctionCall": {"method_name": "ft_transfer",
|
||||
"args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIn0=",
|
||||
"gas": 1, "deposit": "1"}}]),
|
||||
);
|
||||
assert!(calls_pool(&direct, &pools));
|
||||
assert!(calls_pool(&paid, &pools));
|
||||
assert!(!calls_pool(&elsewhere, &pools));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cursor_never_shares_the_intents_scan_key() {
|
||||
assert_ne!(cursor_key("nep141:qtc.omft.near"), "nep141:qtc.omft.near");
|
||||
}
|
||||
}
|
||||
@@ -32,9 +32,11 @@ use crate::state::ChainRuntime;
|
||||
|
||||
/// What this reads, for the pool response to carry.
|
||||
pub const READS: &str = "Ref DCL pools holding the asset, read from the pool contract once a \
|
||||
minute. Swaps are those a solver made through NEAR Intents — found on intents.near's own \
|
||||
shard and fetched whole from a public endpoint. A swap made by calling the pool directly is \
|
||||
not read; the indexed totals beside the pool's own counters show how much that leaves out.";
|
||||
minute. Swaps are those a solver made through NEAR Intents, found on intents.near's own \
|
||||
shard, and those made by calling the pool directly from an account on the pool's or the \
|
||||
token's shard, found on a node of ours tracking both since it synced; each is fetched whole \
|
||||
from a public endpoint. A direct call signed on any other shard, or before that node synced, \
|
||||
is not read; the indexed totals beside the pool's own counters show how much that leaves out.";
|
||||
|
||||
/// How long to wait before fetching a queued transaction, in NEAR blocks — long
|
||||
/// enough for the pool's receipt, which runs a block or two after the relay's.
|
||||
|
||||
@@ -47,6 +47,27 @@ impl ChunkTransaction {
|
||||
== Some(method)
|
||||
})
|
||||
}
|
||||
|
||||
/// Where each `ft_transfer_call` in it sends its tokens: the `receiver_id`
|
||||
/// inside the call's arguments, not the transaction's own receiver, which
|
||||
/// is the token contract. A transfer whose arguments do not decode is
|
||||
/// skipped rather than guessed at.
|
||||
pub fn ft_transfer_call_receivers(&self) -> Vec<String> {
|
||||
use base64::Engine as _;
|
||||
self.actions
|
||||
.iter()
|
||||
.filter_map(|a| a.get("FunctionCall"))
|
||||
.filter(|f| f.get("method_name").and_then(Value::as_str) == Some("ft_transfer_call"))
|
||||
.filter_map(|f| f.get("args").and_then(Value::as_str))
|
||||
.filter_map(|b| base64::engine::general_purpose::STANDARD.decode(b).ok())
|
||||
.filter_map(|bytes| serde_json::from_slice::<Value>(&bytes).ok())
|
||||
.filter_map(|v| {
|
||||
v.get("receiver_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// One shard's chunk at one height.
|
||||
@@ -516,6 +537,26 @@ pub async fn oneclick_lists(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn a_transfer_call_is_read_for_where_it_sends_the_tokens() {
|
||||
// `{"receiver_id":"dclv2.ref-labs.near","amount":"1","msg":""}`, as the
|
||||
// RPC renders a FunctionCall's args: base64.
|
||||
let tx: super::ChunkTransaction = serde_json::from_value(serde_json::json!({
|
||||
"hash": "h", "signer_id": "liq.selfevident.near", "receiver_id": "qtc.omft.near",
|
||||
"actions": [
|
||||
{"FunctionCall": {"method_name": "ft_transfer_call",
|
||||
"args": "eyJyZWNlaXZlcl9pZCI6ImRjbHYyLnJlZi1sYWJzLm5lYXIiLCJhbW91bnQiOiIxIiwibXNnIjoiIn0=",
|
||||
"gas": 1, "deposit": "1"}},
|
||||
{"FunctionCall": {"method_name": "ft_transfer", "args": "e30=", "gas": 1, "deposit": "1"}},
|
||||
{"FunctionCall": {"method_name": "ft_transfer_call", "args": "not base64!", "gas": 1, "deposit": "1"}}
|
||||
]
|
||||
})).unwrap();
|
||||
assert_eq!(
|
||||
tx.ft_transfer_call_receivers(),
|
||||
vec!["dclv2.ref-labs.near".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user