Files
observer/crates/blackbeard-api/src/refdcl.rs
T
grenadeandClaude Opus 5.5 025116fd2d
ci / gate (push) Successful in 4m21s
deploy / validate (push) Skipped
deploy / build (push) Successful in 9m9s
deploy / deploy-web (push) Successful in 10s
deploy / deploy-api (push) Successful in 26s
feat(api): log the pool's non-swap events, to learn whether it logs orders
#41 asks whether `dcl.ref` logs limit-order events — which would say who
posts and withdraws each bid, where the book alone only shows that it
changed. The contract's source is not public where it was looked for, so
the chain is asked instead: `refdcl::fetch` already reads every direct
call in full, and now logs the first sighting of each non-swap `dcl.ref`
event per process with a sample of its data.

Refs #41

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj
2026-10-06 19:49:13 +03:00

624 lines
24 KiB
Rust

//! The tracked asset's Ref Finance DCL pools: their swaps and their ladders.
//!
//! QTC's first liquidity on NEAR was a single-sided sell ladder in a Ref DCL
//! pool, and solvers restock from it through `intents.near`
//! (blackbeard/observer#37). The pool contract is on a shard our node does not
//! track, so this works in two halves:
//!
//! - **Swaps.** The intents scan, reading `intents.near`'s own shard, queues any
//! relay transaction whose Verifier outcome sends a solver's tokens to a
//! tracked pool contract (`ref_pending`). A few blocks later — once the
//! pool's receipt has executed — `fetch` asks a public endpoint, which sees
//! every shard, for that one transaction and records its `dcl.ref` swaps.
//! What this misses is a swap made by a whitelisted account calling the pool
//! directly, outside NEAR Intents; the coverage figures say how much.
//! - **State.** `pools` reads each pool holding the asset once a minute — its
//! point, totals, whitelist, cumulative counters and liquidity ladder — and
//! sets the indexed swap totals beside the pool's own counters, so the blind
//! spot is a measured share rather than a disclaimer.
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use blackbeard_core::dcl;
use blackbeard_data::near::NearClient;
use blackbeard_data::store::{RefSwapRecord, Store};
use blackbeard_entities::{BigUintDec, ChainId, RefPool, RefPools, RefRung};
use serde_json::{Value, json};
use crate::config::IntentsConfig;
use crate::state::ChainRuntime;
/// What this reads, for the pool response to carry.
pub const READS: &str = "Ref DCL pools holding the asset — liquidity and limit orders — read \
from the pool contract once a minute. Swaps are those a solver made through NEAR Intents, found on intents.near's own \
shard, and those made by calling the pool directly from an account on the pool's or the \
token's shard, found on a node of ours tracking both since it synced; each is fetched whole \
from a public endpoint. A direct call signed on any other shard, or before that node synced, \
is not read; the indexed totals beside the pool's own counters show how much that leaves out.";
/// How long to wait before fetching a queued transaction, in NEAR blocks — long
/// enough for the pool's receipt, which runs a block or two after the relay's.
const FETCH_AFTER_BLOCKS: u64 = 20;
/// Queued transactions taken per pass.
const FETCH_BATCH: i64 = 50;
/// How often the queue is worked when it is empty.
const FETCH_INTERVAL: Duration = Duration::from_secs(10);
/// Attempts before a queued transaction is given up on.
const FETCH_ATTEMPTS: u32 = 30;
/// How often pool state is read.
const POOL_INTERVAL: Duration = Duration::from_secs(60);
/// Pool discovery runs every this many state reads.
const DISCOVER_EVERY: u32 = 10;
/// `list_pools` page size.
const LIST_PAGE: u64 = 200;
/// Sizes, in whole units of the asset, whose sale is quoted on every pool read
/// (blackbeard/observer#41): what a seller of that much would be paid now.
pub const QUOTE_SIZES: [u64; 4] = [10, 100, 1_000, 5_000];
/// The widest point range DCL allows, for `get_liquidity_range`.
const POINT_BOUND: i32 = 800_000;
/// Start the fetch and pool tasks.
pub fn spawn(chain: Arc<ChainRuntime>, store: Store, near: NearClient, config: IntentsConfig) {
if config.dcl_contracts.is_empty() || config.token_contract().is_none() {
return;
}
tokio::spawn(fetch(
Arc::clone(&chain),
store.clone(),
near.clone(),
config.clone(),
));
tokio::spawn(pools(chain, store, near, config));
}
/// The pool contracts followed, as a set.
pub fn contracts(config: &IntentsConfig) -> BTreeSet<String> {
config.dcl_contracts.iter().cloned().collect()
}
/// Whether a pool id names the asset's contract as one of its two tokens.
fn holds_asset(pool_id: &str, asset_contract: &str) -> bool {
pool_id.split('|').take(2).any(|t| t == asset_contract)
}
/// Whether a relay's withdrawal to a pool is about the asset: sending it, or an
/// instruction that names a pool holding it.
pub fn concerns_asset(w: &dcl::PoolWithdrawal, asset_contract: &str) -> bool {
w.token == asset_contract || w.msg.contains(asset_contract)
}
async fn fetch(chain: Arc<ChainRuntime>, store: Store, near: NearClient, config: IntentsConfig) {
let id = chain.id();
let pools = contracts(&config);
let Some(asset_contract) = config.token_contract().map(str::to_owned) else {
return;
};
let mut backlog = false;
loop {
// Straight on while the last batch was full: the wait is for an empty
// queue, never for one with work in it. After launch it slept ten
// seconds between batches of twenty while 1,574 relay swaps waited.
if !backlog {
tokio::time::sleep(FETCH_INTERVAL).await;
}
let tip = match near.heights().await {
Ok((tip, _)) => tip,
Err(e) => {
tracing::debug!(chain = %id, error = %e, "near tip unreadable for ref fetch");
continue;
}
};
let due = match store
.due_ref_pending(&id, tip.saturating_sub(FETCH_AFTER_BLOCKS), FETCH_BATCH)
.await
{
Ok(d) => {
backlog = d.len() as i64 == FETCH_BATCH;
d
}
Err(e) => {
tracing::warn!(chain = %id, error = %e, "ref queue unreadable");
continue;
}
};
for (tx_hash, signer, height, trader, attempts) in due {
match fetch_one(
&near,
&id,
&config,
&pools,
&asset_contract,
&tx_hash,
&signer,
height,
&trader,
)
.await
{
Ok(Some(swaps)) => {
if !swaps.is_empty() {
if let Err(e) = store.record_ref_swaps(&swaps).await {
tracing::warn!(chain = %id, tx = %tx_hash, error = %e, "ref swaps not recorded");
continue;
}
tracing::info!(chain = %id, tx = %tx_hash, swaps = swaps.len(), "ref pool swaps recorded");
}
let _ = store.drop_ref_pending(&id, &tx_hash).await;
}
// The pool's receipt is not in the outcome yet, or the endpoint
// failed: try again later, up to a limit.
Ok(None) | Err(_) if attempts + 1 < FETCH_ATTEMPTS => {
let _ = store.bump_ref_pending(&id, &tx_hash).await;
}
other => {
tracing::warn!(chain = %id, tx = %tx_hash, outcome = ?other.err(),
"ref pool swap given up on after repeated fetches");
let _ = store.drop_ref_pending(&id, &tx_hash).await;
}
}
}
}
}
/// The swaps in one queued transaction. `Ok(None)` while the pool's receipt
/// has not executed yet.
#[allow(clippy::too_many_arguments)]
async fn fetch_one(
near: &NearClient,
chain: &ChainId,
config: &IntentsConfig,
pools: &BTreeSet<String>,
asset_contract: &str,
tx_hash: &str,
signer: &str,
height: u64,
trader: &str,
) -> Result<Option<Vec<RefSwapRecord>>, blackbeard_data::DataError> {
let outcomes = near.tx_outcomes_everywhere(tx_hash, signer).await?;
let pool_outcomes: Vec<_> = outcomes
.iter()
.filter(|o| pools.contains(&o.executor_id))
.collect();
if pool_outcomes.is_empty() {
return Ok(None);
}
let (_, at) = near.block_at(json!(height)).await?;
let mut out = Vec::new();
// A failed receipt still reports its logs; only a successful one swapped.
for o in pool_outcomes.into_iter().filter(|o| o.succeeded) {
note_other_events(tx_hash, &o.logs);
for (i, s) in dcl::dcl_swaps(&o.logs)
.into_iter()
.filter(|s| holds_asset(&s.pool_id, asset_contract))
.enumerate()
{
let side = if s.token_out == asset_contract {
"buy"
} else {
"sell"
};
out.push(RefSwapRecord {
chain: chain.clone(),
asset: config.asset.clone(),
near_height: height,
tx_hash: tx_hash.to_owned(),
receipt_id: o.id.clone(),
swap_index: i as u32,
at,
contract: o.executor_id.clone(),
pool_id: s.pool_id,
trader: trader.to_owned(),
swapper: s.swapper,
token_in: s.token_in,
token_out: s.token_out,
amount_in: s.amount_in.to_string(),
amount_out: s.amount_out.to_string(),
total_fee: s.total_fee.to_string(),
protocol_fee: s.protocol_fee.to_string(),
side: side.to_owned(),
});
}
}
Ok(Some(out))
}
async fn pools(chain: Arc<ChainRuntime>, store: Store, near: NearClient, config: IntentsConfig) {
let id = chain.id();
let Some(asset_contract) = config.token_contract().map(str::to_owned) else {
return;
};
// Seeded from the pools swaps have been recorded in, which certainly hold
// the asset: a restart while the public endpoint is rate-limiting would
// otherwise leave the page with no pools until a full listing gets through
// (2026-10-06: twenty minutes of 429s after one deploy). Discovery still
// runs, and is what finds a pool nobody has swapped in yet.
let mut known: Vec<(String, String)> = store
.ref_known_pools(&id, &config.asset)
.await
.map_err(|e| tracing::warn!(chain = %id, error = %e, "recorded pools unreadable"))
.unwrap_or_default();
let mut discovered = false;
let mut round = 0u32;
loop {
if round % DISCOVER_EVERY == 0 || !discovered {
match discover(&near, &config, &asset_contract).await {
Ok(found) => {
known = found;
discovered = true;
}
Err(e) => tracing::warn!(chain = %id, error = %e, "ref pools not listed"),
}
}
round = round.wrapping_add(1);
// Until a discovery has succeeded there is nothing known to read, and
// reading nothing would publish "no pool holds the asset" — a claim
// made from a failed request. The slot stays unread instead, which the
// page says in those words (2026-10-06: a run of 429s did exactly
// this). A discovery that succeeds and finds none is published as none;
// pools seeded from recorded swaps are read meanwhile.
if !discovered && known.is_empty() {
tokio::time::sleep(POOL_INTERVAL).await;
continue;
}
match read_pools(&chain, &store, &near, &config, &asset_contract, &known).await {
Ok(p) => *chain.ref_pools.write().unwrap_or_else(|e| e.into_inner()) = Some(p),
Err(e) => tracing::warn!(chain = %id, error = %e, "ref pools unreadable"),
}
tokio::time::sleep(POOL_INTERVAL).await;
}
}
/// Every `(contract, pool_id)` holding the asset, by paging `list_pools`.
async fn discover(
near: &NearClient,
config: &IntentsConfig,
asset_contract: &str,
) -> Result<Vec<(String, String)>, blackbeard_data::DataError> {
let mut out = Vec::new();
for contract in &config.dcl_contracts {
let mut from = 0u64;
loop {
let page = near
.view_anywhere(
contract,
"list_pools",
json!({"from_index": from, "limit": LIST_PAGE}),
)
.await?;
let page = page.as_array().cloned().unwrap_or_default();
for p in &page {
if let Some(pool_id) = p.get("pool_id").and_then(Value::as_str) {
if holds_asset(pool_id, asset_contract) {
out.push((contract.clone(), pool_id.to_owned()));
}
}
}
if (page.len() as u64) < LIST_PAGE {
break;
}
from += LIST_PAGE;
}
}
Ok(out)
}
/// Write one pool read to the bid-side history (blackbeard/observer#41): the
/// snapshot, its quotes, and the order book when it changed. A failure here is
/// logged and the pass goes on — the live view must not depend on the record.
#[allow(clippy::too_many_arguments)]
async fn record_history(
store: &Store,
near: &NearClient,
chain: &blackbeard_entities::ChainId,
contract: &str,
pool_id: &str,
at: chrono::DateTime<chrono::Utc>,
state: &Value,
order_book: &Value,
asset_contract: &str,
counter_contract: &str,
asset_decimals: u8,
) {
let mut quotes = serde_json::Map::new();
for size in QUOTE_SIZES {
let amount = u128::from(size) * 10u128.pow(u32::from(asset_decimals));
match near
.view_anywhere(
contract,
"quote",
json!({"pool_ids": [pool_id], "input_token": asset_contract,
"output_token": counter_contract, "input_amount": amount.to_string(),
"tag": null}),
)
.await
{
Ok(q) => {
if let Some(out) = q["amount"].as_str() {
quotes.insert(amount.to_string(), Value::String(out.to_owned()));
}
}
// Left out rather than written as zero: an unanswered quote is not
// a pool that would pay nothing.
Err(e) => tracing::debug!(%pool_id, size, error = %e, "quote unreadable"),
}
}
let text = |k: &str| state[k].as_str().unwrap_or("0").to_owned();
let snapshot = blackbeard_data::store::PoolSnapshot {
chain: chain.clone(),
pool_id: pool_id.to_owned(),
at,
current_point: state["current_point"].as_i64().unwrap_or(0) as i32,
total_x: text("total_x"),
total_y: text("total_y"),
total_order_x: text("total_order_x"),
total_order_y: text("total_order_y"),
volume_x_in: text("volume_x_in"),
volume_x_out: text("volume_x_out"),
volume_y_in: text("volume_y_in"),
volume_y_out: text("volume_y_out"),
quotes: Value::Object(quotes),
};
if let Err(e) = store.record_pool_snapshot(&snapshot).await {
tracing::warn!(%pool_id, error = %e, "pool snapshot not recorded");
}
// Sorted by point so equal books compare equal in the store.
let mut levels: Vec<Value> = order_book
.as_object()
.into_iter()
.flat_map(|o| o.values())
.filter_map(|o| {
Some(
json!({"point": o["point"].as_i64()?, "amount_x": o["amount_x"].as_str()?,
"amount_y": o["amount_y"].as_str()?}),
)
})
.collect();
levels.sort_by_key(|l| l["point"].as_i64());
if let Err(e) = store
.record_pool_book(chain, pool_id, at, &Value::Array(levels))
.await
{
tracing::warn!(%pool_id, error = %e, "pool book not recorded");
}
}
/// Log the first sighting of each non-swap `dcl.ref` event this process sees,
/// with a sample of its data: whether the pool logs limit orders, and in what
/// shape, is found out from the chain rather than assumed (blackbeard/observer#41).
fn note_other_events(tx_hash: &str, logs: &[String]) {
static SEEN: std::sync::Mutex<BTreeSet<String>> = std::sync::Mutex::new(BTreeSet::new());
for (name, data) in dcl::dcl_other_events(logs) {
let first = SEEN
.lock()
.unwrap_or_else(|p| p.into_inner())
.insert(name.clone());
if first {
let mut sample = data.to_string();
sample.truncate(600);
tracing::info!(event = %name, tx = %tx_hash, %sample, "dcl.ref event other than swap");
}
}
}
fn num(v: &Value, k: &str) -> u128 {
v.get(k)
.and_then(Value::as_str)
.and_then(|s| s.parse().ok())
.unwrap_or(0)
}
async fn read_pools(
chain: &Arc<ChainRuntime>,
store: &Store,
near: &NearClient,
config: &IntentsConfig,
asset_contract: &str,
known: &[(String, String)],
) -> Result<RefPools, blackbeard_data::DataError> {
let id = chain.id();
let asset_meta = store.near_token(&config.asset).await?;
let totals = store.ref_swap_totals(&id, &config.asset).await?;
// One instant for the whole pass: a snapshot and the book written beside
// it share it, which is what lets a book change be paired with the swap
// volume at the same moment.
let read_at = chrono::Utc::now();
let mut pools = Vec::new();
for (contract, pool_id) in known {
let state = near
.view_anywhere(contract, "get_pool", json!({"pool_id": pool_id}))
.await?;
let ranges = near
.view_anywhere(
contract,
"get_liquidity_range",
json!({"pool_id": pool_id, "left_point": -POINT_BOUND, "right_point": POINT_BOUND}),
)
.await?;
// The limit-order book. One call over the whole point range answered
// for nine levels on 2026-10-06; a view that grows past the gas limit
// fails loudly here rather than returning part of the book.
let order_book = near
.view_anywhere(
contract,
"get_pointorder_range",
json!({"pool_id": pool_id, "left_point": -POINT_BOUND, "right_point": POINT_BOUND}),
)
.await?;
let token_x = state["token_x"].as_str().unwrap_or_default().to_owned();
let token_y = state["token_y"].as_str().unwrap_or_default().to_owned();
let asset_is_y = token_y == asset_contract;
let counter_contract = if asset_is_y { &token_x } else { &token_y };
let counter_token = format!("nep141:{counter_contract}");
crate::intents::remember_token(near, store, &counter_token).await;
let counter_meta = store.near_token(&counter_token).await?;
let (Some((_, ad)), Some((counter_symbol, cd))) = (asset_meta.clone(), counter_meta) else {
continue;
};
let (dx, dy) = if asset_is_y { (cd, ad) } else { (ad, cd) };
record_history(
store,
near,
&id,
contract,
pool_id,
read_at,
&state,
&order_book,
asset_contract,
counter_contract,
ad,
)
.await;
let current = state["current_point"].as_i64().unwrap_or(0) as i32;
let liquidity_x = num(&state, "liquidity_x");
let ranges: Vec<dcl::Range> = ranges
.as_object()
.into_iter()
.flat_map(|o| o.values())
.filter_map(|r| {
Some(dcl::Range {
left: r["left_point"].as_i64()? as i32,
right: r["right_point"].as_i64()? as i32,
liquidity: r["amount_l"].as_str()?.parse().ok()?,
})
})
.collect();
// The asset's price in the counter at a point: y is priced in x by the
// point directly; an x asset is its inverse.
let asset_price = |p: i32| {
let y_in_x = dcl::price_y_in_x(p, dx, dy);
if asset_is_y { y_in_x } else { 1.0 / y_in_x }
};
let scale = |v: f64, d: u8| v / 10f64.powi(i32::from(d));
let orders: Vec<dcl::Order> = order_book
.as_object()
.into_iter()
.flat_map(|o| o.values())
.filter_map(|o| {
Some(dcl::Order {
point: o["point"].as_i64()? as i32,
x: o["amount_x"].as_str()?.parse().ok()?,
y: o["amount_y"].as_str()?.parse().ok()?,
})
})
.collect();
// Liquidity and orders side by side, each marked with its source: an
// order can be cancelled at any moment, liquidity only withdrawn by its
// provider, and a reader deciding when to sell needs to know which.
let parts = dcl::ladder(&ranges, current, liquidity_x)
.into_iter()
.map(|r| (r, "liquidity"))
.chain(dcl::order_rungs(&orders).into_iter().map(|r| (r, "order")));
let mut rungs: Vec<RefRung> = parts
.map(|(r, source)| {
let (asset, counter, counter_to_clear, asset_to_clear) = if asset_is_y {
(
scale(r.y, ad),
scale(r.x, cd),
scale(r.x_to_clear, cd),
scale(r.y_to_clear, ad),
)
} else {
(
scale(r.x, ad),
scale(r.y, cd),
scale(r.y_to_clear, cd),
scale(r.x_to_clear, ad),
)
};
let (a, b) = (asset_price(r.left), asset_price(r.right));
RefRung {
price_from: a.min(b),
price_to: a.max(b),
asset,
counter,
counter_to_clear,
asset_to_clear,
kind: if asset > 0.0 && counter > 0.0 {
"current"
} else if asset > 0.0 {
"asks"
} else {
"bids"
}
.to_owned(),
source: source.to_owned(),
}
})
.collect();
rungs.sort_by(|a, b| a.price_from.total_cmp(&b.price_from));
let (asset_held, counter_held, asset_out, asset_in) = if asset_is_y {
(
num(&state, "total_y"),
num(&state, "total_x"),
num(&state, "volume_y_out"),
num(&state, "volume_y_in"),
)
} else {
(
num(&state, "total_x"),
num(&state, "total_y"),
num(&state, "volume_x_out"),
num(&state, "volume_x_in"),
)
};
let mut indexed_out = 0u128;
let mut indexed_in = 0u128;
let mut indexed_swaps = 0u64;
for (pid, tin, tout, ain, aout, n) in totals.iter().filter(|t| &t.0 == pool_id) {
let _ = pid;
if tout == asset_contract {
indexed_out = indexed_out.saturating_add(aout.parse().unwrap_or(0));
}
if tin == asset_contract {
indexed_in = indexed_in.saturating_add(ain.parse().unwrap_or(0));
}
indexed_swaps += n;
}
pools.push(RefPool {
contract: contract.clone(),
pool_id: pool_id.clone(),
counter_token,
counter_symbol: Some(counter_symbol),
counter_decimals: cd,
fee: state["fee"].as_u64().unwrap_or(0) as u32,
state: state["state"].as_str().unwrap_or_default().to_owned(),
current_point: current,
price: asset_price(current),
asset_held: BigUintDec(asset_held.to_string()),
counter_held: BigUintDec(counter_held.to_string()),
rungs,
whitelist: serde_json::from_value(state["whitelist"].clone()).unwrap_or_default(),
asset_out: BigUintDec(asset_out.to_string()),
asset_in: BigUintDec(asset_in.to_string()),
indexed_asset_out: BigUintDec(indexed_out.to_string()),
indexed_asset_in: BigUintDec(indexed_in.to_string()),
indexed_swaps,
});
}
Ok(RefPools {
chain: id,
asset: config.asset.clone(),
asset_symbol: asset_meta.as_ref().map(|m| m.0.clone()),
asset_decimals: asset_meta.map(|m| m.1),
checked_at: chrono::Utc::now(),
pools,
reads: READS.to_owned(),
})
}