From 49f4d4b69b83137f5ae881c688ddac07901ef412 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Sat, 3 Oct 2026 02:02:38 +0300 Subject: [PATCH] feat(api): index swaps settled by NEAR Intents, off by default The price of a token on NEAR Intents is the ratio inside a trade that `intents.near` actually executed; there is no pool, oracle or stored price, and a 1Click quote is an offer. This reads `execute_intents` transactions on the contract's shard, keeps the `token_diff` legs whose receipt succeeded and whose intent is in `intents_executed`, and records one trade per asset/counter pair per outcome, counting each unit once whichever side the taker was on. - core: event parsing, trade building, shard lookup, price and VWAP, tested against the logs of mainnet tx 85uy9uCF (USDT on Tron for ZEC). - data: a NEAR client over the existing RpcClient, live then archival; migration 0009 (intents_trade, intents_scan, near_token). - api: a contiguous-cursor scan and a supply/listing poll per chain, and GET /v1/chains/{chain}/intents/{swaps,price}. Prices are in counter-token units; no USD until a stablecoin is named by contract. Gated on `[chains.intents]`, which the config template leaves commented out: FastNEAR's free RPC sustains about two requests a second against ~1.7 needed to follow the tip, and neardata allows 30 blocks a minute. It waits for our own node. A test renders the template and fails if the block is enabled. Verified end to end against mainnet with ZEC configured: the trade at NEAR 218,261,895 was recorded and served by both routes. Refs #27 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj --- ...7ae3e55e7fb3b5dcffeea482a52f33624de5c.json | 28 ++ ...317f54b0081aa5f4f3caa5cef2a73321835d2.json | 116 +++++ ...e1c872e31f9ba103b27fdec00f52f1a9b7417.json | 29 ++ ...df386ee582a83346e5c560dfcec5b00b3f26c.json | 29 ++ ...7e065e0ccdf5c4aeb56690c9c15667b2a7314.json | 17 + ...e3098c659e998aaf0ce413c93080949db73f3.json | 16 + ...cf618a3cb4e4cca9bfe817fbf52686de85a9c.json | 114 +++++ CLAUDE.md | 34 ++ Cargo.lock | 1 + Cargo.toml | 2 + asset/config/config.toml.tmpl | 16 + crates/blackbeard-api/src/config.rs | 101 ++++ crates/blackbeard-api/src/intents.rs | 349 ++++++++++++++ crates/blackbeard-api/src/main.rs | 1 + crates/blackbeard-api/src/registry.rs | 4 + crates/blackbeard-api/src/routes.rs | 260 +++++++++++ crates/blackbeard-api/src/state.rs | 6 + crates/blackbeard-core/src/intents.rs | 440 ++++++++++++++++++ crates/blackbeard-core/src/lib.rs | 1 + .../tests/intents-zec-usdt-218261895.json | 10 + crates/blackbeard-data/Cargo.toml | 1 + .../migrations/0009_near_intents.sql | 72 +++ crates/blackbeard-data/src/lib.rs | 1 + crates/blackbeard-data/src/near.rs | 356 ++++++++++++++ crates/blackbeard-data/src/store.rs | 276 +++++++++++ crates/blackbeard-entities/src/intents.rs | 157 +++++++ crates/blackbeard-entities/src/lib.rs | 2 + web/src/api/generated/CounterPrice.ts | 53 +++ web/src/api/generated/IntentsListing.ts | 27 ++ web/src/api/generated/IntentsPrice.ts | 46 ++ web/src/api/generated/IntentsSwap.ts | 71 +++ web/src/api/generated/IntentsSwaps.ts | 34 ++ 32 files changed, 2670 insertions(+) create mode 100644 .sqlx/query-07052e42232e1816532a2652cf17ae3e55e7fb3b5dcffeea482a52f33624de5c.json create mode 100644 .sqlx/query-0c9cb8089edbad6caa8b933e334317f54b0081aa5f4f3caa5cef2a73321835d2.json create mode 100644 .sqlx/query-31bf30fc5a538346f8187e3f069e1c872e31f9ba103b27fdec00f52f1a9b7417.json create mode 100644 .sqlx/query-52a060c30ca48a63d2fe9dafaa4df386ee582a83346e5c560dfcec5b00b3f26c.json create mode 100644 .sqlx/query-6b025ce59057975f0feb1e7f7687e065e0ccdf5c4aeb56690c9c15667b2a7314.json create mode 100644 .sqlx/query-7b002140797907f9344085c0f17e3098c659e998aaf0ce413c93080949db73f3.json create mode 100644 .sqlx/query-b02824ff4cc6414dc701b019123cf618a3cb4e4cca9bfe817fbf52686de85a9c.json create mode 100644 crates/blackbeard-api/src/intents.rs create mode 100644 crates/blackbeard-core/src/intents.rs create mode 100644 crates/blackbeard-core/tests/intents-zec-usdt-218261895.json create mode 100644 crates/blackbeard-data/migrations/0009_near_intents.sql create mode 100644 crates/blackbeard-data/src/near.rs create mode 100644 crates/blackbeard-entities/src/intents.rs create mode 100644 web/src/api/generated/CounterPrice.ts create mode 100644 web/src/api/generated/IntentsListing.ts create mode 100644 web/src/api/generated/IntentsPrice.ts create mode 100644 web/src/api/generated/IntentsSwap.ts create mode 100644 web/src/api/generated/IntentsSwaps.ts diff --git a/.sqlx/query-07052e42232e1816532a2652cf17ae3e55e7fb3b5dcffeea482a52f33624de5c.json b/.sqlx/query-07052e42232e1816532a2652cf17ae3e55e7fb3b5dcffeea482a52f33624de5c.json new file mode 100644 index 0000000..e75ff34 --- /dev/null +++ b/.sqlx/query-07052e42232e1816532a2652cf17ae3e55e7fb3b5dcffeea482a52f33624de5c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "select symbol, decimals from near_token where token_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "symbol", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "decimals", + "type_info": "Int2" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "07052e42232e1816532a2652cf17ae3e55e7fb3b5dcffeea482a52f33624de5c" +} diff --git a/.sqlx/query-0c9cb8089edbad6caa8b933e334317f54b0081aa5f4f3caa5cef2a73321835d2.json b/.sqlx/query-0c9cb8089edbad6caa8b933e334317f54b0081aa5f4f3caa5cef2a73321835d2.json new file mode 100644 index 0000000..8623899 --- /dev/null +++ b/.sqlx/query-0c9cb8089edbad6caa8b933e334317f54b0081aa5f4f3caa5cef2a73321835d2.json @@ -0,0 +1,116 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select chain, asset, near_height, trade_index, at, tx_hash, receipt_id,\n counter_token, asset_amount::text as \"asset_amount!\",\n counter_amount::text as \"counter_amount!\", side, taker, referral,\n accounts, intent_hashes, legs\n from intents_trade\n where chain = $1 and asset = $2 and (near_height, trade_index) > ($3, $4)\n order by near_height, trade_index\n limit $5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "chain", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "asset", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "near_height", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "trade_index", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "tx_hash", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "receipt_id", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "counter_token", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "asset_amount!", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "counter_amount!", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "side", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "taker", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "referral", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "accounts", + "type_info": "TextArray" + }, + { + "ordinal": 14, + "name": "intent_hashes", + "type_info": "TextArray" + }, + { + "ordinal": 15, + "name": "legs", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + null, + null, + true, + true, + true, + false, + false, + false + ] + }, + "hash": "0c9cb8089edbad6caa8b933e334317f54b0081aa5f4f3caa5cef2a73321835d2" +} diff --git a/.sqlx/query-31bf30fc5a538346f8187e3f069e1c872e31f9ba103b27fdec00f52f1a9b7417.json b/.sqlx/query-31bf30fc5a538346f8187e3f069e1c872e31f9ba103b27fdec00f52f1a9b7417.json new file mode 100644 index 0000000..a34da28 --- /dev/null +++ b/.sqlx/query-31bf30fc5a538346f8187e3f069e1c872e31f9ba103b27fdec00f52f1a9b7417.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "select low, high from intents_scan where chain = $1 and asset = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "low", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "high", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "31bf30fc5a538346f8187e3f069e1c872e31f9ba103b27fdec00f52f1a9b7417" +} diff --git a/.sqlx/query-52a060c30ca48a63d2fe9dafaa4df386ee582a83346e5c560dfcec5b00b3f26c.json b/.sqlx/query-52a060c30ca48a63d2fe9dafaa4df386ee582a83346e5c560dfcec5b00b3f26c.json new file mode 100644 index 0000000..01abc0f --- /dev/null +++ b/.sqlx/query-52a060c30ca48a63d2fe9dafaa4df386ee582a83346e5c560dfcec5b00b3f26c.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into intents_trade (\n chain, asset, near_height, trade_index, at, tx_hash, receipt_id,\n counter_token, asset_amount, counter_amount, side, taker, referral,\n accounts, intent_hashes, legs\n ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::numeric, $10::text::numeric,\n $11, $12, $13, $14, $15, $16)\n on conflict (chain, asset, near_height, trade_index) do update set\n at = excluded.at, tx_hash = excluded.tx_hash,\n receipt_id = excluded.receipt_id, counter_token = excluded.counter_token,\n asset_amount = excluded.asset_amount,\n counter_amount = excluded.counter_amount, side = excluded.side,\n taker = excluded.taker, referral = excluded.referral,\n accounts = excluded.accounts, intent_hashes = excluded.intent_hashes,\n legs = excluded.legs\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Timestamptz", + "Text", + "Text", + "Text", + "Text", + "Text", + "Text", + "Text", + "Text", + "TextArray", + "TextArray", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "52a060c30ca48a63d2fe9dafaa4df386ee582a83346e5c560dfcec5b00b3f26c" +} diff --git a/.sqlx/query-6b025ce59057975f0feb1e7f7687e065e0ccdf5c4aeb56690c9c15667b2a7314.json b/.sqlx/query-6b025ce59057975f0feb1e7f7687e065e0ccdf5c4aeb56690c9c15667b2a7314.json new file mode 100644 index 0000000..042560b --- /dev/null +++ b/.sqlx/query-6b025ce59057975f0feb1e7f7687e065e0ccdf5c4aeb56690c9c15667b2a7314.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into intents_scan (chain, asset, low, high) values ($1, $2, $3, $4)\n on conflict (chain, asset) do update\n set low = least(intents_scan.low, excluded.low),\n high = greatest(intents_scan.high, excluded.high),\n updated_at = now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6b025ce59057975f0feb1e7f7687e065e0ccdf5c4aeb56690c9c15667b2a7314" +} diff --git a/.sqlx/query-7b002140797907f9344085c0f17e3098c659e998aaf0ce413c93080949db73f3.json b/.sqlx/query-7b002140797907f9344085c0f17e3098c659e998aaf0ce413c93080949db73f3.json new file mode 100644 index 0000000..e2243fd --- /dev/null +++ b/.sqlx/query-7b002140797907f9344085c0f17e3098c659e998aaf0ce413c93080949db73f3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into near_token (token_id, symbol, decimals) values ($1, $2, $3)\n on conflict (token_id) do update\n set symbol = excluded.symbol, decimals = excluded.decimals, fetched_at = now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "7b002140797907f9344085c0f17e3098c659e998aaf0ce413c93080949db73f3" +} diff --git a/.sqlx/query-b02824ff4cc6414dc701b019123cf618a3cb4e4cca9bfe817fbf52686de85a9c.json b/.sqlx/query-b02824ff4cc6414dc701b019123cf618a3cb4e4cca9bfe817fbf52686de85a9c.json new file mode 100644 index 0000000..8b637ff --- /dev/null +++ b/.sqlx/query-b02824ff4cc6414dc701b019123cf618a3cb4e4cca9bfe817fbf52686de85a9c.json @@ -0,0 +1,114 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select chain as \"chain!\", asset as \"asset!\", near_height as \"near_height!\",\n trade_index as \"trade_index!\", at as \"at!\", tx_hash as \"tx_hash!\",\n receipt_id as \"receipt_id!\", counter_token as \"counter_token!\",\n asset_amount::text as \"asset_amount!\",\n counter_amount::text as \"counter_amount!\", side, taker, referral,\n accounts as \"accounts!\", intent_hashes as \"intent_hashes!\", legs as \"legs!\"\n from (\n select *, row_number() over (\n partition by counter_token order by near_height desc, trade_index desc\n ) as n\n from intents_trade\n where chain = $1 and asset = $2\n ) ranked\n where n <= $3\n order by counter_token, near_height desc, trade_index desc\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "chain!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "asset!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "near_height!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "trade_index!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "tx_hash!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "receipt_id!", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "counter_token!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "asset_amount!", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "counter_amount!", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "side", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "taker", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "referral", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "accounts!", + "type_info": "TextArray" + }, + { + "ordinal": 14, + "name": "intent_hashes!", + "type_info": "TextArray" + }, + { + "ordinal": 15, + "name": "legs!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + null, + null, + true, + true, + true, + false, + false, + false + ] + }, + "hash": "b02824ff4cc6414dc701b019123cf618a3cb4e4cca9bfe817fbf52686de85a9c" +} diff --git a/CLAUDE.md b/CLAUDE.md index 322df65..f13b222 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -477,6 +477,40 @@ infer "is this a name or an address?" from the string: several real node names begin with `q`, so `RecentBlock` and `LeaderboardRow` carry the `AttributionSource` and the UI reads that. +## NEAR Intents is read from logs, and every free source is rate-limited + +`/intents/*` prices QTC by what `intents.near` actually settled (#27). There is +no pool or stored price on NEAR Intents; a price is the ratio inside an executed +`token_diff`, and nothing else is one — not a 1Click quote, and not the swap +screen in wallet 1.7.1 (144), whose "rate" is a hard-coded `50.0` in a demo flow. + +**Both free block sources throttle, and neither says so until it does.** +neardata: 30 blocks a minute, against NEAR's ~100. FastNEAR RPC: about two +requests a second sustained, 429 after ~20 in a burst, recovering in ~2 s — +and following one shard at the tip needs ~1.7. Measured 2026-10-03. That is why +the integration is commented out in `asset/config/config.toml.tmpl` until our +own node serves `rpc_urls`, and a test in `config.rs` fails if it is enabled +without deleting that assertion. + +**A failed receipt keeps its logs.** NEAR rolls back its state but still reports +what it logged, so a `token_diff` is a trade only when the outcome succeeded +*and* its `intent_hash` is in `intents_executed`. Either check alone admits +trades that never happened. + +**A transaction lands in its signer's shard, not its receiver's.** Scanning +`intents.near`'s shard sees `execute_intents` only because the 1Click relay signs +as `intents.near`. Intents arriving as cross-contract receipts are not read; the +price response says so in `reads` rather than leaving it to be found. + +**A chunk call for a block with no new chunk returns the previous one.** +`height_included` is the check; without it a quiet shard's trades are counted +once per block until it produces again. + +**A token's symbol is not its identity.** `tron-d28a…omft.near` calls itself +`USDT` — it is USDT on Tron — and the issue that started this read the contract +name and called it TRON. Anyone can deploy a token called USDC, so a stablecoin +is named by contract id, never by `ft_metadata.symbol`. + ## The frontend's types are generated `web/src/api/generated/` is written by `ts-rs` from `blackbeard-entities` when diff --git a/Cargo.lock b/Cargo.lock index 853accd..55406c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,6 +306,7 @@ dependencies = [ name = "blackbeard-data" version = "0.1.0" dependencies = [ + "base64", "bigdecimal", "blackbeard-core", "blackbeard-entities", diff --git a/Cargo.toml b/Cargo.toml index c179152..eb036d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,8 @@ blackbeard-core = { path = "crates/blackbeard-core", version = "=0.1.0" } blackbeard-data = { path = "crates/blackbeard-data", version = "=0.1.0" } anyhow = "1" +# Only for NEAR view calls, whose arguments travel base64-encoded. +base64 = "0.22" axum = { version = "0.8", features = ["ws", "macros"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } clap = { version = "4", features = ["derive", "env"] } diff --git a/asset/config/config.toml.tmpl b/asset/config/config.toml.tmpl index 7b2706b..05def80 100644 --- a/asset/config/config.toml.tmpl +++ b/asset/config/config.toml.tmpl @@ -130,6 +130,22 @@ target_block_time_seconds = 12.0 # the window — the warm start simply finds fewer blocks than it asks for. warm_start_blocks = 100800 max_gap_fill_blocks = 5000 +# QTC's settled swaps on NEAR Intents (blackbeard/observer#27). OFF until our own +# NEAR RPC node is up: with this block absent nothing NEAR-related starts and the +# /intents routes answer 404 `no_intents`. FastNEAR's free RPC throttles at about +# two requests a second sustained, and following the tip alone needs ~1.7. +# +# `start_height` is how far back the backfill walks. Nothing can trade before the +# token holds any supply on NEAR, and on 2026-10-03 `qtc.omft.near` held none, so +# the height at which this is first enabled is a sound floor while that stays +# true. `rpc_urls` should name our node; the archive covers anything older. +# Enabling it also means deleting the assertion in config.rs's +# `the_committed_config_parses_and_leaves_intents_off`. +# +# [chains.intents] +# asset = "nep141:qtc.omft.near" +# start_height = 218262584 +# rpc_urls = ["http://:3030"] [[chains]] id = "heisenberg" diff --git a/crates/blackbeard-api/src/config.rs b/crates/blackbeard-api/src/config.rs index 3c188e7..db2a1eb 100644 --- a/crates/blackbeard-api/src/config.rs +++ b/crates/blackbeard-api/src/config.rs @@ -185,6 +185,49 @@ pub struct ChainConfig { /// the oldest part of the gap is forgone. #[serde(default = "default_max_gap_fill")] pub max_gap_fill_blocks: u64, + /// Where this chain's token is traded on NEAR Intents, if anywhere. + /// + /// Optional, and absent for every chain whose token has no bridged form. + /// See blackbeard/observer#27. + #[serde(default)] + pub intents: Option, +} + +/// Following one token's settled swaps on NEAR Intents. +/// +/// The asset is configuration rather than a constant because it was proven +/// against a token that already trades (ZEC) before QTC had a single swap, +/// and because the wallet's own asset id was a placeholder when this was +/// written. Changing it starts a fresh scan; it never inherits another +/// asset's cursor. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IntentsConfig { + /// The token, as `intents.near` names it: `nep141:`. + pub asset: String, + /// The NEAR height the backfill walks back to and no further — the token + /// contract's deployment, since nothing can trade before that. + pub start_height: u64, + /// The settlement contract. + #[serde(default = "default_intents_contract")] + pub contract: String, + /// NEAR JSON-RPC endpoints for the tip. These keep about a day. + #[serde(default = "default_near_rpc")] + pub rpc_urls: Vec, + /// Archival NEAR JSON-RPC endpoints, for anything older. + #[serde(default = "default_near_archive")] + pub archive_urls: Vec, + /// 1Click's API, read only for whether it lists the asset. Never for a + /// price: a quote is an offer, not a trade. + #[serde(default = "default_oneclick")] + pub oneclick_url: String, +} + +impl IntentsConfig { + /// The asset's token contract, from its multi-token id. + pub fn token_contract(&self) -> Option<&str> { + self.asset.strip_prefix("nep141:") + } } impl ChainConfig { @@ -241,6 +284,21 @@ fn default_warm_start_blocks() -> usize { fn default_max_gap_fill() -> u64 { 5_000 } +fn default_intents_contract() -> String { + "intents.near".into() +} +fn default_near_rpc() -> Vec { + vec!["https://rpc.mainnet.fastnear.com".into()] +} +fn default_near_archive() -> Vec { + vec![ + "https://archival-rpc.mainnet.fastnear.com".into(), + "https://archival-rpc.mainnet.near.org".into(), + ] +} +fn default_oneclick() -> String { + "https://1click.chaindefuser.com".into() +} fn default_max_tracked_chains() -> usize { 16 } @@ -298,6 +356,22 @@ impl Config { chain.id ); } + if let Some(intents) = &chain.intents { + if intents.token_contract().is_none() { + anyhow::bail!( + "chain `{}` tracks intents asset `{}`, which is not a `nep141:` id; supply \ + and decimals are read from the token contract it names", + chain.id, + intents.asset + ); + } + if intents.rpc_urls.is_empty() || intents.archive_urls.is_empty() { + anyhow::bail!( + "chain `{}` tracks intents but names no NEAR rpc or archive endpoint", + chain.id + ); + } + } if chain.target_block_time_seconds <= 0.0 { anyhow::bail!( "chain `{}` has a target block time of {}; hashrate is difficulty divided by it", @@ -340,9 +414,36 @@ mod tests { target_block_time_seconds: 6.0, warm_start_blocks: 10, max_gap_fill_blocks: 10, + intents: None, } } + /// The deployed config must not reach NEAR until the intents block is + /// uncommented on purpose: the free endpoints throttle below what following + /// the tip needs, so the integration waits for our own node + /// (blackbeard/observer#27). Delete the second assertion in the same commit + /// that enables it. + /// + /// Read from the template the deploy renders, with its `{{…}}` + /// placeholders filled, because that template is the only config committed. + #[test] + fn the_deployed_config_parses_and_leaves_intents_off() { + use figment::providers::{Format, Toml}; + let rendered = include_str!("../../../asset/config/config.toml.tmpl") + .replace("{{BIND_ADDRESS}}", "127.0.0.1") + .replace("{{TARGET_FQDN}}", "host.example"); + assert!( + !rendered.contains("{{"), + "a placeholder this test does not fill" + ); + let config: Config = figment::Figment::new() + .merge(Toml::string(&rendered)) + .extract() + .expect("the rendered template parses"); + assert!(!config.chains.is_empty()); + assert!(config.chains.iter().all(|c| c.intents.is_none())); + } + /// A config naming one endpoint is still a valid config. Every existing /// deployment writes `rpc_url`, and requiring them all to change would be /// this feature breaking the thing it exists to make more reliable. diff --git a/crates/blackbeard-api/src/intents.rs b/crates/blackbeard-api/src/intents.rs new file mode 100644 index 0000000..ad9fb90 --- /dev/null +++ b/crates/blackbeard-api/src/intents.rs @@ -0,0 +1,349 @@ +//! Following a chain's token through NEAR Intents. +//! +//! Two tasks per chain that configures `intents`: +//! +//! - **`scan`** reads `intents.near`'s shard one NEAR block at a time and +//! records every settled trade of the asset. One contiguous cursor, extended +//! upward to the final tip first and then downward to `start_height`, for the +//! reasons `ingest::backfill` gives. +//! - **`listing`** reads whether there is anything to trade at all: the token's +//! supply on NEAR, what `intents.near` holds, and whether 1Click lists it. +//! +//! What is read: `execute_intents` transactions in the chunk of the shard +//! `intents.near` lives on. The 1Click relay signs those as `intents.near` +//! itself, so they land in that shard and settle in the block that includes +//! them. An intent that arrives as a cross-contract receipt instead — an +//! `ft_transfer_call` carrying intents in its message — lands from another +//! shard's chunk and is **not read**. That is the known blind spot, and it is +//! stated in every price response rather than left to be discovered. +//! blackbeard/observer#27. + +use std::sync::Arc; +use std::time::Duration; + +use blackbeard_core::intents as core; +use blackbeard_data::near::{NearClient, oneclick_lists}; +use blackbeard_data::store::{IntentTrade, Store}; +use blackbeard_entities::{BigUintDec, IntentsListing}; +use serde_json::json; + +use crate::config::IntentsConfig; +use crate::state::ChainRuntime; + +/// What the indexer reads, for the price response to carry. See the module +/// docs; this is the sentence a reader of the API needs from them. +pub const READS: &str = "execute_intents transactions submitted to intents.near by its own relay, \ + which is how 1Click settles. Intents reaching the contract as cross-contract receipts are not \ + read, so a trade made that way is missing rather than misreported."; + +/// NEAR blocks per scan batch. Also the most progress a restart can lose. +const BATCH: u64 = 64; + +/// Pause between blocks when walking back through history. +/// +/// The archive is someone else's free service and the history is not going +/// anywhere. FastNEAR's free tier was measured at about two requests a second +/// sustained, and 429s after ~20 in a burst; one block a second leaves room for +/// the transaction lookups a block with trades in it adds. +const BACKFILL_PACE: Duration = Duration::from_secs(1); + +/// Pause when the scan has caught up with the tip. NEAR finalises a block +/// every ~0.6 s; checking twice a block is plenty. +const TIP_WAIT: Duration = Duration::from_secs(1); + +/// Pause after a failure before trying again. +const RETRY: Duration = Duration::from_secs(15); + +/// How often supply and listing are re-read. +const LISTING_INTERVAL: Duration = Duration::from_secs(300); + +/// Every NEAR call's timeout. The archive is the slow one. +const NEAR_TIMEOUT: Duration = Duration::from_secs(20); + +/// Start both tasks, if the chain tracks an asset. +pub fn spawn(chain: Arc, store: Store) { + let Some(config) = chain.spec.intents.clone() else { + return; + }; + let near = match NearClient::new( + config.rpc_urls.clone(), + config.archive_urls.clone(), + NEAR_TIMEOUT, + ) { + Ok(n) => n, + Err(e) => { + tracing::error!(chain = %chain.id(), error = %e, "intents tracking not started"); + return; + } + }; + tracing::info!(chain = %chain.id(), asset = %config.asset, "following the asset on NEAR Intents"); + tokio::spawn(listing( + Arc::clone(&chain), + store.clone(), + near.clone(), + config.clone(), + )); + tokio::spawn(scan(chain, store, near, config)); +} + +/// Walk NEAR blocks, recording trades. +async fn scan(chain: Arc, store: Store, near: NearClient, config: IntentsConfig) { + let id = chain.id(); + loop { + // The cursor references the chain row, which the chain's own poll + // writes once its node has answered. + if chain.read().current_runtime().is_none() { + tokio::time::sleep(RETRY).await; + continue; + } + + let (tip, live_from) = match near.heights().await { + Ok(h) => h, + Err(e) => { + tracing::warn!(chain = %id, error = %e, "near tip unreadable"); + tokio::time::sleep(RETRY).await; + continue; + } + }; + let scan = match store.intents_scan(&id, &config.asset).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(chain = %id, error = %e, "intents scan cursor unreadable"); + tokio::time::sleep(RETRY).await; + continue; + } + }; + // An empty cursor starts at the tip: what is trading now matters more + // than what traded last week, and both arrive in the end. + let (low, high) = scan.unwrap_or((tip, tip.saturating_sub(1))); + + let (heights, descending): (Vec, bool) = if high < tip { + ((high + 1..=tip.min(high + BATCH)).collect(), false) + } else if low > config.start_height { + let floor = low.saturating_sub(BATCH).max(config.start_height); + ((floor..low).rev().collect(), true) + } else { + tokio::time::sleep(TIP_WAIT).await; + continue; + }; + + // Per batch, not once: resharding renumbers shards, and a layout read + // at startup would follow the wrong one after it while every call + // kept succeeding. + let shard = match near.shard_layout(heights[0]).await { + Ok(layout) => core::shard_for( + &config.contract, + &layout.boundary_accounts, + &layout.shard_ids, + ), + Err(e) => { + tracing::warn!(chain = %id, error = %e, "near shard layout unreadable"); + tokio::time::sleep(RETRY).await; + continue; + } + }; + let Some(shard) = shard else { + tracing::error!(chain = %id, contract = %config.contract, "contract maps to no shard"); + tokio::time::sleep(RETRY).await; + continue; + }; + + // Record each block as it is read, and the cursor up to the last one + // that succeeded: a failure part way leaves the interval contiguous. + let mut reached = None; + for &height in &heights { + match read_block(&near, &store, &config, height, shard, live_from).await { + Ok(trades) => { + if !trades.is_empty() { + let rows: Vec = trades + .into_iter() + .map(|t| IntentTrade { + chain: id.clone(), + ..t + }) + .collect(); + if let Err(e) = store.record_intent_trades(&rows).await { + tracing::warn!(chain = %id, near_height = height, error = %e, + "intents trades not recorded"); + break; + } + tracing::info!(chain = %id, near_height = height, trades = rows.len(), + "intents trades recorded"); + } + reached = Some(height); + } + Err(e) => { + tracing::warn!(chain = %id, near_height = height, error = %e, + "near block unreadable; will retry"); + break; + } + } + if descending { + tokio::time::sleep(BACKFILL_PACE).await; + } + } + + if let Some(reached) = reached { + let (new_low, new_high) = if descending { + (reached, high) + } else { + (low.min(heights[0]), reached) + }; + if let Err(e) = store + .set_intents_scan(&id, &config.asset, new_low, new_high) + .await + { + tracing::warn!(chain = %id, error = %e, "intents scan cursor not saved"); + } + } else { + tokio::time::sleep(RETRY).await; + } + } +} + +/// Every trade of the asset settled by `execute_intents` transactions included +/// at `height`. Chain id is left for the caller to fill in. +async fn read_block( + near: &NearClient, + store: &Store, + config: &IntentsConfig, + height: u64, + shard: u64, + live_from: u64, +) -> Result, blackbeard_data::DataError> { + let Some(chunk) = near.chunk(height, shard, live_from).await? else { + return Ok(Vec::new()); + }; + + let mut out = Vec::new(); + for tx in chunk + .transactions + .iter() + .filter(|t| t.receiver_id == config.contract && t.calls("execute_intents")) + { + for outcome in near.tx_outcomes(&tx.hash, &tx.signer_id).await? { + // A failed receipt still reports the logs it wrote before failing. + if outcome.executor_id != config.contract || !outcome.succeeded { + continue; + } + let (legs, malformed) = core::executed_legs(&outcome.logs); + if malformed > 0 { + tracing::warn!(tx = %tx.hash, malformed, + "token_diff entries that did not parse; the event format may have changed"); + } + let (trades, unpriced) = core::trades(&legs, &config.asset); + if unpriced > 0 { + tracing::debug!(tx = %tx.hash, unpriced, "legs of the asset with no single price"); + } + if trades.is_empty() { + continue; + } + let (_, at) = near.block_at(json!(outcome.block_hash)).await?; + for t in trades { + remember_token(near, store, &t.counter).await; + out.push(IntentTrade { + chain: blackbeard_entities::ChainId(String::new()), + asset: config.asset.clone(), + near_height: height, + trade_index: out.len() as u32, + at, + tx_hash: tx.hash.clone(), + receipt_id: outcome.id.clone(), + asset_amount: t.asset_amount.to_string(), + counter_amount: t.counter_amount.to_string(), + side: t.side.map(|s| s.as_str().to_owned()), + taker: t.taker.clone(), + referral: t.referral.clone(), + accounts: t.accounts.clone(), + intent_hashes: t.intent_hashes.clone(), + legs: legs_json(&t.legs), + counter_token: t.counter, + }); + } + } + } + Ok(out) +} + +/// The legs as the contract logged them, amounts as strings. +fn legs_json(legs: &[core::Leg]) -> serde_json::Value { + serde_json::Value::Array( + legs.iter() + .map(|l| { + json!({ + "intent_hash": l.intent_hash, + "account_id": l.account_id, + "diff": l.diff.iter().map(|(k, v)| (k.clone(), json!(v.to_string()))).collect::>(), + "referral": l.referral, + "fees_collected": l.fees.iter().map(|(k, v)| (k.clone(), json!(v.to_string()))).collect::>(), + }) + }) + .collect(), + ) +} + +/// Cache a token's decimals the first time it is seen. +/// +/// Best effort: a token whose metadata cannot be read is left out of the cache +/// and its trades show without a price, which is honest, rather than failing +/// the block, which would stall the scan on one odd token. +async fn remember_token(near: &NearClient, store: &Store, token: &str) { + if matches!(store.near_token(token).await, Ok(Some(_))) { + return; + } + match near.token_metadata(token).await { + Ok(Some(meta)) => { + if let Err(e) = store + .save_near_token(token, &meta.symbol, meta.decimals) + .await + { + tracing::warn!(token, error = %e, "token metadata not cached"); + } + } + Ok(None) => tracing::debug!(token, "not a nep141 token; its trades stay unpriced"), + Err(e) => tracing::warn!(token, error = %e, "token metadata unreadable"), + } +} + +/// Read supply and listing, now and every few minutes. +async fn listing(chain: Arc, store: Store, near: NearClient, config: IntentsConfig) { + let Some(contract) = config.token_contract().map(str::to_owned) else { + return; + }; + loop { + remember_token(&near, &store, &config.asset).await; + + let amount = |v: serde_json::Value| v.as_str().map(|s| BigUintDec(s.to_owned())); + let total_supply = near + .view(&contract, "ft_total_supply", json!({})) + .await + .map_err(|e| tracing::warn!(error = %e, "ft_total_supply unreadable")) + .ok() + .and_then(amount); + let in_intents = near + .view( + &contract, + "ft_balance_of", + json!({"account_id": config.contract}), + ) + .await + .map_err(|e| tracing::warn!(error = %e, "ft_balance_of unreadable")) + .ok() + .and_then(amount); + let oneclick_listed = oneclick_lists(&config.oneclick_url, &config.asset, NEAR_TIMEOUT) + .await + .map_err(|e| tracing::warn!(error = %e, "1Click token list unreadable")) + .ok(); + + *chain + .intents_listing + .write() + .unwrap_or_else(|p| p.into_inner()) = Some(IntentsListing { + checked_at: chrono::Utc::now(), + total_supply, + in_intents, + oneclick_listed, + }); + tokio::time::sleep(LISTING_INTERVAL).await; + } +} diff --git a/crates/blackbeard-api/src/main.rs b/crates/blackbeard-api/src/main.rs index f9f3dc6..16d45a0 100644 --- a/crates/blackbeard-api/src/main.rs +++ b/crates/blackbeard-api/src/main.rs @@ -10,6 +10,7 @@ mod config; mod ingest; +mod intents; mod registry; mod routes; mod state; diff --git a/crates/blackbeard-api/src/registry.rs b/crates/blackbeard-api/src/registry.rs index 40a75ee..bc0dcc3 100644 --- a/crates/blackbeard-api/src/registry.rs +++ b/crates/blackbeard-api/src/registry.rs @@ -53,6 +53,8 @@ pub struct ChainSpec { pub warm_start_blocks: usize, /// Cap on a single gap fill. pub max_gap_fill_blocks: u64, + /// NEAR Intents tracking for this chain's token, if configured. + pub intents: Option, } impl ChainSpec { @@ -72,6 +74,7 @@ impl ChainSpec { target_block_time_seconds: chain.target_block_time_seconds, warm_start_blocks: chain.warm_start_blocks, max_gap_fill_blocks: chain.max_gap_fill_blocks, + intents: chain.intents.clone(), } } } @@ -138,6 +141,7 @@ impl Registry { let runtime = Arc::new(ChainRuntime::new(spec, rpc)); crate::ingest::warm_start(&runtime, &store, config.server.ticker_blocks).await; crate::ingest::spawn(Arc::clone(&runtime), store.clone(), &config); + crate::intents::spawn(Arc::clone(&runtime), store.clone()); by_slug.insert(runtime.spec.slug.clone(), Arc::clone(&runtime)); tracked.push(runtime); } diff --git a/crates/blackbeard-api/src/routes.rs b/crates/blackbeard-api/src/routes.rs index 4f4e028..d9c2b1d 100644 --- a/crates/blackbeard-api/src/routes.rs +++ b/crates/blackbeard-api/src/routes.rs @@ -62,6 +62,8 @@ pub fn router(state: AppState, allowed_origins: &[String]) -> Router { .route("/v1/chains/{chain}/vesting", get(vesting)) .route("/v1/chains/{chain}/distribution", get(distribution)) .route("/v1/chains/{chain}/wormhole", get(wormhole)) + .route("/v1/chains/{chain}/intents/swaps", get(intents_swaps)) + .route("/v1/chains/{chain}/intents/price", get(intents_price)) .route("/v1/chains/{chain}/nodes", get(nodes)) .route("/v1/chains/{chain}/nodes/{peer}", get(node)) .route("/v1/chains/{chain}/accounts", get(accounts)) @@ -1692,6 +1694,264 @@ async fn wormhole_deposits( })) } +/// Swaps per page, by default and at most. +const SWAP_PAGE: i64 = 200; +const SWAP_PAGE_MAX: i64 = 1000; + +/// Trades per counter token a price is taken over, by default and at most. +/// +/// A count, not a duration — the same rule as the mining windows. On a thin +/// market a duration would put a single trade from hours ago behind a label +/// that sounds current; a count says exactly what it is. +const PRICE_TRADES: i64 = 50; +const PRICE_TRADES_MAX: i64 = 1000; + +#[derive(Debug, Deserialize)] +struct SwapsQuery { + /// `near_height:trade_index`, exclusive. + after: Option, + limit: Option, +} + +#[derive(Debug, Deserialize)] +struct PriceQuery { + /// Latest trades per counter token to summarise. + trades: Option, +} + +fn no_intents(chain: &str) -> Failure { + Failure( + StatusCode::NOT_FOUND, + ApiError::new( + "no_intents", + format!("`{chain}` does not track a token on NEAR Intents"), + ), + ) +} + +fn parse_trade_cursor(raw: &str) -> Result<(u64, u32), Failure> { + raw.split_once(':') + .and_then(|(h, i)| Some((h.parse().ok()?, i.parse().ok()?))) + .ok_or_else(|| { + Failure( + StatusCode::BAD_REQUEST, + ApiError::new( + "malformed_cursor", + format!("`{raw}` is not a `near_height:trade_index` cursor"), + ), + ) + }) +} + +/// Token decimals and symbols from the cache, for every token named. +async fn near_tokens( + store: &blackbeard_data::store::Store, + tokens: Vec, +) -> Result, Failure> { + let mut out = std::collections::HashMap::new(); + for token in tokens { + if out.contains_key(&token) { + continue; + } + if let Some(meta) = store + .near_token(&token) + .await + .map_err(database_unavailable)? + { + out.insert(token, meta); + } + } + Ok(out) +} + +/// Settled swaps of the chain's token on NEAR Intents, oldest first. +/// +/// Trades only: every row is a `token_diff` pair that `intents.near` executed +/// in a receipt that succeeded. blackbeard/observer#27. +async fn intents_swaps( + State(state): State, + Path(chain): Path, + Query(query): Query, +) -> Result, Failure> { + let runtime = state.chain(&chain).ok_or_else(|| unknown_chain(&chain))?; + let config = runtime + .spec + .intents + .clone() + .ok_or_else(|| no_intents(&chain))?; + let id = runtime.id(); + let after = match query.after.as_deref() { + Some(raw) => parse_trade_cursor(raw)?, + None => (0, 0), + }; + let limit = query.limit.unwrap_or(SWAP_PAGE).clamp(1, SWAP_PAGE_MAX); + + let rows = state + .store + .intent_trades(&id, &config.asset, after, limit) + .await + .map_err(database_unavailable)?; + let scan = state + .store + .intents_scan(&id, &config.asset) + .await + .map_err(database_unavailable)?; + let named: Vec = std::iter::once(config.asset.clone()) + .chain(rows.iter().map(|r| r.counter_token.clone())) + .collect(); + let tokens = near_tokens(&state.store, named).await?; + let asset_decimals = tokens.get(&config.asset).map(|t| t.1); + + let full = rows.len() as i64 == limit; + let next = rows + .last() + .filter(|_| full) + .map(|r| format!("{}:{}", r.near_height, r.trade_index)); + let swaps = rows + .into_iter() + .map(|r| { + let counter = tokens.get(&r.counter_token); + let price = match ( + asset_decimals, + counter, + r.asset_amount.parse::(), + r.counter_amount.parse::(), + ) { + (Some(ad), Some((_, cd)), Ok(a), Ok(c)) => { + Some(blackbeard_core::intents::price(a, ad, c, *cd)) + } + _ => None, + }; + blackbeard_entities::IntentsSwap { + near_height: r.near_height, + trade_index: r.trade_index, + at: r.at, + tx_hash: r.tx_hash, + receipt_id: r.receipt_id, + counter_symbol: counter.map(|c| c.0.clone()), + counter_token: r.counter_token, + asset_amount: BigUintDec(r.asset_amount), + counter_amount: BigUintDec(r.counter_amount), + price, + side: r.side, + taker: r.taker, + referral: r.referral, + accounts: r.accounts, + intent_hashes: r.intent_hashes, + } + }) + .collect(); + + Ok(Json(blackbeard_entities::IntentsSwaps { + chain: id, + asset: config.asset, + swaps, + next, + indexed_from: scan.map(|(low, _)| low), + indexed_to: scan.map(|(_, high)| high), + })) +} + +/// The chain's token priced by what NEAR Intents settled, per counter token. +/// +/// In counter-token units only. No USD until a stablecoin pairing exists to +/// derive one from — identified by contract, never by the symbol a token gave +/// itself — and never from a quote. +async fn intents_price( + State(state): State, + Path(chain): Path, + Query(query): Query, +) -> Result, Failure> { + let runtime = state.chain(&chain).ok_or_else(|| unknown_chain(&chain))?; + let config = runtime + .spec + .intents + .clone() + .ok_or_else(|| no_intents(&chain))?; + let id = runtime.id(); + let window = query + .trades + .unwrap_or(PRICE_TRADES) + .clamp(1, PRICE_TRADES_MAX); + + let rows = state + .store + .latest_intent_trades(&id, &config.asset, window) + .await + .map_err(database_unavailable)?; + let scan = state + .store + .intents_scan(&id, &config.asset) + .await + .map_err(database_unavailable)?; + let named: Vec = std::iter::once(config.asset.clone()) + .chain(rows.iter().map(|r| r.counter_token.clone())) + .collect(); + let tokens = near_tokens(&state.store, named).await?; + let asset_decimals = tokens.get(&config.asset).map(|t| t.1); + + // Rows arrive grouped by counter, newest first within each. + let mut counters: Vec = Vec::new(); + for group in rows.chunk_by(|a, b| a.counter_token == b.counter_token) { + let amounts: Vec<(u128, u128)> = group + .iter() + .filter_map(|r| Some((r.asset_amount.parse().ok()?, r.counter_amount.parse().ok()?))) + .collect(); + // Exact sums. A u128 holds 3.4e38; a thousand trades of any real + // token's entire supply does not reach it, and if one ever did the + // counter is left out rather than reported wrapped. + let sum = |pick: fn(&(u128, u128)) -> u128| { + amounts + .iter() + .try_fold(0u128, |acc, t| acc.checked_add(pick(t))) + }; + let (Some(asset_volume), Some(counter_volume)) = (sum(|t| t.0), sum(|t| t.1)) else { + tracing::warn!(counter = %group[0].counter_token, "volume overflowed a u128; omitted"); + continue; + }; + let newest = &group[0]; + let oldest = &group[group.len() - 1]; + let counter = tokens.get(&newest.counter_token); + let decimals = counter.map(|c| c.1); + let priced = asset_decimals.zip(decimals); + counters.push(blackbeard_entities::CounterPrice { + counter_token: newest.counter_token.clone(), + symbol: counter.map(|c| c.0.clone()), + decimals, + trades: group.len() as u32, + last_price: priced.and_then(|(ad, cd)| { + let (a, c) = amounts.first()?; + Some(blackbeard_core::intents::price(*a, ad, *c, cd)) + }), + last_at: newest.at, + last_near_height: newest.near_height, + vwap: priced.and_then(|(ad, cd)| blackbeard_core::intents::vwap(&amounts, ad, cd)), + asset_volume: BigUintDec(asset_volume.to_string()), + counter_volume: BigUintDec(counter_volume.to_string()), + first_at: oldest.at, + }); + } + counters.sort_by_key(|c| std::cmp::Reverse(c.last_at)); + + let listing = runtime + .intents_listing + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + + Ok(Json(blackbeard_entities::IntentsPrice { + chain: id, + asset: config.asset, + asset_decimals, + window_trades: window as u32, + counters, + listing, + reads: crate::intents::READS.to_owned(), + indexed_from: scan.map(|(low, _)| low), + indexed_to: scan.map(|(_, high)| high), + })) +} + /// Paging cursor for the block index. #[derive(Debug, Deserialize)] struct BeforeQuery { diff --git a/crates/blackbeard-api/src/state.rs b/crates/blackbeard-api/src/state.rs index fae3f92..1eea822 100644 --- a/crates/blackbeard-api/src/state.rs +++ b/crates/blackbeard-api/src/state.rs @@ -178,6 +178,10 @@ pub struct ChainRuntime { /// The vesting schedules and what they have released, cached the same way. pub vesting: tokio::sync::RwLock>, + /// The intents asset's supply and listing, as last read. `std` lock, never + /// held across an await: the reader clones it out and the writer swaps a + /// whole value in. + pub intents_listing: RwLock>, /// Blocks awaiting their telemetry attribution. pub pending_attributions: std::sync::Mutex>, /// Live subscriber count per window, so the recompute timer can skip a @@ -223,6 +227,7 @@ impl ChainRuntime { roles: tokio::sync::RwLock::new(None), concentration: tokio::sync::RwLock::new(None), vesting: tokio::sync::RwLock::new(None), + intents_listing: RwLock::new(None), pending_attributions: std::sync::Mutex::new(std::collections::VecDeque::new()), subscribers: Default::default(), } @@ -521,6 +526,7 @@ mod tests { target_block_time_seconds: 6.0, warm_start_blocks: 100, max_gap_fill_blocks: 100, + intents: None, }, RpcClient::new("http://127.0.0.1:9944", Duration::from_secs(5)).unwrap(), )) diff --git a/crates/blackbeard-core/src/intents.rs b/crates/blackbeard-core/src/intents.rs new file mode 100644 index 0000000..f6041b1 --- /dev/null +++ b/crates/blackbeard-core/src/intents.rs @@ -0,0 +1,440 @@ +//! Swaps settled by NEAR Intents, read from `intents.near`'s own event logs. +//! +//! NEAR Intents has no pool, no oracle and no stored price. `intents.near` is a +//! settlement contract: signed `token_diff` intents are executed atomically, +//! and a price exists only as the ratio between two amounts that actually +//! changed hands. That ratio is what this module extracts, and nothing else — +//! a quote from 1Click is an offer, not a trade, and never comes through here. +//! See blackbeard/observer#27. +//! +//! Every executed intent logs NEP-297 events, `EVENT_JSON:{…}`, in the receipt +//! outcome that executed it. Two filters decide what counts, and both are +//! needed: +//! +//! - **The receipt succeeded.** NEAR rolls state back on a failed receipt but +//! still reports the logs it wrote first, so a `token_diff` in a failed +//! outcome describes a trade that did not happen. That check is the caller's, +//! because the status is on the outcome, not in the logs. +//! - **Its `intent_hash` is in `intents_executed`** in the same outcome. A +//! transaction can carry intents that do not execute. +//! +//! No I/O here: the logs are handed in, and the tests run against bytes +//! captured off mainnet. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +/// Prefix NEP-297 puts on every structured log line. +pub const EVENT_PREFIX: &str = "EVENT_JSON:"; + +/// One `token_diff` entry: one signer's side of an executed intent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Leg { + /// The intent this diff belongs to. + pub intent_hash: String, + /// Who signed it — a solver's named account or a user's implicit one. + pub account_id: String, + /// Signed amount per multi-token id, in that token's base units: negative + /// is what the signer gave, positive what they received. + pub diff: BTreeMap, + /// The front end that submitted it, when it said. Optional in the + /// protocol, which is why it can only ever *suggest* who the taker was. + pub referral: Option, + /// Protocol fee taken, per token, in base units. + pub fees: BTreeMap, +} + +/// Which way a trade went, from the taker's side, in terms of the tracked +/// asset. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Side { + /// The taker received the asset. + Buy, + /// The taker gave the asset up. + Sell, +} + +impl Side { + /// `buy` or `sell`, as stored. + pub fn as_str(self) -> &'static str { + match self { + Side::Buy => "buy", + Side::Sell => "sell", + } + } +} + +/// The tracked asset against one counter token, within one receipt outcome. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Trade { + /// The other token, as a multi-token id (`nep141:…`). + pub counter: String, + /// The tracked asset that changed hands, in base units. + /// + /// The sum of every *positive* delta of the asset across the legs, so each + /// unit is counted once whichever side the taker was on. Positive rather + /// than negative because what a receiver got is net of the protocol fee — + /// the amount that actually arrived. + pub asset_amount: u128, + /// The counter token that changed hands, counted the same way. + pub counter_amount: u128, + /// The taker's side, when exactly one leg carried a `referral`. Inferred: + /// the field is optional, and solvers can be implicit accounts too. + pub side: Option, + /// The leg with the referral, when there was exactly one. + pub taker: Option, + /// That leg's referral. + pub referral: Option, + /// Every account with a leg in this trade, in log order. + pub accounts: Vec, + /// Every intent in this trade, in log order. + pub intent_hashes: Vec, + /// The legs themselves, so the figures above stay checkable. + pub legs: Vec, +} + +/// Every `token_diff` leg in one receipt outcome's logs whose intent executed. +/// +/// The caller must already know the outcome succeeded; see the module docs. +/// A log line that is not a NEP-297 event, or an event this does not read, is +/// skipped. A `token_diff` entry with an amount that does not parse is skipped +/// too and counted in the second value, so a format change shows up as a +/// number rather than as quietly fewer trades. +pub fn executed_legs(logs: &[String]) -> (Vec, usize) { + let mut legs = Vec::new(); + let mut executed = BTreeSet::new(); + let mut malformed = 0; + + for line in logs { + let Some(event) = parse_event(line) else { + continue; + }; + if event.get("standard").and_then(Value::as_str) != Some("dip4") { + continue; + } + let Some(data) = event.get("data").and_then(Value::as_array) else { + continue; + }; + match event.get("event").and_then(Value::as_str) { + Some("intents_executed") => { + for entry in data { + if let Some(hash) = entry.get("intent_hash").and_then(Value::as_str) { + executed.insert(hash.to_owned()); + } + } + } + Some("token_diff") => { + for entry in data { + match leg(entry) { + Some(l) => legs.push(l), + None => malformed += 1, + } + } + } + _ => {} + } + } + + legs.retain(|l| executed.contains(&l.intent_hash)); + (legs, malformed) +} + +/// The JSON of one `EVENT_JSON:` log line. +fn parse_event(line: &str) -> Option { + serde_json::from_str(line.strip_prefix(EVENT_PREFIX)?).ok() +} + +fn leg(entry: &Value) -> Option { + let diff = entry + .get("diff")? + .as_object()? + .iter() + .map(|(token, amount)| Some((token.clone(), amount.as_str()?.parse::().ok()?))) + .collect::>>()?; + let fees = match entry.get("fees_collected") { + None | Some(Value::Null) => BTreeMap::new(), + Some(v) => v + .as_object()? + .iter() + .map(|(token, amount)| Some((token.clone(), amount.as_str()?.parse::().ok()?))) + .collect::>>()?, + }; + Some(Leg { + intent_hash: entry.get("intent_hash")?.as_str()?.to_owned(), + account_id: entry.get("account_id")?.as_str()?.to_owned(), + diff, + referral: entry + .get("referral") + .and_then(Value::as_str) + .map(str::to_owned), + fees, + }) +} + +/// Group one outcome's legs into trades of `asset` against each counter token. +/// +/// Only a leg whose diff names exactly two tokens — the asset and one other — +/// is a pricing leg. One with three or more is a bundle with no single price +/// in it, and one naming only the asset is a transfer; both are left out and +/// counted in the second value, so they are visible rather than silently +/// absent. +/// +/// A swap is *not* assumed to be two legs. One user intent can be filled by +/// several solvers, so legs are grouped by counter token and summed, and a +/// trade is one asset/counter pair within one outcome. +pub fn trades(legs: &[Leg], asset: &str) -> (Vec, usize) { + let mut by_counter: BTreeMap> = BTreeMap::new(); + let mut unpriced = 0; + for leg in legs.iter().filter(|l| l.diff.contains_key(asset)) { + let others: Vec<&String> = leg.diff.keys().filter(|t| *t != asset).collect(); + match others.as_slice() { + [counter] => by_counter.entry((*counter).clone()).or_default().push(leg), + _ => unpriced += 1, + } + } + + let mut out = Vec::new(); + for (counter, legs) in by_counter { + let positive = |token: &str| -> u128 { + legs.iter() + .filter_map(|l| l.diff.get(token).copied()) + .filter(|d| *d > 0) + .map(|d| d as u128) + .sum() + }; + let asset_amount = positive(asset); + let counter_amount = positive(&counter); + if asset_amount == 0 || counter_amount == 0 { + unpriced += legs.len(); + continue; + } + + let referred: Vec<&&Leg> = legs.iter().filter(|l| l.referral.is_some()).collect(); + let (side, taker, referral) = match referred.as_slice() { + [one] => ( + one.diff + .get(asset) + .map(|d| if *d < 0 { Side::Sell } else { Side::Buy }), + Some(one.account_id.clone()), + one.referral.clone(), + ), + _ => (None, None, None), + }; + + out.push(Trade { + asset_amount, + counter_amount, + side, + taker, + referral, + accounts: legs.iter().map(|l| l.account_id.clone()).collect(), + intent_hashes: legs.iter().map(|l| l.intent_hash.clone()).collect(), + legs: legs.iter().map(|l| (*l).clone()).collect(), + counter, + }); + } + (out, unpriced) +} + +/// The shard an account lives on, under a shard layout. +/// +/// Accounts are split into contiguous ranges by `boundary_accounts`, compared +/// as byte strings; an account belongs to the range whose lower boundary is the +/// greatest one not above it. `shard_ids` lists the shard for each range in +/// order — and shard ids are *not* range indices: resharding renumbers them, so +/// a layout read once and kept would quietly start reading the wrong shard. +pub fn shard_for(account: &str, boundary_accounts: &[String], shard_ids: &[u64]) -> Option { + let index = boundary_accounts + .iter() + .take_while(|b| b.as_bytes() <= account.as_bytes()) + .count(); + shard_ids.get(index).copied() +} + +/// Base units as a token amount, for display arithmetic only. +/// +/// The stored amounts are exact; this is where precision is given up, and it is +/// only ever given up on the way to a ratio. +pub fn scaled(amount: u128, decimals: u8) -> f64 { + amount as f64 / 10f64.powi(i32::from(decimals)) +} + +/// Counter tokens per one unit of the asset. +pub fn price( + asset_amount: u128, + asset_decimals: u8, + counter_amount: u128, + counter_decimals: u8, +) -> f64 { + scaled(counter_amount, counter_decimals) / scaled(asset_amount, asset_decimals) +} + +/// Volume-weighted price over a set of trades, `(asset_amount, counter_amount)`. +/// +/// The ratio of the sums, not the mean of the ratios: a dust trade at a silly +/// price moves a mean as much as a large one does, and moves this by its size. +pub fn vwap(trades: &[(u128, u128)], asset_decimals: u8, counter_decimals: u8) -> Option { + let asset: f64 = trades.iter().map(|t| scaled(t.0, asset_decimals)).sum(); + let counter: f64 = trades.iter().map(|t| scaled(t.1, counter_decimals)).sum(); + (asset > 0.0).then(|| counter / asset) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ZEC: &str = "nep141:zec.omft.near"; + /// USDT on Tron, bridged. Its `ft_metadata.symbol` is `USDT`; the issue + /// that motivated this read the contract name and called it TRON, which is + /// the case for never taking identity from a name. + const USDT: &str = "nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near"; + + /// The `intents.near` receipt outcome of mainnet tx `85uy9uCF…`, block + /// 218,261,895: a 1Click swap of USDT (on Tron) for ZEC, captured from + /// neardata. + fn fixture() -> Vec { + serde_json::from_str(include_str!("../tests/intents-zec-usdt-218261895.json")).unwrap() + } + + #[test] + fn both_legs_of_a_real_swap_are_read_and_both_executed() { + let (legs, malformed) = executed_legs(&fixture()); + assert_eq!(malformed, 0); + assert_eq!(legs.len(), 2); + assert_eq!(legs[0].account_id, "blockzen-1-main.near"); + assert_eq!(legs[0].diff[ZEC], -702_585_891); + assert_eq!(legs[0].fees[ZEC], 703); + assert_eq!(legs[1].referral.as_deref(), Some("1click-swapkit")); + assert_eq!(legs[1].diff[USDT], -9_036_300_000); + } + + #[test] + fn a_diff_whose_intent_did_not_execute_is_not_a_trade() { + let logs: Vec = fixture() + .into_iter() + .filter(|l| !l.contains("\"intents_executed\"")) + .collect(); + let (legs, _) = executed_legs(&logs); + assert!(legs.is_empty()); + } + + #[test] + fn a_real_swap_is_one_trade_counted_once() { + let (legs, _) = executed_legs(&fixture()); + let (trades, unpriced) = trades(&legs, ZEC); + assert_eq!(unpriced, 0); + assert_eq!(trades.len(), 1); + let t = &trades[0]; + assert_eq!(t.counter, USDT); + // What the user received, net of the 703 fee — not both sides added. + assert_eq!(t.asset_amount, 702_585_188); + // What the solver received, net of the 9,037 fee. + assert_eq!(t.counter_amount, 9_036_290_963); + assert_eq!(t.side, Some(Side::Buy)); + assert_eq!(t.referral.as_deref(), Some("1click-swapkit")); + assert_eq!(t.intent_hashes.len(), 2); + + // ZEC has 8 decimals and USDT 6: about 1,286 USDT per ZEC. + let p = price(t.asset_amount, 8, t.counter_amount, 6); + assert!((p - 1286.15).abs() < 0.01, "{p}"); + } + + #[test] + fn the_same_trade_from_the_other_asset_is_the_other_side() { + let (legs, _) = executed_legs(&fixture()); + let (trades, _) = trades(&legs, USDT); + assert_eq!(trades[0].side, Some(Side::Sell)); + assert_eq!(trades[0].asset_amount, 9_036_290_963); + } + + #[test] + fn an_asset_not_in_the_swap_yields_nothing() { + let (legs, _) = executed_legs(&fixture()); + assert!(trades(&legs, "nep141:qtc.omft.near").0.is_empty()); + } + + #[test] + fn several_solvers_filling_one_intent_are_summed_not_paired() { + let leg = |who: &str, a: i128, c: i128, referral: Option<&str>| Leg { + intent_hash: format!("h-{who}"), + account_id: who.into(), + diff: [(ZEC.to_string(), a), (USDT.to_string(), c)].into(), + referral: referral.map(Into::into), + fees: BTreeMap::new(), + }; + let legs = vec![ + leg("user", -300, 3000, Some("quantus")), + leg("solver-a", 100, -1000, None), + leg("solver-b", 200, -2000, None), + ]; + let (trades, unpriced) = trades(&legs, ZEC); + assert_eq!(unpriced, 0); + assert_eq!(trades.len(), 1); + assert_eq!(trades[0].asset_amount, 300); + assert_eq!(trades[0].counter_amount, 3000); + assert_eq!(trades[0].side, Some(Side::Sell)); + assert_eq!(trades[0].taker.as_deref(), Some("user")); + } + + #[test] + fn a_bundle_of_three_tokens_has_no_price_and_says_so() { + let legs = vec![Leg { + intent_hash: "h".into(), + account_id: "a".into(), + diff: [ + (ZEC.to_string(), 1), + (USDT.to_string(), -1), + ("nep141:x".to_string(), -1), + ] + .into(), + referral: None, + fees: BTreeMap::new(), + }]; + let (trades, unpriced) = trades(&legs, ZEC); + assert!(trades.is_empty()); + assert_eq!(unpriced, 1); + } + + #[test] + fn without_exactly_one_referral_the_side_is_not_guessed() { + let (mut legs, _) = executed_legs(&fixture()); + legs[1].referral = None; + let (trades, _) = trades(&legs, ZEC); + assert_eq!(trades[0].side, None); + assert_eq!(trades[0].taker, None); + } + + #[test] + fn intents_near_is_on_shard_seven_under_layout_v3() { + // `EXPERIMENTAL_protocol_config.shard_layout.V3`, mainnet, 2026-10-02. + let boundaries: Vec = [ + "650", + "aurora", + "aurora-0", + "earn.kaiching", + "game.hot.tg", + "game.hot.tg-0", + "kkuuue2akv_1630967379.near", + "tge-lockup.sweat", + "wallet.ka", + ] + .map(String::from) + .into(); + let ids = [10, 11, 1, 8, 9, 6, 7, 4, 12, 13]; + assert_eq!(shard_for("intents.near", &boundaries, &ids), Some(7)); + // A boundary belongs to the range it opens, not the one it closes. + assert_eq!(shard_for("aurora", &boundaries, &ids), Some(1)); + assert_eq!(shard_for("0", &boundaries, &ids), Some(10)); + assert_eq!(shard_for("zzz", &boundaries, &ids), Some(13)); + } + + #[test] + fn vwap_weights_by_size() { + // 1 unit at 10 and 9 units at 20: the mean of ratios is 15, the volume + // weighted price 19. + let v = vwap(&[(1, 10), (9, 180)], 0, 0).unwrap(); + assert!((v - 19.0).abs() < 1e-9); + assert_eq!(vwap(&[], 0, 0), None); + } +} diff --git a/crates/blackbeard-core/src/lib.rs b/crates/blackbeard-core/src/lib.rs index df64480..1459839 100644 --- a/crates/blackbeard-core/src/lib.rs +++ b/crates/blackbeard-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod concentration; pub mod digest; pub mod geo; pub mod hashrate; +pub mod intents; pub mod pallet; pub mod runtime; pub mod scale; diff --git a/crates/blackbeard-core/tests/intents-zec-usdt-218261895.json b/crates/blackbeard-core/tests/intents-zec-usdt-218261895.json new file mode 100644 index 0000000..fa643f6 --- /dev/null +++ b/crates/blackbeard-core/tests/intents-zec-usdt-218261895.json @@ -0,0 +1,10 @@ +[ + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"token_diff\",\"data\":[{\"intent_hash\":\"812iSWw5jq51fAku4UzSYx1t3aBejoyv23mFMbHpBhK6\",\"account_id\":\"blockzen-1-main.near\",\"diff\":{\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\":\"9036290963\",\"nep141:zec.omft.near\":\"-702585891\"},\"fees_collected\":{\"nep141:zec.omft.near\":\"703\"}}]}", + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"token_diff\",\"data\":[{\"intent_hash\":\"GZEag38RnD9UooNUEwE4CGUks6gv3zRzKXeJz1izcb8S\",\"account_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"diff\":{\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\":\"-9036300000\",\"nep141:zec.omft.near\":\"702585188\"},\"referral\":\"1click-swapkit\",\"fees_collected\":{\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\":\"9037\"}}]}", + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"transfer\",\"data\":[{\"intent_hash\":\"GZEag38RnD9UooNUEwE4CGUks6gv3zRzKXeJz1izcb8S\",\"account_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"receiver_id\":\"143f2bb439575c286f4f585163f0e5fb4e464f8ce57ec06ae80741407d3cfb31\",\"tokens\":{\"nep141:zec.omft.near\":\"975706\"}}]}", + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"transfer\",\"data\":[{\"intent_hash\":\"GZEag38RnD9UooNUEwE4CGUks6gv3zRzKXeJz1izcb8S\",\"account_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"receiver_id\":\"2f09a4dc4b29cfdfe5d7a14afb04f0aa6c68e0b54f3bb606b168919b68085059\",\"tokens\":{\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\":\"63700000\"}}]}", + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"ft_withdraw\",\"data\":[{\"intent_hash\":\"GZEag38RnD9UooNUEwE4CGUks6gv3zRzKXeJz1izcb8S\",\"account_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"token\":\"zec.omft.near\",\"receiver_id\":\"omni.bridge.near\",\"amount\":\"701609482\",\"msg\":\"{\\\"recipient\\\":\\\"zcash:t1ZDvmCzfadaYBMMkaLjtwiCy2ThivFUeFg\\\",\\\"fee\\\":\\\"0\\\",\\\"native_token_fee\\\":\\\"0\\\",\\\"external_id\\\":\\\"44a44e4e-ca73-41fe-b8b8-07a235d209ca\\\",\\\"msg\\\":\\\"{\\\\\\\"MaxGasFee\\\\\\\":\\\\\\\"22000\\\\\\\"}\\\"}\",\"min_gas\":\"37400000000000\"}]}", + "EVENT_JSON:{\"standard\":\"dip4\",\"version\":\"0.4.3\",\"event\":\"intents_executed\",\"data\":[{\"intent_hash\":\"812iSWw5jq51fAku4UzSYx1t3aBejoyv23mFMbHpBhK6\",\"account_id\":\"blockzen-1-main.near\",\"nonce\":\"opzQM8Om4DlM6R4UD2xIzSnjfZnHhw5f6jXQ51GJY+Q=\"},{\"intent_hash\":\"GZEag38RnD9UooNUEwE4CGUks6gv3zRzKXeJz1izcb8S\",\"account_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"nonce\":\"Vij2xgAlKBKzgIfZV+7V2hieyzKhDhK14/M2W/bsLlA=\"}]}", + "EVENT_JSON:{\"standard\":\"nep245\",\"version\":\"1.0.0\",\"event\":\"mt_transfer\",\"data\":[{\"old_owner_id\":\"blockzen-1-main.near\",\"new_owner_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"token_ids\":[\"nep141:zec.omft.near\"],\"amounts\":[\"701609482\"]},{\"old_owner_id\":\"blockzen-1-main.near\",\"new_owner_id\":\"7066024d3f20f94de601c003163367873cca78507eeca4df66d9be645f197f05\",\"token_ids\":[\"nep141:zec.omft.near\"],\"amounts\":[\"703\"]},{\"old_owner_id\":\"blockzen-1-main.near\",\"new_owner_id\":\"143f2bb439575c286f4f585163f0e5fb4e464f8ce57ec06ae80741407d3cfb31\",\"token_ids\":[\"nep141:zec.omft.near\"],\"amounts\":[\"975706\"]},{\"old_owner_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"new_owner_id\":\"2f09a4dc4b29cfdfe5d7a14afb04f0aa6c68e0b54f3bb606b168919b68085059\",\"token_ids\":[\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\"],\"amounts\":[\"63700000\"]},{\"old_owner_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"new_owner_id\":\"7066024d3f20f94de601c003163367873cca78507eeca4df66d9be645f197f05\",\"token_ids\":[\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\"],\"amounts\":[\"9037\"]},{\"old_owner_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"new_owner_id\":\"blockzen-1-main.near\",\"token_ids\":[\"nep141:tron-d28a265909efecdcee7c5028585214ea0b96f015.omft.near\"],\"amounts\":[\"9036290963\"]}]}", + "EVENT_JSON:{\"standard\":\"nep245\",\"version\":\"1.0.0\",\"event\":\"mt_burn\",\"data\":[{\"owner_id\":\"42f3f2161b85004ad5b937d1fea9729925450e15ce61698b746214101d178171\",\"token_ids\":[\"nep141:zec.omft.near\"],\"amounts\":[\"701609482\"],\"memo\":\"withdraw\"}]}" +] \ No newline at end of file diff --git a/crates/blackbeard-data/Cargo.toml b/crates/blackbeard-data/Cargo.toml index 98857d4..21d1c1f 100644 --- a/crates/blackbeard-data/Cargo.toml +++ b/crates/blackbeard-data/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true blackbeard-core.workspace = true blackbeard-entities.workspace = true +base64.workspace = true bigdecimal.workspace = true chrono.workspace = true futures-util.workspace = true diff --git a/crates/blackbeard-data/migrations/0009_near_intents.sql b/crates/blackbeard-data/migrations/0009_near_intents.sql new file mode 100644 index 0000000..3d6a0fe --- /dev/null +++ b/crates/blackbeard-data/migrations/0009_near_intents.sql @@ -0,0 +1,72 @@ +-- Swaps settled by NEAR Intents, for the price of a chain's token there. +-- blackbeard/observer#27. +-- +-- NEAR Intents has no pool and no stored price: a price exists only as the +-- ratio between two amounts that actually changed hands in `intents.near`, so +-- that is all this holds. Quotes are offers and never land here. +-- +-- Keyed by the tracked `asset` as well as the chain, because the asset is +-- configuration: the indexer was proven against ZEC before QTC had a single +-- trade, and switching it must start a fresh scan rather than inherit one that +-- was looking for something else. + +-- One asset/counter pair settled within one receipt outcome. +-- +-- `near_height` is the block the `execute_intents` transaction was included in. +-- The relay signs as `intents.near` itself, so its first receipt executes in +-- that same block and the trade is settled there; `at` is that block's own +-- timestamp, the NEAR counterpart of `authored_at`. +create table intents_trade ( + chain text not null references chain (id) on delete cascade, + asset text not null, + near_height bigint not null, + trade_index integer not null, + at timestamptz not null, + tx_hash text not null, + receipt_id text not null, + counter_token text not null, + -- Base units, exact. The sum of positive deltas across the trade's legs, + -- so each unit is counted once whichever side the taker was on. + asset_amount numeric not null, + counter_amount numeric not null, + -- Inferred from the one leg carrying a `referral`, when there is exactly + -- one. Null means "could not tell", never "neither". + side text check (side in ('buy', 'sell')), + taker text, + referral text, + accounts text[] not null, + intent_hashes text[] not null, + -- The legs as the contract logged them, so every figure above stays + -- checkable against the source. + legs jsonb not null, + primary key (chain, asset, near_height, trade_index) +); + +create index intents_trade_by_counter + on intents_trade (chain, asset, counter_token, near_height desc, trade_index desc); + +-- How far the scan has read, as one contiguous interval of NEAR heights. +-- +-- The same shape as `event_scan` and for the same reason: a block with no trade +-- and a block never read look identical in `intents_trade`, and nearly every +-- block has no trade. `high` follows the tip; `low` walks back towards the +-- configured start. +create table intents_scan ( + chain text not null references chain (id) on delete cascade, + asset text not null, + low bigint not null, + high bigint not null, + updated_at timestamptz not null default now(), + primary key (chain, asset) +); + +-- A NEP-141 token's own `ft_metadata`, so decimals are asked for once. +-- +-- `symbol` is the token's self-description and is display only: anyone can +-- deploy a token that calls itself USDC, so nothing may treat it as identity. +create table near_token ( + token_id text primary key, + symbol text not null, + decimals smallint not null, + fetched_at timestamptz not null default now() +); diff --git a/crates/blackbeard-data/src/lib.rs b/crates/blackbeard-data/src/lib.rs index 4960a8d..a3680dc 100644 --- a/crates/blackbeard-data/src/lib.rs +++ b/crates/blackbeard-data/src/lib.rs @@ -8,6 +8,7 @@ #![deny(missing_docs)] +pub mod near; pub mod rpc; pub mod store; pub mod telemetry; diff --git a/crates/blackbeard-data/src/near.rs b/crates/blackbeard-data/src/near.rs new file mode 100644 index 0000000..29ba0f1 --- /dev/null +++ b/crates/blackbeard-data/src/near.rs @@ -0,0 +1,356 @@ +//! Reading NEAR mainnet, for the swaps `intents.near` settles. +//! +//! Plain JSON-RPC against public endpoints, through the same [`RpcClient`] the +//! Quantus side uses — so failover means the same thing here: a transport +//! failure moves to the next endpoint, a JSON-RPC error is the node answering +//! and stops. See blackbeard/observer#27 for how the source was chosen: +//! +//! - **neardata** returns a whole block with every outcome's logs in one +//! request, which would be ideal, but its free tier allows 30 blocks a minute +//! against NEAR's ~100. It falls behind the tip immediately. +//! - **`chunk` per block on one shard** is about 10/s on FastNEAR's free RPC, +//! so following one shard is comfortable. That is what this reads. +//! +//! Two clients, because a NEAR RPC node keeps about a day: `rpc.mainnet.*` held +//! ~21 hours when measured. Anything older goes to an archival endpoint, and so +//! does anything the live one says it does not know. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::DataError; +use crate::rpc::RpcClient; + +/// A transaction in a chunk, reduced to what decides whether it is read. +#[derive(Debug, Clone, Deserialize)] +pub struct ChunkTransaction { + /// Base58 transaction hash. + pub hash: String, + /// Who signed it — also the account `tx_status` needs to find it. + pub signer_id: String, + /// The contract it calls. + pub receiver_id: String, + /// Its actions, as the RPC renders them. + pub actions: Vec, +} + +impl ChunkTransaction { + /// Whether any action is a function call to `method`. + pub fn calls(&self, method: &str) -> bool { + self.actions.iter().any(|a| { + a.get("FunctionCall") + .and_then(|f| f.get("method_name")) + .and_then(Value::as_str) + == Some(method) + }) + } +} + +/// One shard's chunk at one height. +#[derive(Debug, Clone)] +pub struct Chunk { + /// Transactions included in it. + pub transactions: Vec, +} + +/// One receipt's execution, from `tx_status`. +#[derive(Debug, Clone)] +pub struct ReceiptOutcome { + /// Receipt id. + pub id: String, + /// The account that executed it. + pub executor_id: String, + /// Block it executed in. + pub block_hash: String, + /// Its log lines, `EVENT_JSON:` ones included. + pub logs: Vec, + /// True for `SuccessValue` and `SuccessReceiptId`. A failed receipt still + /// reports the logs it wrote before failing, and its state changes were + /// rolled back — so this is the first thing a reader must check. + pub succeeded: bool, +} + +/// A shard layout: which shard each account range is on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardLayout { + /// Lower bounds of every range but the first. + pub boundary_accounts: Vec, + /// Shard id for each range, in range order. + pub shard_ids: Vec, +} + +/// Live and archival NEAR endpoints. +#[derive(Debug, Clone)] +pub struct NearClient { + live: RpcClient, + archive: RpcClient, +} + +/// What a NEAR node calls "I have no such block", in its error's `cause.name`. +/// +/// Said both for a height that was never produced — NEAR skips heights — and +/// for one the node has pruned. Only the endpoint's retention tells them apart, +/// which is why a live node saying it is followed by asking an archive. +fn unknown(e: &DataError) -> bool { + matches!(e, DataError::Rpc { message, .. } + if message.contains("UNKNOWN_BLOCK") + || message.contains("UNKNOWN_CHUNK") + || message.contains("UNKNOWN_TRANSACTION") + || message.contains("GARBAGE_COLLECTED_BLOCK")) +} + +impl NearClient { + /// A client over `live` endpoints, falling back to `archive` ones. + pub fn new( + live: Vec, + archive: Vec, + timeout: Duration, + ) -> Result { + Ok(Self { + live: RpcClient::with_endpoints(live, timeout)?, + archive: RpcClient::with_endpoints(archive, timeout)?, + }) + } + + /// Ask the live endpoints, and the archive when they do not know. + async fn deep(&self, method: &str, params: Value) -> Result { + match self.live.call(method, params.clone()).await { + Err(e) if unknown(&e) => self.archive.call(method, params).await, + other => other, + } + } + + /// The latest final height, and the oldest the live endpoint still holds. + pub async fn heights(&self) -> Result<(u64, u64), DataError> { + let status = self.live.call("status", json!([])).await?; + let sync = &status["sync_info"]; + let field = |name: &str| { + sync[name].as_u64().ok_or_else(|| DataError::Malformed { + method: "status".into(), + message: format!("no {name}"), + }) + }; + // `latest_block_height` is the head, not finality. Reading only final + // blocks keeps a fork from writing a trade that did not happen, and two + // or three blocks is the whole cost. + let block = self + .live + .call("block", json!({"finality": "final"})) + .await?; + let final_height = + block["header"]["height"] + .as_u64() + .ok_or_else(|| DataError::Malformed { + method: "block".into(), + message: "no header.height".into(), + })?; + Ok((final_height, field("earliest_block_height")?)) + } + + /// The shard layout in force at `height`. + /// + /// Read per batch rather than once: resharding renumbers shards, and a + /// layout cached across one would follow the wrong shard while every call + /// kept succeeding. + pub async fn shard_layout(&self, height: u64) -> Result { + let config = self + .deep("EXPERIMENTAL_protocol_config", json!({"block_id": height})) + .await?; + let layout = config["shard_layout"] + .as_object() + .and_then(|o| o.values().next()) + .ok_or_else(|| DataError::Malformed { + method: "EXPERIMENTAL_protocol_config".into(), + message: "no shard_layout".into(), + })?; + let boundary_accounts = serde_json::from_value(layout["boundary_accounts"].clone())?; + let shard_ids = serde_json::from_value(layout["shard_ids"].clone())?; + Ok(ShardLayout { + boundary_accounts, + shard_ids, + }) + } + + /// The chunk `shard` produced at `height`, if it produced one. + /// + /// `None` for a height that was never produced, and for a block that + /// carried no new chunk for this shard: the RPC then returns the shard's + /// *previous* chunk, and reading it again would count its trades twice. + /// `height_included` is what tells them apart. + /// + /// `live_from` is the oldest height the live endpoint holds. At or above + /// it, the live endpoint's "unknown block" means the height was skipped; + /// below it, it means pruned, and the archive is asked. + pub async fn chunk( + &self, + height: u64, + shard: u64, + live_from: u64, + ) -> Result, DataError> { + let params = json!({"block_id": height, "shard_id": shard}); + let result = if height >= live_from { + self.live.call("chunk", params).await + } else { + self.archive.call("chunk", params).await + }; + let chunk = match result { + Ok(c) => c, + Err(e) if unknown(&e) => return Ok(None), + Err(e) => return Err(e), + }; + if chunk["header"]["height_included"].as_u64() != Some(height) { + return Ok(None); + } + Ok(Some(Chunk { + transactions: serde_json::from_value(chunk["transactions"].clone())?, + })) + } + + /// Every receipt outcome of a transaction, once it has executed. + pub async fn tx_outcomes( + &self, + hash: &str, + signer: &str, + ) -> Result, DataError> { + let status = self + .deep( + "EXPERIMENTAL_tx_status", + json!({"tx_hash": hash, "sender_account_id": signer, "wait_until": "EXECUTED"}), + ) + .await?; + let outcomes = + status["receipts_outcome"] + .as_array() + .ok_or_else(|| DataError::Malformed { + method: "EXPERIMENTAL_tx_status".into(), + message: "no receipts_outcome".into(), + })?; + Ok(outcomes + .iter() + .filter_map(|o| { + let outcome = &o["outcome"]; + let status = outcome["status"].as_object()?; + Some(ReceiptOutcome { + id: o["id"].as_str()?.to_owned(), + executor_id: outcome["executor_id"].as_str()?.to_owned(), + block_hash: o["block_hash"].as_str()?.to_owned(), + logs: serde_json::from_value(outcome["logs"].clone()).ok()?, + succeeded: status.contains_key("SuccessValue") + || status.contains_key("SuccessReceiptId"), + }) + }) + .collect()) + } + + /// A block's height and timestamp, by hash or height. + pub async fn block_at(&self, block_id: Value) -> Result<(u64, DateTime), DataError> { + let block = self.deep("block", json!({"block_id": block_id})).await?; + let header = &block["header"]; + let malformed = |what: &str| DataError::Malformed { + method: "block".into(), + message: format!("no header.{what}"), + }; + let height = header["height"] + .as_u64() + .ok_or_else(|| malformed("height"))?; + // Nanoseconds, as a string — past what an f64 holds exactly. + let nanos: i64 = header["timestamp_nanosec"] + .as_str() + .and_then(|s| s.parse().ok()) + .or_else(|| header["timestamp"].as_i64()) + .ok_or_else(|| malformed("timestamp"))?; + Ok((height, DateTime::from_timestamp_nanos(nanos))) + } + + /// Call a view method at the final block and decode its JSON result. + pub async fn view( + &self, + contract: &str, + method: &str, + args: Value, + ) -> Result { + use base64::Engine as _; + let result = self + .live + .call( + "query", + json!({ + "request_type": "call_function", + "finality": "final", + "account_id": contract, + "method_name": method, + "args_base64": base64::engine::general_purpose::STANDARD.encode(args.to_string()), + }), + ) + .await?; + let bytes: Vec = serde_json::from_value(result["result"].clone())?; + Ok(serde_json::from_slice(&bytes)?) + } +} + +/// A NEP-141 token's display facts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokenMetadata { + /// Its own `ft_metadata.symbol`. Self-reported: anyone can deploy a token + /// calling itself USDC, so this is a label and never an identity. + pub symbol: String, + /// Decimal places in its base unit. + pub decimals: u8, +} + +impl NearClient { + /// `ft_metadata` for a multi-token id, when it is a NEP-141 one. + /// + /// `None` for `nep245:` and `nep171:` ids, which have no `ft_metadata` to + /// ask — those counters are left unpriced rather than given a guessed + /// decimal count. + pub async fn token_metadata(&self, token_id: &str) -> Result, DataError> { + let Some(contract) = token_id.strip_prefix("nep141:") else { + return Ok(None); + }; + let meta = self.view(contract, "ft_metadata", json!({})).await?; + let malformed = |what: &str| DataError::Malformed { + method: "ft_metadata".into(), + message: format!("{contract}: no {what}"), + }; + Ok(Some(TokenMetadata { + symbol: meta["symbol"] + .as_str() + .ok_or_else(|| malformed("symbol"))? + .to_owned(), + decimals: meta["decimals"] + .as_u64() + .and_then(|d| u8::try_from(d).ok()) + .ok_or_else(|| malformed("decimals"))?, + })) + } +} + +/// Whether 1Click's public token list names `asset`. +/// +/// Off-chain, and only ever a listing: it says a swap can be *requested*, not +/// that one has settled. Read for the "is there a market yet" panel and for +/// nothing else — never for a price, because 1Click's prices are its own and a +/// quote is an offer. +pub async fn oneclick_lists( + base_url: &str, + asset: &str, + timeout: Duration, +) -> Result { + let url = format!("{}/v0/tokens", base_url.trim_end_matches('/')); + let tokens: Vec = reqwest::Client::builder() + .timeout(timeout) + .build()? + .get(&url) + .send() + .await? + .error_for_status()? + .json() + .await?; + Ok(tokens + .iter() + .any(|t| t.get("assetId").and_then(Value::as_str) == Some(asset))) +} diff --git a/crates/blackbeard-data/src/store.rs b/crates/blackbeard-data/src/store.rs index e28712b..55e9fcf 100644 --- a/crates/blackbeard-data/src/store.rs +++ b/crates/blackbeard-data/src/store.rs @@ -257,6 +257,46 @@ impl From<&crate::telemetry::NodeMetadata> for NodeTelemetry { } } +/// One asset/counter pair settled by NEAR Intents, as stored. +/// +/// Amounts are decimal strings in base units, because they are `u128` on NEAR +/// and `numeric` here, and nothing between the two should round them. +#[derive(Debug, Clone)] +pub struct IntentTrade { + /// Which chain's token this tracks. + pub chain: ChainId, + /// The tracked asset, as a multi-token id. + pub asset: String, + /// NEAR block the `execute_intents` transaction was included in. + pub near_height: u64, + /// Position among the trades read from that block. + pub trade_index: u32, + /// That block's timestamp. + pub at: DateTime, + /// The transaction. + pub tx_hash: String, + /// The receipt whose outcome logged the trade. + pub receipt_id: String, + /// The other token. + pub counter_token: String, + /// Asset that changed hands, base units. + pub asset_amount: String, + /// Counter token that changed hands, base units. + pub counter_amount: String, + /// `buy` or `sell` from the inferred taker's side, if it could be told. + pub side: Option, + /// The leg carrying the referral, when exactly one did. + pub taker: Option, + /// That leg's referral. + pub referral: Option, + /// Every account with a leg in it. + pub accounts: Vec, + /// Every intent in it. + pub intent_hashes: Vec, + /// The legs as logged. + pub legs: serde_json::Value, +} + /// A connected Postgres store. #[derive(Debug, Clone)] pub struct Store { @@ -2703,4 +2743,240 @@ impl Store { sqlx::query!("select 1 as ok").fetch_one(&self.pool).await?; Ok(()) } + + /// Persist trades settled by NEAR Intents. Idempotent on `(chain, asset, + /// near_height, trade_index)`, so re-reading a block rewrites rather than + /// doubles. One statement per trade: there are a handful a day. + pub async fn record_intent_trades(&self, trades: &[IntentTrade]) -> Result<(), DataError> { + let mut tx = self.pool.begin().await?; + for t in trades { + sqlx::query!( + r#" + insert into intents_trade ( + chain, asset, near_height, trade_index, at, tx_hash, receipt_id, + counter_token, asset_amount, counter_amount, side, taker, referral, + accounts, intent_hashes, legs + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::numeric, $10::text::numeric, + $11, $12, $13, $14, $15, $16) + on conflict (chain, asset, near_height, trade_index) do update set + at = excluded.at, tx_hash = excluded.tx_hash, + receipt_id = excluded.receipt_id, counter_token = excluded.counter_token, + asset_amount = excluded.asset_amount, + counter_amount = excluded.counter_amount, side = excluded.side, + taker = excluded.taker, referral = excluded.referral, + accounts = excluded.accounts, intent_hashes = excluded.intent_hashes, + legs = excluded.legs + "#, + t.chain.as_str(), + t.asset, + t.near_height as i64, + t.trade_index as i32, + t.at, + t.tx_hash, + t.receipt_id, + t.counter_token, + t.asset_amount, + t.counter_amount, + t.side, + t.taker, + t.referral, + &t.accounts, + &t.intent_hashes, + t.legs, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// The NEAR heights the intents scan has read for an asset, `(low, high)`. + /// + /// Not `max(near_height) from intents_trade`, for the reason `event_scan` + /// is not: almost no block has a trade in it. + pub async fn intents_scan( + &self, + chain: &ChainId, + asset: &str, + ) -> Result, DataError> { + let row = sqlx::query!( + "select low, high from intents_scan where chain = $1 and asset = $2", + chain.as_str(), + asset, + ) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| (r.low.max(0) as u64, r.high.max(0) as u64))) + } + + /// Record how far the intents scan has reached. Only ever grows; the + /// statement enforces it, as `set_event_scan` does. + pub async fn set_intents_scan( + &self, + chain: &ChainId, + asset: &str, + low: u64, + high: u64, + ) -> Result<(), DataError> { + sqlx::query!( + r#" + insert into intents_scan (chain, asset, low, high) values ($1, $2, $3, $4) + on conflict (chain, asset) do update + set low = least(intents_scan.low, excluded.low), + high = greatest(intents_scan.high, excluded.high), + updated_at = now() + "#, + chain.as_str(), + asset, + low as i64, + high as i64, + ) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Trades after a `(near_height, trade_index)` cursor, oldest first. + pub async fn intent_trades( + &self, + chain: &ChainId, + asset: &str, + after: (u64, u32), + limit: i64, + ) -> Result, DataError> { + let rows = sqlx::query!( + r#" + select chain, asset, near_height, trade_index, at, tx_hash, receipt_id, + counter_token, asset_amount::text as "asset_amount!", + counter_amount::text as "counter_amount!", side, taker, referral, + accounts, intent_hashes, legs + from intents_trade + where chain = $1 and asset = $2 and (near_height, trade_index) > ($3, $4) + order by near_height, trade_index + limit $5 + "#, + chain.as_str(), + asset, + after.0 as i64, + after.1 as i32, + limit, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| IntentTrade { + chain: ChainId(r.chain), + asset: r.asset, + near_height: r.near_height.max(0) as u64, + trade_index: r.trade_index.max(0) as u32, + at: r.at, + tx_hash: r.tx_hash, + receipt_id: r.receipt_id, + counter_token: r.counter_token, + asset_amount: r.asset_amount, + counter_amount: r.counter_amount, + side: r.side, + taker: r.taker, + referral: r.referral, + accounts: r.accounts, + intent_hashes: r.intent_hashes, + legs: r.legs, + }) + .collect()) + } + + /// The latest `per_counter` trades against each counter token, newest + /// first within each. + /// + /// A count per counter rather than a time range: a window here is a + /// number of trades, as a window elsewhere on this site is a number of + /// blocks. On a thin market a time range would put one trade from hours ago + /// behind a label that sounds current. + pub async fn latest_intent_trades( + &self, + chain: &ChainId, + asset: &str, + per_counter: i64, + ) -> Result, DataError> { + let rows = sqlx::query!( + r#" + select chain as "chain!", asset as "asset!", near_height as "near_height!", + trade_index as "trade_index!", at as "at!", tx_hash as "tx_hash!", + receipt_id as "receipt_id!", counter_token as "counter_token!", + asset_amount::text as "asset_amount!", + counter_amount::text as "counter_amount!", side, taker, referral, + accounts as "accounts!", intent_hashes as "intent_hashes!", legs as "legs!" + from ( + select *, row_number() over ( + partition by counter_token order by near_height desc, trade_index desc + ) as n + from intents_trade + where chain = $1 and asset = $2 + ) ranked + where n <= $3 + order by counter_token, near_height desc, trade_index desc + "#, + chain.as_str(), + asset, + per_counter, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| IntentTrade { + chain: ChainId(r.chain), + asset: r.asset, + near_height: r.near_height.max(0) as u64, + trade_index: r.trade_index.max(0) as u32, + at: r.at, + tx_hash: r.tx_hash, + receipt_id: r.receipt_id, + counter_token: r.counter_token, + asset_amount: r.asset_amount, + counter_amount: r.counter_amount, + side: r.side, + taker: r.taker, + referral: r.referral, + accounts: r.accounts, + intent_hashes: r.intent_hashes, + legs: r.legs, + }) + .collect()) + } + + /// A NEP-141 token's cached `ft_metadata`, `(symbol, decimals)`. + pub async fn near_token(&self, token_id: &str) -> Result, DataError> { + let row = sqlx::query!( + "select symbol, decimals from near_token where token_id = $1", + token_id + ) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| (r.symbol, r.decimals.clamp(0, 255) as u8))) + } + + /// Cache a token's `ft_metadata`. + pub async fn save_near_token( + &self, + token_id: &str, + symbol: &str, + decimals: u8, + ) -> Result<(), DataError> { + sqlx::query!( + r#" + insert into near_token (token_id, symbol, decimals) values ($1, $2, $3) + on conflict (token_id) do update + set symbol = excluded.symbol, decimals = excluded.decimals, fetched_at = now() + "#, + token_id, + symbol, + i16::from(decimals), + ) + .execute(&self.pool) + .await?; + Ok(()) + } } diff --git a/crates/blackbeard-entities/src/intents.rs b/crates/blackbeard-entities/src/intents.rs new file mode 100644 index 0000000..7296f3e --- /dev/null +++ b/crates/blackbeard-entities/src/intents.rs @@ -0,0 +1,157 @@ +//! A chain's token as NEAR Intents settles it. +//! +//! There is no price on NEAR Intents except the ratio inside a trade that +//! actually settled, so that is all this reports. Every figure is in units of +//! the counter token — no USD is derived until a stablecoin pairing exists to +//! derive it from, and a quote, which is an offer and not a trade, never +//! appears here. See blackbeard/observer#27. +//! +//! What is read is `execute_intents` transactions signed by `intents.near`, +//! which is how the 1Click relay submits. An intent reaching the contract as a +//! cross-contract receipt instead is not read; that is the indexer's known +//! blind spot and the reason [`IntentsPrice::reads`] exists. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::{BigUintDec, ChainId}; + +/// One settled trade of the asset against one counter token. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "IntentsSwap.ts")] +pub struct IntentsSwap { + /// NEAR block it settled in. + #[ts(type = "number")] + pub near_height: u64, + /// Position among the trades in that block. With the height, the cursor. + pub trade_index: u32, + /// That block's own timestamp. + pub at: DateTime, + /// The `execute_intents` transaction. + pub tx_hash: String, + /// The receipt whose outcome logged it. + pub receipt_id: String, + /// The other token, as a multi-token id (`nep141:…`). + pub counter_token: String, + /// That token's own `ft_metadata.symbol`. A label it chose for itself — + /// never an identity, because anyone can deploy a token called USDC. + pub counter_symbol: Option, + /// The asset that changed hands, in base units. + pub asset_amount: BigUintDec, + /// The counter token that changed hands, in base units. + pub counter_amount: BigUintDec, + /// Counter tokens per one whole unit of the asset. Absent when either + /// token's decimals are unknown. + pub price: Option, + /// `buy` or `sell`, from the taker's side. **Inferred** from the one leg + /// that carried a `referral`; absent when that could not be told. + pub side: Option, + /// The account inferred to be the taker. + pub taker: Option, + /// The front end the taker came through, as it named itself. + pub referral: Option, + /// Every account with a leg in the trade — solvers by their named + /// accounts, nothing more is claimed about who they are. + pub accounts: Vec, + /// Every intent in the trade. + pub intent_hashes: Vec, +} + +/// A page of swaps, oldest first. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "IntentsSwaps.ts")] +pub struct IntentsSwaps { + /// Which chain's token. + pub chain: ChainId, + /// The tracked asset, as a multi-token id. + pub asset: String, + /// The swaps. + pub swaps: Vec, + /// Cursor for the next page, `near_height:trade_index`, when this one was + /// full. Absent when there is nothing more *in the indexed range*. + pub next: Option, + /// Lowest NEAR height read. A swap below it is missing from this answer, + /// not from the chain. + #[ts(type = "number")] + pub indexed_from: Option, + /// Highest NEAR height read. + #[ts(type = "number")] + pub indexed_to: Option, +} + +/// The asset's price against one counter token, over its latest trades. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "CounterPrice.ts")] +pub struct CounterPrice { + /// The counter token, as a multi-token id. + pub counter_token: String, + /// Its self-reported symbol. Display only. + pub symbol: Option, + /// Its decimals, when it is a NEP-141 token that said. + pub decimals: Option, + /// How many trades the figures below cover. Every figure is only as good + /// as this count, and one trade is one trade however it is labelled. + pub trades: u32, + /// The latest trade's price. + pub last_price: Option, + /// When it settled. + pub last_at: DateTime, + /// Where it settled. + #[ts(type = "number")] + pub last_near_height: u64, + /// Volume-weighted over the trades counted — the ratio of the summed + /// amounts, so a dust trade at a silly price moves it by its size. + pub vwap: Option, + /// Asset traded over those trades, base units. + pub asset_volume: BigUintDec, + /// Counter token traded over those trades, base units. + pub counter_volume: BigUintDec, + /// When the oldest of them settled. + pub first_at: DateTime, +} + +/// Whether there is anything to trade yet, read rather than assumed. +/// +/// Until the asset exists on NEAR there can be no swap, and saying so with +/// the evidence beats an empty panel — or a price from somewhere else. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "IntentsListing.ts")] +pub struct IntentsListing { + /// When these were read. + pub checked_at: DateTime, + /// The token contract's `ft_total_supply`: everything bridged to NEAR. + pub total_supply: Option, + /// What `intents.near` holds of it: what can be traded at all. + pub in_intents: Option, + /// Whether 1Click's public token list names the asset. Off-chain, and only + /// a listing — it says swaps can be *requested*, not that any settled. + pub oneclick_listed: Option, +} + +/// The asset's price on NEAR Intents, per counter token. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "IntentsPrice.ts")] +pub struct IntentsPrice { + /// Which chain's token. + pub chain: ChainId, + /// The tracked asset, as a multi-token id. + pub asset: String, + /// Its decimals, from its own `ft_metadata`. + pub asset_decimals: Option, + /// Trades per counter token the figures are taken over, at most. + pub window_trades: u32, + /// One entry per counter token ever traded in the indexed range, most + /// recently traded first. + pub counters: Vec, + /// Supply and listing, when they have been read. + pub listing: Option, + /// What the indexer reads, in words — and so what it cannot see. + pub reads: String, + /// Lowest NEAR height read. + #[ts(type = "number")] + pub indexed_from: Option, + /// Highest NEAR height read. + #[ts(type = "number")] + pub indexed_to: Option, +} diff --git a/crates/blackbeard-entities/src/lib.rs b/crates/blackbeard-entities/src/lib.rs index dba9e8f..b343cea 100644 --- a/crates/blackbeard-entities/src/lib.rs +++ b/crates/blackbeard-entities/src/lib.rs @@ -20,6 +20,7 @@ mod block; mod call; mod chain; mod error; +mod intents; mod miner; mod reversible; mod roles; @@ -38,6 +39,7 @@ pub use chain::{ WormholeDeposit, WormholeDeposits, WormholeSummary, }; pub use error::{ApiError, EntityError}; +pub use intents::{CounterPrice, IntentsListing, IntentsPrice, IntentsSwap, IntentsSwaps}; pub use miner::{ AttributionSource, LeaderboardRow, MinerDetail, MinerId, MinerSeriesPoint, NodeActivity, NodeIndex, NodeInfo, NodeRow, diff --git a/web/src/api/generated/CounterPrice.ts b/web/src/api/generated/CounterPrice.ts new file mode 100644 index 0000000..f4890a1 --- /dev/null +++ b/web/src/api/generated/CounterPrice.ts @@ -0,0 +1,53 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BigUintDec } from "./BigUintDec"; + +/** + * The asset's price against one counter token, over its latest trades. + */ +export type CounterPrice = { +/** + * The counter token, as a multi-token id. + */ +counter_token: string, +/** + * Its self-reported symbol. Display only. + */ +symbol: string | null, +/** + * Its decimals, when it is a NEP-141 token that said. + */ +decimals: number | null, +/** + * How many trades the figures below cover. Every figure is only as good + * as this count, and one trade is one trade however it is labelled. + */ +trades: number, +/** + * The latest trade's price. + */ +last_price: number | null, +/** + * When it settled. + */ +last_at: string, +/** + * Where it settled. + */ +last_near_height: number, +/** + * Volume-weighted over the trades counted — the ratio of the summed + * amounts, so a dust trade at a silly price moves it by its size. + */ +vwap: number | null, +/** + * Asset traded over those trades, base units. + */ +asset_volume: BigUintDec, +/** + * Counter token traded over those trades, base units. + */ +counter_volume: BigUintDec, +/** + * When the oldest of them settled. + */ +first_at: string, }; diff --git a/web/src/api/generated/IntentsListing.ts b/web/src/api/generated/IntentsListing.ts new file mode 100644 index 0000000..82a0bf7 --- /dev/null +++ b/web/src/api/generated/IntentsListing.ts @@ -0,0 +1,27 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BigUintDec } from "./BigUintDec"; + +/** + * Whether there is anything to trade yet, read rather than assumed. + * + * Until the asset exists on NEAR there can be no swap, and saying so with + * the evidence beats an empty panel — or a price from somewhere else. + */ +export type IntentsListing = { +/** + * When these were read. + */ +checked_at: string, +/** + * The token contract's `ft_total_supply`: everything bridged to NEAR. + */ +total_supply: BigUintDec | null, +/** + * What `intents.near` holds of it: what can be traded at all. + */ +in_intents: BigUintDec | null, +/** + * Whether 1Click's public token list names the asset. Off-chain, and only + * a listing — it says swaps can be *requested*, not that any settled. + */ +oneclick_listed: boolean | null, }; diff --git a/web/src/api/generated/IntentsPrice.ts b/web/src/api/generated/IntentsPrice.ts new file mode 100644 index 0000000..ef2ba9f --- /dev/null +++ b/web/src/api/generated/IntentsPrice.ts @@ -0,0 +1,46 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ChainId } from "./ChainId"; +import type { CounterPrice } from "./CounterPrice"; +import type { IntentsListing } from "./IntentsListing"; + +/** + * The asset's price on NEAR Intents, per counter token. + */ +export type IntentsPrice = { +/** + * Which chain's token. + */ +chain: ChainId, +/** + * The tracked asset, as a multi-token id. + */ +asset: string, +/** + * Its decimals, from its own `ft_metadata`. + */ +asset_decimals: number | null, +/** + * Trades per counter token the figures are taken over, at most. + */ +window_trades: number, +/** + * One entry per counter token ever traded in the indexed range, most + * recently traded first. + */ +counters: Array, +/** + * Supply and listing, when they have been read. + */ +listing: IntentsListing | null, +/** + * What the indexer reads, in words — and so what it cannot see. + */ +reads: string, +/** + * Lowest NEAR height read. + */ +indexed_from: number, +/** + * Highest NEAR height read. + */ +indexed_to: number, }; diff --git a/web/src/api/generated/IntentsSwap.ts b/web/src/api/generated/IntentsSwap.ts new file mode 100644 index 0000000..9158d57 --- /dev/null +++ b/web/src/api/generated/IntentsSwap.ts @@ -0,0 +1,71 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BigUintDec } from "./BigUintDec"; + +/** + * One settled trade of the asset against one counter token. + */ +export type IntentsSwap = { +/** + * NEAR block it settled in. + */ +near_height: number, +/** + * Position among the trades in that block. With the height, the cursor. + */ +trade_index: number, +/** + * That block's own timestamp. + */ +at: string, +/** + * The `execute_intents` transaction. + */ +tx_hash: string, +/** + * The receipt whose outcome logged it. + */ +receipt_id: string, +/** + * The other token, as a multi-token id (`nep141:…`). + */ +counter_token: string, +/** + * That token's own `ft_metadata.symbol`. A label it chose for itself — + * never an identity, because anyone can deploy a token called USDC. + */ +counter_symbol: string | null, +/** + * The asset that changed hands, in base units. + */ +asset_amount: BigUintDec, +/** + * The counter token that changed hands, in base units. + */ +counter_amount: BigUintDec, +/** + * Counter tokens per one whole unit of the asset. Absent when either + * token's decimals are unknown. + */ +price: number | null, +/** + * `buy` or `sell`, from the taker's side. **Inferred** from the one leg + * that carried a `referral`; absent when that could not be told. + */ +side: string | null, +/** + * The account inferred to be the taker. + */ +taker: string | null, +/** + * The front end the taker came through, as it named itself. + */ +referral: string | null, +/** + * Every account with a leg in the trade — solvers by their named + * accounts, nothing more is claimed about who they are. + */ +accounts: Array, +/** + * Every intent in the trade. + */ +intent_hashes: Array, }; diff --git a/web/src/api/generated/IntentsSwaps.ts b/web/src/api/generated/IntentsSwaps.ts new file mode 100644 index 0000000..f08cc3b --- /dev/null +++ b/web/src/api/generated/IntentsSwaps.ts @@ -0,0 +1,34 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ChainId } from "./ChainId"; +import type { IntentsSwap } from "./IntentsSwap"; + +/** + * A page of swaps, oldest first. + */ +export type IntentsSwaps = { +/** + * Which chain's token. + */ +chain: ChainId, +/** + * The tracked asset, as a multi-token id. + */ +asset: string, +/** + * The swaps. + */ +swaps: Array, +/** + * Cursor for the next page, `near_height:trade_index`, when this one was + * full. Absent when there is nothing more *in the indexed range*. + */ +next: string | null, +/** + * Lowest NEAR height read. A swap below it is missing from this answer, + * not from the chain. + */ +indexed_from: number, +/** + * Highest NEAR height read. + */ +indexed_to: number, };