From 9becf2629e7c2d64d01d612d51480e0854588587 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Tue, 6 Oct 2026 17:42:57 +0300 Subject: [PATCH] fix(data): count each visible transfer once, not once per pallet Every visible transfer emits `Balances::Transfer` and `Wormhole::NativeTransferred` with the same payload in the same extrinsic (193,686 of 193,691 on mainnet), and `account_flows`, `flows_for` and `wormhole_routes` added the two: every account page's sent and received, the distribution table's `sent on`, and the wormhole page's visible routes were doubled. A `Balances::Transfer` now counts only when it has no such twin, which keeps the five that stand alone. Checked on production data: the bridge custody account's `received` goes from 372,003.992 over 30 receipts to 186,001.996 over 15. Closes #42 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj --- ...4f79ef31cf33677396ddd6f2ecc356ff7b045.json | 59 ------------------- ...baade8105b98a527720265f9c24bfccb581f.json} | 4 +- ...7f89dd46a8d958a1def09ae2d61438cb0e9e.json} | 4 +- ...fa185dc937f899ed914fcd4c788ab644de1ff.json | 59 +++++++++++++++++++ crates/blackbeard-data/src/store.rs | 41 ++++++++++++- 5 files changed, 101 insertions(+), 66 deletions(-) delete mode 100644 .sqlx/query-717abd3ad62607561855d8fbe6c4f79ef31cf33677396ddd6f2ecc356ff7b045.json rename .sqlx/{query-fbede02f645d92cce9e02673db479234db0bcb647411773c6416bb072267f35d.json => query-74634b489079a40bdf7bdac45a70baade8105b98a527720265f9c24bfccb581f.json} (66%) rename .sqlx/{query-6e5b0988898e0f185fa26915206b1946d73ea0908ada9212941a1e456dcb7567.json => query-ee4201fd3cc1ee671ca159ad37897f89dd46a8d958a1def09ae2d61438cb0e9e.json} (63%) create mode 100644 .sqlx/query-f0c7775b07654af481c7feedf2bfa185dc937f899ed914fcd4c788ab644de1ff.json diff --git a/.sqlx/query-717abd3ad62607561855d8fbe6c4f79ef31cf33677396ddd6f2ecc356ff7b045.json b/.sqlx/query-717abd3ad62607561855d8fbe6c4f79ef31cf33677396ddd6f2ecc356ff7b045.json deleted file mode 100644 index 176db76..0000000 --- a/.sqlx/query-717abd3ad62607561855d8fbe6c4f79ef31cf33677396ddd6f2ecc356ff7b045.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n with moves as (\n select fields->>'from' as src,\n fields->>'to' as dst,\n (fields->>'amount')::numeric as amount\n from chain_event\n where chain = $1\n and accounts && $2::text[]\n and ( (pallet = 'Wormhole' and variant = 'NativeTransferred')\n or (pallet = 'Balances' and variant = 'Transfer'))\n ),\n wanted as (select unnest($2::text[]) as account)\n select w.account as \"account!\",\n coalesce((select sum(amount) from moves where src = w.account), 0) as \"sent!\",\n coalesce((select sum(amount) from moves where dst = w.account), 0) as \"received!\",\n (select count(*) from moves where src = w.account) as \"sends!\",\n (select count(*) from moves where dst = w.account) as \"receipts!\",\n (select count(distinct dst) from moves where src = w.account) as \"recipients!\",\n (select count(distinct src) from moves where dst = w.account) as \"senders!\"\n from wanted w\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "account!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "sent!", - "type_info": "Numeric" - }, - { - "ordinal": 2, - "name": "received!", - "type_info": "Numeric" - }, - { - "ordinal": 3, - "name": "sends!", - "type_info": "Int8" - }, - { - "ordinal": 4, - "name": "receipts!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "recipients!", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "senders!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "717abd3ad62607561855d8fbe6c4f79ef31cf33677396ddd6f2ecc356ff7b045" -} diff --git a/.sqlx/query-fbede02f645d92cce9e02673db479234db0bcb647411773c6416bb072267f35d.json b/.sqlx/query-74634b489079a40bdf7bdac45a70baade8105b98a527720265f9c24bfccb581f.json similarity index 66% rename from .sqlx/query-fbede02f645d92cce9e02673db479234db0bcb647411773c6416bb072267f35d.json rename to .sqlx/query-74634b489079a40bdf7bdac45a70baade8105b98a527720265f9c24bfccb581f.json index d72b69e..15bbb4f 100644 --- a/.sqlx/query-fbede02f645d92cce9e02673db479234db0bcb647411773c6416bb072267f35d.json +++ b/.sqlx/query-74634b489079a40bdf7bdac45a70baade8105b98a527720265f9c24bfccb581f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n select\n coalesce(sum((fields->>'amount')::numeric)\n filter (where fields->>'from' = $2), 0) as \"sent!\",\n coalesce(sum((fields->>'amount')::numeric)\n filter (where fields->>'to' = $2), 0) as \"received!\",\n count(*) filter (where fields->>'from' = $2) as \"sends!\",\n count(*) filter (where fields->>'to' = $2) as \"receipts!\",\n count(distinct fields->>'to')\n filter (where fields->>'from' = $2) as \"recipients!\",\n count(distinct fields->>'from')\n filter (where fields->>'to' = $2) as \"senders!\"\n from chain_event\n where chain = $1\n and accounts @> array[$2]\n and ( (pallet = 'Wormhole' and variant = 'NativeTransferred')\n or (pallet = 'Balances' and variant = 'Transfer'))\n ", + "query": "\n select\n coalesce(sum((fields->>'amount')::numeric)\n filter (where fields->>'from' = $2), 0) as \"sent!\",\n coalesce(sum((fields->>'amount')::numeric)\n filter (where fields->>'to' = $2), 0) as \"received!\",\n count(*) filter (where fields->>'from' = $2) as \"sends!\",\n count(*) filter (where fields->>'to' = $2) as \"receipts!\",\n count(distinct fields->>'to')\n filter (where fields->>'from' = $2) as \"recipients!\",\n count(distinct fields->>'from')\n filter (where fields->>'to' = $2) as \"senders!\"\n from chain_event\n where chain = $1\n and accounts @> array[$2]\n and ( (pallet = 'Wormhole' and variant = 'NativeTransferred')\n or (pallet = 'Balances' and variant = 'Transfer'\n -- the same transfer's other event: counted once (#42)\n and not exists (\n select 1 from chain_event twin\n where twin.chain = chain_event.chain\n and twin.height = chain_event.height\n and twin.extrinsic_index is not distinct from chain_event.extrinsic_index\n and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred'\n and twin.fields->>'from' = chain_event.fields->>'from'\n and twin.fields->>'to' = chain_event.fields->>'to'\n and twin.fields->>'amount' = chain_event.fields->>'amount')))\n ", "describe": { "columns": [ { @@ -49,5 +49,5 @@ null ] }, - "hash": "fbede02f645d92cce9e02673db479234db0bcb647411773c6416bb072267f35d" + "hash": "74634b489079a40bdf7bdac45a70baade8105b98a527720265f9c24bfccb581f" } diff --git a/.sqlx/query-6e5b0988898e0f185fa26915206b1946d73ea0908ada9212941a1e456dcb7567.json b/.sqlx/query-ee4201fd3cc1ee671ca159ad37897f89dd46a8d958a1def09ae2d61438cb0e9e.json similarity index 63% rename from .sqlx/query-6e5b0988898e0f185fa26915206b1946d73ea0908ada9212941a1e456dcb7567.json rename to .sqlx/query-ee4201fd3cc1ee671ca159ad37897f89dd46a8d958a1def09ae2d61438cb0e9e.json index 1751f32..d2f12e2 100644 --- a/.sqlx/query-6e5b0988898e0f185fa26915206b1946d73ea0908ada9212941a1e456dcb7567.json +++ b/.sqlx/query-ee4201fd3cc1ee671ca159ad37897f89dd46a8d958a1def09ae2d61438cb0e9e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n select\n coalesce(x.pallet || '::' || x.call, '') as \"produced_by!\",\n (e.fields->>'from' = $2) as \"sender_hidden!\",\n count(*) as \"transfers!\",\n count(distinct e.fields->>'to') as \"recipients!\",\n coalesce(sum((e.fields->>'amount')::numeric), 0) as \"amount!\"\n from chain_event e\n left join chain_extrinsic x\n on x.chain = e.chain\n and x.height = e.height\n and x.extrinsic_index = e.extrinsic_index\n where e.chain = $1\n and ( (e.pallet = 'Wormhole' and e.variant = 'NativeTransferred')\n or (e.pallet = 'Balances' and e.variant = 'Transfer'))\n group by 1, 2\n order by count(*) desc\n ", + "query": "\n select\n coalesce(x.pallet || '::' || x.call, '') as \"produced_by!\",\n (e.fields->>'from' = $2) as \"sender_hidden!\",\n count(*) as \"transfers!\",\n count(distinct e.fields->>'to') as \"recipients!\",\n coalesce(sum((e.fields->>'amount')::numeric), 0) as \"amount!\"\n from chain_event e\n left join chain_extrinsic x\n on x.chain = e.chain\n and x.height = e.height\n and x.extrinsic_index = e.extrinsic_index\n where e.chain = $1\n and ( (e.pallet = 'Wormhole' and e.variant = 'NativeTransferred')\n or (e.pallet = 'Balances' and e.variant = 'Transfer'\n -- the same transfer's other event: counted once (#42)\n and not exists (\n select 1 from chain_event twin\n where twin.chain = e.chain\n and twin.height = e.height\n and twin.extrinsic_index is not distinct from e.extrinsic_index\n and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred'\n and twin.fields->>'from' = e.fields->>'from'\n and twin.fields->>'to' = e.fields->>'to'\n and twin.fields->>'amount' = e.fields->>'amount')))\n group by 1, 2\n order by count(*) desc\n ", "describe": { "columns": [ { @@ -43,5 +43,5 @@ null ] }, - "hash": "6e5b0988898e0f185fa26915206b1946d73ea0908ada9212941a1e456dcb7567" + "hash": "ee4201fd3cc1ee671ca159ad37897f89dd46a8d958a1def09ae2d61438cb0e9e" } diff --git a/.sqlx/query-f0c7775b07654af481c7feedf2bfa185dc937f899ed914fcd4c788ab644de1ff.json b/.sqlx/query-f0c7775b07654af481c7feedf2bfa185dc937f899ed914fcd4c788ab644de1ff.json new file mode 100644 index 0000000..fa19a6a --- /dev/null +++ b/.sqlx/query-f0c7775b07654af481c7feedf2bfa185dc937f899ed914fcd4c788ab644de1ff.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "\n with moves as (\n select fields->>'from' as src,\n fields->>'to' as dst,\n (fields->>'amount')::numeric as amount\n from chain_event\n where chain = $1\n and accounts && $2::text[]\n and ( (pallet = 'Wormhole' and variant = 'NativeTransferred')\n or (pallet = 'Balances' and variant = 'Transfer'\n -- the same transfer's other event: counted once (#42)\n and not exists (\n select 1 from chain_event twin\n where twin.chain = chain_event.chain\n and twin.height = chain_event.height\n and twin.extrinsic_index is not distinct from chain_event.extrinsic_index\n and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred'\n and twin.fields->>'from' = chain_event.fields->>'from'\n and twin.fields->>'to' = chain_event.fields->>'to'\n and twin.fields->>'amount' = chain_event.fields->>'amount')))\n ),\n wanted as (select unnest($2::text[]) as account)\n select w.account as \"account!\",\n coalesce((select sum(amount) from moves where src = w.account), 0) as \"sent!\",\n coalesce((select sum(amount) from moves where dst = w.account), 0) as \"received!\",\n (select count(*) from moves where src = w.account) as \"sends!\",\n (select count(*) from moves where dst = w.account) as \"receipts!\",\n (select count(distinct dst) from moves where src = w.account) as \"recipients!\",\n (select count(distinct src) from moves where dst = w.account) as \"senders!\"\n from wanted w\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "account!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "sent!", + "type_info": "Numeric" + }, + { + "ordinal": 2, + "name": "received!", + "type_info": "Numeric" + }, + { + "ordinal": 3, + "name": "sends!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "receipts!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "recipients!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "senders!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "f0c7775b07654af481c7feedf2bfa185dc937f899ed914fcd4c788ab644de1ff" +} diff --git a/crates/blackbeard-data/src/store.rs b/crates/blackbeard-data/src/store.rs index d42e909..886e071 100644 --- a/crates/blackbeard-data/src/store.rs +++ b/crates/blackbeard-data/src/store.rs @@ -2190,6 +2190,11 @@ impl Store { /// direction is a fact in the payload rather than something inferred from /// an account appearing nearby. /// + /// **Every visible transfer emits both**, with the same payload in the same + /// extrinsic (193,686 of 193,691 `Balances::Transfer` on mainnet). Adding + /// the two doubled every sum and count here until #42; a `Balances::Transfer` + /// now counts only when it has no such twin. + /// /// **A nonce cannot answer this question on this chain.** Wormhole /// transfers are dispatched unsigned — `verify_private_batch` and /// `verify_public_batch` have no signer — so an address can move a great @@ -2221,7 +2226,17 @@ impl Store { where chain = $1 and accounts @> array[$2] and ( (pallet = 'Wormhole' and variant = 'NativeTransferred') - or (pallet = 'Balances' and variant = 'Transfer')) + or (pallet = 'Balances' and variant = 'Transfer' + -- the same transfer's other event: counted once (#42) + and not exists ( + select 1 from chain_event twin + where twin.chain = chain_event.chain + and twin.height = chain_event.height + and twin.extrinsic_index is not distinct from chain_event.extrinsic_index + and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred' + and twin.fields->>'from' = chain_event.fields->>'from' + and twin.fields->>'to' = chain_event.fields->>'to' + and twin.fields->>'amount' = chain_event.fields->>'amount'))) "#, chain.as_str(), account, @@ -2263,7 +2278,17 @@ impl Store { and x.extrinsic_index = e.extrinsic_index where e.chain = $1 and ( (e.pallet = 'Wormhole' and e.variant = 'NativeTransferred') - or (e.pallet = 'Balances' and e.variant = 'Transfer')) + or (e.pallet = 'Balances' and e.variant = 'Transfer' + -- the same transfer's other event: counted once (#42) + and not exists ( + select 1 from chain_event twin + where twin.chain = e.chain + and twin.height = e.height + and twin.extrinsic_index is not distinct from e.extrinsic_index + and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred' + and twin.fields->>'from' = e.fields->>'from' + and twin.fields->>'to' = e.fields->>'to' + and twin.fields->>'amount' = e.fields->>'amount'))) group by 1, 2 order by count(*) desc "#, @@ -2500,7 +2525,17 @@ impl Store { where chain = $1 and accounts && $2::text[] and ( (pallet = 'Wormhole' and variant = 'NativeTransferred') - or (pallet = 'Balances' and variant = 'Transfer')) + or (pallet = 'Balances' and variant = 'Transfer' + -- the same transfer's other event: counted once (#42) + and not exists ( + select 1 from chain_event twin + where twin.chain = chain_event.chain + and twin.height = chain_event.height + and twin.extrinsic_index is not distinct from chain_event.extrinsic_index + and twin.pallet = 'Wormhole' and twin.variant = 'NativeTransferred' + and twin.fields->>'from' = chain_event.fields->>'from' + and twin.fields->>'to' = chain_event.fields->>'to' + and twin.fields->>'amount' = chain_event.fields->>'amount'))) ), wanted as (select unnest($2::text[]) as account) select w.account as "account!",