From 36bea6bb7fb244fd41f7cce4001a9f4568249c5c Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Fri, 9 Oct 2026 12:41:56 +0300 Subject: [PATCH] feat(infra): our NEAR nodes public by shard, s.near.blackbeard.observer near.blackbeard.observer reaches only frankie, and a NEAR node refuses any account on a shard it does not track. The wallet's NEAR Intents account (blackbeard/wallet#71) needs reads on every shard its route touches, so each shard one of our nodes tracks now has its own name on oolon: s4/s8 to gargravarr, s7/s13 to frankie, s10/s12 to quadbrat. One vhost and one certificate lineage cover them all, the shard-to-node map sits in blackbeard-nodes.conf with a request ceiling per node, and infra-setup.sh --role nodes provisions them (NEAR_SHARDS). Already applied: DNS, the certificate (to 2027-01-07) and the vhost, all answering status. Every name answers for an account on its shard except s13. That needs a node for wrap.near: adding it to frankie's tracked accounts stalled the node, as CLAUDE.md now records, and was reverted. Refs blackbeard/wallet#73 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NUx7SPonBnB3ofU4mH41WT --- CLAUDE.md | 20 ++++++ asset/nginx/blackbeard-nodes.conf | 47 ++++++++++++ .../near-shards.blackbeard.observer.conf | 72 +++++++++++++++++++ script/infra-setup.sh | 45 +++++++++--- 4 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 asset/nginx/near-shards.blackbeard.observer.conf diff --git a/CLAUDE.md b/CLAUDE.md index 60bfd32..ec1f9aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -526,6 +526,26 @@ hanzalova WAN forwards 24567 to gargravarr, and neard tells peers its own listen port, so the forwarded port and the node's must match. neard's metrics are on the RPC port at `/metrics`, scraped by `asset/prometheus/blackbeard-neard.yml`. +**Adding a tracked account to a running node stalls it.** On 2026-10-09 (blackbeard/wallet#73) +`wrap.near` was added to frankie's `tracked_shards_config.Accounts` and neard restarted. +The node did not schedule shard 13 for a later epoch. It stopped applying blocks +at the restart height, showing "Downloading blocks 0.00%" with the gap growing, +and the swap indexer on bob stopped with it. Restoring `config.json.bak-*` and +restarting recovered it in about 2 minutes; the outage was 12:25–12:38. A new +shard means a fresh node (an empty data dir syncs every tracked shard at once), +never an edit to a live one. + +**The nodes are public by shard.** `s.near.blackbeard.observer` (s4, s7, s8, +s10, s12; s13 is named but has no node yet and answers `UNAVAILABLE_SHARD`) are +served on oolon by `asset/nginx/near-shards.blackbeard.observer.conf`, with the +shard-to-node map and a per-node request ceiling in `blackbeard-nodes.conf`, and +installed by `infra-setup.sh --role nodes` (`NEAR_SHARDS`). oolon reaches the +nodes directly on 24403. Two things were needed for that: +- a `near-rpc` firewalld rich rule for 10.3.1.101 on frankie and gargravarr + (quadbrat's zone already opens high ports); +- `frankie`/`gargravarr.hanzalova.internal` host overrides on the kosherinata + router, which had been missing. + **On a one-shard node, `tx_status` must not wait for execution.** Our node tracks only intents.near's shard. `EXPERIMENTAL_tx_status` with `EXECUTED` waits for every receipt in the transaction, including the withdrawals on diff --git a/asset/nginx/blackbeard-nodes.conf b/asset/nginx/blackbeard-nodes.conf index 4666811..3de5845 100644 --- a/asset/nginx/blackbeard-nodes.conf +++ b/asset/nginx/blackbeard-nodes.conf @@ -86,3 +86,50 @@ map "$request_method $uri" $near_allowed { "GET /status" 1; "GET /health" 1; } + +# NEAR by shard: `s.near.blackbeard.observer`, one name per shard one of +# our nodes tracks (blackbeard/wallet#73). near.blackbeard.observer above only +# reaches frankie, and a NEAR node answers UNAVAILABLE_SHARD about any account +# on a shard it does not track; the wallet reads the shard layout from a node, +# works out an account's shard, and asks that shard's name. Shard ids are the +# mainnet layout V3's and change when NEAR reshards: then this map, the names +# in near-shards.blackbeard.observer.conf and NEAR_SHARDS in infra-setup.sh +# move together. +# +# The nodes are reached directly across the mesh, as the Quantus node is, on +# the near-rpc port (architecture port-allocations.md). Each node's firewalld +# admits this proxy's mesh address (10.3.1.101) with a `near-rpc` rich rule, +# and the kosherinata router carries their .hanzalova.internal records. +# +# frankie shards 7, 13 intents.near, wrap.near +# gargravarr shards 4, 8 qtc.omft.near and the registrar, Ref DCL +# quadbrat shards 10, 12 native USDC, Ref v1 +upstream near_frankie { + server frankie.hanzalova.internal:24403 max_fails=3 fail_timeout=30s; + keepalive 8; +} +upstream near_gargravarr { + server gargravarr.hanzalova.internal:24403 max_fails=3 fail_timeout=30s; + keepalive 8; +} +upstream near_quadbrat { + server quadbrat.hanzalova.internal:24403 max_fails=3 fail_timeout=30s; + keepalive 8; +} + +map $host $near_shard_node { + default ""; + s4.near.blackbeard.observer near_gargravarr; + s7.near.blackbeard.observer near_frankie; + s8.near.blackbeard.observer near_gargravarr; + s10.near.blackbeard.observer near_quadbrat; + s12.near.blackbeard.observer near_quadbrat; + s13.near.blackbeard.observer near_frankie; +} + +# The ceiling across all clients is per NODE, not per name: two shard names on +# one node share its budget, and the observer's own readers (pool_rpc_urls on +# gargravarr, the swap indexer on frankie through near.internal) keep the rest. +# Per-client limits reuse near_rpc and near_conn, so one client's budget covers +# near.blackbeard.observer and every shard name together. +limit_req_zone $near_shard_node zone=near_node_total:1m rate=30r/s; diff --git a/asset/nginx/near-shards.blackbeard.observer.conf b/asset/nginx/near-shards.blackbeard.observer.conf new file mode 100644 index 0000000..10da2a3 --- /dev/null +++ b/asset/nginx/near-shards.blackbeard.observer.conf @@ -0,0 +1,72 @@ +# Public JSON-RPC endpoints for our NEAR mainnet nodes, one name per shard, on +# the oolon edge proxy (kosherinata). blackbeard/wallet#73. +# +# Cert: one Let's Encrypt lineage, near-shards.blackbeard.observer, naming +# every shard host (certbot + Cloudflare DNS-01, architecture/external-tls.md). +# Installed by script/infra-setup.sh --role nodes, never by CI. +# +# listen 127.0.0.1:14443 + proxy_protocol, NOT 443: an nginx stream SNI router +# owns TCP 443 on this host (architecture/reverse-proxies.md §5). +# +# Which node serves which name is $near_shard_node in conf.d/blackbeard-nodes.conf. +# Each node is non-validating and tracks only its shards. A transaction that +# crosses shards (the wallet's all do) is accepted and included by the node of +# the signer's shard, but no single node here can report its full outcome: the +# wallet judges each step by the state it leaves, not by tx_status. + +server { + listen 127.0.0.1:14443 ssl proxy_protocol; + http2 on; + server_name s4.near.blackbeard.observer s7.near.blackbeard.observer + s8.near.blackbeard.observer s10.near.blackbeard.observer + s12.near.blackbeard.observer s13.near.blackbeard.observer; + + ssl_certificate /etc/letsencrypt/live/near-shards.blackbeard.observer/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/near-shards.blackbeard.observer/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + absolute_redirect off; + + access_log /var/log/nginx/near-shards.blackbeard.observer.access.log blackbeard_json; + + # Same refusal as near.blackbeard.observer (conf.d/blackbeard-deny.conf). + if ($blackbeard_denied) { + rewrite ^ /.denied last; + } + + location = /.denied { + internal; + default_type text/plain; + return 403 "near.blackbeard.observer does not serve requests from hosting-provider networks.\n"; + } + + # Per client, shared with near.blackbeard.observer; per node across all + # clients (zones in conf.d/blackbeard-nodes.conf). + limit_req zone=near_rpc burst=20 nodelay; + limit_req zone=near_node_total burst=60 nodelay; + limit_conn near_conn 8; + limit_req_status 429; + limit_conn_status 429; + + # An ML-DSA-65 signed transaction is about 5.6 kB; base64 in JSON, under 8. + client_max_body_size 64k; + + proxy_read_timeout 60s; + proxy_buffering off; + + location / { + if ($near_allowed = 0) { + return 405; + } + proxy_pass http://$near_shard_node; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy same-origin always; +} diff --git a/script/infra-setup.sh b/script/infra-setup.sh index 5cf6235..62fff77 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -58,6 +58,12 @@ SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}" # `near` is not a wallet chain: it is our NEAR RPC node (blackbeard/observer#29), # published the same way and throttled harder in its own vhost. NODE_CHAINS="${NODE_CHAINS:-quantus planck near}" +# The NEAR shards our nodes track, published as `s.near.blackbeard.observer` +# (blackbeard/wallet#73): one vhost (near-shards.blackbeard.observer.conf), one +# certificate lineage naming them all, and the shard-to-node map in +# blackbeard-nodes.conf. All three change together when NEAR reshards. +NEAR_SHARDS="${NEAR_SHARDS:-4 7 8 10 12 13}" +NEAR_SHARDS_LINEAGE="near-shards.${PUBLIC_NAME}" CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}" PUBKEY="" @@ -475,24 +481,29 @@ sys.exit(0 if d["success"] else 1)' REMOTE } +# node_cert LINEAGE [NAME...]: one certificate lineage for the names given, or +# for LINEAGE alone. --expand lets a lineage gain a name (a new shard) in place. node_cert() { - local name="$1" + local lineage="$1"; shift + local names="${*:-$lineage}" # Same recipe as role_cert; the deploy hook and timer are host-wide and # already handled there. - ssh "$EDGE_HOST" "NAME='$name' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE' - if sudo test -d "/etc/letsencrypt/live/${NAME}"; then - echo "certificate lineage ${NAME} already exists" + ssh "$EDGE_HOST" "LINEAGE='$lineage' NAMES='$names' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE' + if sudo test -d "/etc/letsencrypt/live/${LINEAGE}"; then + echo "certificate lineage ${LINEAGE} already exists" fi + domains=() + for n in $NAMES; do domains+=(-d "$n"); done sudo certbot certonly \ -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive \ - --cert-name "$NAME" \ + --cert-name "$LINEAGE" \ --key-type ecdsa \ --dns-cloudflare \ --dns-cloudflare-credentials /root/.certbot-internal \ --dns-cloudflare-propagation-seconds 60 \ - --keep-until-expiring \ - -d "$NAME" - sudo test -f "/etc/letsencrypt/live/${NAME}/fullchain.pem" \ + --keep-until-expiring --expand \ + "${domains[@]}" + sudo test -f "/etc/letsencrypt/live/${LINEAGE}/fullchain.pem" \ || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; } REMOTE } @@ -513,14 +524,26 @@ role_nodes() { node_cert "$name" done + local shard shard_names=() + for shard in $NEAR_SHARDS; do shard_names+=("s${shard}.near.${PUBLIC_NAME}"); done + [ -f "$REPO_ROOT/asset/nginx/${NEAR_SHARDS_LINEAGE}.conf" ] \ + || { echo "no vhost asset/nginx/${NEAR_SHARDS_LINEAGE}.conf" >&2; return 1; } + for name in "${shard_names[@]}"; do + info "$EDGE_HOST: public DNS for $name" + node_dns "$name" + done + info "$EDGE_HOST: Let's Encrypt certificate $NEAR_SHARDS_LINEAGE for ${shard_names[*]}" + node_cert "$NEAR_SHARDS_LINEAGE" "${shard_names[@]}" + info "$EDGE_HOST: node upstreams, throttle zones and vhosts" rsync -a --rsync-path 'sudo rsync' \ "$REPO_ROOT/asset/nginx/blackbeard-nodes.conf" \ "$EDGE_HOST:/etc/nginx/conf.d/blackbeard-nodes.conf" local files=() for name in "${names[@]}"; do files+=("$REPO_ROOT/asset/nginx/${name}.conf"); done + files+=("$REPO_ROOT/asset/nginx/${NEAR_SHARDS_LINEAGE}.conf") rsync -a --rsync-path 'sudo rsync' "${files[@]}" "$EDGE_HOST:/etc/nginx/sites-available/" - ssh "$EDGE_HOST" "NAMES='${names[*]}' bash -euo pipefail" <<'REMOTE' + ssh "$EDGE_HOST" "NAMES='${names[*]} ${NEAR_SHARDS_LINEAGE}' bash -euo pipefail" <<'REMOTE' # Enable, test, and only then reload. A vhost that fails the test is # disabled again before leaving, so a typo here never blocks the next # reload of every other site on this proxy. @@ -539,12 +562,12 @@ role_nodes() { INNER REMOTE - for name in "${names[@]}"; do + for name in "${names[@]}" "${shard_names[@]}"; do info "$name answers" # Each node is asked something its own RPC understands: NEAR has no # system_chain, and a Substrate node has no status method. local method=system_chain params='[]' - case "$name" in near.*) method=status params='{}' ;; esac + case "$name" in near.*|s*.near.*) method=status params='{}' ;; esac curl -sS --max-time 15 -H 'content-type: application/json' \ -d "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$method\",\"params\":$params}" \ "https://${name}" | head -c 400 && echo