rob thijssen
110fbc3631
Cargo workspace plus a Vite frontend, following ~/git/architecture/generic.md. Every block header carries its author's wormhole reward preimage in a `pow_` PreRuntime digest, so authorship for the whole network is derivable from headers alone — no indexer, no registration, no way for a miner to be left out. That decoding, the hashrate maths and the telemetry name attribution live in blackbeard-core with no I/O at all, so the parts that are easy to get subtly wrong are exercised by unit tests rather than only against a live chain. The browser holds one WebSocket: snapshot on subscribe, deltas thereafter. The head stream is itself a push (chain_subscribeNewHeads), so a block reaches the page the moment the node imports it. Messages are serialised once per broadcast, and leaderboards are recomputed only for windows a socket is actually watching. No RxJS — useSyncExternalStore is React's own contract for this. Verified against the live Planck testnet: 12/12 headers decoded, telemetry names attributed (quanpool-planck, baba-gorchitsa, …), warm start restoring 84 blocks and 5 held names across a restart. Three findings worth recording, all in CLAUDE.md: - substrate-telemetry sends its JSON in *binary* frames. A text-only client connects, subscribes, reports healthy and receives nothing at all — and a Python probe hides it, because json.loads accepts bytes. - Difficulty is a little-endian U512; decoding it big-endian gives a number wrong by ~10^150 that still renders fine. - Planck's real block interval is ~13-15s against a 6s target with enormous variance, so a measured interval needs 20 tip samples before it is publishable. Deploy assets, the Gitea Actions workflow and script/infra-setup.sh are included; port 25864 is registered in architecture/port-allocations.md. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MSDYiibCtELsrjQq6KXnoi
344 lines
16 KiB
Bash
Executable File
344 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# One-time host provisioning for blackbeard.observer.
|
|
#
|
|
# Run by an OPERATOR from a workstation with full sudo ssh to the targets — not
|
|
# by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no
|
|
# rights to create accounts, read certificate keys, or reload nginx on a shared
|
|
# edge proxy.
|
|
#
|
|
# ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
|
|
#
|
|
# Idempotent, and it skips past unreachable hosts so one offline node does not
|
|
# block the rest. Re-run it whenever the deploy gains a new file to ship: every
|
|
# deploy job preflights the target's sudoers against the grants below and fails
|
|
# up front naming what is missing, so the two cannot silently drift.
|
|
#
|
|
# Roles (all run by default; pass --role to narrow):
|
|
#
|
|
# api the host running blackbeard-api beside quantus-node
|
|
# edge the site's nginx proxy: vhosts, webroot, internal cert
|
|
# database Postgres roles, database, and the pg_ident CN mapping
|
|
#
|
|
# Conventions: architecture/generic.md §8-§11, deployment-gitea-actions.md.
|
|
|
|
set -euo pipefail
|
|
|
|
# --- infra truth, matching .gitea/workflows/deploy.yaml -----------------------
|
|
API_HOST="${API_HOST:-bob.hanzalova.internal}"
|
|
API_PORT="${API_PORT:-25864}"
|
|
EDGE_HOST="${EDGE_HOST:-hanzalova.internal}"
|
|
PG_PRIMARY="${PG_PRIMARY:-magrathea.kosherinata.internal}"
|
|
# The standby needs the same ident mapping: pg_ident.conf contents are NOT
|
|
# replicated, and a failover to a server missing it locks the app out.
|
|
PG_STANDBY="${PG_STANDBY:-frankie.hanzalova.internal}"
|
|
PG_VERSION="${PG_VERSION:-18}"
|
|
WEBROOT="${WEBROOT:-/var/www/blackbeard.observer}"
|
|
PUBLIC_NAME="${PUBLIC_NAME:-blackbeard.observer}"
|
|
INTERNAL_NAME="${INTERNAL_NAME:-blackbeard.internal}"
|
|
DB_NAME="${DB_NAME:-blackbeard}"
|
|
DB_ROLE="${DB_ROLE:-blackbeard_rw}"
|
|
|
|
PUBKEY=""
|
|
ROLES="api edge database"
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
info() { printf '\033[36m==\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
usage() {
|
|
sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit "${1:-0}"
|
|
}
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--pubkey) PUBKEY="$2"; shift 2 ;;
|
|
--role) ROLES="$2"; shift 2 ;;
|
|
-h|--help) usage 0 ;;
|
|
*) die "unknown argument: $1 (try --help)" ;;
|
|
esac
|
|
done
|
|
|
|
has_role() { [[ " $ROLES " == *" $1 "* ]]; }
|
|
|
|
# Reachability is checked once per host and the result reused, so an offline
|
|
# host produces one clear message rather than a failure per step.
|
|
reachable() {
|
|
local host="$1"
|
|
if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true; then
|
|
return 0
|
|
fi
|
|
warn "$host is unreachable — skipping its steps"
|
|
return 1
|
|
}
|
|
|
|
# --- the scoped sudoers grants ------------------------------------------------
|
|
#
|
|
# These strings are the single source of truth for what CI may do on each host.
|
|
# The deploy workflow's preflight extracts the paths from THIS FILE and compares
|
|
# them against `sudo -n -l` on the target, so adding a file to the deploy means
|
|
# adding a line here and re-running this script — nothing else keeps them in
|
|
# step.
|
|
#
|
|
# `:` and `=` are reserved in sudoers and must be escaped inside command
|
|
# arguments, or visudo rejects the file.
|
|
|
|
api_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/blackbeard/config.toml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/blackbeard.conf
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert-reload.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/blackbeard-api.xml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/setfacl -m u\:blackbeard\:r /etc/pki/tls/private/*
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 25864
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/sudo -u blackbeard /usr/local/bin/blackbeard-api --config /etc/blackbeard/config.toml --check
|
|
EOF
|
|
}
|
|
|
|
edge_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/blackbeard.observer/
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/blackbeard.observer
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
|
|
EOF
|
|
}
|
|
|
|
# --- gitea_ci -----------------------------------------------------------------
|
|
|
|
provision_gitea_ci() {
|
|
local host="$1" sudoers_body="$2"
|
|
|
|
[ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)"
|
|
[ -f "$PUBKEY" ] || die "$PUBKEY does not exist"
|
|
|
|
info "$host: gitea_ci account"
|
|
# A real shell, never nologin. The deploy runs `ssh gitea_ci@host <command>`;
|
|
# a nologin shell authenticates the key and then refuses the command with
|
|
# "This account is currently not available", which reads as an auth problem
|
|
# rather than a shell one. Every pre-existing gitea_ci on the fleet has bash,
|
|
# and one provisioned with nologin can never be deployed to — so repair it
|
|
# rather than leaving it.
|
|
ssh "$host" sudo bash -euo pipefail <<'REMOTE'
|
|
if ! id gitea_ci >/dev/null 2>&1; then
|
|
useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
|
|
fi
|
|
current=$(getent passwd gitea_ci | cut -d: -f7)
|
|
if [ "$current" != "/bin/bash" ]; then
|
|
echo "repairing gitea_ci shell: $current -> /bin/bash"
|
|
usermod --shell /bin/bash gitea_ci
|
|
fi
|
|
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
|
# Lets the deploy capture `journalctl -u <unit>` after a restart without
|
|
# a sudoers entry for it.
|
|
usermod -aG systemd-journal gitea_ci
|
|
REMOTE
|
|
|
|
rsync -a --chown gitea_ci:gitea_ci --chmod 0600 --rsync-path 'sudo rsync' \
|
|
"$PUBKEY" "$host:/var/lib/gitea_ci/.ssh/authorized_keys"
|
|
|
|
info "$host: scoped sudoers"
|
|
# Named <app>_gitea_ci, not bare gitea_ci, so several apps can drop their own
|
|
# files on a shared host without clobbering each other.
|
|
printf '%s\n' "$sudoers_body" | \
|
|
ssh "$host" 'sudo tee /etc/sudoers.d/blackbeard_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/blackbeard_gitea_ci'
|
|
# Verify before leaving: a syntax error in a sudoers drop-in can lock every
|
|
# sudo on the host, not just this one.
|
|
ssh "$host" sudo visudo -cf /etc/sudoers.d/blackbeard_gitea_ci
|
|
}
|
|
|
|
# --- roles --------------------------------------------------------------------
|
|
|
|
role_api() {
|
|
reachable "$API_HOST" || return 0
|
|
provision_gitea_ci "$API_HOST" "$(api_sudoers)"
|
|
|
|
info "$API_HOST: service account and directories"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/systemd/blackbeard.sysusers.conf" \
|
|
"$API_HOST:/etc/sysusers.d/blackbeard.conf"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<'REMOTE'
|
|
systemd-sysusers
|
|
install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
install -d -o blackbeard -g blackbeard -m 0750 /var/lib/blackbeard
|
|
# The mTLS credential for Postgres. The key is not world-readable; the
|
|
# service account is granted read access here and again on every deploy,
|
|
# because a certificate rotation replaces the file and drops the ACL.
|
|
setfacl -m "u:blackbeard:r" "/etc/pki/tls/private/$(hostname -f).pem"
|
|
REMOTE
|
|
|
|
info "$API_HOST: SELinux port label"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<REMOTE
|
|
if semanage port -l | grep -qE "^http_port_t.*\b$API_PORT\b"; then
|
|
echo "port $API_PORT already labelled http_port_t"
|
|
else
|
|
semanage port -a -t http_port_t -p tcp "$API_PORT"
|
|
fi
|
|
REMOTE
|
|
}
|
|
|
|
role_edge() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)"
|
|
|
|
info "$EDGE_HOST: webroot"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -o root -g root -m 0755 "$WEBROOT"
|
|
# An unlabelled webroot makes nginx return 403 for every file, with
|
|
# nothing in the nginx error log to explain it.
|
|
restorecon -R "$WEBROOT"
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: internal certificate for $INTERNAL_NAME"
|
|
# Minted from the internal step-ca and renewed by a templated step@ unit
|
|
# (architecture/internal-tls.md). Certs are issued with a 24-hour expiry, so
|
|
# the timer is not optional.
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -m 0755 /etc/nginx/tls/cert
|
|
install -d -m 0700 /etc/nginx/tls/key
|
|
if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
|
|
echo "$INTERNAL_NAME certificate already present"
|
|
else
|
|
echo "MISSING: /etc/nginx/tls/cert/$INTERNAL_NAME.pem" >&2
|
|
echo "Mint it with the lair provisioner per architecture/internal-tls.md §4," >&2
|
|
echo "then re-run this script. The vhost is NOT installed until it exists:" >&2
|
|
echo "nginx -t fails on a missing ssl_certificate and blocks every reload." >&2
|
|
exit 1
|
|
fi
|
|
systemctl enable --now "step@$(basename "$INTERNAL_NAME" .internal).timer" || \
|
|
echo "step@ renewal timer not armed — renew $INTERNAL_NAME manually until it is"
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: nginx configuration"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/blackbeard-upstream.conf" \
|
|
"$EDGE_HOST:/etc/nginx/conf.d/blackbeard-upstream.conf"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \
|
|
"$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \
|
|
"$EDGE_HOST:/etc/nginx/sites-available/"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
# sites-enabled holds only symlinks, and relative ones.
|
|
ln -sfn "../sites-available/$PUBLIC_NAME.conf" "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
|
|
ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
|
|
|
|
# The vhosts use \$connection_upgrade for the WebSocket upgrade; without
|
|
# the map the socket silently degrades to a hanging request and the page
|
|
# shows "reconnecting" forever with nothing in any log.
|
|
if ! grep -rqs 'connection_upgrade' /etc/nginx/conf.d/ /etc/nginx/nginx.conf; then
|
|
cat > /etc/nginx/conf.d/websocket-upgrade.conf <<'MAP'
|
|
map \$http_upgrade \$connection_upgrade {
|
|
default upgrade;
|
|
'' close;
|
|
}
|
|
MAP
|
|
echo "installed the connection_upgrade map"
|
|
else
|
|
echo "connection_upgrade map already present"
|
|
fi
|
|
|
|
# nginx -t parses without binding, so it catches syntax and missing
|
|
# certs but not a port owned across http{} and stream{}. Verify the
|
|
# reload landed rather than trusting the test.
|
|
nginx -t
|
|
systemctl reload nginx
|
|
sleep 1
|
|
systemctl is-active --quiet nginx || { echo "nginx did not come back after reload" >&2; exit 1; }
|
|
REMOTE
|
|
|
|
cat <<EOF
|
|
|
|
$EDGE_HOST is configured, but two steps are NOT automated here:
|
|
|
|
1. The public certificate for $PUBLIC_NAME (Let's Encrypt, certbot,
|
|
Cloudflare DNS-01) — see architecture/external-tls.md.
|
|
|
|
2. Split-horizon DNS for $INTERNAL_NAME, on BOTH site routers. A record on
|
|
only one router NXDOMAINs everywhere else:
|
|
|
|
for site in hanzalova kosherinata; do
|
|
opn-cli --config ~/.opn-cli/\$site.yml unbound host create \\
|
|
--hostname blackbeard --domain internal --rr A --server <edge mesh ip>
|
|
done
|
|
|
|
`create` only saves; POST /api/unbound/service/reconfigure on each router
|
|
to apply.
|
|
|
|
A public DNS record for $PUBLIC_NAME goes in Cloudflare, unproxied, CNAMEd to
|
|
the site indirection name rather than carrying a site address directly —
|
|
architecture/public-dns.md.
|
|
|
|
EOF
|
|
}
|
|
|
|
role_database() {
|
|
reachable "$PG_PRIMARY" || return 0
|
|
|
|
info "$PG_PRIMARY: roles and database"
|
|
# Roles and databases are created on the PRIMARY only — replication carries
|
|
# them to the standby.
|
|
ssh "$PG_PRIMARY" sudo -u postgres bash -euo pipefail <<REMOTE
|
|
if psql -tAc "select 1 from pg_database where datname = '$DB_NAME'" | grep -q 1; then
|
|
echo "database $DB_NAME already exists"
|
|
else
|
|
# createdb, not the bootstrap SQL: create database cannot run inside
|
|
# a transaction or a DO block.
|
|
psql -c "create role $DB_ROLE with login" || true
|
|
createdb -O "$DB_ROLE" "$DB_NAME"
|
|
fi
|
|
REMOTE
|
|
rsync -a --rsync-path 'sudo rsync' "$REPO_ROOT/asset/sql/bootstrap.sql" \
|
|
"$PG_PRIMARY:/tmp/blackbeard-bootstrap.sql"
|
|
ssh "$PG_PRIMARY" "sudo -u postgres psql -v ON_ERROR_STOP=1 -f /tmp/blackbeard-bootstrap.sql && sudo rm -f /tmp/blackbeard-bootstrap.sql"
|
|
|
|
# The CN → role mapping, on BOTH servers. pg_ident.conf contents are not
|
|
# replicated, and a failover to a server missing this mapping locks the app
|
|
# out entirely — with an authentication error that looks like a certificate
|
|
# problem.
|
|
local api_fqdn
|
|
api_fqdn=$(ssh "$API_HOST" hostname -f 2>/dev/null || echo "$API_HOST")
|
|
for server in "$PG_PRIMARY" "$PG_STANDBY"; do
|
|
reachable "$server" || continue
|
|
info "$server: pg_ident mapping for $api_fqdn -> $DB_ROLE"
|
|
printf 'cert_cn %s %s\n' "$api_fqdn" "$DB_ROLE" | \
|
|
ssh "$server" "sudo install -d -m 0700 -o postgres -g postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d && sudo tee /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf > /dev/null"
|
|
ssh "$server" "sudo chown postgres:postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf"
|
|
# Reload, not restart: pg_ident is re-read on SIGHUP.
|
|
ssh "$server" "sudo systemctl reload postgresql-$PG_VERSION"
|
|
done
|
|
|
|
info "verifying the mapping from $API_HOST"
|
|
if reachable "$API_HOST"; then
|
|
ssh "$API_HOST" "sudo -u blackbeard psql 'host=$PG_PRIMARY port=5432 dbname=$DB_NAME user=$DB_ROLE sslmode=verify-full sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem sslcert=/etc/pki/tls/misc/\$(hostname -f).pem sslkey=/etc/pki/tls/private/\$(hostname -f).pem' -tAc 'select current_user'" \
|
|
|| warn "the app host could not authenticate to $PG_PRIMARY — check the CN mapping and the key ACL"
|
|
fi
|
|
}
|
|
|
|
# --- run ----------------------------------------------------------------------
|
|
|
|
info "roles: $ROLES"
|
|
has_role api && role_api
|
|
has_role edge && role_edge
|
|
has_role database && role_database
|
|
info "done"
|