grenadeandClaude Opus 5.5
dc68fe3be3
Through the existing nodes role: a Cloudflare CNAME to the site indirection, a Let's Encrypt cert by DNS-01 on oolon, and a vhost beside the Quantus and Planck endpoints. The upstream is https://near.internal rather than frankie, so next week's move to new hardware changes one internal vhost and nothing here, and frankie's firewall still admits only the hanzalova proxy and bob. Throttled harder than the Quantus endpoint, because the node is a four-core stopgap and the swap indexer reads it too (via near.internal, outside these limits): 5 r/s per client with a burst of 20, 8 connections per client, and a 30 r/s ceiling across all clients so many addresses together cannot starve it. Only POST / and GET /status, /health pass; bodies over 64 KiB are refused. Hetzner and OVH hosting networks get the site's 403, and the name has its own JSON log. Checked from outside after install: 405 for other methods and paths, 413 for a 100 KB body, 32 of 40 parallel requests answered 429, and the log carries the real client address. /health answers 503 because the node is not synced; that is neard's own answer, passed through. Closes #29 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015pkwBsR54NRmZyU4QVhvfj
630 lines
31 KiB
Bash
Executable File
630 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# One-time host provisioning for blackbeard.observer.
|
|
#
|
|
# Run by an OPERATOR from a workstation with full sudo ssh to the targets — not
|
|
# by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no
|
|
# rights to create accounts, read certificate keys, or reload nginx on a shared
|
|
# edge proxy.
|
|
#
|
|
# ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
|
|
#
|
|
# Idempotent, and it skips past unreachable hosts so one offline node does not
|
|
# block the rest. Re-run it whenever the deploy gains a new file to ship: every
|
|
# deploy job preflights the target's sudoers against the grants below and fails
|
|
# up front naming what is missing, so the two cannot silently drift.
|
|
#
|
|
# Roles (all run by default; pass --role to narrow):
|
|
#
|
|
# api the host running blackbeard-api beside quantus-node
|
|
# dns the public apex CNAME on Cloudflare
|
|
# cert the Let's Encrypt certificate for the public name
|
|
# edge the site's nginx proxy: vhosts, webroot, internal cert
|
|
# database Postgres roles, database, and the pg_ident CN mapping
|
|
# nodes the public node endpoints <chain>.blackbeard.observer for the
|
|
# blackbeard wallet: DNS, certificate and vhost per chain name
|
|
# (blackbeard/wallet #53)
|
|
#
|
|
# `dns` and `cert` run BEFORE `edge`, and in that order: certbot uses a DNS-01
|
|
# challenge, and nginx -t fails on a missing ssl_certificate — which blocks
|
|
# every reload on the proxy, not just this vhost.
|
|
#
|
|
# Conventions: architecture/generic.md §8-§11, deployment-gitea-actions.md.
|
|
|
|
set -euo pipefail
|
|
|
|
# --- infra truth, matching .gitea/workflows/deploy.yaml -----------------------
|
|
API_HOST="${API_HOST:-bob.hanzalova.internal}"
|
|
API_PORT="${API_PORT:-25864}"
|
|
EDGE_HOST="${EDGE_HOST:-oolon.kosherinata.internal}"
|
|
PG_PRIMARY="${PG_PRIMARY:-magrathea.kosherinata.internal}"
|
|
# The standby needs the same ident mapping: pg_ident.conf contents are NOT
|
|
# replicated, and a failover to a server missing it locks the app out.
|
|
PG_STANDBY="${PG_STANDBY:-frankie.hanzalova.internal}"
|
|
PG_VERSION="${PG_VERSION:-18}"
|
|
WEBROOT="${WEBROOT:-/var/www/blackbeard.observer}"
|
|
PUBLIC_NAME="${PUBLIC_NAME:-blackbeard.observer}"
|
|
INTERNAL_NAME="${INTERNAL_NAME:-blackbeard.internal}"
|
|
DB_NAME="${DB_NAME:-blackbeard}"
|
|
DB_ROLE="${DB_ROLE:-blackbeard_rw}"
|
|
# The per-site indirection name the public record CNAMEs to. `bl` is the DC
|
|
# (oolon), `nh` the office (hanzalova) — architecture/public-dns.md §2. A vhost
|
|
# record must never carry a site address directly: the address changes and every
|
|
# hardcoded A record then has to be hunted down across zones.
|
|
SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}"
|
|
# The chains the wallet reaches through this proxy, one public name each:
|
|
# `<chain>.blackbeard.observer`. Their upstreams live in
|
|
# asset/nginx/blackbeard-nodes.conf; each needs a vhost file of the same name.
|
|
# `near` is not a wallet chain: it is our NEAR RPC node (blackbeard/observer#29),
|
|
# published the same way and throttled harder in its own vhost.
|
|
NODE_CHAINS="${NODE_CHAINS:-quantus planck near}"
|
|
CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}"
|
|
|
|
PUBKEY=""
|
|
ROLES="api dns cert edge database nodes"
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
info() { printf '\033[36m==\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
usage() {
|
|
sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit "${1:-0}"
|
|
}
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--pubkey) PUBKEY="$2"; shift 2 ;;
|
|
--role) ROLES="$2"; shift 2 ;;
|
|
-h|--help) usage 0 ;;
|
|
*) die "unknown argument: $1 (try --help)" ;;
|
|
esac
|
|
done
|
|
|
|
has_role() { [[ " $ROLES " == *" $1 "* ]]; }
|
|
|
|
# Reachability is checked once per host and the result reused, so an offline
|
|
# host produces one clear message rather than a failure per step.
|
|
reachable() {
|
|
local host="$1"
|
|
if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true; then
|
|
return 0
|
|
fi
|
|
warn "$host is unreachable — skipping its steps"
|
|
return 1
|
|
}
|
|
|
|
# --- the scoped sudoers grants ------------------------------------------------
|
|
#
|
|
# These strings are the single source of truth for what CI may do on each host.
|
|
# The deploy workflow's preflight extracts the paths from THIS FILE and compares
|
|
# them against `sudo -n -l` on the target, so adding a file to the deploy means
|
|
# adding a line here and re-running this script — nothing else keeps them in
|
|
# step.
|
|
#
|
|
# `:` and `=` are reserved in sudoers and must be escaped inside command
|
|
# arguments, or visudo rejects the file.
|
|
|
|
api_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/blackbeard/config.toml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/blackbeard.conf
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert-reload.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/blackbeard-api.xml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/setfacl -m u\:blackbeard\:r /etc/pki/tls/private/*
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 25864
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active blackbeard-api.service
|
|
# Runas is (blackbeard), not (root): the deploy validates the config AS the
|
|
# service account, which is the only way to prove the account can actually read
|
|
# the config and the certificate key it names. A (root) grant would not permit
|
|
# `sudo -u blackbeard` at all, and a root-run check would pass on a config the
|
|
# service cannot read.
|
|
gitea_ci ALL=(blackbeard) NOPASSWD: /usr/local/bin/blackbeard-api --config /etc/blackbeard/config.toml --check
|
|
EOF
|
|
}
|
|
|
|
edge_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/blackbeard.observer/
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/blackbeard.observer
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
|
|
EOF
|
|
}
|
|
|
|
# --- gitea_ci -----------------------------------------------------------------
|
|
|
|
provision_gitea_ci() {
|
|
local host="$1" sudoers_body="$2"
|
|
|
|
[ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)"
|
|
[ -f "$PUBKEY" ] || die "$PUBKEY does not exist"
|
|
|
|
info "$host: gitea_ci account"
|
|
# A real shell, never nologin. The deploy runs `ssh gitea_ci@host <command>`;
|
|
# a nologin shell authenticates the key and then refuses the command with
|
|
# "This account is currently not available", which reads as an auth problem
|
|
# rather than a shell one. Every pre-existing gitea_ci on the fleet has bash,
|
|
# and one provisioned with nologin can never be deployed to — so repair it
|
|
# rather than leaving it.
|
|
ssh "$host" sudo bash -euo pipefail <<'REMOTE'
|
|
if ! id gitea_ci >/dev/null 2>&1; then
|
|
useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
|
|
fi
|
|
current=$(getent passwd gitea_ci | cut -d: -f7)
|
|
if [ "$current" != "/bin/bash" ]; then
|
|
echo "repairing gitea_ci shell: $current -> /bin/bash"
|
|
usermod --shell /bin/bash gitea_ci
|
|
fi
|
|
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
|
# Lets the deploy capture `journalctl -u <unit>` after a restart without
|
|
# a sudoers entry for it.
|
|
usermod -aG systemd-journal gitea_ci
|
|
REMOTE
|
|
|
|
rsync -a --chown gitea_ci:gitea_ci --chmod 0600 --rsync-path 'sudo rsync' \
|
|
"$PUBKEY" "$host:/var/lib/gitea_ci/.ssh/authorized_keys"
|
|
|
|
info "$host: scoped sudoers"
|
|
# Named <app>_gitea_ci, not bare gitea_ci, so several apps can drop their own
|
|
# files on a shared host without clobbering each other.
|
|
printf '%s\n' "$sudoers_body" | \
|
|
ssh "$host" 'sudo tee /etc/sudoers.d/blackbeard_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/blackbeard_gitea_ci'
|
|
# Verify before leaving: a syntax error in a sudoers drop-in can lock every
|
|
# sudo on the host, not just this one.
|
|
ssh "$host" sudo visudo -cf /etc/sudoers.d/blackbeard_gitea_ci
|
|
}
|
|
|
|
# --- roles --------------------------------------------------------------------
|
|
|
|
role_api() {
|
|
reachable "$API_HOST" || return 0
|
|
provision_gitea_ci "$API_HOST" "$(api_sudoers)"
|
|
|
|
info "$API_HOST: service account and directories"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/systemd/blackbeard.sysusers.conf" \
|
|
"$API_HOST:/etc/sysusers.d/blackbeard.conf"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<'REMOTE'
|
|
systemd-sysusers
|
|
install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
install -d -o blackbeard -g blackbeard -m 0750 /var/lib/blackbeard
|
|
# The mTLS credential for Postgres. The key is not world-readable; the
|
|
# service account is granted read access here and again on every deploy,
|
|
# because a certificate rotation replaces the file and drops the ACL.
|
|
setfacl -m "u:blackbeard:r" "/etc/pki/tls/private/$(hostname -f).pem"
|
|
REMOTE
|
|
|
|
info "$API_HOST: SELinux port label"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<REMOTE
|
|
if semanage port -l | grep -qE "^http_port_t.*\b$API_PORT\b"; then
|
|
echo "port $API_PORT already labelled http_port_t"
|
|
else
|
|
semanage port -a -t http_port_t -p tcp "$API_PORT"
|
|
fi
|
|
REMOTE
|
|
}
|
|
|
|
role_edge() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)"
|
|
|
|
info "$EDGE_HOST: webroot"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -o root -g root -m 0755 "$WEBROOT"
|
|
# An unlabelled webroot makes nginx return 403 for every file, with
|
|
# nothing in the nginx error log to explain it.
|
|
restorecon -R "$WEBROOT"
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: internal certificate for $INTERNAL_NAME"
|
|
# Minted from the internal step-ca and renewed by a templated step@ unit
|
|
# (architecture/internal-tls.md). Certs are issued with a 24-hour expiry, so
|
|
# the timer is not optional.
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -m 0755 /etc/nginx/tls/cert
|
|
install -d -m 0700 /etc/nginx/tls/key
|
|
if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
|
|
echo "$INTERNAL_NAME certificate present"
|
|
systemctl enable --now "step@blackbeard.timer" || \
|
|
echo "step@ renewal timer not armed — renew $INTERNAL_NAME manually until it is"
|
|
else
|
|
# Not fatal. The mesh vhost is a convenience; the public site is the
|
|
# deliverable, and nginx -t fails on a missing ssl_certificate —
|
|
# which would block every reload on this proxy, not just ours.
|
|
echo "NOTE: no internal certificate for $INTERNAL_NAME; the mesh vhost will be skipped."
|
|
echo " Mint it with the lair provisioner (architecture/internal-tls.md §4)"
|
|
echo " and re-run --role edge to enable it."
|
|
fi
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: nginx configuration"
|
|
# Shipped together: the log format in the upstream file reads
|
|
# $blackbeard_denied, which only blackbeard-deny.conf defines, and nginx -t
|
|
# rejects a config naming a variable nothing defines. The deny list is
|
|
# regenerated here by script/deny-networks.py and never edited on the proxy
|
|
# (blackbeard/observer#24).
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/blackbeard-deny.conf" \
|
|
"$REPO_ROOT/asset/nginx/blackbeard-upstream.conf" \
|
|
"$EDGE_HOST:/etc/nginx/conf.d/"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \
|
|
"$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \
|
|
"$EDGE_HOST:/etc/nginx/sites-available/"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
# sites-enabled holds only symlinks, and relative ones.
|
|
ln -sfn "../sites-available/$PUBLIC_NAME.conf" "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
|
|
if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
|
|
ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
|
|
else
|
|
rm -f "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
|
|
fi
|
|
|
|
# The vhosts use \$connection_upgrade for the WebSocket upgrade; without
|
|
# the map the socket silently degrades to a hanging request and the page
|
|
# shows "reconnecting" forever with nothing in any log.
|
|
if ! grep -rqs 'connection_upgrade' /etc/nginx/conf.d/ /etc/nginx/nginx.conf; then
|
|
cat > /etc/nginx/conf.d/websocket-upgrade.conf <<'MAP'
|
|
map \$http_upgrade \$connection_upgrade {
|
|
default upgrade;
|
|
'' close;
|
|
}
|
|
MAP
|
|
echo "installed the connection_upgrade map"
|
|
else
|
|
echo "connection_upgrade map already present"
|
|
fi
|
|
|
|
# nginx -t parses without binding, so it catches syntax and missing
|
|
# certs but not a port owned across http{} and stream{}. Verify the
|
|
# reload landed rather than trusting the test.
|
|
nginx -t
|
|
systemctl reload nginx
|
|
sleep 1
|
|
systemctl is-active --quiet nginx || { echo "nginx did not come back after reload" >&2; exit 1; }
|
|
REMOTE
|
|
|
|
cat <<EOF
|
|
|
|
$EDGE_HOST is configured. One step is NOT automated here:
|
|
|
|
Split-horizon DNS for $INTERNAL_NAME, on BOTH site routers. A record on
|
|
only one router NXDOMAINs everywhere else:
|
|
|
|
for site in hanzalova kosherinata; do
|
|
opn-cli --config ~/.opn-cli/\$site.yml unbound host create \\
|
|
--hostname blackbeard --domain internal --rr A --server <edge mesh ip>
|
|
done
|
|
|
|
\`create\` only saves; POST /api/unbound/service/reconfigure on each router
|
|
to apply.
|
|
|
|
EOF
|
|
}
|
|
|
|
# The Cloudflare API token lives on the edge proxy and is never copied off it:
|
|
# it can rewrite DNS for every zone on the account, including MX records for
|
|
# domains whose mail we host (architecture/public-dns.md §1). Every call that
|
|
# needs it therefore runs ON the proxy.
|
|
role_dns() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
info "$EDGE_HOST: public DNS for $PUBLIC_NAME"
|
|
|
|
ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE'
|
|
TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ")
|
|
[ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; }
|
|
api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }
|
|
|
|
zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${PUBLIC_NAME}" | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
|
|
[ -n "$zid" ] || { echo "zone $PUBLIC_NAME is not on this Cloudflare account" >&2; exit 1; }
|
|
|
|
# List the name's FULL record set first, all types. Filtering by the one
|
|
# type you expect and finding nothing does not mean the name is free —
|
|
# it may be a CNAME, and adding an A beside it is a conflict at best
|
|
# (architecture/public-dns.md §4).
|
|
existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${PUBLIC_NAME}")
|
|
echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
print(f"existing records for the apex: {len(rs)}")
|
|
for r in rs:
|
|
print(" %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"]))
|
|
'
|
|
current=$(echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
cn = [r for r in rs if r["type"] == "CNAME"]
|
|
print(cn[0]["content"] if cn else "")
|
|
')
|
|
if [ "$current" = "$SITE" ]; then
|
|
echo "apex already CNAMEs to $SITE — nothing to do"
|
|
exit 0
|
|
fi
|
|
if [ -n "$current" ]; then
|
|
echo "apex already CNAMEs to $current, not $SITE." >&2
|
|
echo "Refusing to repoint an existing record; change it deliberately." >&2
|
|
exit 1
|
|
fi
|
|
if echo "$existing" | grep -q '"type"'; then
|
|
# Records exist but none is a CNAME. A CNAME cannot coexist with
|
|
# other types at a name, so this needs a human decision.
|
|
echo "apex carries non-CNAME records; refusing to add a CNAME beside them." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Cloudflare flattens an apex CNAME, answering with an A — which is what
|
|
# makes this convention work at a zone apex at all. It is
|
|
# Cloudflare-specific, not portable DNS.
|
|
api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records" -H 'Content-Type: application/json' --data "$(python3 -c '
|
|
import json, os
|
|
print(json.dumps({"type":"CNAME","name":os.environ["PUBLIC_NAME"],"content":os.environ["SITE"],
|
|
"ttl":1,"proxied":False,"comment":"Quantus-Network/blackbeard.observer"}))')" | python3 -c '
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"]
|
|
else "FAILED %s" % d["errors"])
|
|
import sys as s
|
|
s.exit(0 if d["success"] else 1)
|
|
'
|
|
REMOTE
|
|
}
|
|
|
|
role_cert() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
info "$EDGE_HOST: Let's Encrypt certificate for $PUBLIC_NAME"
|
|
|
|
# DNS-01 via the same Cloudflare credential, ECDSA, per
|
|
# architecture/external-tls.md §1. --keep-until-expiring makes this a no-op
|
|
# when the cert is still valid, so the role is safe to run every time.
|
|
#
|
|
# `sudo test`, not bare `test`: /etc/letsencrypt/live is root-only 0700 and
|
|
# an unprivileged check silently returns false, concluding the cert is
|
|
# missing and re-issuing on every run.
|
|
ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE'
|
|
if sudo test -d "/etc/letsencrypt/live/${PUBLIC_NAME}"; then
|
|
echo "certificate lineage ${PUBLIC_NAME} already exists"
|
|
fi
|
|
sudo certbot certonly -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive --cert-name "$PUBLIC_NAME" --key-type ecdsa --dns-cloudflare --dns-cloudflare-credentials /root/.certbot-internal --dns-cloudflare-propagation-seconds 60 --keep-until-expiring -d "$PUBLIC_NAME"
|
|
sudo test -f "/etc/letsencrypt/live/${PUBLIC_NAME}/fullchain.pem" || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; }
|
|
|
|
# One deploy hook per host, not per cert. Never silence it: a reload is
|
|
# only a request, and nginx exits 0 having kept its old certificates
|
|
# whenever it cannot re-acquire a socket. On a 90-day public cert a
|
|
# stuck reload takes months to become visible.
|
|
hook=/etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
|
|
if ! sudo test -x "$hook"; then
|
|
sudo install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
|
|
printf '%s\n' '#!/bin/sh' 'systemctl reload nginx || logger -t reload-nginx -p daemon.err \' ' "nginx reload failed after certbot renewal; cert on disk is newer than the one served"' | sudo tee "$hook" > /dev/null
|
|
sudo chmod +x "$hook"
|
|
echo "installed the certbot deploy hook"
|
|
else
|
|
echo "certbot deploy hook already present"
|
|
fi
|
|
systemctl is-enabled certbot-renew.timer >/dev/null 2>&1 && echo "certbot-renew.timer is enabled" || echo "WARNING: certbot-renew.timer is not enabled on this host"
|
|
REMOTE
|
|
}
|
|
|
|
# --- public node endpoints ----------------------------------------------------
|
|
#
|
|
# One name per chain for the wallet's node connections, on this same edge
|
|
# proxy: DNS CNAME to the site indirection, a Let's Encrypt certificate, and a
|
|
# vhost that proxies JSON-RPC and WebSocket to the chain's upstream with
|
|
# per-client throttling. The same three steps role_dns/role_cert/role_edge do
|
|
# for the site, per name; kept separate so the site and the endpoints can be
|
|
# rolled independently.
|
|
node_dns() {
|
|
local name="$1"
|
|
ssh "$EDGE_HOST" "ZONE='$PUBLIC_NAME' NAME='$name' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE'
|
|
TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ")
|
|
[ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; }
|
|
api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }
|
|
zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${ZONE}" \
|
|
| python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
|
|
[ -n "$zid" ] || { echo "zone $ZONE is not on this Cloudflare account" >&2; exit 1; }
|
|
# The name's FULL record set first (architecture/public-dns.md §4).
|
|
existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${NAME}")
|
|
echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
print(f"existing records for {sys.argv[1]}: {len(rs)}")
|
|
for r in rs:
|
|
print(" %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"]))
|
|
' "$NAME"
|
|
current=$(echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
cn = [r for r in rs if r["type"] == "CNAME"]
|
|
print(cn[0]["content"] if cn else "")')
|
|
if [ "$current" = "$SITE" ]; then echo "$NAME already CNAMEs to $SITE"; exit 0; fi
|
|
if [ -n "$current" ]; then
|
|
echo "$NAME already CNAMEs to $current, not $SITE; refusing to repoint it." >&2; exit 1
|
|
fi
|
|
if echo "$existing" | grep -q '"type"'; then
|
|
echo "$NAME carries non-CNAME records; refusing to add a CNAME beside them." >&2; exit 1
|
|
fi
|
|
api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records" \
|
|
-H 'Content-Type: application/json' \
|
|
--data "$(python3 -c '
|
|
import json, os
|
|
print(json.dumps({"type":"CNAME","name":os.environ["NAME"],"content":os.environ["SITE"],
|
|
"ttl":1,"proxied":False,"comment":"blackbeard.observer node endpoint (blackbeard/observer infra-setup --role nodes)"}))')" \
|
|
| python3 -c '
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"] else "FAILED %s" % d["errors"])
|
|
sys.exit(0 if d["success"] else 1)'
|
|
REMOTE
|
|
}
|
|
|
|
node_cert() {
|
|
local name="$1"
|
|
# Same recipe as role_cert; the deploy hook and timer are host-wide and
|
|
# already handled there.
|
|
ssh "$EDGE_HOST" "NAME='$name' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE'
|
|
if sudo test -d "/etc/letsencrypt/live/${NAME}"; then
|
|
echo "certificate lineage ${NAME} already exists"
|
|
fi
|
|
sudo certbot certonly \
|
|
-m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive \
|
|
--cert-name "$NAME" \
|
|
--key-type ecdsa \
|
|
--dns-cloudflare \
|
|
--dns-cloudflare-credentials /root/.certbot-internal \
|
|
--dns-cloudflare-propagation-seconds 60 \
|
|
--keep-until-expiring \
|
|
-d "$NAME"
|
|
sudo test -f "/etc/letsencrypt/live/${NAME}/fullchain.pem" \
|
|
|| { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; }
|
|
REMOTE
|
|
}
|
|
|
|
role_nodes() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
local names=() chain
|
|
for chain in $NODE_CHAINS; do
|
|
names+=("${chain}.${PUBLIC_NAME}")
|
|
[ -f "$REPO_ROOT/asset/nginx/${chain}.${PUBLIC_NAME}.conf" ] \
|
|
|| { echo "no vhost asset/nginx/${chain}.${PUBLIC_NAME}.conf" >&2; return 1; }
|
|
done
|
|
|
|
for name in "${names[@]}"; do
|
|
info "$EDGE_HOST: public DNS for $name"
|
|
node_dns "$name"
|
|
info "$EDGE_HOST: Let's Encrypt certificate for $name"
|
|
node_cert "$name"
|
|
done
|
|
|
|
info "$EDGE_HOST: node upstreams, throttle zones and vhosts"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/blackbeard-nodes.conf" \
|
|
"$EDGE_HOST:/etc/nginx/conf.d/blackbeard-nodes.conf"
|
|
local files=()
|
|
for name in "${names[@]}"; do files+=("$REPO_ROOT/asset/nginx/${name}.conf"); done
|
|
rsync -a --rsync-path 'sudo rsync' "${files[@]}" "$EDGE_HOST:/etc/nginx/sites-available/"
|
|
ssh "$EDGE_HOST" "NAMES='${names[*]}' bash -euo pipefail" <<'REMOTE'
|
|
# Enable, test, and only then reload. A vhost that fails the test is
|
|
# disabled again before leaving, so a typo here never blocks the next
|
|
# reload of every other site on this proxy.
|
|
sudo bash -euo pipefail <<INNER
|
|
for name in $NAMES; do
|
|
ln -sfn "../sites-available/\$name.conf" "/etc/nginx/sites-enabled/\$name.conf"
|
|
done
|
|
if ! nginx -t; then
|
|
for name in $NAMES; do rm -f "/etc/nginx/sites-enabled/\$name.conf"; done
|
|
echo "nginx -t failed; the node vhosts are disabled again" >&2
|
|
exit 1
|
|
fi
|
|
systemctl reload nginx
|
|
sleep 1
|
|
systemctl is-active --quiet nginx || { echo "nginx did not come back after reload" >&2; exit 1; }
|
|
INNER
|
|
REMOTE
|
|
|
|
for name in "${names[@]}"; do
|
|
info "$name answers"
|
|
# Each node is asked something its own RPC understands: NEAR has no
|
|
# system_chain, and a Substrate node has no status method.
|
|
local method=system_chain params='[]'
|
|
case "$name" in near.*) method=status params='{}' ;; esac
|
|
curl -sS --max-time 15 -H 'content-type: application/json' \
|
|
-d "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$method\",\"params\":$params}" \
|
|
"https://${name}" | head -c 400 && echo
|
|
done
|
|
}
|
|
|
|
role_database() {
|
|
reachable "$PG_PRIMARY" || return 0
|
|
|
|
info "$PG_PRIMARY: roles and database"
|
|
# Roles and databases are created on the PRIMARY only — replication carries
|
|
# them to the standby.
|
|
ssh "$PG_PRIMARY" sudo -u postgres bash -euo pipefail <<REMOTE
|
|
if psql -tAc "select 1 from pg_database where datname = '$DB_NAME'" | grep -q 1; then
|
|
echo "database $DB_NAME already exists"
|
|
else
|
|
# createdb, not the bootstrap SQL: create database cannot run inside
|
|
# a transaction or a DO block.
|
|
psql -c "create role $DB_ROLE with login" || true
|
|
createdb -O "$DB_ROLE" "$DB_NAME"
|
|
fi
|
|
REMOTE
|
|
rsync -a --rsync-path 'sudo rsync' "$REPO_ROOT/asset/sql/bootstrap.sql" \
|
|
"$PG_PRIMARY:/tmp/blackbeard-bootstrap.sql"
|
|
ssh "$PG_PRIMARY" "sudo -u postgres psql -v ON_ERROR_STOP=1 -f /tmp/blackbeard-bootstrap.sql && sudo rm -f /tmp/blackbeard-bootstrap.sql"
|
|
|
|
# The CN → role mapping, on BOTH servers. pg_ident.conf contents are not
|
|
# replicated, and a failover to a server missing this mapping locks the app
|
|
# out entirely — with an authentication error that looks like a certificate
|
|
# problem.
|
|
local api_fqdn
|
|
api_fqdn=$(ssh "$API_HOST" hostname -f 2>/dev/null || echo "$API_HOST")
|
|
for server in "$PG_PRIMARY" "$PG_STANDBY"; do
|
|
reachable "$server" || continue
|
|
info "$server: pg_ident mapping for $api_fqdn -> $DB_ROLE"
|
|
printf 'cert_cn %s %s\n' "$api_fqdn" "$DB_ROLE" | \
|
|
ssh "$server" "sudo install -d -m 0700 -o postgres -g postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d && sudo tee /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf > /dev/null"
|
|
ssh "$server" "sudo chown postgres:postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf"
|
|
# Reload, not restart: pg_ident is re-read on SIGHUP.
|
|
ssh "$server" "sudo systemctl reload postgresql-$PG_VERSION"
|
|
done
|
|
|
|
info "verifying the mapping from $API_HOST"
|
|
if reachable "$API_HOST"; then
|
|
# `psql` is not installed on an app host and there is no reason for it
|
|
# to be, so its absence must not be reported as an authentication
|
|
# failure — that reads as a broken mapping and sends you looking in the
|
|
# wrong place. When it is missing, the deploy's own health probe is the
|
|
# verification: the daemon connects with exactly these credentials.
|
|
if ssh "$API_HOST" "command -v psql >/dev/null"; then
|
|
ssh "$API_HOST" "sudo -u blackbeard psql 'host=$PG_PRIMARY port=5432 dbname=$DB_NAME user=$DB_ROLE sslmode=verify-full sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem sslcert=/etc/pki/tls/misc/\$(hostname -f).pem sslkey=/etc/pki/tls/private/\$(hostname -f).pem' -tAc 'select current_user'" \
|
|
|| warn "the app host could not authenticate to $PG_PRIMARY — check the CN mapping and the key ACL"
|
|
else
|
|
info "psql absent on $API_HOST; the deploy's health probe verifies the connection instead"
|
|
fi
|
|
fi
|
|
|
|
# Not a warning to be dismissed. pg_ident.conf contents are NOT replicated,
|
|
# so a standby that never received this mapping will refuse the app the
|
|
# moment it is promoted — with an authentication error that looks like a
|
|
# certificate problem and arrives during an outage.
|
|
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "$PG_STANDBY" true 2>/dev/null; then
|
|
warn "$PG_STANDBY did not receive the pg_ident mapping."
|
|
warn "Re-run: ./script/infra-setup.sh --role database once it is reachable."
|
|
warn "Until then a failover to it will lock blackbeard-api out of the database."
|
|
fi
|
|
}
|
|
|
|
# --- run ----------------------------------------------------------------------
|
|
|
|
info "roles: $ROLES"
|
|
has_role api && role_api
|
|
# dns before cert (DNS-01 needs the zone), cert before edge (nginx -t fails on
|
|
# a missing ssl_certificate and blocks every reload on the proxy).
|
|
has_role dns && role_dns
|
|
has_role cert && role_cert
|
|
has_role edge && role_edge
|
|
has_role database && role_database
|
|
# The node endpoints after the site: they share its zone, its certbot and
|
|
# its nginx, and nothing in the site depends on them.
|
|
has_role nodes && role_nodes
|
|
info "done"
|