diff --git a/asset/nginx/quantus.internal.conf b/asset/nginx/quantus.internal.conf new file mode 100644 index 0000000..db51da4 --- /dev/null +++ b/asset/nginx/quantus.internal.conf @@ -0,0 +1,56 @@ +# Mesh-only vhost for the Quantus node's JSON-RPC, fronted by the hanzalova edge +# proxy (office site). Gives tooling a `wss://` endpoint, which matters because +# subxt — and therefore quantus-cli — refuses plain `ws://` to any host but +# localhost (InsecureUrl), so without this every client needs an ssh tunnel. +# +# Cert: internal `lair` CA, minted from the operator box and renewed by +# step@quantus.timer (architecture internal-tls.md). Operator-installed, NOT +# CI-deployed: the runner has no rights on the edge proxy +# (reverse-proxies.md §4, and the same split lair/mail uses). +# ln -sf ../sites-available/quantus.internal.conf /etc/nginx/sites-enabled/ +# +# listen 14443 + proxy_protocol, NOT 443: the nginx stream SNI router owns TCP +# 443 on this host and forwards every non-passthrough name to the local https +# tier (reverse-proxies.md §5). Copy this from a currently-ENABLED vhost, not +# from sites-available — several of those are stale (see the repo readme). +# +# NOTE ON EXPOSURE: this is an unauthenticated RPC. `--rpc-methods safe` on the +# node blocks the dangerous calls, but anyone who reaches this name can read all +# chain state and submit signed extrinsics. It is mesh-only by construction — +# `.internal` has no public record and must never get one. + +upstream quantus_rpc_up { + server bob.hanzalova.internal:9944; + keepalive 8; +} + +server { + listen 127.0.0.1:14443 ssl proxy_protocol; + http2 on; + server_name quantus.internal; + + ssl_certificate /etc/nginx/tls/cert/quantus.internal.pem; + ssl_certificate_key /etc/nginx/tls/key/quantus.internal.pem; + ssl_protocols TLSv1.3; + ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem; + + # Substrate subscriptions (chainHead_follow, new heads) are long-lived and + # can sit idle between blocks. The 60s default would drop them mid-session + # and surface to the client as an unexplained disconnect. + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + # RPC bodies are small; a large limit here only widens the DoS surface. + client_max_body_size 2m; + + location / { + proxy_pass http://quantus_rpc_up; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/readme.md b/readme.md index 6d34d29..3343fd8 100644 --- a/readme.md +++ b/readme.md @@ -138,6 +138,48 @@ Still to decide before launch day, none of which the deploy can settle for you: Sync at launch is trivial — a new chain starts at genesis — so the long testnet sync is not a preview of mainnet day. +## `wss://quantus.internal` + +`subxt` — and therefore `quantus-cli` — refuses plain `ws://` to any host but +localhost (`InsecureUrl`), so without TLS every client needs an ssh tunnel: + +```sh +ssh -f -N -L 19944:127.0.0.1:9944 +quantus balance --node-url ws://127.0.0.1:19944 --address +``` + +The tunnel-free answer is a vhost on the site's edge proxy, per +`architecture/reverse-proxies.md` and `internal-tls.md`: + +```sh +./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub # includes the edge role +``` + +It mints an internal-CA cert for `quantus.internal`, installs +`asset/nginx/quantus.internal.conf`, arms `step@quantus.timer` for renewal, and +reloads nginx. **Operator-run, never CI-deployed** — the runner has no rights to +read certificate keys or reload nginx on a shared edge proxy. + +Split-horizon DNS is a separate manual step (the record lives in the site +router's Unbound, not here): + +```sh +opn-cli --config ~/.opn-cli/.yml unbound host create \ + --hostname quantus --domain internal --server +``` + +Two traps worth knowing: + +- **`listen 127.0.0.1:14443 ssl proxy_protocol;`, not `listen 443`.** A stream + SNI router owns TCP 443 on the edge proxies and hands non-passthrough names to + the local https tier. Copy the `listen` line from a currently **enabled** + vhost — on hanzalova, 11 of 19 enabled vhosts are regular files that have + drifted from their `sites-available` counterparts since that migration, so the + `sites-available` copy can be stale and wrong. +- The RPC behind this is **unauthenticated**. `--rpc-methods safe` blocks the + dangerous calls, but anyone reaching the name reads all state and can submit + signed extrinsics. `.internal` is mesh-only and must never get a public record. + ## Reaching the JSON-RPC from other machines Loopback by default, which is the right posture for a validator. Opening it diff --git a/script/infra-setup.sh b/script/infra-setup.sh index 9da9b5b..55e0b44 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -11,6 +11,10 @@ # metrics (--metrics-hosts) the fleet Prometheus/Grafana host: a scrape # config drop-in and a provisioned dashboard # -> deploy.yaml deploy-metrics +# edge (--edge-hosts) the site reverse proxy: an internal TLS cert and an +# nginx vhost giving `wss://quantus.internal`. NOT +# CI-deployed — the runner has no rights on the edge +# proxy, deliberately (reverse-proxies.md §4). # # The roles are separate deliberately: the miner host holds a GPU and an # outbound-only process, and a compromise of the miner deploy must not be able @@ -42,6 +46,14 @@ ADMIN_USER="${ADMIN_USER:-$USER}" NODE_HOSTS="${NODE_HOSTS-bob.hanzalova.internal}" MINER_HOSTS="${MINER_HOSTS-benjy.hanzalova.internal quadbrat.hanzalova.internal}" METRICS_HOSTS="${METRICS_HOSTS-golgafrinchans.kosherinata.internal}" +# The site reverse proxy that fronts quantus.internal. Empty to skip the edge role. +EDGE_HOSTS="${EDGE_HOSTS-hanzalova.internal}" +# Vanity name served by the edge proxy. Dot-free short name: the step@ renewal +# unit is a template instance and %i cannot carry dots (internal-tls.md §1). +EDGE_NAME="${EDGE_NAME:-quantus}" +# step-ca JWK provisioner password, shipped to the edge host just long enough to +# mint the first cert and removed even on failure (internal-tls.md §4). +STEP_PROVISIONER="${STEP_PROVISIONER:-${HOME}/.step/secrets/provisioner}" PUBKEY="${PUBKEY:-${HOME}/.ssh/id_gitea_ci.pub}" CHAIN="${CHAIN:-planck}" NODE_VERSION="${NODE_VERSION:-0.10.0}" @@ -52,6 +64,7 @@ while [[ $# -gt 0 ]]; do --node-hosts) NODE_HOSTS="$2"; shift 2 ;; --miner-hosts) MINER_HOSTS="$2"; shift 2 ;; --metrics-hosts) METRICS_HOSTS="$2"; shift 2 ;; + --edge-hosts) EDGE_HOSTS="$2"; shift 2 ;; --admin) ADMIN_USER="$2"; shift 2 ;; --node-version) NODE_VERSION="$2"; shift 2 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; @@ -322,6 +335,78 @@ SUDO REMOTE } + +# The site reverse proxy. Gives tooling a `wss://quantus.internal` endpoint, +# which matters because subxt — and therefore quantus-cli — refuses plain `ws://` +# to anything but localhost, so without this every client needs an ssh tunnel. +# +# Operator-run rather than CI-deployed on purpose: the runner's rights never +# extend to reading certificate keys or reloading nginx on a shared edge proxy. +# Same split lair/mail uses for its edge role. +provision_edge() { + local host="$1" name="$2" + echo "== ${host} (edge: ${name}.internal) ==" + if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then + echo " ! unreachable as ${ADMIN_USER} — skipping" >&2 + return 1 + fi + + local cert="/etc/nginx/tls/cert/${name}.internal.pem" + local key="/etc/nginx/tls/key/${name}.internal.pem" + + # verify checks chain and expiry, not the name — good enough to decide whether + # to mint, and it keeps re-runs from burning a provisioner use each time. + local state + state=$(ssh "${ADMIN_USER}@${host}" "[ -f ${cert} ] && sudo step certificate verify ${cert} \ + --roots /etc/pki/ca-trust/source/anchors/root-internal.pem >/dev/null 2>&1 \ + && echo valid || echo missing") + + if [ "$state" = valid ]; then + echo " = ${name}.internal cert already valid" + else + [[ -s "$STEP_PROVISIONER" ]] || { + echo " ! provisioner password not found at ${STEP_PROVISIONER}" >&2 + return 1 + } + rsync -az --rsync-path='sudo rsync' --chmod=0600 \ + "$STEP_PROVISIONER" "${ADMIN_USER}@${host}:/tmp/${name}-provisioner" + ssh "${ADMIN_USER}@${host}" " + sudo mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key + rc=0 + sudo step ca certificate --force \ + --provisioner lair \ + --provisioner-password-file /tmp/${name}-provisioner \ + --ca-url https://ca.internal \ + --root /etc/pki/ca-trust/source/anchors/root-internal.pem \ + --san ${name}.internal \ + ${name}.internal ${cert} ${key} || rc=\$? + sudo rm -f /tmp/${name}-provisioner + [ \$rc -eq 0 ] || { echo 'mint failed' >&2; exit \$rc; } + sudo chown root:root ${cert} ${key} + sudo chmod 644 ${cert}; sudo chmod 640 ${key} + sudo setfacl -m u:nginx:r ${key}" + echo " + minted ${name}.internal" + fi + + # Ship the vhost and enable it as a SYMLINK. Several vhosts on this host are + # regular files that have drifted from sites-available since the SNI-router + # migration; a symlink keeps this one honest. + rsync -az --rsync-path='sudo rsync' --chmod=F0644 \ + "asset/nginx/${name}.internal.conf" \ + "${ADMIN_USER}@${host}:/etc/nginx/sites-available/${name}.internal.conf" + ssh "${ADMIN_USER}@${host}" " + sudo ln -sfn ../sites-available/${name}.internal.conf \ + /etc/nginx/sites-enabled/${name}.internal.conf + sudo nginx -t" + ssh "${ADMIN_USER}@${host}" "sudo systemctl reload nginx.service" + ssh "${ADMIN_USER}@${host}" "sudo systemctl enable --now step@${name}.timer" + echo " = vhost enabled, nginx reloaded, step@${name}.timer armed" + + echo " NOTE: split-horizon DNS is a separate step —" + echo " opn-cli --config ~/.opn-cli/.yml unbound host create \\" + echo " --hostname ${name} --domain internal --server " +} + # The node's libp2p identity. It must exist BEFORE the first start — without it # the node exits with NetworkKeyNotFound(".../network/secret_dilithium") rather # than generating one. It is also the node's stable identity on the network, so @@ -377,6 +462,9 @@ done for h in $METRICS_HOSTS; do provision_metrics "$h" || rc=1 done +for h in $EDGE_HOSTS; do + provision_edge "$h" "$EDGE_NAME" || rc=1 +done echo echo "done. remaining operator steps:"