#!/usr/bin/env bash # # One-time host provisioning for the qapi console. # # Run by an OPERATOR from a workstation with full sudo ssh to the target — not # by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no # rights to create accounts, read certificate keys, or reload nginx on a shared # edge proxy. # # ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub # # Idempotent, and it skips past an unreachable host rather than failing halfway. # Re-run it whenever the deploy gains a new file to ship: the deploy job # preflights the target's sudoers against the grants below and fails up front # naming what is missing, so the two cannot silently drift. # # Roles (all run by default; pass --role to narrow): # # dns the public CNAME on Cloudflare # cert the Let's Encrypt certificate for the public name # edge the nginx proxy: gitea_ci, vhosts, webroot, internal cert # # They run in that order and the order matters: certbot uses a DNS-01 challenge, # and `nginx -t` fails on a missing ssl_certificate — which blocks every reload # on this proxy, not just this vhost. # # Conventions: architecture/deployment-gitea-actions.md §2, reverse-proxies.md, # external-tls.md, public-dns.md. set -euo pipefail # --- infra truth, matching .gitea/workflows/deploy.yaml ----------------------- EDGE_HOST="${EDGE_HOST:-oolon.kosherinata.internal}" PUBLIC_NAME="${PUBLIC_NAME:-qapi.blackbeard.observer}" INTERNAL_NAME="${INTERNAL_NAME:-qapi.internal}" WEBROOT="${WEBROOT:-/var/www/qapi.blackbeard.observer}" # The zone apex, which is not the record name: this is a subdomain of an # existing zone, so the Cloudflare lookup is by apex and the record by full name. ZONE="${ZONE:-blackbeard.observer}" # The per-site indirection name the public record CNAMEs to. `bl` is the DC # (oolon), `nh` the office (hanzalova) — architecture/public-dns.md §2. A vhost # record must never carry a site address directly: the address changes and every # hardcoded A record then has to be hunted down across zones. SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}" CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}" PUBKEY="" ROLES="dns cert edge" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" info() { printf '\033[36m==\033[0m %s\n' "$*"; } warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; } die() { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; } usage() { sed -n '3,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-0}" } while [ $# -gt 0 ]; do case "$1" in --pubkey) PUBKEY="$2"; shift 2 ;; --role) ROLES="$2"; shift 2 ;; -h|--help) usage 0 ;; *) die "unknown argument: $1 (try --help)" ;; esac done has_role() { [[ " $ROLES " == *" $1 "* ]]; } _reachable="" reachable() { local host="$1" if [ -z "$_reachable" ]; then if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true 2>/dev/null; then _reachable=yes else _reachable=no warn "$host is unreachable; skipping every role that needs it" fi fi [ "$_reachable" = yes ] } # --- the scoped sudoers grants ------------------------------------------------ # # These strings are the single source of truth for what CI may do on the edge # proxy. The deploy workflow's preflight extracts them from THIS FILE and # compares against `sudo -n -l` on the target, so adding a file to the deploy # means adding a line here and re-running this script — nothing else keeps them # in step. # # Note what is absent: no certbot, no access to /etc/letsencrypt, no # sites-available write, no ability to create users. CI ships a static bundle # into one directory and asks nginx to reload. Everything that touches keys or # the shared proxy's configuration is an operator action, here. # # `:` and `=` are reserved in sudoers and must be escaped inside command # arguments, or visudo rejects the file. edge_sudoers() { cat <<'EOF' gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/qapi.blackbeard.observer/ gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/qapi.blackbeard.observer gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx EOF } # --- gitea_ci ----------------------------------------------------------------- provision_gitea_ci() { local host="$1" sudoers_body="$2" [ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)" [ -f "$PUBKEY" ] || die "$PUBKEY does not exist" info "$host: gitea_ci account" # A real shell, never nologin. The deploy runs `ssh gitea_ci@host `; # a nologin shell authenticates the key and then refuses the command with # "This account is currently not available", which reads as an auth problem # rather than a shell one. ssh "$host" sudo bash -euo pipefail <<'REMOTE' if ! id gitea_ci >/dev/null 2>&1; then useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci fi current=$(getent passwd gitea_ci | cut -d: -f7) if [ "$current" != "/bin/bash" ]; then echo "repairing gitea_ci shell: $current -> /bin/bash" usermod --shell /bin/bash gitea_ci fi install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh usermod -aG systemd-journal gitea_ci REMOTE # Append rather than overwrite: oolon is a SHARED proxy and other projects' # keys are already in this file. Overwriting it would break every other # deploy on the fleet, silently, until each one next ran. info "$host: authorized_keys (appending; this host is shared)" local key key="$(cat "$PUBKEY")" ssh "$host" sudo bash -euo pipefail <> "\$f" echo "runner key appended" fi chown gitea_ci:gitea_ci "\$f" chmod 0600 "\$f" REMOTE info "$host: scoped sudoers" # Named _gitea_ci, not bare gitea_ci, so several apps can drop their own # files on this shared host without clobbering each other. printf '%s\n' "$sudoers_body" | \ ssh "$host" 'sudo tee /etc/sudoers.d/qapi_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/qapi_gitea_ci' # Verify before leaving: a syntax error in a sudoers drop-in can lock every # sudo on the host, not just this one. ssh "$host" sudo visudo -cf /etc/sudoers.d/qapi_gitea_ci } # --- roles -------------------------------------------------------------------- role_dns() { reachable "$EDGE_HOST" || return 0 info "$EDGE_HOST: public DNS for $PUBLIC_NAME" # Run from the proxy, not from a workstation. The token there can rewrite # DNS for every domain on the account — including MX records for domains # whose mail we host — so copying it off the host spreads a credential with # that blast radius for no benefit (architecture/public-dns.md §1). ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' ZONE='$ZONE' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE' TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ") [ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; } api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; } zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${ZONE}" \ | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")') [ -n "$zid" ] || { echo "zone $ZONE is not on this Cloudflare account" >&2; exit 1; } # List the name's FULL record set first, all types. Filtering by the one # type you expect and finding nothing does not mean the name is free — # it may be a CNAME, and adding an A beside it is a conflict at best # (architecture/public-dns.md §4). existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${PUBLIC_NAME}") echo "$existing" | python3 -c ' import sys, json rs = json.load(sys.stdin)["result"] print("existing records for this name: %d" % len(rs)) for r in rs: print(" %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"])) ' current=$(echo "$existing" | python3 -c ' import sys, json rs = json.load(sys.stdin)["result"] cn = [r for r in rs if r["type"] == "CNAME"] print(cn[0]["content"] if cn else "") ') if [ "$current" = "$SITE" ]; then echo "already CNAMEs to $SITE — nothing to do" exit 0 fi if [ -n "$current" ]; then echo "already CNAMEs to $current, not $SITE." >&2 echo "Refusing to repoint an existing record; change it deliberately." >&2 exit 1 fi if echo "$existing" | grep -q '"type"'; then # A CNAME cannot coexist with other types at a name. echo "name carries non-CNAME records; refusing to add a CNAME beside them." >&2 exit 1 fi api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records" \ -H 'Content-Type: application/json' \ --data "$(python3 -c ' import json, os print(json.dumps({"type":"CNAME","name":os.environ["PUBLIC_NAME"],"content":os.environ["SITE"], "ttl":1,"proxied":False,"comment":"Quantus-Network/papi-console (qapi console)"}))')" \ | python3 -c ' import sys, json d = json.load(sys.stdin) print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"] else "FAILED %s" % d["errors"]) sys.exit(0 if d["success"] else 1) ' REMOTE } role_cert() { reachable "$EDGE_HOST" || return 0 info "$EDGE_HOST: Let's Encrypt certificate for $PUBLIC_NAME" # DNS-01 via the same Cloudflare credential, ECDSA, per # architecture/external-tls.md §1. --keep-until-expiring makes this a no-op # when the cert is still valid, so the role is safe to run every time. # # `sudo test`, not bare `test`: /etc/letsencrypt/live is root-only 0700 and # an unprivileged check silently returns false, concluding the cert is # missing and re-issuing on every run. ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE' if sudo test -d "/etc/letsencrypt/live/${PUBLIC_NAME}"; then echo "certificate lineage ${PUBLIC_NAME} already exists" fi sudo certbot certonly \ -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive \ --cert-name "$PUBLIC_NAME" \ --key-type ecdsa \ --dns-cloudflare \ --dns-cloudflare-credentials /root/.certbot-internal \ --dns-cloudflare-propagation-seconds 60 \ --keep-until-expiring \ -d "$PUBLIC_NAME" sudo test -f "/etc/letsencrypt/live/${PUBLIC_NAME}/fullchain.pem" \ || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; } # One deploy hook per host, not per cert; oolon already has one for its # other lineages, so this is normally a no-op. hook=/etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh if sudo test -x "$hook"; then echo "certbot deploy hook already present" else sudo install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy printf '%s\n' '#!/bin/sh' \ 'systemctl reload nginx || logger -t reload-nginx -p daemon.err \' \ ' "nginx reload failed after certbot renewal; cert on disk is newer than the one served"' \ | sudo tee "$hook" > /dev/null sudo chmod +x "$hook" echo "installed the certbot deploy hook" fi systemctl is-enabled certbot-renew.timer >/dev/null 2>&1 \ && echo "certbot-renew.timer is enabled" \ || echo "WARNING: certbot-renew.timer is not enabled on this host" REMOTE } role_edge() { reachable "$EDGE_HOST" || return 0 provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)" info "$EDGE_HOST: webroot" ssh "$EDGE_HOST" sudo bash -euo pipefail </dev/null \ || echo "step@qapi.timer not armed — renew $INTERNAL_NAME manually until it is" else # Not fatal. The mesh vhost is a convenience for operators; the # public site is the deliverable, and nginx -t fails on a missing # ssl_certificate — which would block every reload on this shared # proxy, not just ours. echo "NOTE: no internal certificate for $INTERNAL_NAME; the mesh vhost will be skipped." echo " Mint it with the lair provisioner (architecture/internal-tls.md §4)" echo " and re-run --role edge to enable it. Until then, reach the" echo " console from the mesh with curl --resolve, or from off-mesh." fi REMOTE info "$EDGE_HOST: nginx configuration" rsync -a --rsync-path 'sudo rsync' \ "$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \ "$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \ "$EDGE_HOST:/etc/nginx/sites-available/" ssh "$EDGE_HOST" sudo bash -euo pipefail <&2 rm -f "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf" fi if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf" else rm -f "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf" fi # nginx -t parses without binding, so it catches syntax and missing cert # files and nothing else. A pass is not evidence the reload landed — # hence the served-certificate check below. nginx -t systemctl reload nginx REMOTE info "$EDGE_HOST: what is actually being served" # The proof that matters. `nginx -t` passing and `systemctl reload` exiting # 0 are both compatible with nginx still serving whatever it loaded last # (architecture/reverse-proxies.md §4). Ask the listener. # # --resolve to loopback: from the mesh the public name resolves to the # site's WAN address and dead-ends on the OPNsense LAN interface. Pinning it # to 127.0.0.1 still traverses the real path — the :443 stream router, SNI, # the https tier, this vhost. ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' WEBROOT='$WEBROOT' bash -euo pipefail" <<'REMOTE' # `sudo test`: /etc/letsencrypt/live is root-only 0700, so an # unprivileged check silently returns false. if ! sudo test -d "/etc/letsencrypt/live/$PUBLIC_NAME"; then echo "skipped: no certificate yet" exit 0 fi echo | openssl s_client -connect 127.0.0.1:443 -servername "$PUBLIC_NAME" 2>/dev/null \ | openssl x509 -noout -subject -enddate -ext subjectAltName 2>/dev/null \ || echo "could not read the served certificate" code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 20 \ --resolve "$PUBLIC_NAME:443:127.0.0.1" "https://$PUBLIC_NAME/" || echo 000) echo "GET / -> $code" case "$code" in 200) ;; 403|404) # Distinguish the two causes rather than guessing. An empty # webroot answers 403 under try_files with autoindex off, and # that is the expected state before the first deploy — not a # labelling fault, which is what this used to claim. if sudo test -f "$WEBROOT/index.html"; then echo "the bundle is present but not served; check the SELinux label:" >&2 ls -ldZ "$WEBROOT" >&2 exit 1 fi echo "webroot is empty — expected until the deploy workflow has run" ;; *) echo "unexpected status $code" >&2; exit 1 ;; esac REMOTE } # --- run ---------------------------------------------------------------------- info "roles: $ROLES" # dns before cert (the DNS-01 challenge needs the zone; the record itself is not # required, but publishing it first means the verification below can be real), # cert before edge (nginx -t fails on a missing ssl_certificate). has_role dns && role_dns has_role cert && role_cert has_role edge && role_edge info "done"