qapi/script/infra-setup.sh

#!/usr/bin/env bash
#
# One-time host provisioning for the qapi console.
#
# Run by an OPERATOR from a workstation with full sudo ssh to the target — not
# by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no
# rights to create accounts, read certificate keys, or reload nginx on a shared
# edge proxy.
#
#   ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
#
# Idempotent, and it skips past an unreachable host rather than failing halfway.
# Re-run it whenever the deploy gains a new file to ship: the deploy job
# preflights the target's sudoers against the grants below and fails up front
# naming what is missing, so the two cannot silently drift.
#
# Roles (all run by default; pass --role to narrow):
#
#   dns   the public CNAME on Cloudflare
#   cert  the Let's Encrypt certificate for the public name
#   edge  the nginx proxy: gitea_ci, vhosts, webroot, internal cert
#
# They run in that order and the order matters: certbot uses a DNS-01 challenge,
# and `nginx -t` fails on a missing ssl_certificate — which blocks every reload
# on this proxy, not just this vhost.
#
# Conventions: architecture/deployment-gitea-actions.md §2, reverse-proxies.md,
# external-tls.md, public-dns.md.

set -euo pipefail

# --- infra truth, matching .gitea/workflows/deploy.yaml -----------------------
EDGE_HOST="${EDGE_HOST:-oolon.kosherinata.internal}"
PUBLIC_NAME="${PUBLIC_NAME:-qapi.blackbeard.observer}"
INTERNAL_NAME="${INTERNAL_NAME:-qapi.internal}"
WEBROOT="${WEBROOT:-/var/www/qapi.blackbeard.observer}"
# The zone apex, which is not the record name: this is a subdomain of an
# existing zone, so the Cloudflare lookup is by apex and the record by full name.
ZONE="${ZONE:-blackbeard.observer}"
# The per-site indirection name the public record CNAMEs to. `bl` is the DC
# (oolon), `nh` the office (hanzalova) — architecture/public-dns.md §2. A vhost
# record must never carry a site address directly: the address changes and every
# hardcoded A record then has to be hunted down across zones.
SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}"
CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}"

PUBKEY=""
ROLES="dns cert edge"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

info() { printf '\033[36m==\033[0m %s\n' "$*"; }
warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; }

usage() {
    sed -n '3,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
    exit "${1:-0}"
}

while [ $# -gt 0 ]; do
    case "$1" in
        --pubkey) PUBKEY="$2"; shift 2 ;;
        --role)   ROLES="$2"; shift 2 ;;
        -h|--help) usage 0 ;;
        *) die "unknown argument: $1 (try --help)" ;;
    esac
done

has_role() { [[ " $ROLES " == *" $1 "* ]]; }

_reachable=""
reachable() {
    local host="$1"
    if [ -z "$_reachable" ]; then
        if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true 2>/dev/null; then
            _reachable=yes
        else
            _reachable=no
            warn "$host is unreachable; skipping every role that needs it"
        fi
    fi
    [ "$_reachable" = yes ]
}

# --- the scoped sudoers grants ------------------------------------------------
#
# These strings are the single source of truth for what CI may do on the edge
# proxy. The deploy workflow's preflight extracts them from THIS FILE and
# compares against `sudo -n -l` on the target, so adding a file to the deploy
# means adding a line here and re-running this script — nothing else keeps them
# in step.
#
# Note what is absent: no certbot, no access to /etc/letsencrypt, no
# sites-available write, no ability to create users. CI ships a static bundle
# into one directory and asks nginx to reload. Everything that touches keys or
# the shared proxy's configuration is an operator action, here.
#
# `:` and `=` are reserved in sudoers and must be escaped inside command
# arguments, or visudo rejects the file.

edge_sudoers() {
    cat <<'EOF'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/qapi.blackbeard.observer/
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/qapi.blackbeard.observer
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
EOF
}

# --- gitea_ci -----------------------------------------------------------------

provision_gitea_ci() {
    local host="$1" sudoers_body="$2"

    [ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)"
    [ -f "$PUBKEY" ] || die "$PUBKEY does not exist"

    info "$host: gitea_ci account"
    # A real shell, never nologin. The deploy runs `ssh gitea_ci@host <command>`;
    # a nologin shell authenticates the key and then refuses the command with
    # "This account is currently not available", which reads as an auth problem
    # rather than a shell one.
    ssh "$host" sudo bash -euo pipefail <<'REMOTE'
        if ! id gitea_ci >/dev/null 2>&1; then
            useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
        fi
        current=$(getent passwd gitea_ci | cut -d: -f7)
        if [ "$current" != "/bin/bash" ]; then
            echo "repairing gitea_ci shell: $current -> /bin/bash"
            usermod --shell /bin/bash gitea_ci
        fi
        install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
        usermod -aG systemd-journal gitea_ci
REMOTE

    # Append rather than overwrite: oolon is a SHARED proxy and other projects'
    # keys are already in this file. Overwriting it would break every other
    # deploy on the fleet, silently, until each one next ran.
    info "$host: authorized_keys (appending; this host is shared)"
    local key
    key="$(cat "$PUBKEY")"
    ssh "$host" sudo bash -euo pipefail <<REMOTE
        f=/var/lib/gitea_ci/.ssh/authorized_keys
        touch "\$f"
        if grep -qxF '$key' "\$f"; then
            echo "runner key already present"
        else
            printf '%s\n' '$key' >> "\$f"
            echo "runner key appended"
        fi
        chown gitea_ci:gitea_ci "\$f"
        chmod 0600 "\$f"
REMOTE

    info "$host: scoped sudoers"
    # Named <app>_gitea_ci, not bare gitea_ci, so several apps can drop their own
    # files on this shared host without clobbering each other.
    printf '%s\n' "$sudoers_body" | \
        ssh "$host" 'sudo tee /etc/sudoers.d/qapi_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/qapi_gitea_ci'
    # Verify before leaving: a syntax error in a sudoers drop-in can lock every
    # sudo on the host, not just this one.
    ssh "$host" sudo visudo -cf /etc/sudoers.d/qapi_gitea_ci
}

# --- roles --------------------------------------------------------------------

role_dns() {
    reachable "$EDGE_HOST" || return 0
    info "$EDGE_HOST: public DNS for $PUBLIC_NAME"

    # Run from the proxy, not from a workstation. The token there can rewrite
    # DNS for every domain on the account — including MX records for domains
    # whose mail we host — so copying it off the host spreads a credential with
    # that blast radius for no benefit (architecture/public-dns.md §1).
    ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' ZONE='$ZONE' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE'
        TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ")
        [ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; }
        api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }

        zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${ZONE}" \
            | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
        [ -n "$zid" ] || { echo "zone $ZONE is not on this Cloudflare account" >&2; exit 1; }

        # List the name's FULL record set first, all types. Filtering by the one
        # type you expect and finding nothing does not mean the name is free —
        # it may be a CNAME, and adding an A beside it is a conflict at best
        # (architecture/public-dns.md §4).
        existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${PUBLIC_NAME}")
        echo "$existing" | python3 -c '
import sys, json
rs = json.load(sys.stdin)["result"]
print("existing records for this name: %d" % len(rs))
for r in rs:
    print("  %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"]))
'
        current=$(echo "$existing" | python3 -c '
import sys, json
rs = json.load(sys.stdin)["result"]
cn = [r for r in rs if r["type"] == "CNAME"]
print(cn[0]["content"] if cn else "")
')
        if [ "$current" = "$SITE" ]; then
            echo "already CNAMEs to $SITE — nothing to do"
            exit 0
        fi
        if [ -n "$current" ]; then
            echo "already CNAMEs to $current, not $SITE." >&2
            echo "Refusing to repoint an existing record; change it deliberately." >&2
            exit 1
        fi
        if echo "$existing" | grep -q '"type"'; then
            # A CNAME cannot coexist with other types at a name.
            echo "name carries non-CNAME records; refusing to add a CNAME beside them." >&2
            exit 1
        fi

        api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records" \
            -H 'Content-Type: application/json' \
            --data "$(python3 -c '
import json, os
print(json.dumps({"type":"CNAME","name":os.environ["PUBLIC_NAME"],"content":os.environ["SITE"],
                  "ttl":1,"proxied":False,"comment":"Quantus-Network/papi-console (qapi console)"}))')" \
        | python3 -c '
import sys, json
d = json.load(sys.stdin)
print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"]
      else "FAILED %s" % d["errors"])
sys.exit(0 if d["success"] else 1)
'
REMOTE
}

role_cert() {
    reachable "$EDGE_HOST" || return 0
    info "$EDGE_HOST: Let's Encrypt certificate for $PUBLIC_NAME"

    # DNS-01 via the same Cloudflare credential, ECDSA, per
    # architecture/external-tls.md §1. --keep-until-expiring makes this a no-op
    # when the cert is still valid, so the role is safe to run every time.
    #
    # `sudo test`, not bare `test`: /etc/letsencrypt/live is root-only 0700 and
    # an unprivileged check silently returns false, concluding the cert is
    # missing and re-issuing on every run.
    ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE'
        if sudo test -d "/etc/letsencrypt/live/${PUBLIC_NAME}"; then
            echo "certificate lineage ${PUBLIC_NAME} already exists"
        fi
        sudo certbot certonly \
            -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive \
            --cert-name "$PUBLIC_NAME" \
            --key-type ecdsa \
            --dns-cloudflare \
            --dns-cloudflare-credentials /root/.certbot-internal \
            --dns-cloudflare-propagation-seconds 60 \
            --keep-until-expiring \
            -d "$PUBLIC_NAME"
        sudo test -f "/etc/letsencrypt/live/${PUBLIC_NAME}/fullchain.pem" \
            || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; }

        # One deploy hook per host, not per cert; oolon already has one for its
        # other lineages, so this is normally a no-op.
        hook=/etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
        if sudo test -x "$hook"; then
            echo "certbot deploy hook already present"
        else
            sudo install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
            printf '%s\n' '#!/bin/sh' \
                'systemctl reload nginx || logger -t reload-nginx -p daemon.err \' \
                '    "nginx reload failed after certbot renewal; cert on disk is newer than the one served"' \
                | sudo tee "$hook" > /dev/null
            sudo chmod +x "$hook"
            echo "installed the certbot deploy hook"
        fi
        systemctl is-enabled certbot-renew.timer >/dev/null 2>&1 \
            && echo "certbot-renew.timer is enabled" \
            || echo "WARNING: certbot-renew.timer is not enabled on this host"
REMOTE
}

role_edge() {
    reachable "$EDGE_HOST" || return 0
    provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)"

    info "$EDGE_HOST: webroot"
    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        install -d -o root -g root -m 0755 "$WEBROOT"
        # An unlabelled webroot makes nginx return 403 for every file, with
        # nothing in the nginx error log to explain it.
        restorecon -R "$WEBROOT"
REMOTE

    info "$EDGE_HOST: internal certificate for $INTERNAL_NAME"
    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        install -d -m 0755 /etc/nginx/tls/cert
        install -d -m 0700 /etc/nginx/tls/key
        if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
            echo "$INTERNAL_NAME certificate present"
            systemctl enable --now "step@qapi.timer" 2>/dev/null \
                || echo "step@qapi.timer not armed — renew $INTERNAL_NAME manually until it is"
        else
            # Not fatal. The mesh vhost is a convenience for operators; the
            # public site is the deliverable, and nginx -t fails on a missing
            # ssl_certificate — which would block every reload on this shared
            # proxy, not just ours.
            echo "NOTE: no internal certificate for $INTERNAL_NAME; the mesh vhost will be skipped."
            echo "      Mint it with the lair provisioner (architecture/internal-tls.md §4)"
            echo "      and re-run --role edge to enable it. Until then, reach the"
            echo "      console from the mesh with curl --resolve, or from off-mesh."
        fi
REMOTE

    info "$EDGE_HOST: nginx configuration"
    rsync -a --rsync-path 'sudo rsync' \
        "$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \
        "$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \
        "$EDGE_HOST:/etc/nginx/sites-available/"

    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        # sites-enabled holds only symlinks, and relative ones.
        if [ -d "/etc/letsencrypt/live/$PUBLIC_NAME" ]; then
            ln -sfn "../sites-available/$PUBLIC_NAME.conf" "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
        else
            # Gate on the cert: enabling a vhost whose ssl_certificate does
            # not exist fails the nginx config test and blocks every reload on
            # this proxy, not just ours.
            #
            # No backticks anywhere in this heredoc: it is UNQUOTED so that
            # the names interpolate, which also means bash performs command
            # substitution on it. A backticked "nginx -t" in a comment here ran
            # nginx on the operator workstation instead of being a comment.
            echo "no certificate for $PUBLIC_NAME yet; leaving the public vhost disabled" >&2
            rm -f "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
        fi

        if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
            ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
        else
            rm -f "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
        fi

        # nginx -t parses without binding, so it catches syntax and missing cert
        # files and nothing else. A pass is not evidence the reload landed —
        # hence the served-certificate check below.
        nginx -t
        systemctl reload nginx
REMOTE

    info "$EDGE_HOST: what is actually being served"
    # The proof that matters. `nginx -t` passing and `systemctl reload` exiting
    # 0 are both compatible with nginx still serving whatever it loaded last
    # (architecture/reverse-proxies.md §4). Ask the listener.
    #
    # --resolve to loopback: from the mesh the public name resolves to the
    # site's WAN address and dead-ends on the OPNsense LAN interface. Pinning it
    # to 127.0.0.1 still traverses the real path — the :443 stream router, SNI,
    # the https tier, this vhost.
    ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' WEBROOT='$WEBROOT' bash -euo pipefail" <<'REMOTE'
        # `sudo test`: /etc/letsencrypt/live is root-only 0700, so an
        # unprivileged check silently returns false.
        if ! sudo test -d "/etc/letsencrypt/live/$PUBLIC_NAME"; then
            echo "skipped: no certificate yet"
            exit 0
        fi
        echo | openssl s_client -connect 127.0.0.1:443 -servername "$PUBLIC_NAME" 2>/dev/null \
            | openssl x509 -noout -subject -enddate -ext subjectAltName 2>/dev/null \
            || echo "could not read the served certificate"
        code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 20 \
            --resolve "$PUBLIC_NAME:443:127.0.0.1" "https://$PUBLIC_NAME/" || echo 000)
        echo "GET / -> $code"
        case "$code" in
            200) ;;
            403|404)
                # Distinguish the two causes rather than guessing. An empty
                # webroot answers 403 under try_files with autoindex off, and
                # that is the expected state before the first deploy — not a
                # labelling fault, which is what this used to claim.
                if sudo test -f "$WEBROOT/index.html"; then
                    echo "the bundle is present but not served; check the SELinux label:" >&2
                    ls -ldZ "$WEBROOT" >&2
                    exit 1
                fi
                echo "webroot is empty — expected until the deploy workflow has run"
                ;;
            *)  echo "unexpected status $code" >&2; exit 1 ;;
        esac
REMOTE
}

# --- run ----------------------------------------------------------------------

info "roles: $ROLES"
# dns before cert (the DNS-01 challenge needs the zone; the record itself is not
# required, but publishing it first means the verification below can be real),
# cert before edge (nginx -t fails on a missing ssl_certificate).
has_role dns  && role_dns
has_role cert && role_cert
has_role edge && role_edge
info "done"