rustingface/script/infra-setup.sh

#!/usr/bin/env bash
#
# One-time host provisioning for rustingface.
#
# Run by an operator from a workstation with full sudo (not by CI, and not as
# the scoped gitea_ci account). Idempotent: re-running with nothing to change
# is a no-op. Skips past unreachable hosts so one offline node does not block
# the rest.
#
#   script/infra-setup.sh              # provision everything
#   script/infra-setup.sh --skip-minio # leave the bucket and credentials alone
#   script/infra-setup.sh --skip-dns   # leave public DNS alone
#
# What it does:
#   1. bob      — gitea_ci user, runner key, scoped sudoers, SELinux port label
#   2. caveman  — the rustingface bucket and a scoped MinIO service account
#   3. hanzalova— rf.internal cert (internal CA) and rustingface.com cert
#                 (Let's Encrypt), both vhosts, the webroot, the renewal
#                 timers, and the gitea_ci account CI uses to ship the frontend
#
# Public DNS is created here if absent, from the proxy where the Cloudflare
# token already lives. It is never copied off that host: the token can rewrite
# DNS for every zone on the account, including MX records for domains whose
# mail we host (public-dns.md §1). A name that already points somewhere else is
# reported and left alone.
#
# Application config is NOT shipped here. The deploy workflow renders it from
# Gitea secrets on every deploy.

set -euo pipefail

readonly APP=rustingface
readonly PORT=20482
readonly SERVICE_HOST=bob.hanzalova.internal
readonly PROXY_HOST=hanzalova.internal
readonly MINIO_HOST=caveman.kosherinata.internal
readonly MINIO_URL=http://caveman.kosherinata.internal:9000
readonly BUCKET=rustingface
readonly VHOST=rf
readonly WEBROOT=/var/www/rustingface
readonly PUBLIC_NAME=rustingface.com
readonly SITE_INDIRECTION=nh.thgttg.com          # hanzalova's WAN address
readonly CERTBOT_CREDENTIALS=/root/.certbot-internal
readonly RUNNER_PUBKEY="${RUNNER_PUBKEY:-$HOME/.ssh/id_gitea_ci.pub}"
readonly PROVISIONER_PASSWORD="${PROVISIONER_PASSWORD:-$HOME/.step/secrets/provisioner}"
readonly ROOT_CA=/etc/pki/ca-trust/source/anchors/root-internal.pem

SKIP_MINIO=0
SKIP_DNS=0
for arg in "$@"; do
    case "$arg" in
        --skip-minio) SKIP_MINIO=1 ;;
        --skip-dns) SKIP_DNS=1 ;;
        -h|--help) sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
        *) echo "unknown argument: $arg" >&2; exit 2 ;;
    esac
done

info() { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
ok()   { printf '    \033[32mok\033[0m %s\n' "$*"; }
warn() { printf '    \033[33m!!\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31m!!\033[0m %s\n' "$*" >&2; exit 1; }

# Returns non-zero (visibly) when a host cannot be reached, so the caller can
# skip it deliberately rather than by swallowing an error.
reachable() {
    local host=$1
    if ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \
           "$host" true; then
        return 0
    fi
    warn "$host is unreachable; skipping it. Re-run this script when it is back."
    return 1
}

# ---------------------------------------------------------------------------
# 1. The service host: bob
# ---------------------------------------------------------------------------

provision_service_host() {
    info "service host $SERVICE_HOST"

    [ -f "$RUNNER_PUBKEY" ] || die "runner public key not found at $RUNNER_PUBKEY.
The keypair is maintained at ~/.ssh/id_gitea_ci on the operator workstation and is
shared by every project's deploy. Copy it from there; do not generate a new one, or
every other project's deploy breaks."

    # The gitea_ci account, its key, and journal access. Creating the user is
    # idempotent via `useradd` guarded on `id`.
    ssh "$SERVICE_HOST" 'sudo bash -s' <<'REMOTE'
set -euo pipefail
if id gitea_ci >/dev/null 2>&1; then
    echo "    gitea_ci exists"
else
    useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
    echo "    created gitea_ci"
fi
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
usermod -aG systemd-journal gitea_ci
REMOTE
    ok "gitea_ci account"

    # shellcheck disable=SC2029  # the key must expand locally
    ssh "$SERVICE_HOST" "sudo bash -c '
        umask 077
        touch /var/lib/gitea_ci/.ssh/authorized_keys
        grep -qxF \"$(cat "$RUNNER_PUBKEY")\" /var/lib/gitea_ci/.ssh/authorized_keys \
            || echo \"$(cat "$RUNNER_PUBKEY")\" >> /var/lib/gitea_ci/.ssh/authorized_keys
        chown -R gitea_ci:gitea_ci /var/lib/gitea_ci/.ssh
        chmod 600 /var/lib/gitea_ci/.ssh/authorized_keys'"
    ok "runner key installed"

    # Scoped sudoers: exactly the commands the deploy runs, pinned to literal
    # destinations, and verified before it can lock anyone out.
    ssh "$SERVICE_HOST" 'sudo bash -s' <<REMOTE
set -euo pipefail
cat > /etc/sudoers.d/.${APP}_gitea_ci.new <<'SUDOERS'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/config.toml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-access-key
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-secret-key
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/hf-token
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/client-tokens
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/${APP}.conf
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/${APP}.xml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g ${APP} -m 0750 /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chown -R root\:${APP} /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token /etc/${APP}/client-tokens
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0755 /usr/local/bin/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/${APP} /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=${APP}
# The post-deploy health check, run as the service account rather than root:
# it proves the credentials the service will actually use can reach and write
# to the bucket, which \`systemctl is-active\` cannot.
gitea_ci ALL=(${APP}) NOPASSWD: /usr/local/bin/${APP} --config /etc/${APP}/config.toml doctor
SUDOERS
chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new
# Verify before moving it into place: a typo in a sudoers file that is already
# installed can lock the host out of sudo entirely.
visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new
mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci
REMOTE
    ok "scoped sudoers installed and visudo-verified"

    # SELinux: the daemon cannot bind an unlabelled port. This must happen
    # before the first service start, which is why it lives here and not in
    # the deploy workflow.
    ssh "$SERVICE_HOST" "sudo bash -s" <<REMOTE
set -euo pipefail
if semanage port -l | grep -Eq "^http_port_t .*\\b${PORT}\\b"; then
    echo "    tcp/${PORT} already labelled http_port_t"
else
    semanage port -a -t http_port_t -p tcp ${PORT} \
        || semanage port -m -t http_port_t -p tcp ${PORT}
    echo "    labelled tcp/${PORT} http_port_t"
fi
REMOTE
    ok "SELinux port label"
}

# ---------------------------------------------------------------------------
# 2. Object storage: caveman
# ---------------------------------------------------------------------------

provision_bucket() {
    info "object storage $MINIO_HOST"

    if ! ssh "$MINIO_HOST" 'command -v mc'; then
        warn "mc (the MinIO client) is not installed on $MINIO_HOST.
Create the bucket and a scoped service account by hand, then re-run with --skip-minio:

  mc alias set local $MINIO_URL <root-user> <root-password>
  mc mb local/$BUCKET
  mc admin user svcacct add local <root-user> --name rustingface

Put the resulting access key and secret in the Gitea repo secrets
S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY."
        return 0
    fi

    if ssh "$MINIO_HOST" "mc --version >/dev/null && test -d /srv/minio/$BUCKET"; then
        ok "bucket $BUCKET exists"
        return 0
    fi

    warn "bucket and service-account creation needs the MinIO root credentials, which
this script deliberately does not read -- they are root-equivalent for every bucket on
that host, not just this one. Run there as an operator (architecture object-storage.md §3):

  set -a; source <(sudo cat /etc/minio/minio.env); set +a
  export MC_CONFIG_DIR=\$(mktemp -d); trap 'rm -rf \"\$MC_CONFIG_DIR\"' EXIT
  mc alias set local http://127.0.0.1:9000 \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\"
  mc mb --ignore-existing local/$BUCKET
  mc admin user svcacct add local \"\$MINIO_ROOT_USER\" --name $APP \\
      --access-key '$APP-<8 random>' --policy <policy scoped to $BUCKET>

then set the Gitea repo secrets S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY.
Re-run this script with --skip-minio once that is done."
}

# ---------------------------------------------------------------------------
# Public DNS for the WAN-facing name
# ---------------------------------------------------------------------------

# Idempotent: creates the records if absent, reports them if already correct,
# and refuses to touch a name that resolves somewhere else.
#
# The API calls run on the proxy, where the Cloudflare token already lives.
# That token can rewrite DNS for every zone on the account, including MX
# records for domains whose mail we host (public-dns.md §1), so it is never
# copied off the host.
provision_public_dns() {
    info "public DNS (${PUBLIC_NAME})"

    if [ "$SKIP_DNS" -eq 1 ]; then
        ok "skipped (--skip-dns)"
        return 0
    fi

    # The script is piped in from a file rather than embedded as a heredoc:
    # quoting shell through a heredoc has silently mangled this project's
    # scripts more than once, and a file can be linted and run on its own.
    # It executes on the proxy because that is where the Cloudflare token is,
    # and the token is never copied off that host.
    ssh "$PROXY_HOST" \
        "sudo bash -s ${PUBLIC_NAME} ${SITE_INDIRECTION} ${CERTBOT_CREDENTIALS}" \
        < script/publish-dns.sh

    # Checked from here rather than on the proxy: the operator workstation is
    # on the mesh, and is the audience that has to be able to reach the name.
    if getent hosts "${PUBLIC_NAME}" >/dev/null; then
        ok "${PUBLIC_NAME} resolves to $(getent hosts "${PUBLIC_NAME}" | awk '{print $1}' | head -1)"
    else
        warn "${PUBLIC_NAME} does not resolve here yet. A record created moments ago takes
time to propagate, and a resolver that queried the name while it was missing caches the
NXDOMAIN: clear it with 'resolvectl flush-caches' or wait out the negative TTL."
    fi
}

# ---------------------------------------------------------------------------
# 3. The edge proxy: hanzalova
# ---------------------------------------------------------------------------

provision_proxy() {
    info "edge proxy $PROXY_HOST"

    local cert=/etc/nginx/tls/cert/${VHOST}.internal.pem
    local key=/etc/nginx/tls/key/${VHOST}.internal.pem

    # `step certificate verify` checks chain and expiry, not the name, which is
    # enough to decide whether a fresh mint is needed.
    local state
    state=$(ssh "$PROXY_HOST" "if sudo test -f $cert && sudo step certificate verify $cert --roots $ROOT_CA; then echo valid; else echo missing; fi" \
        | tail -1)

    if [ "$state" != valid ]; then
        [ -f "$PROVISIONER_PASSWORD" ] \
            || die "provisioner password not found at $PROVISIONER_PASSWORD; cannot mint ${VHOST}.internal"
        rsync -az --rsync-path='sudo rsync' --chmod=0600 \
            "$PROVISIONER_PASSWORD" "$PROXY_HOST:/tmp/${VHOST}-provisioner"
        # Remove the credential whether or not the mint succeeds, then
        # propagate the failure.
        ssh "$PROXY_HOST" "
            set -euo pipefail
            sudo mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key
            rc=0
            sudo step ca certificate --force \
                --provisioner lair \
                --provisioner-password-file /tmp/${VHOST}-provisioner \
                --ca-url https://ca.internal \
                --root $ROOT_CA \
                --san ${VHOST}.internal \
                ${VHOST}.internal $cert $key || rc=\$?
            sudo rm -f /tmp/${VHOST}-provisioner
            [ \$rc -eq 0 ] || { echo 'minting ${VHOST}.internal failed' >&2; exit \$rc; }
            sudo chown root:root $cert $key
            sudo chmod 0644 $cert
            sudo chmod 0640 $key
            sudo setfacl -m u:nginx:r $key"
        ok "minted ${VHOST}.internal"
    else
        ok "${VHOST}.internal cert is present and valid"
    fi

    ssh "$PROXY_HOST" "sudo systemctl enable --now step@${VHOST}.timer"
    ok "renewal timer enabled"

    # The public cert. Idempotent: --keep-until-expiring makes a repeat run a
    # no-op while the lineage is still valid. DNS-01 means this works before
    # the name resolves anywhere, so the cert can exist before the record does.
    local have_public
    have_public=$(ssh "$PROXY_HOST" \
        "sudo test -d /etc/letsencrypt/live/${PUBLIC_NAME} && echo yes || echo no" | tail -1)
    if [ "$have_public" = no ]; then
        if ! ssh "$PROXY_HOST" "sudo test -f ${CERTBOT_CREDENTIALS}"; then
            die "${CERTBOT_CREDENTIALS} is missing on ${PROXY_HOST}; cannot issue ${PUBLIC_NAME}"
        fi
        ssh "$PROXY_HOST" "sudo certbot certonly \
            -m ops@${PUBLIC_NAME} --agree-tos --no-eff-email --noninteractive \
            --cert-name ${PUBLIC_NAME} \
            --key-type ecdsa \
            --dns-cloudflare \
            --dns-cloudflare-credentials ${CERTBOT_CREDENTIALS} \
            --dns-cloudflare-propagation-seconds 60 \
            --keep-until-expiring \
            -d ${PUBLIC_NAME} -d www.${PUBLIC_NAME}"
        ok "issued ${PUBLIC_NAME}"
    else
        ok "${PUBLIC_NAME} cert is present"
    fi

    # certbot renews on its own timer, but a reload is only a request and a
    # failed one leaves the old cert on the wire for up to 90 days. The hook
    # must therefore leave a trace rather than swallow the failure.
    ssh "$PROXY_HOST" 'sudo bash -s' <<'REMOTE'
set -euo pipefail
install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK'
#!/bin/sh
systemctl reload nginx || logger -t reload-nginx -p daemon.err \
    "nginx reload failed after certbot renewal; cert on disk is newer than the one served"
HOOK
chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
REMOTE
    ok "certbot deploy hook installed"

    # The vhosts reference cert paths, so they are only installed once both
    # certs exist: nginx -t fails on a missing ssl_certificate, and that
    # blocks every unrelated reload on this shared proxy.
    rsync -az --rsync-path='sudo rsync' --mkpath \
        asset/nginx/rustingface-upstream.conf \
        "$PROXY_HOST:/etc/nginx/conf.d/rustingface-upstream.conf"
    for conf in ${VHOST}.internal.conf ${PUBLIC_NAME}.conf; do
        rsync -az --rsync-path='sudo rsync' --mkpath \
            "asset/nginx/${conf}" "$PROXY_HOST:/etc/nginx/sites-available/${conf}"
        ssh "$PROXY_HOST" "sudo ln -sfn ../sites-available/${conf} /etc/nginx/sites-enabled/${conf}"
    done
    ssh "$PROXY_HOST" "
        set -euo pipefail
        sudo nginx -t
        sudo systemctl reload nginx"
    ok "both vhosts installed and nginx reloaded"

    # A reload is a request, not a guarantee: nginx keeps its old cycle if it
    # cannot rebind, and then serves the certificate it loaded then. Compare
    # what is actually on the wire against what is on disk, by serial.
    #
    # Old workers finish their in-flight connections before exiting, so for a
    # second or two after a reload either cycle may answer. Retry rather than
    # report a mismatch that is really just the handover in progress.
    local served disk
    disk=$(ssh "$PROXY_HOST" "sudo openssl x509 -noout -serial -in $cert")
    for _ in $(seq 1 10); do
        served=$(echo | openssl s_client -servername ${VHOST}.internal -connect ${PROXY_HOST}:443 2>/dev/null \
            | openssl x509 -noout -serial || echo "serial=unavailable")
        [ "$served" = "$disk" ] && break
        sleep 2
    done
    if [ "$served" = "$disk" ]; then
        ok "the certificate on the wire matches the one on disk"
    else
        warn "nginx is serving a different certificate than the one on disk
  wire: $served
  disk: $disk
A reload was requested and reported success, but nginx kept its previous cycle.
With 24h certs this puts an expired certificate on the wire within a day.
Investigate the bind failure in the nginx error log, then restart nginx."
    fi

    # CI ships the built frontend here, so this host needs its own scoped
    # deploy account. Its whitelist is narrower than the service host's: a
    # webroot rsync, a relabel, and an nginx config test plus reload.
    ssh "$PROXY_HOST" 'sudo bash -s' <<'REMOTE'
set -euo pipefail
if id gitea_ci >/dev/null 2>&1; then
    echo "    gitea_ci exists"
else
    useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
    echo "    created gitea_ci"
fi
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
usermod -aG systemd-journal gitea_ci
REMOTE

    # shellcheck disable=SC2029  # the key must expand locally
    ssh "$PROXY_HOST" "sudo bash -c '
        umask 077
        touch /var/lib/gitea_ci/.ssh/authorized_keys
        grep -qxF \"$(cat "$RUNNER_PUBKEY")\" /var/lib/gitea_ci/.ssh/authorized_keys \
            || echo \"$(cat "$RUNNER_PUBKEY")\" >> /var/lib/gitea_ci/.ssh/authorized_keys
        chown -R gitea_ci:gitea_ci /var/lib/gitea_ci/.ssh
        chmod 600 /var/lib/gitea_ci/.ssh/authorized_keys'"
    ok "gitea_ci account and runner key on the proxy"

    ssh "$PROXY_HOST" 'sudo bash -s' <<REMOTE
set -euo pipefail
install -d -o root -g root -m 0755 ${WEBROOT}
# /var/www is httpd_sys_content_t by default; relabel so a freshly created
# directory inherits it rather than picking up the parent's runtime context.
restorecon -R ${WEBROOT}

cat > /etc/sudoers.d/.${APP}_gitea_ci.new <<'SUDOERS'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * ${WEBROOT}/
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g root -m 0755 ${WEBROOT}
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R ${WEBROOT}
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
# The deploy's health check asks whether the public certificate exists
# before probing the public vhost; /etc/letsencrypt/live is root-only, so
# an unprivileged test silently returns false and would skip the check.
gitea_ci ALL=(root) NOPASSWD: /usr/bin/test -d /etc/letsencrypt/live/rustingface.com
SUDOERS
chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new
visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new
mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci
REMOTE
    ok "webroot ${WEBROOT} and scoped sudoers on the proxy"

    provision_public_dns

    info "mesh DNS"
    if getent hosts "${VHOST}.internal" >/dev/null; then
        ok "${VHOST}.internal resolves"
        return 0
    fi
    echo "    Add the split-horizon record on both routers:"
    echo "      opn-cli --config ~/.opn-cli/hanzalova.yml unbound host create \\"
    echo "          --hostname ${VHOST} --domain internal --server 10.6.0.46 \\"
    echo "          --description 'rustingface via hanzalova reverse proxy (bob backend)'"
    echo "      opn-cli --config ~/.opn-cli/kosherinata.yml unbound host create \\"
    echo "          --hostname ${VHOST} --domain internal --server 10.6.0.46 \\"
    echo "          --description 'rustingface via hanzalova reverse proxy (bob backend)'"
    echo "    opn-cli 'create' only stages the record; apply it on each router"
    echo "    (reverse-proxies.md §2). Verify with: getent hosts ${VHOST}.internal"
    echo "    Never give ${VHOST}.internal a public / Cloudflare record."
}

# ---------------------------------------------------------------------------

main() {
    cd "$(dirname "$0")/.."

    if reachable "$SERVICE_HOST"; then
        provision_service_host
    fi

    if [ "$SKIP_MINIO" -eq 0 ] && reachable "$MINIO_HOST"; then
        provision_bucket
    fi

    if reachable "$PROXY_HOST"; then
        provision_proxy
    fi

    info "done"
    cat <<'NEXT'
    Repo secrets this deploy needs: RSYNC_SSH_KEY, S3_ACCESS_KEY_ID,
    S3_SECRET_ACCESS_KEY, CLIENT_TOKENS, and HF_TOKEN (optional, gated repos
    only). Push to main to deploy.

    Order matters for the public name: land a deploy that enforces auth
    BEFORE publishing DNS. The certificate and vhost above are inert while the
    name does not resolve, which is the only thing keeping the bucket private
    until then.
NEXT
}

main "$@"