rustingface/script/infra-setup.sh

#!/usr/bin/env bash
#
# One-time host provisioning for rustingface.
#
# Run by an operator from a workstation with full sudo (not by CI, and not as
# the scoped gitea_ci account). Idempotent: re-running with nothing to change
# is a no-op. Skips past unreachable hosts so one offline node does not block
# the rest.
#
#   script/infra-setup.sh              # provision everything
#   script/infra-setup.sh --skip-minio # leave the bucket and credentials alone
#
# What it does:
#   1. bob      — gitea_ci user, runner key, scoped sudoers, SELinux port label
#   2. caveman  — the rustingface bucket and a scoped MinIO service account
#   3. hanzalova— rf.internal cert, nginx vhost, step@ renewal timer
#
# Application config is NOT shipped here. The deploy workflow renders it from
# Gitea secrets on every deploy.

set -euo pipefail

readonly APP=rustingface
readonly PORT=20482
readonly SERVICE_HOST=bob.hanzalova.internal
readonly PROXY_HOST=hanzalova.internal
readonly MINIO_HOST=caveman.kosherinata.internal
readonly MINIO_URL=http://caveman.kosherinata.internal:9000
readonly BUCKET=rustingface
readonly VHOST=rf
readonly RUNNER_PUBKEY="${RUNNER_PUBKEY:-$HOME/.ssh/id_gitea_ci.pub}"
readonly PROVISIONER_PASSWORD="${PROVISIONER_PASSWORD:-$HOME/.step/secrets/provisioner}"
readonly ROOT_CA=/etc/pki/ca-trust/source/anchors/root-internal.pem

SKIP_MINIO=0
for arg in "$@"; do
    case "$arg" in
        --skip-minio) SKIP_MINIO=1 ;;
        -h|--help) sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
        *) echo "unknown argument: $arg" >&2; exit 2 ;;
    esac
done

info() { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
ok()   { printf '    \033[32mok\033[0m %s\n' "$*"; }
warn() { printf '    \033[33m!!\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31m!!\033[0m %s\n' "$*" >&2; exit 1; }

# Returns non-zero (visibly) when a host cannot be reached, so the caller can
# skip it deliberately rather than by swallowing an error.
reachable() {
    local host=$1
    if ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \
           "$host" true; then
        return 0
    fi
    warn "$host is unreachable; skipping it. Re-run this script when it is back."
    return 1
}

# ---------------------------------------------------------------------------
# 1. The service host: bob
# ---------------------------------------------------------------------------

provision_service_host() {
    info "service host $SERVICE_HOST"

    [ -f "$RUNNER_PUBKEY" ] || die "runner public key not found at $RUNNER_PUBKEY.
The keypair is maintained at ~/.ssh/id_gitea_ci on the operator workstation and is
shared by every project's deploy. Copy it from there; do not generate a new one, or
every other project's deploy breaks."

    # The gitea_ci account, its key, and journal access. Creating the user is
    # idempotent via `useradd` guarded on `id`.
    ssh "$SERVICE_HOST" 'sudo bash -s' <<'REMOTE'
set -euo pipefail
if id gitea_ci >/dev/null 2>&1; then
    echo "    gitea_ci exists"
else
    useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
    echo "    created gitea_ci"
fi
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
usermod -aG systemd-journal gitea_ci
REMOTE
    ok "gitea_ci account"

    # shellcheck disable=SC2029  # the key must expand locally
    ssh "$SERVICE_HOST" "sudo bash -c '
        umask 077
        touch /var/lib/gitea_ci/.ssh/authorized_keys
        grep -qxF \"$(cat "$RUNNER_PUBKEY")\" /var/lib/gitea_ci/.ssh/authorized_keys \
            || echo \"$(cat "$RUNNER_PUBKEY")\" >> /var/lib/gitea_ci/.ssh/authorized_keys
        chown -R gitea_ci:gitea_ci /var/lib/gitea_ci/.ssh
        chmod 600 /var/lib/gitea_ci/.ssh/authorized_keys'"
    ok "runner key installed"

    # Scoped sudoers: exactly the commands the deploy runs, pinned to literal
    # destinations, and verified before it can lock anyone out.
    ssh "$SERVICE_HOST" 'sudo bash -s' <<REMOTE
set -euo pipefail
cat > /etc/sudoers.d/.${APP}_gitea_ci.new <<'SUDOERS'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/config.toml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-access-key
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-secret-key
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/hf-token
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/${APP}.conf
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/${APP}.xml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g ${APP} -m 0750 /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chown -R root\:${APP} /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0755 /usr/local/bin/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/${APP} /etc/${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active ${APP}.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=${APP}
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=${APP}
# The post-deploy health check, run as the service account rather than root:
# it proves the credentials the service will actually use can reach and write
# to the bucket, which \`systemctl is-active\` cannot.
gitea_ci ALL=(${APP}) NOPASSWD: /usr/local/bin/${APP} --config /etc/${APP}/config.toml doctor
SUDOERS
chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new
# Verify before moving it into place: a typo in a sudoers file that is already
# installed can lock the host out of sudo entirely.
visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new
mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci
REMOTE
    ok "scoped sudoers installed and visudo-verified"

    # SELinux: the daemon cannot bind an unlabelled port. This must happen
    # before the first service start, which is why it lives here and not in
    # the deploy workflow.
    ssh "$SERVICE_HOST" "sudo bash -s" <<REMOTE
set -euo pipefail
if semanage port -l | grep -Eq "^http_port_t .*\\b${PORT}\\b"; then
    echo "    tcp/${PORT} already labelled http_port_t"
else
    semanage port -a -t http_port_t -p tcp ${PORT} \
        || semanage port -m -t http_port_t -p tcp ${PORT}
    echo "    labelled tcp/${PORT} http_port_t"
fi
REMOTE
    ok "SELinux port label"
}

# ---------------------------------------------------------------------------
# 2. Object storage: caveman
# ---------------------------------------------------------------------------

provision_bucket() {
    info "object storage $MINIO_HOST"

    if ! ssh "$MINIO_HOST" 'command -v mc'; then
        warn "mc (the MinIO client) is not installed on $MINIO_HOST.
Create the bucket and a scoped service account by hand, then re-run with --skip-minio:

  mc alias set local $MINIO_URL <root-user> <root-password>
  mc mb local/$BUCKET
  mc admin user svcacct add local <root-user> --name rustingface

Put the resulting access key and secret in the Gitea repo secrets
S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY."
        return 0
    fi

    if ssh "$MINIO_HOST" "mc --version >/dev/null && test -d /srv/minio/$BUCKET"; then
        ok "bucket $BUCKET exists"
        return 0
    fi

    warn "bucket and service-account creation needs the MinIO root credentials, which
this script deliberately does not read -- they are root-equivalent for every bucket on
that host, not just this one. Run there as an operator (architecture object-storage.md §3):

  set -a; source <(sudo cat /etc/minio/minio.env); set +a
  export MC_CONFIG_DIR=\$(mktemp -d); trap 'rm -rf \"\$MC_CONFIG_DIR\"' EXIT
  mc alias set local http://127.0.0.1:9000 \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\"
  mc mb --ignore-existing local/$BUCKET
  mc admin user svcacct add local \"\$MINIO_ROOT_USER\" --name $APP \\
      --access-key '$APP-<8 random>' --policy <policy scoped to $BUCKET>

then set the Gitea repo secrets S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY.
Re-run this script with --skip-minio once that is done."
}

# ---------------------------------------------------------------------------
# 3. The edge proxy: hanzalova
# ---------------------------------------------------------------------------

provision_proxy() {
    info "edge proxy $PROXY_HOST"

    local cert=/etc/nginx/tls/cert/${VHOST}.internal.pem
    local key=/etc/nginx/tls/key/${VHOST}.internal.pem

    # `step certificate verify` checks chain and expiry, not the name, which is
    # enough to decide whether a fresh mint is needed.
    local state
    state=$(ssh "$PROXY_HOST" "if sudo test -f $cert && sudo step certificate verify $cert --roots $ROOT_CA; then echo valid; else echo missing; fi" \
        | tail -1)

    if [ "$state" != valid ]; then
        [ -f "$PROVISIONER_PASSWORD" ] \
            || die "provisioner password not found at $PROVISIONER_PASSWORD; cannot mint ${VHOST}.internal"
        rsync -az --rsync-path='sudo rsync' --chmod=0600 \
            "$PROVISIONER_PASSWORD" "$PROXY_HOST:/tmp/${VHOST}-provisioner"
        # Remove the credential whether or not the mint succeeds, then
        # propagate the failure.
        ssh "$PROXY_HOST" "
            set -euo pipefail
            sudo mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key
            rc=0
            sudo step ca certificate --force \
                --provisioner lair \
                --provisioner-password-file /tmp/${VHOST}-provisioner \
                --ca-url https://ca.internal \
                --root $ROOT_CA \
                --san ${VHOST}.internal \
                ${VHOST}.internal $cert $key || rc=\$?
            sudo rm -f /tmp/${VHOST}-provisioner
            [ \$rc -eq 0 ] || { echo 'minting ${VHOST}.internal failed' >&2; exit \$rc; }
            sudo chown root:root $cert $key
            sudo chmod 0644 $cert
            sudo chmod 0640 $key
            sudo setfacl -m u:nginx:r $key"
        ok "minted ${VHOST}.internal"
    else
        ok "${VHOST}.internal cert is present and valid"
    fi

    ssh "$PROXY_HOST" "sudo systemctl enable --now step@${VHOST}.timer"
    ok "renewal timer enabled"

    # The vhost references the cert paths, so it is only installed after the
    # cert exists: nginx -t fails on a missing ssl_certificate and that blocks
    # every unrelated reload on the host.
    rsync -az --rsync-path='sudo rsync' --mkpath \
        asset/nginx/${VHOST}.internal.conf \
        "$PROXY_HOST:/etc/nginx/sites-available/${VHOST}.internal.conf"
    ssh "$PROXY_HOST" "
        set -euo pipefail
        sudo ln -sfn ../sites-available/${VHOST}.internal.conf \
            /etc/nginx/sites-enabled/${VHOST}.internal.conf
        sudo nginx -t
        sudo systemctl reload nginx"
    ok "vhost installed and nginx reloaded"

    # A reload is a request, not a guarantee: nginx keeps its old cycle if it
    # cannot rebind, and then serves the certificate it loaded then. Compare
    # what is actually on the wire against what is on disk, by serial.
    #
    # Old workers finish their in-flight connections before exiting, so for a
    # second or two after a reload either cycle may answer. Retry rather than
    # report a mismatch that is really just the handover in progress.
    local served disk
    disk=$(ssh "$PROXY_HOST" "sudo openssl x509 -noout -serial -in $cert")
    for _ in $(seq 1 10); do
        served=$(echo | openssl s_client -servername ${VHOST}.internal -connect ${PROXY_HOST}:443 2>/dev/null \
            | openssl x509 -noout -serial || echo "serial=unavailable")
        [ "$served" = "$disk" ] && break
        sleep 2
    done
    if [ "$served" = "$disk" ]; then
        ok "the certificate on the wire matches the one on disk"
    else
        warn "nginx is serving a different certificate than the one on disk
  wire: $served
  disk: $disk
A reload was requested and reported success, but nginx kept its previous cycle.
With 24h certs this puts an expired certificate on the wire within a day.
Investigate the bind failure in the nginx error log, then restart nginx."
    fi

    info "DNS"
    if getent hosts "${VHOST}.internal" >/dev/null; then
        ok "${VHOST}.internal resolves"
        return 0
    fi
    echo "    Add the split-horizon record on both routers:"
    echo "      opn-cli --config ~/.opn-cli/hanzalova.yml unbound host create \\"
    echo "          --hostname ${VHOST} --domain internal --server 10.6.0.46 \\"
    echo "          --description 'rustingface via hanzalova reverse proxy (bob backend)'"
    echo "      opn-cli --config ~/.opn-cli/kosherinata.yml unbound host create \\"
    echo "          --hostname ${VHOST} --domain internal --server 10.6.0.46 \\"
    echo "          --description 'rustingface via hanzalova reverse proxy (bob backend)'"
    echo "    Never give ${VHOST}.internal a public / Cloudflare record."
}

# ---------------------------------------------------------------------------

main() {
    cd "$(dirname "$0")/.."

    if reachable "$SERVICE_HOST"; then
        provision_service_host
    fi

    if [ "$SKIP_MINIO" -eq 0 ] && reachable "$MINIO_HOST"; then
        provision_bucket
    fi

    if reachable "$PROXY_HOST"; then
        provision_proxy
    fi

    info "done"
    echo "    Next: set the Gitea repo secrets (RSYNC_SSH_KEY, S3_ACCESS_KEY_ID,"
    echo "    S3_SECRET_ACCESS_KEY, HF_TOKEN) and push to main to deploy."
}

main "$@"