Merge pull request 'fix(ci): share one nginx-vhost renderer; drop unused deploy.sh' (#5) from fix/render-drift-remove-deploy-sh into main

.gitea/workflows/deploy.yml

@@ -6,10 +6,11 @@ name: deploy
 #   - web            (prerendered static SPA + nginx vhost) -> oolon
 #
 # This workflow is the source of infra truth (hosts, ports, paths live in `env`
-# below — there is no manifest.yml in this model). It replaces script/deploy.sh:
-# instead of an operator running it from a workstation with `pass`, a Gitea
-# Actions runner deploys as the dedicated `gitea_ci` user over SSH, with secrets
-# from the repo settings and scoped sudo (see asset/sudoers.d/ + script/infra-setup.sh).
+# below — there is no manifest.yml in this model). A Gitea Actions runner deploys
+# as the dedicated `gitea_ci` user over SSH, with secrets from the repo settings
+# and scoped sudo (see asset/sudoers.d/ + script/infra-setup.sh). The nginx vhost
+# is rendered by script/render-site-conf.py, shared with refresh.yml so the two
+# never drift on what they substitute.
 #
 # The api/worker binaries are pure-rustls (no openssl), so they build as fully
 # static musl — a runner newer than the target host can't produce an unloadable
@@ -338,22 +339,10 @@ jobs:
               gitea_ci@"${WEB_HOST}" 'hostname -f'
 
       - name: Render nginx vhost
-        run: |
-          mkdir -p rendered
-          python3 - <<'PY'
-          import os
-          t = open("asset/nginx/site.conf.tmpl").read()
-          subs = {
-              "SERVER_NAME": os.environ["SERVER_NAME"],
-              "DOCROOT": os.environ["WEB_ROOT"],
-              "WEB_LISTEN": os.environ["WEB_LISTEN"],
-              "API_UPSTREAM_SCHEME": os.environ["API_UPSTREAM_SCHEME"],
-              "API_UPSTREAM_ADDR": os.environ["API_UPSTREAM_ADDR"],
-          }
-          for k, v in subs.items():
-              t = t.replace("{{%s}}" % k, v)
-          open("rendered/site.conf", "w").write(t)
-          PY
+        # Shared with refresh.yml so the two pipelines can't drift on what they
+        # substitute; fails loudly if any {{PLACEHOLDER}} lacks an env value or
+        # survives substitution, so a broken vhost never reaches oolon.
+        run: python3 script/render-site-conf.py rendered/site.conf
 
       - name: Sync static site (prerendered)
         run: |

.gitea/workflows/refresh.yml

@@ -23,6 +23,11 @@ env:
   WEB_HOST: oolon.kosherinata.internal
   SERVER_NAME: rob.tn
   WEB_ROOT: /var/www/rob.tn
+  # TCP 443 on oolon is owned by the stream SNI router; this vhost sits behind it
+  # on 127.0.0.1:14443 with PROXY protocol. Must match deploy.yml's WEB_LISTEN —
+  # render-site-conf.py fails the build if it is missing rather than shipping a
+  # literal placeholder (see the render step below and deploy.yml for the history).
+  WEB_LISTEN: 127.0.0.1:14443 ssl proxy_protocol
   API_PORT: "42424"
   API_UPSTREAM_SCHEME: http
   API_UPSTREAM_ADDR: nikola.kosherinata.internal:42424
@@ -67,16 +72,13 @@ jobs:
               gitea_ci@"${WEB_HOST}" 'hostname -f'
 
       - name: Render nginx vhost
-        run: |
-          mkdir -p rendered
-          python3 - <<'PY'
-          import os
-          t = open("asset/nginx/site.conf.tmpl").read()
-          for k in ("SERVER_NAME", "API_UPSTREAM_SCHEME", "API_UPSTREAM_ADDR"):
-              t = t.replace("{{%s}}" % k, os.environ[k])
-          t = t.replace("{{DOCROOT}}", os.environ["WEB_ROOT"])
-          open("rendered/site.conf", "w").write(t)
-          PY
+        # Shared with deploy.yml so the two pipelines can't drift on what they
+        # substitute; fails loudly if any {{PLACEHOLDER}} lacks an env value or
+        # survives substitution, so a broken vhost never reaches oolon. (This
+        # step is exactly where the drift bit: WEB_LISTEN was added to the
+        # template and deploy.yml but not here, and the nightly refresh shipped
+        # a literal `listen {{WEB_LISTEN}};` that froze every reload on oolon.)
+        run: python3 script/render-site-conf.py rendered/site.conf
 
       - name: Sync static site (prerendered)
         run: |

.gitignore

@@ -14,3 +14,7 @@
 # rendered configs (templates committed, rendered output never)
 /asset/config/*.toml
 !/asset/config/*.toml.tmpl
+
+# python bytecode (script/render-site-conf.py)
+__pycache__/
+*.pyc

CLAUDE.md

@@ -107,6 +107,11 @@ Actions, so the worker poller's tokens use the `QUERY_` prefix).
 Nginx reverse-proxies `/api/` to the API host and serves the per-route static
 files via `try_files $uri $uri/ /index.html`.
 
-`./script/deploy.sh` is the legacy operator-driven path (workstation + `pass`);
-it still works and the Gitea workflow supersedes it. Remove it once the workflow
-is validated on the live hosts.
+Both workflows render the nginx vhost through the shared `script/render-site-conf.py`
+rather than an inline substitution per workflow. It requires every `{{PLACEHOLDER}}`
+in `asset/nginx/site.conf.tmpl` to have a matching env var and refuses to emit a
+file with any placeholder left unrendered — so a variable added to the template
+but forgotten in one workflow's `env:` fails that build instead of shipping a
+broken vhost to the edge. (The former per-workflow renderers drifted exactly this
+way once: `WEB_LISTEN` reached the template and `deploy.yml` but not `refresh.yml`,
+and the nightly refresh froze every reload on oolon.)

asset/nginx/site.conf.tmpl

@@ -15,7 +15,7 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/{{SERVER_NAME}}/privkey.pem;
     ssl_protocols TLSv1.2 TLSv1.3;
 
-    root {{DOCROOT}};
+    root {{WEB_ROOT}};
     index index.html;
 
     # Compress text responses on the wire. text/html is always compressed when

readme.md

@@ -16,7 +16,7 @@ crates/
 ui/                   # vite + react + swc + typescript frontend
 asset/                # systemd, nginx, firewalld, manifest.yml
 script/
-  deploy.sh           # manifest-driven deploy to prod
+  render-site-conf.py # render the nginx vhost from env (shared by both workflows)
   hg-ingest.sh        # one-shot local hg clone + psql ingest
   certify.sh          # letsencrypt cert management
   teardown.sh         # service removal
@@ -83,14 +83,14 @@ migrations live in `crates/moments-data/migrations/` and run automatically on wo
 
 ## deployment
 
-```sh
-./script/deploy.sh <env> all          # api + worker + web
-./script/deploy.sh <env> api worker   # subset
-./script/deploy.sh <env> default      # api + web only (worker untouched)
-./script/deploy.sh <env> all --dry-run
-```
+deployment is driven by Gitea Actions, not an operator workstation:
 
-concrete hosts, ports, and the site's `server_name` live in `asset/manifest.yml`. the shape of the deployment:
+- `.gitea/workflows/deploy.yml` — on push to `main` (or manual dispatch): lint/test gate, build the api + worker as static musl binaries and the prerendered web bundle, then deploy each component over SSH as the `gitea_ci` user with scoped sudo (`asset/sudoers.d/`).
+- `.gitea/workflows/refresh.yml` — daily `schedule:` (or manual): rebuilds and redeploys only the web tier, re-baking the prerendered crawler snapshot without bouncing the api/worker.
+
+both workflows carry the infra truth (hosts, ports, paths) in their `env:` blocks and render the nginx vhost through the shared `script/render-site-conf.py`, which fails the build if any template placeholder is unset rather than shipping it. one-time per-host provisioning (the `gitea_ci` user, its `authorized_keys`, the scoped sudoers drop-in) is `script/infra-setup.sh`.
+
+the shape of the deployment:
 
 | component | notes |
 |-----------|-------|
@@ -101,7 +101,7 @@ concrete hosts, ports, and the site's `server_name` live in `asset/manifest.yml`
 
 postgres roles `moments_rw` and `moments_ro` must exist on the primary, with `pg_ident.conf.d/<host>.conf` mapping the api host's fqdn to `moments_ro` and the worker host's fqdn to `moments_rw`. see `asset/sql/bootstrap-moments.sql`, `asset/postgres/ident.conf.tmpl`, and `script/db-perms.sh`.
 
-secrets are resolved at deploy time via `pass`. the mapping of env-var name to pass-store path lives under `worker.secrets` in `manifest.yml`; `deploy.sh` iterates the map, fetches each secret, and substitutes the matching `{{NAME}}` placeholder in `worker.env.tmpl`.
+the worker's poller tokens are Gitea repo Actions secrets (`QUERY_GITHUB_TOKEN`, `QUERY_GITEA_TOKEN` — the bare `GITHUB_TOKEN`/`GITEA_TOKEN` names are reserved by Actions). `deploy.yml`'s deploy-worker job substitutes them into the matching `{{NAME}}` placeholders in `worker.env.tmpl` at deploy time; secrets come from the runner environment and never touch a command line.
 
 ## environment variables
 

script/deploy.sh

@@ -1,559 +0,0 @@
-#!/usr/bin/env bash
-#
-# moments deployment script.
-#
-#   ./script/deploy.sh <environment> [component...]
-#   ./script/deploy.sh prod api worker web
-#   ./script/deploy.sh prod all
-#
-# Builds artifacts locally, resolves secrets from `pass`, renders config
-# templates, rsyncs everything to the target hosts, and reloads systemd /
-# nginx / firewalld / SELinux state idempotently.
-
-set -euo pipefail
-shopt -s nullglob
-
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-manifest="${repo_root}/asset/manifest.yml"
-dry_run=0
-
-usage() {
-    cat <<EOF >&2
-usage: $(basename "$0") <environment> [component...] [--dry-run]
-       $(basename "$0") prod api worker web
-       $(basename "$0") prod all
-       $(basename "$0") prod default        # api + web (worker isn't restarted unless asked)
-EOF
-    exit 2
-}
-
-log()  { printf '\033[1;34m[deploy]\033[0m %s\n' "$*" >&2; }
-warn() { printf '\033[1;33m[deploy]\033[0m %s\n' "$*" >&2; }
-die()  { printf '\033[1;31m[deploy]\033[0m %s\n' "$*" >&2; exit 1; }
-
-run() {
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m %s\n' "$*" >&2
-    else
-        "$@"
-    fi
-}
-
-ssh_run() {
-    local host="$1"; shift
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m ssh %s -- %s\n' "$host" "$*" >&2
-    else
-        ssh -o BatchMode=yes "$host" "$@"
-    fi
-}
-
-# Ensure /tmp on the remote is world-writable + sticky (mode 1777). Some
-# hosts in this fleet have had /tmp reset to root-owned 0755 by an
-# unrelated configuration step, which silently breaks the rsync of the
-# deploy stage dir under our unprivileged user. Check the mode first so a
-# correctly-configured host doesn't incur a needless sudo call.
-ensure_tmp_writable() {
-    local host="$1"
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m ssh %s -- stat /tmp; chmod 1777 if needed\n' "$host" >&2
-        return 0
-    fi
-    local mode
-    mode="$(ssh -o BatchMode=yes "$host" 'stat -c %a /tmp')" || {
-        warn "could not stat /tmp on $host"
-        return 1
-    }
-    if [[ "$mode" != "1777" ]]; then
-        warn "/tmp on $host is mode $mode; fixing to 1777"
-        ssh -o BatchMode=yes "$host" 'sudo chmod 1777 /tmp' || {
-            warn "failed to chmod /tmp on $host"
-            return 1
-        }
-    fi
-}
-
-[[ $# -ge 1 ]] || usage
-environment="$1"; shift
-components=()
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --dry-run) dry_run=1 ;;
-        *)         components+=("$1") ;;
-    esac
-    shift
-done
-
-[[ -f "$manifest" ]]                    || die "manifest not found: $manifest"
-command -v yq     >/dev/null 2>&1       || die "yq is required"
-command -v pass   >/dev/null 2>&1       || die "pass is required"
-command -v rsync  >/dev/null 2>&1       || die "rsync is required"
-command -v cargo  >/dev/null 2>&1       || die "cargo is required"
-command -v podman >/dev/null 2>&1       || die "podman is required (used for the deploy build container)"
-
-# Rust binaries are built inside a Debian container so the resulting ELF
-# links against an older glibc than this workstation's. Building natively
-# on f44 (glibc 2.43) produces binaries that won't load on f42 / f43
-# servers — the dynamic loader refuses them outright. Debian bookworm's
-# glibc 2.36 is older than every Fedora release we deploy to, so its
-# binaries are forward-compatible.
-#
-# The artifacts land in target/deploy/release/ so a native `cargo build`
-# in this checkout (for tests, clippy, dev runs) doesn't compete with
-# the container for incremental state, and vice-versa.
-rust_build_image="docker.io/library/rust:1-bookworm"
-rust_target_dir="${repo_root}/target/deploy"
-
-# Resolve component list ----------------------------------------------------
-
-env_path=".environments.${environment}"
-yq --exit-status "${env_path}" "$manifest" >/dev/null \
-    || die "environment '$environment' not found in manifest"
-
-mapfile -t all_components < <(yq --raw-output "${env_path}.components | keys | .[]" "$manifest")
-
-if [[ ${#components[@]} -eq 0 ]]; then
-    usage
-fi
-case "${components[0]:-}" in
-    all)     components=("${all_components[@]}") ;;
-    default) components=(api web) ;;
-esac
-
-# Build artifacts -----------------------------------------------------------
-
-needs_rust=0
-needs_web=0
-for c in "${components[@]}"; do
-    case "$c" in
-        api|worker) needs_rust=1 ;;
-        web)        needs_web=1 ;;
-    esac
-done
-
-if (( needs_rust )); then
-    log "cargo build --release in ${rust_build_image} (api, worker)"
-    install --directory "$rust_target_dir"
-    # Named volumes cache the cargo registry and git index across runs so
-    # subsequent builds don't re-fetch every crate. CARGO_TARGET_DIR
-    # redirects build output into the host-mounted target/deploy.
-    # :Z relabels the bind mount for SELinux on Fedora hosts.
-    run podman run --rm \
-        --volume "${repo_root}:/workspace:Z" \
-        --volume moments-deploy-cargo-registry:/usr/local/cargo/registry \
-        --volume moments-deploy-cargo-git:/usr/local/cargo/git \
-        --workdir /workspace \
-        --env CARGO_TARGET_DIR=/workspace/target/deploy \
-        "$rust_build_image" \
-        cargo build --release --bin moments-api --bin moments-worker
-fi
-
-if (( needs_web )); then
-    log "vite build (ui)"
-    run sh -c "cd '${repo_root}/ui' && pnpm install --frozen-lockfile && pnpm run build"
-fi
-
-# Per-component deploy ------------------------------------------------------
-
-deploy_api() {
-    local host="$1"
-    log "api -> $host"
-
-    local bind
-    bind="$(yq --raw-output "${env_path}.components.api.config.bind" "$manifest")"
-    [[ -n "$bind" && "$bind" != "null" ]] || die "api.config.bind missing in manifest"
-    [[ "$bind" == *:* ]] \
-        || die "api.config.bind must be host:port form: '$bind'"
-
-    local api_port
-    api_port="${bind##*:}"
-    [[ "$api_port" =~ ^[0-9]+$ ]] \
-        || die "api.config.bind port is not numeric: '$api_port'"
-
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m render api.env (HOSTNAME=%s, BIND=%s) + firewalld svc (port=%s) + units, stage to %s:/tmp/, install via heredoc, run sysusers/restorecon/semanage/systemctl on %s\n' \
-            "$host" "$bind" "$api_port" "$host" "$host" >&2
-        return 0
-    fi
-
-    local fqdn="$host"
-
-    local stage
-    stage="$(mktemp --directory)"
-    trap "rm --recursive --force '$stage'" RETURN
-
-    install --directory \
-        "$stage/etc/moments" \
-        "$stage/etc/systemd/system" \
-        "$stage/etc/sysusers.d" \
-        "$stage/etc/firewalld/services" \
-        "$stage/usr/local/bin"
-
-    local rendered
-    rendered="$(<"${repo_root}/asset/config/api.env.tmpl")"
-    rendered=${rendered//'{{HOSTNAME}}'/$fqdn}
-    rendered=${rendered//'{{BIND}}'/$bind}
-    printf '%s\n' "$rendered" > "$stage/etc/moments/api.env"
-
-    rendered="$(<"${repo_root}/asset/systemd/moments-api-cert.path")"
-    rendered=${rendered//'{{HOSTNAME}}'/$fqdn}
-    printf '%s\n' "$rendered" > "$stage/etc/systemd/system/moments-api-cert.path"
-
-    rendered="$(<"${repo_root}/asset/firewalld/moments-api.xml.tmpl")"
-    rendered=${rendered//'{{API_PORT}}'/$api_port}
-    printf '%s\n' "$rendered" > "$stage/etc/firewalld/services/moments-api.xml"
-    chmod 0644 "$stage/etc/firewalld/services/moments-api.xml"
-
-    install --mode=0644 "${repo_root}/asset/systemd/moments-api.service"             "$stage/etc/systemd/system/"
-    install --mode=0644 "${repo_root}/asset/systemd/moments-api-cert-reload.service" "$stage/etc/systemd/system/"
-    install --mode=0644 "${repo_root}/asset/systemd/moments.sysusers.conf"           "$stage/etc/sysusers.d/moments.conf"
-    install --mode=0755 "${rust_target_dir}/release/moments-api"                    "$stage/usr/local/bin/moments-api"
-
-    chmod 0640 "$stage/etc/moments/api.env"
-
-    # Stage to a tmpdir on the remote, then `install` each file at its final
-    # path via the heredoc. Never rsync into /, since rsync of staged parent
-    # dirs (etc/, usr/, ...) can leak ownership, ACLs and xattrs onto the
-    # live system dirs.
-    local remote_stage="/tmp/moments-deploy.api.${$}.${RANDOM}"
-
-    ensure_tmp_writable "$host" || return 1
-
-    rsync \
-        --archive \
-        --hard-links \
-        --numeric-ids \
-        --rsh='ssh -o BatchMode=yes' \
-        "$stage/" \
-        "${host}:${remote_stage}/"
-
-    ssh_run "$host" "sudo bash -s -- ${remote_stage@Q} ${api_port@Q}" <<'REMOTE_EOF'
-set -euo pipefail
-remote_stage="$1"
-api_port="$2"
-trap 'rm --recursive --force "$remote_stage"' EXIT
-
-fqdn="$(hostname --fqdn)"
-
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/sysusers.d/moments.conf" \
-    /etc/sysusers.d/moments.conf
-systemd-sysusers /etc/sysusers.d/moments.conf
-
-install --directory --owner=root    --group=moments --mode=0750 /etc/moments
-install --directory --owner=moments --group=moments --mode=0750 /var/lib/moments
-
-install --owner=root --group=moments --mode=0640 \
-    "$remote_stage/etc/moments/api.env" \
-    /etc/moments/api.env
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-api.service" \
-    /etc/systemd/system/moments-api.service
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-api-cert.path" \
-    /etc/systemd/system/moments-api-cert.path
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-api-cert-reload.service" \
-    /etc/systemd/system/moments-api-cert-reload.service
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/firewalld/services/moments-api.xml" \
-    /etc/firewalld/services/moments-api.xml
-install --owner=root --group=root --mode=0755 \
-    "$remote_stage/usr/local/bin/moments-api" \
-    /usr/local/bin/moments-api
-
-# Grant the moments user read access to the host private key for the
-# postgres mTLS connection.
-setfacl --modify=u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true
-
-# Idempotent label: --add fails if the port is already labelled (we suppress
-# that one stderr line); --modify is then a no-op or fixes a stale type.
-semanage port --add    --type=http_port_t --proto=tcp "$api_port" 2>/dev/null \
-    || semanage port --modify --type=http_port_t --proto=tcp "$api_port"
-
-firewall-cmd --reload
-zone="$(firewall-cmd --get-default-zone)"
-if ! firewall-cmd --zone="$zone" --query-service=moments-api >/dev/null 2>&1; then
-    firewall-cmd --permanent --zone="$zone" --add-service=moments-api
-    firewall-cmd            --zone="$zone" --add-service=moments-api
-fi
-
-restorecon -Rv /usr/local/bin/moments-api /etc/moments /var/lib/moments
-
-systemctl daemon-reload
-systemctl enable --now moments-api-cert.path
-systemctl enable --now moments-api.service
-systemctl restart    moments-api.service
-
-# Quietly retry while the service binds; only show curl's diagnostics if
-# every attempt fails. The journalctl tail on failure is the verbose source.
-for i in 1 2 3 4 5 6 7 8 9 10; do
-    if curl --fail --silent "http://${fqdn}:${api_port}/v1/healthz" >/dev/null 2>&1; then
-        echo "moments-api healthy"
-        exit 0
-    fi
-    sleep 1
-done
-echo "moments-api did not become healthy" >&2
-curl --fail --silent --show-error "http://${fqdn}:${api_port}/v1/healthz" >/dev/null || true
-journalctl --unit=moments-api.service --lines=50 --no-pager >&2
-exit 1
-REMOTE_EOF
-}
-
-deploy_worker() {
-    local host="$1"
-    log "worker -> $host"
-
-    # Manifest entries under `worker.secrets` map env-var name -> pass store path.
-    # The script fetches each via `pass` and substitutes the matching {{NAME}}
-    # placeholder in worker.env.tmpl. Adding a new secret is then a manifest +
-    # template change; no script edit required.
-    local -a secret_lines secret_keys
-    mapfile -t secret_lines < <(yq --raw-output \
-        "${env_path}.components.worker.secrets // {} | to_entries | .[] | \"\(.key)=\(.value)\"" \
-        "$manifest")
-    local line
-    for line in "${secret_lines[@]}"; do
-        [[ -n "$line" ]] && secret_keys+=("${line%%=*}")
-    done
-
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m render worker.env (HOSTNAME=%s, secrets [%s] from pass) + units, stage to %s:/tmp/, install via heredoc, run sysusers/restorecon/systemctl on %s\n' \
-            "$host" "${secret_keys[*]:-none}" "$host" "$host" >&2
-        return 0
-    fi
-
-    local fqdn="$host"
-
-    local stage
-    stage="$(mktemp --directory)"
-    trap "rm --recursive --force '$stage'" RETURN
-
-    install --directory \
-        "$stage/etc/moments" \
-        "$stage/etc/systemd/system" \
-        "$stage/etc/sysusers.d" \
-        "$stage/usr/local/bin"
-
-    # Render templates in-memory so secrets never appear on a command line
-    # (sed would expose them to anything that can read /proc/<pid>/cmdline).
-    local rendered
-    rendered="$(<"${repo_root}/asset/config/worker.env.tmpl")"
-    rendered=${rendered//'{{HOSTNAME}}'/$fqdn}
-    local key pass_path value
-    for line in "${secret_lines[@]}"; do
-        [[ -z "$line" ]] && continue
-        key="${line%%=*}"
-        pass_path="${line#*=}"
-        if pass show "$pass_path" >/dev/null 2>&1; then
-            value="$(pass show "$pass_path")"
-        else
-            warn "no secret in pass at '${pass_path}' for ${key}; worker will run without ${key}"
-            value=""
-        fi
-        rendered=${rendered//"{{${key}}}"/$value}
-    done
-    printf '%s\n' "$rendered" > "$stage/etc/moments/worker.env"
-
-    rendered="$(<"${repo_root}/asset/systemd/moments-worker-cert.path")"
-    rendered=${rendered//'{{HOSTNAME}}'/$fqdn}
-    printf '%s\n' "$rendered" > "$stage/etc/systemd/system/moments-worker-cert.path"
-
-    install --mode=0644 "${repo_root}/asset/systemd/moments-worker.service"             "$stage/etc/systemd/system/"
-    install --mode=0644 "${repo_root}/asset/systemd/moments-worker-cert-reload.service" "$stage/etc/systemd/system/"
-    install --mode=0644 "${repo_root}/asset/systemd/moments.sysusers.conf"              "$stage/etc/sysusers.d/moments.conf"
-    install --mode=0755 "${rust_target_dir}/release/moments-worker"                    "$stage/usr/local/bin/moments-worker"
-
-    chmod 0640 "$stage/etc/moments/worker.env"
-
-    # Stage to a tmpdir on the remote, then `install` each file at its final
-    # path via the heredoc. Never rsync into /.
-    local remote_stage="/tmp/moments-deploy.worker.${$}.${RANDOM}"
-
-    ensure_tmp_writable "$host" || return 1
-
-    rsync \
-        --archive \
-        --hard-links \
-        --numeric-ids \
-        --rsh='ssh -o BatchMode=yes' \
-        "$stage/" \
-        "${host}:${remote_stage}/"
-
-    ssh_run "$host" "sudo bash -s -- ${remote_stage@Q}" <<'REMOTE_EOF'
-set -euo pipefail
-remote_stage="$1"
-trap 'rm --recursive --force "$remote_stage"' EXIT
-
-fqdn="$(hostname --fqdn)"
-
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/sysusers.d/moments.conf" \
-    /etc/sysusers.d/moments.conf
-systemd-sysusers /etc/sysusers.d/moments.conf
-
-install --directory --owner=root    --group=moments --mode=0750 /etc/moments
-install --directory --owner=moments --group=moments --mode=0750 /var/lib/moments
-
-install --owner=root --group=moments --mode=0640 \
-    "$remote_stage/etc/moments/worker.env" \
-    /etc/moments/worker.env
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-worker.service" \
-    /etc/systemd/system/moments-worker.service
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-worker-cert.path" \
-    /etc/systemd/system/moments-worker-cert.path
-install --owner=root --group=root --mode=0644 \
-    "$remote_stage/etc/systemd/system/moments-worker-cert-reload.service" \
-    /etc/systemd/system/moments-worker-cert-reload.service
-install --owner=root --group=root --mode=0755 \
-    "$remote_stage/usr/local/bin/moments-worker" \
-    /usr/local/bin/moments-worker
-
-setfacl --modify=u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true
-
-restorecon -Rv /usr/local/bin/moments-worker /etc/moments /var/lib/moments
-
-systemctl daemon-reload
-systemctl enable --now moments-worker-cert.path
-systemctl enable --now moments-worker.service
-systemctl restart    moments-worker.service
-
-if ! systemctl is-active --quiet moments-worker.service; then
-    journalctl --unit=moments-worker.service --lines=50 --no-pager >&2
-    exit 1
-fi
-echo "moments-worker active"
-REMOTE_EOF
-}
-
-deploy_web() {
-    local host="$1"
-    log "web -> $host"
-
-    local server_name web_root api_upstream web_listen
-    server_name="$(yq --raw-output "${env_path}.components.web.config.server_name" "$manifest")"
-    web_root="$(yq    --raw-output "${env_path}.components.web.config.root"        "$manifest")"
-    api_upstream="$(yq --raw-output "${env_path}.components.web.config.api_upstream" "$manifest")"
-    # The edge's stream SNI router owns TCP 443; vhosts sit behind it on 14443.
-    # Binding 443 here collides with the router — `nginx -t` passes, but every
-    # later reload aborts while systemctl still reports success. See the
-    # web.config.listen note in asset/manifest.yml.
-    web_listen="$(yq --raw-output "${env_path}.components.web.config.listen" "$manifest")"
-    [[ -n "$web_listen" && "$web_listen" != "null" ]] \
-        || web_listen='127.0.0.1:14443 ssl proxy_protocol'
-    [[ -n "$server_name"  && "$server_name"  != "null" ]] || die "web.config.server_name missing in manifest"
-    [[ -n "$web_root"     && "$web_root"     != "null" ]] || die "web.config.root missing in manifest"
-    [[ -n "$api_upstream" && "$api_upstream" != "null" ]] || die "web.config.api_upstream missing in manifest"
-    [[ "$web_root" == /* ]] \
-        || die "web.config.root must be an absolute path: '$web_root'"
-    [[ "$api_upstream" == http://*  || "$api_upstream" == https://* ]] \
-        || die "web.config.api_upstream must be a http(s) URL: '$api_upstream'"
-
-    local api_upstream_scheme api_upstream_addr api_upstream_port
-    api_upstream_scheme="${api_upstream%%://*}"
-    api_upstream_addr="${api_upstream#*://}"
-    [[ "$api_upstream_addr" == *:* ]] \
-        || die "web.config.api_upstream must include an explicit port: '$api_upstream'"
-    api_upstream_port="${api_upstream_addr##*:}"
-    [[ "$api_upstream_port" =~ ^[0-9]+$ ]] \
-        || die "extracted upstream port is not numeric: '$api_upstream_port'"
-
-    local site_conf_path="/etc/nginx/conf.d/${server_name}.conf"
-
-    if (( dry_run )); then
-        printf '\033[2m[dry-run]\033[0m render %s (server_name=%s, docroot=%s, upstream=%s://%s) + rsync ui/dist/ to %s:%s/, run nginx -t/reload on %s\n' \
-            "$site_conf_path" "$server_name" "$web_root" \
-            "$api_upstream_scheme" "$api_upstream_addr" \
-            "$host" "$web_root" "$host" >&2
-        return 0
-    fi
-
-    local stage
-    stage="$(mktemp --directory)"
-    trap "rm --recursive --force '$stage'" RETURN
-
-    install --directory "${stage}${web_root}" "$stage/etc/nginx/conf.d"
-
-    rsync --archive "${repo_root}/ui/dist/" "${stage}${web_root}/"
-
-    local rendered
-    rendered="$(<"${repo_root}/asset/nginx/site.conf.tmpl")"
-    rendered=${rendered//'{{SERVER_NAME}}'/$server_name}
-    rendered=${rendered//'{{WEB_LISTEN}}'/$web_listen}
-    rendered=${rendered//'{{DOCROOT}}'/$web_root}
-    rendered=${rendered//'{{API_UPSTREAM_SCHEME}}'/$api_upstream_scheme}
-    rendered=${rendered//'{{API_UPSTREAM_ADDR}}'/$api_upstream_addr}
-    printf '%s\n' "$rendered" > "${stage}${site_conf_path}"
-    chmod 0644 "${stage}${site_conf_path}"
-
-    # Both targets are leaf paths (the docroot itself, and a single named
-    # file) so rsync does not traverse /var or /etc parents — `--chown` is
-    # enough; -A/-X are intentionally absent.
-    rsync \
-        --archive \
-        --hard-links \
-        --numeric-ids \
-        --chown root:root \
-        --rsh='ssh -o BatchMode=yes' \
-        --rsync-path 'sudo rsync' \
-        --delete \
-        "${stage}${web_root}/" \
-        "${host}:${web_root}/"
-    rsync \
-        --archive \
-        --hard-links \
-        --numeric-ids \
-        --chown root:root \
-        --rsh='ssh -o BatchMode=yes' \
-        --rsync-path 'sudo rsync' \
-        "${stage}${site_conf_path}" \
-        "${host}:${site_conf_path}"
-
-    ssh_run "$host" "sudo bash -s -- ${web_root@Q} ${site_conf_path@Q} ${api_upstream_port@Q}" <<'REMOTE_EOF'
-set -euo pipefail
-web_root="$1"
-site_conf_path="$2"
-api_upstream_port="$3"
-
-# Allow nginx to make outbound connections to the moments-api upstream
-# across the WG mesh.
-setsebool -P httpd_can_network_connect on
-
-# Idempotent label: --add fails if the port is already labelled (we suppress
-# that one stderr line); --modify is then a no-op or fixes a stale type.
-semanage port --add    --type=http_port_t --proto=tcp "$api_upstream_port" 2>/dev/null \
-    || semanage port --modify --type=http_port_t --proto=tcp "$api_upstream_port"
-
-restorecon -Rv "$web_root" "$site_conf_path"
-
-if ! nginx -t; then
-    echo "nginx config check failed" >&2
-    exit 1
-fi
-systemctl reload nginx
-echo "nginx reloaded"
-REMOTE_EOF
-}
-
-# Dispatch ------------------------------------------------------------------
-
-failed=()
-for component in "${components[@]}"; do
-    mapfile -t hosts < <(yq --raw-output "${env_path}.components.${component}.hosts[]" "$manifest")
-    for host in "${hosts[@]}"; do
-        case "$component" in
-            api)    deploy_api    "$host" || failed+=("api@$host") ;;
-            worker) deploy_worker "$host" || failed+=("worker@$host") ;;
-            web)    deploy_web    "$host" || failed+=("web@$host") ;;
-            *)      warn "unknown component: $component" ;;
-        esac
-    done
-done
-
-if [[ ${#failed[@]} -gt 0 ]]; then
-    die "failed: ${failed[*]}"
-fi
-log "deploy complete"

script/render-site-conf.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Render the nginx vhost (asset/nginx/site.conf.tmpl) from the environment.
+
+Both .gitea/workflows/deploy.yml (deploy-web) and refresh.yml call this, so the
+two pipelines can never disagree about what they substitute. They did once: the
+WEB_LISTEN placeholder was added to the template and to deploy.yml but not to
+refresh.yml, so the nightly refresh shipped a literal `listen {{WEB_LISTEN}};`
+to oolon. `nginx -t` then failed for the whole edge, and because the file is
+rsynced straight into the live conf.d before it is tested, every vhost's reload
+-- including the step@ cert renewals -- stayed frozen for days while renewed
+certs piled up unserved on disk.
+
+Guard rails, so that can't recur:
+  * every {{PLACEHOLDER}} in the template must have a matching environment
+    variable, or the render fails before anything leaves the runner;
+  * no {{...}} may survive substitution.
+A forgotten or misnamed variable is now a red build, not a broken edge.
+
+usage: render-site-conf.py [OUTPUT]   (default: rendered/site.conf)
+"""
+import os
+import re
+import sys
+
+TEMPLATE = "asset/nginx/site.conf.tmpl"
+PLACEHOLDER = re.compile(r"\{\{(\w+)\}\}")
+
+
+def main() -> int:
+    out = sys.argv[1] if len(sys.argv) > 1 else "rendered/site.conf"
+    with open(TEMPLATE, encoding="utf-8") as fh:
+        text = fh.read()
+
+    names = sorted(set(PLACEHOLDER.findall(text)))
+    missing = [n for n in names if n not in os.environ]
+    if missing:
+        sys.stderr.write(
+            "render-site-conf: no environment value for placeholder(s): "
+            + ", ".join(missing) + "\n")
+        return 1
+
+    for name in names:
+        text = text.replace("{{%s}}" % name, os.environ[name])
+
+    leftover = sorted(set(PLACEHOLDER.findall(text)))
+    if leftover:
+        sys.stderr.write(
+            "render-site-conf: unrendered placeholder(s) after substitution: "
+            + ", ".join(leftover) + "\n")
+        return 1
+
+    os.makedirs(os.path.dirname(out) or ".", exist_ok=True)
+    with open(out, "w", encoding="utf-8") as fh:
+        fh.write(text)
+    sys.stderr.write(
+        "render-site-conf: wrote %s (%d substitutions: %s)\n"
+        % (out, len(names), ", ".join(names)))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())