From 3260bfb35b8a6320c1201100853467f3aab66719 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Sun, 26 Jul 2026 15:52:18 +0300 Subject: [PATCH] fix(ci): share one nginx-vhost renderer; drop unused deploy.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly refresh.yml and deploy.yml each substituted asset/nginx/ site.conf.tmpl with their own inline python. When bb2f5b1 templated the listen line as {{WEB_LISTEN}} (moving the vhost behind oolon's stream SNI router), it added the substitution to the template and deploy.yml but not to refresh.yml. The daily refresh then rsynced a literal `listen {{WEB_LISTEN}};` into /etc/nginx/conf.d/rob.tn.conf, `nginx -t` failed for the whole edge, and — because the file is written into the live conf.d before it is tested — every vhost's reload (including the step@ cert renewals) stayed frozen. Internal vhosts, cichlid.internal among them, served certs that had expired days earlier while the renewed certs sat unused on disk. - Replace both inline renderers with script/render-site-conf.py, shared by deploy.yml and refresh.yml so they cannot drift on what they substitute. - Guard rails: the renderer fails if any {{PLACEHOLDER}} lacks an env value or survives substitution, so a forgotten/misnamed variable is a red build on the runner instead of a broken vhost on the edge. - Add the missing WEB_LISTEN to refresh.yml's env (the immediate drift). - Rename the template's {{DOCROOT}} to {{WEB_ROOT}} so every placeholder maps to the env var of the same name. - Remove script/deploy.sh: the third, unused renderer of the same template (superseded by the Actions workflows) and a standing source of drift. - Docs (readme, CLAUDE.md) updated to the Actions-only deploy path. Known follow-up (needs a sudoers change + infra-setup re-run on oolon, so out of scope here): the rendered vhost is still rsynced straight into the live conf.d and only then `nginx -t`'d, so a valid-but-wrong config could still wedge nginx. Stage-validate-swap with rollback would close that. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01QsH1rcWQYtRVhvaftiKm22 --- .gitea/workflows/deploy.yml | 29 +- .gitea/workflows/refresh.yml | 22 +- .gitignore | 4 + CLAUDE.md | 11 +- asset/nginx/site.conf.tmpl | 2 +- readme.md | 18 +- script/deploy.sh | 559 ----------------------------------- script/render-site-conf.py | 62 ++++ 8 files changed, 105 insertions(+), 602 deletions(-) delete mode 100755 script/deploy.sh create mode 100644 script/render-site-conf.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 19d234c..f44795f 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -6,10 +6,11 @@ name: deploy # - web (prerendered static SPA + nginx vhost) -> oolon # # This workflow is the source of infra truth (hosts, ports, paths live in `env` -# below — there is no manifest.yml in this model). It replaces script/deploy.sh: -# instead of an operator running it from a workstation with `pass`, a Gitea -# Actions runner deploys as the dedicated `gitea_ci` user over SSH, with secrets -# from the repo settings and scoped sudo (see asset/sudoers.d/ + script/infra-setup.sh). +# below — there is no manifest.yml in this model). A Gitea Actions runner deploys +# as the dedicated `gitea_ci` user over SSH, with secrets from the repo settings +# and scoped sudo (see asset/sudoers.d/ + script/infra-setup.sh). The nginx vhost +# is rendered by script/render-site-conf.py, shared with refresh.yml so the two +# never drift on what they substitute. # # The api/worker binaries are pure-rustls (no openssl), so they build as fully # static musl — a runner newer than the target host can't produce an unloadable @@ -338,22 +339,10 @@ jobs: gitea_ci@"${WEB_HOST}" 'hostname -f' - name: Render nginx vhost - run: | - mkdir -p rendered - python3 - <<'PY' - import os - t = open("asset/nginx/site.conf.tmpl").read() - subs = { - "SERVER_NAME": os.environ["SERVER_NAME"], - "DOCROOT": os.environ["WEB_ROOT"], - "WEB_LISTEN": os.environ["WEB_LISTEN"], - "API_UPSTREAM_SCHEME": os.environ["API_UPSTREAM_SCHEME"], - "API_UPSTREAM_ADDR": os.environ["API_UPSTREAM_ADDR"], - } - for k, v in subs.items(): - t = t.replace("{{%s}}" % k, v) - open("rendered/site.conf", "w").write(t) - PY + # Shared with refresh.yml so the two pipelines can't drift on what they + # substitute; fails loudly if any {{PLACEHOLDER}} lacks an env value or + # survives substitution, so a broken vhost never reaches oolon. + run: python3 script/render-site-conf.py rendered/site.conf - name: Sync static site (prerendered) run: | diff --git a/.gitea/workflows/refresh.yml b/.gitea/workflows/refresh.yml index b9063b7..cd2ba52 100644 --- a/.gitea/workflows/refresh.yml +++ b/.gitea/workflows/refresh.yml @@ -23,6 +23,11 @@ env: WEB_HOST: oolon.kosherinata.internal SERVER_NAME: rob.tn WEB_ROOT: /var/www/rob.tn + # TCP 443 on oolon is owned by the stream SNI router; this vhost sits behind it + # on 127.0.0.1:14443 with PROXY protocol. Must match deploy.yml's WEB_LISTEN — + # render-site-conf.py fails the build if it is missing rather than shipping a + # literal placeholder (see the render step below and deploy.yml for the history). + WEB_LISTEN: 127.0.0.1:14443 ssl proxy_protocol API_PORT: "42424" API_UPSTREAM_SCHEME: http API_UPSTREAM_ADDR: nikola.kosherinata.internal:42424 @@ -67,16 +72,13 @@ jobs: gitea_ci@"${WEB_HOST}" 'hostname -f' - name: Render nginx vhost - run: | - mkdir -p rendered - python3 - <<'PY' - import os - t = open("asset/nginx/site.conf.tmpl").read() - for k in ("SERVER_NAME", "API_UPSTREAM_SCHEME", "API_UPSTREAM_ADDR"): - t = t.replace("{{%s}}" % k, os.environ[k]) - t = t.replace("{{DOCROOT}}", os.environ["WEB_ROOT"]) - open("rendered/site.conf", "w").write(t) - PY + # Shared with deploy.yml so the two pipelines can't drift on what they + # substitute; fails loudly if any {{PLACEHOLDER}} lacks an env value or + # survives substitution, so a broken vhost never reaches oolon. (This + # step is exactly where the drift bit: WEB_LISTEN was added to the + # template and deploy.yml but not here, and the nightly refresh shipped + # a literal `listen {{WEB_LISTEN}};` that froze every reload on oolon.) + run: python3 script/render-site-conf.py rendered/site.conf - name: Sync static site (prerendered) run: | diff --git a/.gitignore b/.gitignore index a70f837..f65824a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ # rendered configs (templates committed, rendered output never) /asset/config/*.toml !/asset/config/*.toml.tmpl + +# python bytecode (script/render-site-conf.py) +__pycache__/ +*.pyc diff --git a/CLAUDE.md b/CLAUDE.md index 8894a14..3734954 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,6 +107,11 @@ Actions, so the worker poller's tokens use the `QUERY_` prefix). Nginx reverse-proxies `/api/` to the API host and serves the per-route static files via `try_files $uri $uri/ /index.html`. -`./script/deploy.sh` is the legacy operator-driven path (workstation + `pass`); -it still works and the Gitea workflow supersedes it. Remove it once the workflow -is validated on the live hosts. +Both workflows render the nginx vhost through the shared `script/render-site-conf.py` +rather than an inline substitution per workflow. It requires every `{{PLACEHOLDER}}` +in `asset/nginx/site.conf.tmpl` to have a matching env var and refuses to emit a +file with any placeholder left unrendered — so a variable added to the template +but forgotten in one workflow's `env:` fails that build instead of shipping a +broken vhost to the edge. (The former per-workflow renderers drifted exactly this +way once: `WEB_LISTEN` reached the template and `deploy.yml` but not `refresh.yml`, +and the nightly refresh froze every reload on oolon.) diff --git a/asset/nginx/site.conf.tmpl b/asset/nginx/site.conf.tmpl index 1d62c1f..05594f0 100644 --- a/asset/nginx/site.conf.tmpl +++ b/asset/nginx/site.conf.tmpl @@ -15,7 +15,7 @@ server { ssl_certificate_key /etc/letsencrypt/live/{{SERVER_NAME}}/privkey.pem; ssl_protocols TLSv1.2 TLSv1.3; - root {{DOCROOT}}; + root {{WEB_ROOT}}; index index.html; # Compress text responses on the wire. text/html is always compressed when diff --git a/readme.md b/readme.md index 5bba4e8..10a9708 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ crates/ ui/ # vite + react + swc + typescript frontend asset/ # systemd, nginx, firewalld, manifest.yml script/ - deploy.sh # manifest-driven deploy to prod + render-site-conf.py # render the nginx vhost from env (shared by both workflows) hg-ingest.sh # one-shot local hg clone + psql ingest certify.sh # letsencrypt cert management teardown.sh # service removal @@ -83,14 +83,14 @@ migrations live in `crates/moments-data/migrations/` and run automatically on wo ## deployment -```sh -./script/deploy.sh all # api + worker + web -./script/deploy.sh api worker # subset -./script/deploy.sh default # api + web only (worker untouched) -./script/deploy.sh all --dry-run -``` +deployment is driven by Gitea Actions, not an operator workstation: -concrete hosts, ports, and the site's `server_name` live in `asset/manifest.yml`. the shape of the deployment: +- `.gitea/workflows/deploy.yml` — on push to `main` (or manual dispatch): lint/test gate, build the api + worker as static musl binaries and the prerendered web bundle, then deploy each component over SSH as the `gitea_ci` user with scoped sudo (`asset/sudoers.d/`). +- `.gitea/workflows/refresh.yml` — daily `schedule:` (or manual): rebuilds and redeploys only the web tier, re-baking the prerendered crawler snapshot without bouncing the api/worker. + +both workflows carry the infra truth (hosts, ports, paths) in their `env:` blocks and render the nginx vhost through the shared `script/render-site-conf.py`, which fails the build if any template placeholder is unset rather than shipping it. one-time per-host provisioning (the `gitea_ci` user, its `authorized_keys`, the scoped sudoers drop-in) is `script/infra-setup.sh`. + +the shape of the deployment: | component | notes | |-----------|-------| @@ -101,7 +101,7 @@ concrete hosts, ports, and the site's `server_name` live in `asset/manifest.yml` postgres roles `moments_rw` and `moments_ro` must exist on the primary, with `pg_ident.conf.d/.conf` mapping the api host's fqdn to `moments_ro` and the worker host's fqdn to `moments_rw`. see `asset/sql/bootstrap-moments.sql`, `asset/postgres/ident.conf.tmpl`, and `script/db-perms.sh`. -secrets are resolved at deploy time via `pass`. the mapping of env-var name to pass-store path lives under `worker.secrets` in `manifest.yml`; `deploy.sh` iterates the map, fetches each secret, and substitutes the matching `{{NAME}}` placeholder in `worker.env.tmpl`. +the worker's poller tokens are Gitea repo Actions secrets (`QUERY_GITHUB_TOKEN`, `QUERY_GITEA_TOKEN` — the bare `GITHUB_TOKEN`/`GITEA_TOKEN` names are reserved by Actions). `deploy.yml`'s deploy-worker job substitutes them into the matching `{{NAME}}` placeholders in `worker.env.tmpl` at deploy time; secrets come from the runner environment and never touch a command line. ## environment variables diff --git a/script/deploy.sh b/script/deploy.sh deleted file mode 100755 index 1479933..0000000 --- a/script/deploy.sh +++ /dev/null @@ -1,559 +0,0 @@ -#!/usr/bin/env bash -# -# moments deployment script. -# -# ./script/deploy.sh [component...] -# ./script/deploy.sh prod api worker web -# ./script/deploy.sh prod all -# -# Builds artifacts locally, resolves secrets from `pass`, renders config -# templates, rsyncs everything to the target hosts, and reloads systemd / -# nginx / firewalld / SELinux state idempotently. - -set -euo pipefail -shopt -s nullglob - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -manifest="${repo_root}/asset/manifest.yml" -dry_run=0 - -usage() { - cat <&2 -usage: $(basename "$0") [component...] [--dry-run] - $(basename "$0") prod api worker web - $(basename "$0") prod all - $(basename "$0") prod default # api + web (worker isn't restarted unless asked) -EOF - exit 2 -} - -log() { printf '\033[1;34m[deploy]\033[0m %s\n' "$*" >&2; } -warn() { printf '\033[1;33m[deploy]\033[0m %s\n' "$*" >&2; } -die() { printf '\033[1;31m[deploy]\033[0m %s\n' "$*" >&2; exit 1; } - -run() { - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m %s\n' "$*" >&2 - else - "$@" - fi -} - -ssh_run() { - local host="$1"; shift - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m ssh %s -- %s\n' "$host" "$*" >&2 - else - ssh -o BatchMode=yes "$host" "$@" - fi -} - -# Ensure /tmp on the remote is world-writable + sticky (mode 1777). Some -# hosts in this fleet have had /tmp reset to root-owned 0755 by an -# unrelated configuration step, which silently breaks the rsync of the -# deploy stage dir under our unprivileged user. Check the mode first so a -# correctly-configured host doesn't incur a needless sudo call. -ensure_tmp_writable() { - local host="$1" - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m ssh %s -- stat /tmp; chmod 1777 if needed\n' "$host" >&2 - return 0 - fi - local mode - mode="$(ssh -o BatchMode=yes "$host" 'stat -c %a /tmp')" || { - warn "could not stat /tmp on $host" - return 1 - } - if [[ "$mode" != "1777" ]]; then - warn "/tmp on $host is mode $mode; fixing to 1777" - ssh -o BatchMode=yes "$host" 'sudo chmod 1777 /tmp' || { - warn "failed to chmod /tmp on $host" - return 1 - } - fi -} - -[[ $# -ge 1 ]] || usage -environment="$1"; shift -components=() -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) dry_run=1 ;; - *) components+=("$1") ;; - esac - shift -done - -[[ -f "$manifest" ]] || die "manifest not found: $manifest" -command -v yq >/dev/null 2>&1 || die "yq is required" -command -v pass >/dev/null 2>&1 || die "pass is required" -command -v rsync >/dev/null 2>&1 || die "rsync is required" -command -v cargo >/dev/null 2>&1 || die "cargo is required" -command -v podman >/dev/null 2>&1 || die "podman is required (used for the deploy build container)" - -# Rust binaries are built inside a Debian container so the resulting ELF -# links against an older glibc than this workstation's. Building natively -# on f44 (glibc 2.43) produces binaries that won't load on f42 / f43 -# servers — the dynamic loader refuses them outright. Debian bookworm's -# glibc 2.36 is older than every Fedora release we deploy to, so its -# binaries are forward-compatible. -# -# The artifacts land in target/deploy/release/ so a native `cargo build` -# in this checkout (for tests, clippy, dev runs) doesn't compete with -# the container for incremental state, and vice-versa. -rust_build_image="docker.io/library/rust:1-bookworm" -rust_target_dir="${repo_root}/target/deploy" - -# Resolve component list ---------------------------------------------------- - -env_path=".environments.${environment}" -yq --exit-status "${env_path}" "$manifest" >/dev/null \ - || die "environment '$environment' not found in manifest" - -mapfile -t all_components < <(yq --raw-output "${env_path}.components | keys | .[]" "$manifest") - -if [[ ${#components[@]} -eq 0 ]]; then - usage -fi -case "${components[0]:-}" in - all) components=("${all_components[@]}") ;; - default) components=(api web) ;; -esac - -# Build artifacts ----------------------------------------------------------- - -needs_rust=0 -needs_web=0 -for c in "${components[@]}"; do - case "$c" in - api|worker) needs_rust=1 ;; - web) needs_web=1 ;; - esac -done - -if (( needs_rust )); then - log "cargo build --release in ${rust_build_image} (api, worker)" - install --directory "$rust_target_dir" - # Named volumes cache the cargo registry and git index across runs so - # subsequent builds don't re-fetch every crate. CARGO_TARGET_DIR - # redirects build output into the host-mounted target/deploy. - # :Z relabels the bind mount for SELinux on Fedora hosts. - run podman run --rm \ - --volume "${repo_root}:/workspace:Z" \ - --volume moments-deploy-cargo-registry:/usr/local/cargo/registry \ - --volume moments-deploy-cargo-git:/usr/local/cargo/git \ - --workdir /workspace \ - --env CARGO_TARGET_DIR=/workspace/target/deploy \ - "$rust_build_image" \ - cargo build --release --bin moments-api --bin moments-worker -fi - -if (( needs_web )); then - log "vite build (ui)" - run sh -c "cd '${repo_root}/ui' && pnpm install --frozen-lockfile && pnpm run build" -fi - -# Per-component deploy ------------------------------------------------------ - -deploy_api() { - local host="$1" - log "api -> $host" - - local bind - bind="$(yq --raw-output "${env_path}.components.api.config.bind" "$manifest")" - [[ -n "$bind" && "$bind" != "null" ]] || die "api.config.bind missing in manifest" - [[ "$bind" == *:* ]] \ - || die "api.config.bind must be host:port form: '$bind'" - - local api_port - api_port="${bind##*:}" - [[ "$api_port" =~ ^[0-9]+$ ]] \ - || die "api.config.bind port is not numeric: '$api_port'" - - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m render api.env (HOSTNAME=%s, BIND=%s) + firewalld svc (port=%s) + units, stage to %s:/tmp/, install via heredoc, run sysusers/restorecon/semanage/systemctl on %s\n' \ - "$host" "$bind" "$api_port" "$host" "$host" >&2 - return 0 - fi - - local fqdn="$host" - - local stage - stage="$(mktemp --directory)" - trap "rm --recursive --force '$stage'" RETURN - - install --directory \ - "$stage/etc/moments" \ - "$stage/etc/systemd/system" \ - "$stage/etc/sysusers.d" \ - "$stage/etc/firewalld/services" \ - "$stage/usr/local/bin" - - local rendered - rendered="$(<"${repo_root}/asset/config/api.env.tmpl")" - rendered=${rendered//'{{HOSTNAME}}'/$fqdn} - rendered=${rendered//'{{BIND}}'/$bind} - printf '%s\n' "$rendered" > "$stage/etc/moments/api.env" - - rendered="$(<"${repo_root}/asset/systemd/moments-api-cert.path")" - rendered=${rendered//'{{HOSTNAME}}'/$fqdn} - printf '%s\n' "$rendered" > "$stage/etc/systemd/system/moments-api-cert.path" - - rendered="$(<"${repo_root}/asset/firewalld/moments-api.xml.tmpl")" - rendered=${rendered//'{{API_PORT}}'/$api_port} - printf '%s\n' "$rendered" > "$stage/etc/firewalld/services/moments-api.xml" - chmod 0644 "$stage/etc/firewalld/services/moments-api.xml" - - install --mode=0644 "${repo_root}/asset/systemd/moments-api.service" "$stage/etc/systemd/system/" - install --mode=0644 "${repo_root}/asset/systemd/moments-api-cert-reload.service" "$stage/etc/systemd/system/" - install --mode=0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf" - install --mode=0755 "${rust_target_dir}/release/moments-api" "$stage/usr/local/bin/moments-api" - - chmod 0640 "$stage/etc/moments/api.env" - - # Stage to a tmpdir on the remote, then `install` each file at its final - # path via the heredoc. Never rsync into /, since rsync of staged parent - # dirs (etc/, usr/, ...) can leak ownership, ACLs and xattrs onto the - # live system dirs. - local remote_stage="/tmp/moments-deploy.api.${$}.${RANDOM}" - - ensure_tmp_writable "$host" || return 1 - - rsync \ - --archive \ - --hard-links \ - --numeric-ids \ - --rsh='ssh -o BatchMode=yes' \ - "$stage/" \ - "${host}:${remote_stage}/" - - ssh_run "$host" "sudo bash -s -- ${remote_stage@Q} ${api_port@Q}" <<'REMOTE_EOF' -set -euo pipefail -remote_stage="$1" -api_port="$2" -trap 'rm --recursive --force "$remote_stage"' EXIT - -fqdn="$(hostname --fqdn)" - -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/sysusers.d/moments.conf" \ - /etc/sysusers.d/moments.conf -systemd-sysusers /etc/sysusers.d/moments.conf - -install --directory --owner=root --group=moments --mode=0750 /etc/moments -install --directory --owner=moments --group=moments --mode=0750 /var/lib/moments - -install --owner=root --group=moments --mode=0640 \ - "$remote_stage/etc/moments/api.env" \ - /etc/moments/api.env -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-api.service" \ - /etc/systemd/system/moments-api.service -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-api-cert.path" \ - /etc/systemd/system/moments-api-cert.path -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-api-cert-reload.service" \ - /etc/systemd/system/moments-api-cert-reload.service -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/firewalld/services/moments-api.xml" \ - /etc/firewalld/services/moments-api.xml -install --owner=root --group=root --mode=0755 \ - "$remote_stage/usr/local/bin/moments-api" \ - /usr/local/bin/moments-api - -# Grant the moments user read access to the host private key for the -# postgres mTLS connection. -setfacl --modify=u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true - -# Idempotent label: --add fails if the port is already labelled (we suppress -# that one stderr line); --modify is then a no-op or fixes a stale type. -semanage port --add --type=http_port_t --proto=tcp "$api_port" 2>/dev/null \ - || semanage port --modify --type=http_port_t --proto=tcp "$api_port" - -firewall-cmd --reload -zone="$(firewall-cmd --get-default-zone)" -if ! firewall-cmd --zone="$zone" --query-service=moments-api >/dev/null 2>&1; then - firewall-cmd --permanent --zone="$zone" --add-service=moments-api - firewall-cmd --zone="$zone" --add-service=moments-api -fi - -restorecon -Rv /usr/local/bin/moments-api /etc/moments /var/lib/moments - -systemctl daemon-reload -systemctl enable --now moments-api-cert.path -systemctl enable --now moments-api.service -systemctl restart moments-api.service - -# Quietly retry while the service binds; only show curl's diagnostics if -# every attempt fails. The journalctl tail on failure is the verbose source. -for i in 1 2 3 4 5 6 7 8 9 10; do - if curl --fail --silent "http://${fqdn}:${api_port}/v1/healthz" >/dev/null 2>&1; then - echo "moments-api healthy" - exit 0 - fi - sleep 1 -done -echo "moments-api did not become healthy" >&2 -curl --fail --silent --show-error "http://${fqdn}:${api_port}/v1/healthz" >/dev/null || true -journalctl --unit=moments-api.service --lines=50 --no-pager >&2 -exit 1 -REMOTE_EOF -} - -deploy_worker() { - local host="$1" - log "worker -> $host" - - # Manifest entries under `worker.secrets` map env-var name -> pass store path. - # The script fetches each via `pass` and substitutes the matching {{NAME}} - # placeholder in worker.env.tmpl. Adding a new secret is then a manifest + - # template change; no script edit required. - local -a secret_lines secret_keys - mapfile -t secret_lines < <(yq --raw-output \ - "${env_path}.components.worker.secrets // {} | to_entries | .[] | \"\(.key)=\(.value)\"" \ - "$manifest") - local line - for line in "${secret_lines[@]}"; do - [[ -n "$line" ]] && secret_keys+=("${line%%=*}") - done - - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m render worker.env (HOSTNAME=%s, secrets [%s] from pass) + units, stage to %s:/tmp/, install via heredoc, run sysusers/restorecon/systemctl on %s\n' \ - "$host" "${secret_keys[*]:-none}" "$host" "$host" >&2 - return 0 - fi - - local fqdn="$host" - - local stage - stage="$(mktemp --directory)" - trap "rm --recursive --force '$stage'" RETURN - - install --directory \ - "$stage/etc/moments" \ - "$stage/etc/systemd/system" \ - "$stage/etc/sysusers.d" \ - "$stage/usr/local/bin" - - # Render templates in-memory so secrets never appear on a command line - # (sed would expose them to anything that can read /proc//cmdline). - local rendered - rendered="$(<"${repo_root}/asset/config/worker.env.tmpl")" - rendered=${rendered//'{{HOSTNAME}}'/$fqdn} - local key pass_path value - for line in "${secret_lines[@]}"; do - [[ -z "$line" ]] && continue - key="${line%%=*}" - pass_path="${line#*=}" - if pass show "$pass_path" >/dev/null 2>&1; then - value="$(pass show "$pass_path")" - else - warn "no secret in pass at '${pass_path}' for ${key}; worker will run without ${key}" - value="" - fi - rendered=${rendered//"{{${key}}}"/$value} - done - printf '%s\n' "$rendered" > "$stage/etc/moments/worker.env" - - rendered="$(<"${repo_root}/asset/systemd/moments-worker-cert.path")" - rendered=${rendered//'{{HOSTNAME}}'/$fqdn} - printf '%s\n' "$rendered" > "$stage/etc/systemd/system/moments-worker-cert.path" - - install --mode=0644 "${repo_root}/asset/systemd/moments-worker.service" "$stage/etc/systemd/system/" - install --mode=0644 "${repo_root}/asset/systemd/moments-worker-cert-reload.service" "$stage/etc/systemd/system/" - install --mode=0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf" - install --mode=0755 "${rust_target_dir}/release/moments-worker" "$stage/usr/local/bin/moments-worker" - - chmod 0640 "$stage/etc/moments/worker.env" - - # Stage to a tmpdir on the remote, then `install` each file at its final - # path via the heredoc. Never rsync into /. - local remote_stage="/tmp/moments-deploy.worker.${$}.${RANDOM}" - - ensure_tmp_writable "$host" || return 1 - - rsync \ - --archive \ - --hard-links \ - --numeric-ids \ - --rsh='ssh -o BatchMode=yes' \ - "$stage/" \ - "${host}:${remote_stage}/" - - ssh_run "$host" "sudo bash -s -- ${remote_stage@Q}" <<'REMOTE_EOF' -set -euo pipefail -remote_stage="$1" -trap 'rm --recursive --force "$remote_stage"' EXIT - -fqdn="$(hostname --fqdn)" - -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/sysusers.d/moments.conf" \ - /etc/sysusers.d/moments.conf -systemd-sysusers /etc/sysusers.d/moments.conf - -install --directory --owner=root --group=moments --mode=0750 /etc/moments -install --directory --owner=moments --group=moments --mode=0750 /var/lib/moments - -install --owner=root --group=moments --mode=0640 \ - "$remote_stage/etc/moments/worker.env" \ - /etc/moments/worker.env -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-worker.service" \ - /etc/systemd/system/moments-worker.service -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-worker-cert.path" \ - /etc/systemd/system/moments-worker-cert.path -install --owner=root --group=root --mode=0644 \ - "$remote_stage/etc/systemd/system/moments-worker-cert-reload.service" \ - /etc/systemd/system/moments-worker-cert-reload.service -install --owner=root --group=root --mode=0755 \ - "$remote_stage/usr/local/bin/moments-worker" \ - /usr/local/bin/moments-worker - -setfacl --modify=u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true - -restorecon -Rv /usr/local/bin/moments-worker /etc/moments /var/lib/moments - -systemctl daemon-reload -systemctl enable --now moments-worker-cert.path -systemctl enable --now moments-worker.service -systemctl restart moments-worker.service - -if ! systemctl is-active --quiet moments-worker.service; then - journalctl --unit=moments-worker.service --lines=50 --no-pager >&2 - exit 1 -fi -echo "moments-worker active" -REMOTE_EOF -} - -deploy_web() { - local host="$1" - log "web -> $host" - - local server_name web_root api_upstream web_listen - server_name="$(yq --raw-output "${env_path}.components.web.config.server_name" "$manifest")" - web_root="$(yq --raw-output "${env_path}.components.web.config.root" "$manifest")" - api_upstream="$(yq --raw-output "${env_path}.components.web.config.api_upstream" "$manifest")" - # The edge's stream SNI router owns TCP 443; vhosts sit behind it on 14443. - # Binding 443 here collides with the router — `nginx -t` passes, but every - # later reload aborts while systemctl still reports success. See the - # web.config.listen note in asset/manifest.yml. - web_listen="$(yq --raw-output "${env_path}.components.web.config.listen" "$manifest")" - [[ -n "$web_listen" && "$web_listen" != "null" ]] \ - || web_listen='127.0.0.1:14443 ssl proxy_protocol' - [[ -n "$server_name" && "$server_name" != "null" ]] || die "web.config.server_name missing in manifest" - [[ -n "$web_root" && "$web_root" != "null" ]] || die "web.config.root missing in manifest" - [[ -n "$api_upstream" && "$api_upstream" != "null" ]] || die "web.config.api_upstream missing in manifest" - [[ "$web_root" == /* ]] \ - || die "web.config.root must be an absolute path: '$web_root'" - [[ "$api_upstream" == http://* || "$api_upstream" == https://* ]] \ - || die "web.config.api_upstream must be a http(s) URL: '$api_upstream'" - - local api_upstream_scheme api_upstream_addr api_upstream_port - api_upstream_scheme="${api_upstream%%://*}" - api_upstream_addr="${api_upstream#*://}" - [[ "$api_upstream_addr" == *:* ]] \ - || die "web.config.api_upstream must include an explicit port: '$api_upstream'" - api_upstream_port="${api_upstream_addr##*:}" - [[ "$api_upstream_port" =~ ^[0-9]+$ ]] \ - || die "extracted upstream port is not numeric: '$api_upstream_port'" - - local site_conf_path="/etc/nginx/conf.d/${server_name}.conf" - - if (( dry_run )); then - printf '\033[2m[dry-run]\033[0m render %s (server_name=%s, docroot=%s, upstream=%s://%s) + rsync ui/dist/ to %s:%s/, run nginx -t/reload on %s\n' \ - "$site_conf_path" "$server_name" "$web_root" \ - "$api_upstream_scheme" "$api_upstream_addr" \ - "$host" "$web_root" "$host" >&2 - return 0 - fi - - local stage - stage="$(mktemp --directory)" - trap "rm --recursive --force '$stage'" RETURN - - install --directory "${stage}${web_root}" "$stage/etc/nginx/conf.d" - - rsync --archive "${repo_root}/ui/dist/" "${stage}${web_root}/" - - local rendered - rendered="$(<"${repo_root}/asset/nginx/site.conf.tmpl")" - rendered=${rendered//'{{SERVER_NAME}}'/$server_name} - rendered=${rendered//'{{WEB_LISTEN}}'/$web_listen} - rendered=${rendered//'{{DOCROOT}}'/$web_root} - rendered=${rendered//'{{API_UPSTREAM_SCHEME}}'/$api_upstream_scheme} - rendered=${rendered//'{{API_UPSTREAM_ADDR}}'/$api_upstream_addr} - printf '%s\n' "$rendered" > "${stage}${site_conf_path}" - chmod 0644 "${stage}${site_conf_path}" - - # Both targets are leaf paths (the docroot itself, and a single named - # file) so rsync does not traverse /var or /etc parents — `--chown` is - # enough; -A/-X are intentionally absent. - rsync \ - --archive \ - --hard-links \ - --numeric-ids \ - --chown root:root \ - --rsh='ssh -o BatchMode=yes' \ - --rsync-path 'sudo rsync' \ - --delete \ - "${stage}${web_root}/" \ - "${host}:${web_root}/" - rsync \ - --archive \ - --hard-links \ - --numeric-ids \ - --chown root:root \ - --rsh='ssh -o BatchMode=yes' \ - --rsync-path 'sudo rsync' \ - "${stage}${site_conf_path}" \ - "${host}:${site_conf_path}" - - ssh_run "$host" "sudo bash -s -- ${web_root@Q} ${site_conf_path@Q} ${api_upstream_port@Q}" <<'REMOTE_EOF' -set -euo pipefail -web_root="$1" -site_conf_path="$2" -api_upstream_port="$3" - -# Allow nginx to make outbound connections to the moments-api upstream -# across the WG mesh. -setsebool -P httpd_can_network_connect on - -# Idempotent label: --add fails if the port is already labelled (we suppress -# that one stderr line); --modify is then a no-op or fixes a stale type. -semanage port --add --type=http_port_t --proto=tcp "$api_upstream_port" 2>/dev/null \ - || semanage port --modify --type=http_port_t --proto=tcp "$api_upstream_port" - -restorecon -Rv "$web_root" "$site_conf_path" - -if ! nginx -t; then - echo "nginx config check failed" >&2 - exit 1 -fi -systemctl reload nginx -echo "nginx reloaded" -REMOTE_EOF -} - -# Dispatch ------------------------------------------------------------------ - -failed=() -for component in "${components[@]}"; do - mapfile -t hosts < <(yq --raw-output "${env_path}.components.${component}.hosts[]" "$manifest") - for host in "${hosts[@]}"; do - case "$component" in - api) deploy_api "$host" || failed+=("api@$host") ;; - worker) deploy_worker "$host" || failed+=("worker@$host") ;; - web) deploy_web "$host" || failed+=("web@$host") ;; - *) warn "unknown component: $component" ;; - esac - done -done - -if [[ ${#failed[@]} -gt 0 ]]; then - die "failed: ${failed[*]}" -fi -log "deploy complete" diff --git a/script/render-site-conf.py b/script/render-site-conf.py new file mode 100644 index 0000000..e957c4a --- /dev/null +++ b/script/render-site-conf.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Render the nginx vhost (asset/nginx/site.conf.tmpl) from the environment. + +Both .gitea/workflows/deploy.yml (deploy-web) and refresh.yml call this, so the +two pipelines can never disagree about what they substitute. They did once: the +WEB_LISTEN placeholder was added to the template and to deploy.yml but not to +refresh.yml, so the nightly refresh shipped a literal `listen {{WEB_LISTEN}};` +to oolon. `nginx -t` then failed for the whole edge, and because the file is +rsynced straight into the live conf.d before it is tested, every vhost's reload +-- including the step@ cert renewals -- stayed frozen for days while renewed +certs piled up unserved on disk. + +Guard rails, so that can't recur: + * every {{PLACEHOLDER}} in the template must have a matching environment + variable, or the render fails before anything leaves the runner; + * no {{...}} may survive substitution. +A forgotten or misnamed variable is now a red build, not a broken edge. + +usage: render-site-conf.py [OUTPUT] (default: rendered/site.conf) +""" +import os +import re +import sys + +TEMPLATE = "asset/nginx/site.conf.tmpl" +PLACEHOLDER = re.compile(r"\{\{(\w+)\}\}") + + +def main() -> int: + out = sys.argv[1] if len(sys.argv) > 1 else "rendered/site.conf" + with open(TEMPLATE, encoding="utf-8") as fh: + text = fh.read() + + names = sorted(set(PLACEHOLDER.findall(text))) + missing = [n for n in names if n not in os.environ] + if missing: + sys.stderr.write( + "render-site-conf: no environment value for placeholder(s): " + + ", ".join(missing) + "\n") + return 1 + + for name in names: + text = text.replace("{{%s}}" % name, os.environ[name]) + + leftover = sorted(set(PLACEHOLDER.findall(text))) + if leftover: + sys.stderr.write( + "render-site-conf: unrendered placeholder(s) after substitution: " + + ", ".join(leftover) + "\n") + return 1 + + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + with open(out, "w", encoding="utf-8") as fh: + fh.write(text) + sys.stderr.write( + "render-site-conf: wrote %s (%d substitutions: %s)\n" + % (out, len(names), ", ".join(names))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- 2.52.0