From 110b523fd0ae7e905d59af9bdb15b1fed05b7300 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Sun, 3 May 2026 20:17:17 +0300 Subject: [PATCH] chore(deploy): add manifest, systemd units, nginx config, deploy.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires up the prod deployment per architecture-doc conventions: - api → nikola.kosherinata.internal, loopback bind 127.0.0.1:42424 (less-common port, registered with SELinux as http_port_t). - worker → frootmig.kosherinata.internal, no listening port. - web (static ui/dist + nginx server_name rob.tn) → nikola, with /api/* reverse-proxied to the loopback API. - db → existing magrathea cluster via mTLS, hostname-baked DATABASE_URL rendered into /etc/moments/{api,worker}.env at deploy time. Cert rotation: step-ca renews host certs every 24h; .path units watch /etc/pki/tls/misc/.pem and trigger systemctl restart of the relevant service. Both binaries hold cert state in rustls and read once at startup, so restart is the right reload semantics. deploy.sh contract matches the architecture doc: positional env arg, component list (or `all` / `default`), --dry-run support. Renders config templates from `pass`, rsyncs over ssh+sudo, runs sysusers / restorecon / semanage / systemctl / nginx -t idempotently. Co-Authored-By: Claude Opus 4.7 (1M context) --- asset/config/api.env.tmpl | 9 + asset/config/worker.env.tmpl | 27 ++ asset/manifest.yml | 36 ++ asset/nginx/rob.tn.conf | 65 ++++ asset/systemd/moments-api-cert-reload.service | 6 + asset/systemd/moments-api-cert.path | 13 + asset/systemd/moments-api.service | 34 ++ .../moments-worker-cert-reload.service | 6 + asset/systemd/moments-worker-cert.path | 12 + asset/systemd/moments-worker.service | 34 ++ asset/systemd/moments.sysusers.conf | 2 + readme.md | 34 +- script/deploy.sh | 326 ++++++++++++++++++ 13 files changed, 602 insertions(+), 2 deletions(-) create mode 100644 asset/config/api.env.tmpl create mode 100644 asset/config/worker.env.tmpl create mode 100644 asset/manifest.yml create mode 100644 asset/nginx/rob.tn.conf create mode 100644 asset/systemd/moments-api-cert-reload.service create mode 100644 asset/systemd/moments-api-cert.path create mode 100644 asset/systemd/moments-api.service create mode 100644 asset/systemd/moments-worker-cert-reload.service create mode 100644 asset/systemd/moments-worker-cert.path create mode 100644 asset/systemd/moments-worker.service create mode 100644 asset/systemd/moments.sysusers.conf create mode 100755 script/deploy.sh diff --git a/asset/config/api.env.tmpl b/asset/config/api.env.tmpl new file mode 100644 index 0000000..453e49c --- /dev/null +++ b/asset/config/api.env.tmpl @@ -0,0 +1,9 @@ +# /etc/moments/api.env — rendered by deploy.sh, do not edit on the host. +# {{HOSTNAME}} resolves to the target host's FQDN at deploy time. + +JOURNAL_STREAM=1 +RUST_LOG=info,sqlx=warn,tower_http=info + +BIND_ADDR=127.0.0.1:42424 + +DATABASE_URL=postgres://moments_ro@magrathea.kosherinata.internal:5432/moments?sslmode=verify-full&sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem&sslcert=/etc/pki/tls/misc/{{HOSTNAME}}.pem&sslkey=/etc/pki/tls/private/{{HOSTNAME}}.pem diff --git a/asset/config/worker.env.tmpl b/asset/config/worker.env.tmpl new file mode 100644 index 0000000..625a3ab --- /dev/null +++ b/asset/config/worker.env.tmpl @@ -0,0 +1,27 @@ +# /etc/moments/worker.env — rendered by deploy.sh, do not edit on the host. +# {{HOSTNAME}} resolves to the target host's FQDN at deploy time. +# {{GITHUB_TOKEN}} is resolved from `pass`; the rendered file lives in +# /etc/moments/ chmod 0640 owned by root:moments. + +JOURNAL_STREAM=1 +RUST_LOG=info,sqlx=warn + +DATABASE_URL=postgres://moments_rw@magrathea.kosherinata.internal:5432/moments?sslmode=verify-full&sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem&sslcert=/etc/pki/tls/misc/{{HOSTNAME}}.pem&sslkey=/etc/pki/tls/private/{{HOSTNAME}}.pem + +GITHUB_USER=grenade +GITHUB_TOKEN={{GITHUB_TOKEN}} +POLL_INTERVAL_SECS=600 +SEARCH_POLL_INTERVAL_SECS=86400 + +GITEA_HOST=git.lair.cafe +GITEA_USER=grenade +GITEA_POLL_INTERVAL_SECS=600 + +HG_HOST=hg-edge.mozilla.org +HG_REPOS=build/puppet,build/tools,build/buildbot-configs +HG_AUTHOR_TERMS=thijssen,grenade +HG_POLL_INTERVAL_SECS=86400 + +BUGZILLA_HOST=bugzilla.mozilla.org +BUGZILLA_EMAIL=rthijssen@mozilla.com +BUGZILLA_POLL_INTERVAL_SECS=86400 diff --git a/asset/manifest.yml b/asset/manifest.yml new file mode 100644 index 0000000..d5ac2bd --- /dev/null +++ b/asset/manifest.yml @@ -0,0 +1,36 @@ +app: moments +environments: + prod: + components: + api: + hosts: [nikola.kosherinata.internal] + config: + bind: 127.0.0.1:42424 + db_role: moments_ro + db_host: magrathea.kosherinata.internal + db_port: 5432 + db_name: moments + worker: + hosts: [frootmig.kosherinata.internal] + config: + db_role: moments_rw + db_host: magrathea.kosherinata.internal + db_port: 5432 + db_name: moments + github_user: grenade + gitea_host: git.lair.cafe + gitea_user: grenade + hg_host: hg-edge.mozilla.org + hg_repos: build/puppet,build/tools,build/buildbot-configs + hg_author_terms: thijssen,grenade + bugzilla_host: bugzilla.mozilla.org + bugzilla_email: rthijssen@mozilla.com + secrets: + GITHUB_TOKEN: github.com/grenade/admin-token + # GITEA_TOKEN, BUGZILLA_API_KEY: optional, omit unless required. + web: + hosts: [nikola.kosherinata.internal] + config: + server_name: rob.tn + root: /var/www/moments + api_upstream: http://127.0.0.1:42424 diff --git a/asset/nginx/rob.tn.conf b/asset/nginx/rob.tn.conf new file mode 100644 index 0000000..ed2a0f2 --- /dev/null +++ b/asset/nginx/rob.tn.conf @@ -0,0 +1,65 @@ +# /etc/nginx/conf.d/rob.tn.conf — rob.tn site config for moments. +# +# Static frontend out of /var/www/moments; /api/* reverse-proxied to the +# moments-api binary on loopback. The UI fetches /api/v1/... so the strip +# matches what Vite's dev proxy does (drop the /api prefix before sending +# to axum, whose routes are mounted at /v1/*). + +upstream moments_api { + server 127.0.0.1:42424 max_fails=3 fail_timeout=30s; + keepalive 8; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name rob.tn; + + ssl_certificate /etc/pki/tls/misc/nikola.kosherinata.internal.pem; + ssl_certificate_key /etc/pki/tls/private/nikola.kosherinata.internal.pem; + + # Public forge — visitors are not on the internal mTLS mesh, so no + # client-cert verification here. The X25519MLKEM768 default falls + # back to classical curves for clients that don't speak PQ yet. + ssl_protocols TLSv1.3; + + root /var/www/moments; + index index.html; + + # Static SPA: serve the file if it exists, else fall back to index.html + # so client-side routing works. + location / { + try_files $uri $uri/ /index.html; + add_header Cache-Control "no-cache" always; + } + + # Asset bundles are content-hashed by Vite — safe to cache aggressively. + location ~* \.(js|css|woff2?|ttf|eot|svg|png|jpg|jpeg|gif|ico|webp|avif)$ { + expires 30d; + add_header Cache-Control "public, max-age=2592000, immutable"; + try_files $uri =404; + } + + location /api/ { + # Strip /api so axum sees /v1/events, not /api/v1/events. + rewrite ^/api/(.*)$ /$1 break; + proxy_pass http://moments_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 30s; + proxy_connect_timeout 5s; + } + + access_log /var/log/nginx/rob.tn.access.log; + error_log /var/log/nginx/rob.tn.error.log; +} + +server { + listen 80; + listen [::]:80; + server_name rob.tn; + return 301 https://$host$request_uri; +} diff --git a/asset/systemd/moments-api-cert-reload.service b/asset/systemd/moments-api-cert-reload.service new file mode 100644 index 0000000..b79beaf --- /dev/null +++ b/asset/systemd/moments-api-cert-reload.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart moments-api on host cert change + +[Service] +Type=oneshot +ExecStart=/bin/systemctl restart moments-api.service diff --git a/asset/systemd/moments-api-cert.path b/asset/systemd/moments-api-cert.path new file mode 100644 index 0000000..a845d1b --- /dev/null +++ b/asset/systemd/moments-api-cert.path @@ -0,0 +1,13 @@ +[Unit] +Description=Watch host cert for moments-api +Documentation=https://git.lair.cafe/grenade/architecture + +[Path] +# Hostname is substituted at deploy time. step-ca rotates host certs every +# 24h; rustls reads them at process start, so the API must restart on +# rotation. Read-only public timeline — a few seconds of churn is fine. +PathChanged=/etc/pki/tls/misc/{{HOSTNAME}}.pem +Unit=moments-api-cert-reload.service + +[Install] +WantedBy=multi-user.target diff --git a/asset/systemd/moments-api.service b/asset/systemd/moments-api.service new file mode 100644 index 0000000..7184180 --- /dev/null +++ b/asset/systemd/moments-api.service @@ -0,0 +1,34 @@ +[Unit] +Description=moments read-only HTTP API +Documentation=https://git.lair.cafe/grenade/moments +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=moments +Group=moments +EnvironmentFile=/etc/moments/api.env +ExecStart=/usr/local/bin/moments-api +Restart=on-failure +RestartSec=5s + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +ReadWritePaths=/var/lib/moments +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/asset/systemd/moments-worker-cert-reload.service b/asset/systemd/moments-worker-cert-reload.service new file mode 100644 index 0000000..66e08ea --- /dev/null +++ b/asset/systemd/moments-worker-cert-reload.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart moments-worker on host cert change + +[Service] +Type=oneshot +ExecStart=/bin/systemctl restart moments-worker.service diff --git a/asset/systemd/moments-worker-cert.path b/asset/systemd/moments-worker-cert.path new file mode 100644 index 0000000..dced840 --- /dev/null +++ b/asset/systemd/moments-worker-cert.path @@ -0,0 +1,12 @@ +[Unit] +Description=Watch host cert for moments-worker +Documentation=https://git.lair.cafe/grenade/architecture + +[Path] +# Worker holds a sqlx pool with rustls — restart on cert rotation. The +# poller is idempotent, so dropping mid-poll is safe. +PathChanged=/etc/pki/tls/misc/{{HOSTNAME}}.pem +Unit=moments-worker-cert-reload.service + +[Install] +WantedBy=multi-user.target diff --git a/asset/systemd/moments-worker.service b/asset/systemd/moments-worker.service new file mode 100644 index 0000000..546d52c --- /dev/null +++ b/asset/systemd/moments-worker.service @@ -0,0 +1,34 @@ +[Unit] +Description=moments ingestion worker +Documentation=https://git.lair.cafe/grenade/moments +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=moments +Group=moments +EnvironmentFile=/etc/moments/worker.env +ExecStart=/usr/local/bin/moments-worker +Restart=on-failure +RestartSec=10s + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +ReadWritePaths=/var/lib/moments +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/asset/systemd/moments.sysusers.conf b/asset/systemd/moments.sysusers.conf new file mode 100644 index 0000000..21b676e --- /dev/null +++ b/asset/systemd/moments.sysusers.conf @@ -0,0 +1,2 @@ +#Type Name ID GECOS Home directory Shell +u moments - "moments service account" /var/lib/moments /usr/sbin/nologin diff --git a/readme.md b/readme.md index eb98144..4af6bec 100644 --- a/readme.md +++ b/readme.md @@ -34,8 +34,38 @@ The API expects a Postgres reachable at `DATABASE_URL`. For magrathea, that's an DATABASE_URL=postgres://localhost/moments cargo run -p moments-api ``` -Migrations live in `crates/moments-data/migrations/` and run automatically on API startup. +Migrations live in `crates/moments-data/migrations/` and run automatically on worker startup. The API connects as `moments_ro` and never runs migrations — the worker (as `moments_rw`) is the schema owner. ## Deployment -See `asset/manifest.yml` and `script/deploy.sh`. +```sh +./script/deploy.sh prod all # api + worker + web +./script/deploy.sh prod api worker # subset +./script/deploy.sh prod default # api + web only (worker untouched) +./script/deploy.sh prod all --dry-run +``` + +Topology: + +| Component | Host | Notes | +| --------- | --------------------------------- | --------------------------------------------- | +| api | `nikola.kosherinata.internal` | binds `127.0.0.1:42424`, fronted by local nginx | +| worker | `frootmig.kosherinata.internal` | no listening port; pollers only | +| web | `nikola.kosherinata.internal` | static `ui/dist/` under `/var/www/moments` | +| db | `magrathea.kosherinata.internal` | postgres mTLS, passwordless | + +Postgres roles `moments_rw` and `moments_ro` must exist on the primary, with `pg_ident.conf` mappings in place for `nikola.kosherinata.internal` → `moments_ro` and `frootmig.kosherinata.internal` → `moments_rw`. See `asset/sql/bootstrap-moments.sql` and `asset/postgres/ident.conf.tmpl`. + +Secrets resolved by `deploy.sh` via `pass`: + +- `github.com/grenade/admin-token` — GitHub PAT for events + search APIs (worker only). + +Optional, set if needed in `worker.env`: `GITEA_TOKEN`, `BUGZILLA_API_KEY`. + +### DNS cutover + +`rob.tn` currently resolves to GitHub Pages. After the first successful prod deploy: + +1. Update Cloudflare DNS for `rob.tn` to the WAN IP that fronts `nikola` (unproxied — see architecture doc §11). +2. Confirm `curl -fsS https://rob.tn/api/v1/healthz` returns `ok`. +3. Add an archival notice to the top of [grenade-events-react/readme.md](https://github.com/grenade/grenade-events-react) pointing at this repo, and archive the GitHub repo. diff --git a/script/deploy.sh b/script/deploy.sh new file mode 100755 index 0000000..67e65fb --- /dev/null +++ b/script/deploy.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# +# moments deployment script. +# +# ./script/deploy.sh [component...] +# ./script/deploy.sh prod api worker web +# ./script/deploy.sh prod all +# +# Builds artifacts locally, resolves secrets from `pass`, renders config +# templates, rsyncs everything to the target hosts, and reloads systemd / +# nginx / firewalld / SELinux state idempotently. + +set -euo pipefail +shopt -s nullglob + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +manifest="${repo_root}/asset/manifest.yml" +dry_run=0 + +usage() { + cat <&2 +usage: $(basename "$0") [component...] [--dry-run] + $(basename "$0") prod api worker web + $(basename "$0") prod all + $(basename "$0") prod default # api + web (worker isn't restarted unless asked) +EOF + exit 2 +} + +log() { printf '\033[1;34m[deploy]\033[0m %s\n' "$*" >&2; } +warn() { printf '\033[1;33m[deploy]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m[deploy]\033[0m %s\n' "$*" >&2; exit 1; } + +run() { + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m %s\n' "$*" >&2 + else + "$@" + fi +} + +ssh_run() { + local host="$1"; shift + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m ssh %s -- %s\n' "$host" "$*" >&2 + else + ssh -o BatchMode=yes "$host" "$@" + fi +} + +[[ $# -ge 1 ]] || usage +environment="$1"; shift +components=() +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) dry_run=1 ;; + *) components+=("$1") ;; + esac + shift +done + +[[ -f "$manifest" ]] || die "manifest not found: $manifest" +command -v yq >/dev/null 2>&1 || die "yq is required" +command -v pass >/dev/null 2>&1 || die "pass is required" +command -v rsync >/dev/null 2>&1 || die "rsync is required" +command -v cargo >/dev/null 2>&1 || die "cargo is required" + +# Resolve component list ---------------------------------------------------- + +env_path=".environments.${environment}" +yq -e "${env_path}" "$manifest" >/dev/null \ + || die "environment '$environment' not found in manifest" + +mapfile -t all_components < <(yq -r "${env_path}.components | keys | .[]" "$manifest") + +if [[ ${#components[@]} -eq 0 ]]; then + usage +fi +case "${components[0]:-}" in + all) components=("${all_components[@]}") ;; + default) components=(api web) ;; +esac + +# Build artifacts ----------------------------------------------------------- + +needs_rust=0 +needs_web=0 +for c in "${components[@]}"; do + case "$c" in + api|worker) needs_rust=1 ;; + web) needs_web=1 ;; + esac +done + +if (( needs_rust )); then + log "cargo build --release (api, worker)" + run cargo build --release --bin moments-api --bin moments-worker --manifest-path "${repo_root}/Cargo.toml" +fi + +if (( needs_web )); then + log "vite build (ui)" + run sh -c "cd '${repo_root}/ui' && pnpm install --frozen-lockfile && pnpm run build" +fi + +# Per-component deploy ------------------------------------------------------ + +deploy_api() { + local host="$1" + log "api -> $host" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m render api.env (HOSTNAME=%s) + units, rsync to %s:/, run sysusers/restorecon/semanage/systemctl on %s\n' \ + "$host" "$host" "$host" >&2 + return 0 + fi + + local fqdn + fqdn="$host" + + local stage + stage="$(mktemp -d)" + trap "rm -rf '$stage'" RETURN + + install -d "$stage/etc/moments" "$stage/etc/systemd/system" "$stage/etc/sysusers.d" "$stage/usr/local/bin" + + # Render env file with hostname substitution. + sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/config/api.env.tmpl" \ + > "$stage/etc/moments/api.env" + + sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/systemd/moments-api-cert.path" \ + > "$stage/etc/systemd/system/moments-api-cert.path" + + install -m 0644 "${repo_root}/asset/systemd/moments-api.service" "$stage/etc/systemd/system/" + install -m 0644 "${repo_root}/asset/systemd/moments-api-cert-reload.service" "$stage/etc/systemd/system/" + install -m 0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf" + install -m 0755 "${repo_root}/target/release/moments-api" "$stage/usr/local/bin/moments-api" + + # Permissions on the rendered env: root-owned, moments group readable. + chmod 0640 "$stage/etc/moments/api.env" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2 + else + rsync -aHAX --rsync-path="sudo rsync" "$stage/" "${host}:/" + fi + + ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF' +set -euo pipefail +fqdn="$(hostname -f)" + +systemd-sysusers /etc/sysusers.d/moments.conf + +install -d -o root -g moments -m 0750 /etc/moments +install -d -o moments -g moments -m 0750 /var/lib/moments +chown root:moments /etc/moments/api.env +chmod 0640 /etc/moments/api.env + +# Grant the moments user read access to the host private key — required for +# the postgres mTLS connection. +setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true + +# Label loopback API port. Idempotent — the -m flag turns "already labelled" +# into a no-op. +if ! semanage port -l | awk '{print $1, $3}' | grep -qE "^http_port_t .*42424"; then + semanage port -a -t http_port_t -p tcp 42424 || \ + semanage port -m -t http_port_t -p tcp 42424 +fi + +restorecon -Rv /usr/local/bin/moments-api /etc/moments /var/lib/moments + +systemctl daemon-reload +systemctl enable --now moments-api-cert.path +systemctl enable --now moments-api.service +systemctl restart moments-api.service + +# Health probe. +for i in 1 2 3 4 5 6 7 8 9 10; do + if curl -fsS http://127.0.0.1:42424/v1/healthz >/dev/null; then + echo "moments-api healthy" + exit 0 + fi + sleep 1 +done +echo "moments-api did not become healthy" >&2 +journalctl -u moments-api.service -n 50 --no-pager >&2 +exit 1 +REMOTE_EOF +} + +deploy_worker() { + local host="$1" + log "worker -> $host" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m render worker.env (HOSTNAME=%s, GITHUB_TOKEN from pass) + units, rsync to %s:/, run sysusers/restorecon/systemctl on %s\n' \ + "$host" "$host" "$host" >&2 + return 0 + fi + + local fqdn + fqdn="$host" + + local github_token="" + if pass show github.com/grenade/admin-token >/dev/null 2>&1; then + github_token="$(pass show github.com/grenade/admin-token)" + else + warn "no github admin-token in pass; worker will run without GITHUB_TOKEN" + fi + + local stage + stage="$(mktemp -d)" + trap "rm -rf '$stage'" RETURN + + install -d "$stage/etc/moments" "$stage/etc/systemd/system" "$stage/etc/sysusers.d" "$stage/usr/local/bin" + + sed -e "s|{{HOSTNAME}}|${fqdn}|g" \ + -e "s|{{GITHUB_TOKEN}}|${github_token}|g" \ + "${repo_root}/asset/config/worker.env.tmpl" > "$stage/etc/moments/worker.env" + + sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/systemd/moments-worker-cert.path" \ + > "$stage/etc/systemd/system/moments-worker-cert.path" + + install -m 0644 "${repo_root}/asset/systemd/moments-worker.service" "$stage/etc/systemd/system/" + install -m 0644 "${repo_root}/asset/systemd/moments-worker-cert-reload.service" "$stage/etc/systemd/system/" + install -m 0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf" + install -m 0755 "${repo_root}/target/release/moments-worker" "$stage/usr/local/bin/moments-worker" + + chmod 0640 "$stage/etc/moments/worker.env" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2 + else + rsync -aHAX --rsync-path="sudo rsync" "$stage/" "${host}:/" + fi + + ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF' +set -euo pipefail +fqdn="$(hostname -f)" + +systemd-sysusers /etc/sysusers.d/moments.conf + +install -d -o root -g moments -m 0750 /etc/moments +install -d -o moments -g moments -m 0750 /var/lib/moments +chown root:moments /etc/moments/worker.env +chmod 0640 /etc/moments/worker.env + +setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true + +restorecon -Rv /usr/local/bin/moments-worker /etc/moments /var/lib/moments + +systemctl daemon-reload +systemctl enable --now moments-worker-cert.path +systemctl enable --now moments-worker.service +systemctl restart moments-worker.service + +# Liveness probe — worker doesn't expose a port, so check is-active. +if ! systemctl is-active --quiet moments-worker.service; then + journalctl -u moments-worker.service -n 50 --no-pager >&2 + exit 1 +fi +echo "moments-worker active" +REMOTE_EOF +} + +deploy_web() { + local host="$1" + log "web -> $host" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m rsync ui/dist/ to %s:/var/www/moments/ + nginx config, run nginx -t/reload on %s\n' \ + "$host" "$host" >&2 + return 0 + fi + + local stage + stage="$(mktemp -d)" + trap "rm -rf '$stage'" RETURN + + install -d "$stage/var/www/moments" "$stage/etc/nginx/conf.d" + + rsync -a "${repo_root}/ui/dist/" "$stage/var/www/moments/" + install -m 0644 "${repo_root}/asset/nginx/rob.tn.conf" "$stage/etc/nginx/conf.d/rob.tn.conf" + + if (( dry_run )); then + printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2 + else + rsync -aHAX --delete --rsync-path="sudo rsync" "$stage/var/www/moments/" "${host}:/var/www/moments/" + rsync -aHAX --rsync-path="sudo rsync" "$stage/etc/nginx/conf.d/rob.tn.conf" "${host}:/etc/nginx/conf.d/rob.tn.conf" + fi + + ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF' +set -euo pipefail + +# Allow nginx to talk upstream to the loopback API socket. +setsebool -P httpd_can_network_connect on + +restorecon -Rv /var/www/moments /etc/nginx/conf.d/rob.tn.conf + +if ! nginx -t; then + echo "nginx config check failed" >&2 + exit 1 +fi +systemctl reload nginx +echo "nginx reloaded" +REMOTE_EOF +} + +# Dispatch ------------------------------------------------------------------ + +failed=() +for component in "${components[@]}"; do + mapfile -t hosts < <(yq -r "${env_path}.components.${component}.hosts[]" "$manifest") + for host in "${hosts[@]}"; do + case "$component" in + api) deploy_api "$host" || failed+=("api@$host") ;; + worker) deploy_worker "$host" || failed+=("worker@$host") ;; + web) deploy_web "$host" || failed+=("web@$host") ;; + *) warn "unknown component: $component" ;; + esac + done +done + +if [[ ${#failed[@]} -gt 0 ]]; then + die "failed: ${failed[*]}" +fi +log "deploy complete"