moments/.gitea/workflows/deploy.yml

name: deploy

# Build and roll out all three moments components on a push to main:
#   - moments-api    (static musl binary, read-only API) -> nikola
#   - moments-worker (static musl binary, ingestion daemon) -> frootmig
#   - web            (prerendered static SPA + nginx vhost) -> oolon
#
# This workflow is the source of infra truth (hosts, ports, paths live in `env`
# below — there is no manifest.yml in this model). A Gitea Actions runner deploys
# as the dedicated `gitea_ci` user over SSH, with secrets from the repo settings
# and scoped sudo (see asset/sudoers.d/ + script/infra-setup.sh). The nginx vhost
# is rendered by script/render-site-conf.py, shared with refresh.yml so the two
# never drift on what they substitute.
#
# The api/worker binaries are pure-rustls (no openssl), so they build as fully
# static musl — a runner newer than the target host can't produce an unloadable
# glibc binary (architecture/deployment-gitea-actions.md §6).
#
# Secrets (Gitea -> repo -> Settings -> Actions -> Secrets):
#   RSYNC_SSH_KEY       private SSH key whose pubkey script/infra-setup.sh installed
#   QUERY_GITHUB_TOKEN  github api token for the worker's poller
#   QUERY_GITEA_TOKEN   git.lair.cafe api token for the worker's poller
# (GITHUB_TOKEN / GITEA_TOKEN are reserved Actions names, hence the QUERY_ prefix.)
#
# One-time per-host provisioning (gitea_ci user, authorized_keys, scoped
# sudoers, and the postgres mTLS host cert the api/worker need) is done by
# script/infra-setup.sh — run it once per host before this workflow can succeed.

on:
  push:
    branches: [main]
  workflow_dispatch:

# Serialize deploys; never cancel an in-flight one (a half-applied rollout is
# worse than a slightly stale one).
concurrency:
  group: deploy
  cancel-in-progress: false

env:
  # --- infra truth (mirrors the former asset/manifest.yml) ---
  API_HOST: nikola.kosherinata.internal
  WORKER_HOST: frootmig.kosherinata.internal
  WEB_HOST: oolon.kosherinata.internal
  API_BIND: 0.0.0.0:42424
  API_PORT: "42424"
  SERVER_NAME: rob.tn
  WEB_ROOT: /var/www/rob.tn
  # TCP 443 on the oolon edge is owned by the stream SNI router
  # (/etc/nginx/streams-enabled/sni-router.conf), which ssl_prereads the
  # handshake and forwards to the local https tier on 127.0.0.1:14443 with
  # PROXY protocol. An http vhost that binds 443 itself collides with the
  # router: `nginx -t` still passes (nginx only detects duplicate listeners
  # within a context, not across http{} and stream{}), but every subsequent
  # reload aborts with "bind() to 0.0.0.0:443 failed (98: Address already in
  # use)" while `systemctl reload` still exits 0. That silently froze oolon's
  # running config for a day and stranded every renewed cert on disk.
  WEB_LISTEN: 127.0.0.1:14443 ssl proxy_protocol
  API_UPSTREAM_SCHEME: http
  API_UPSTREAM_ADDR: nikola.kosherinata.internal:42424
  MUSL_TARGET: x86_64-unknown-linux-musl
  # Build-time prerender base: the runner is on the internal mesh and can't
  # resolve/reach the public rob.tn, so fetch the API directly over the mesh
  # (same host the deploy reaches). The api binary serves its routes under /v1.
  # The client bundle still ships the same-origin relative /api/v1 that nginx
  # proxies (VITE_API_BASE only affects the SSR build, not the browser bundle).
  VITE_API_BASE: http://nikola.kosherinata.internal:42424/v1
  DEPLOY_KEY: |
    ${{ secrets.RSYNC_SSH_KEY }}

jobs:
  # The rust runner has cargo + musl but no node; the fedora runner has
  # node + pnpm but no cargo — so the build is split across the two images.
  build-binaries:
    name: Build api + worker (static musl)
    runs-on: rust
    steps:
      - uses: actions/checkout@v4

      # The `rust` runner image provides cargo/clippy/rustfmt, musl-gcc, and the
      # x86_64-unknown-linux-musl std. No package installs at run time.
      - name: Lint + test (deploy gate)
        run: |
          cargo fmt --check --all
          cargo clippy --workspace --all-targets -- -D warnings
          cargo test --workspace

      - name: Build moments-api + moments-worker (static musl release)
        run: cargo build --release --target "${MUSL_TARGET}" -p moments-api -p moments-worker

      - name: Stage binaries
        run: |
          mkdir --parents artifacts
          cp "target/${MUSL_TARGET}/release/moments-api"    artifacts/moments-api
          cp "target/${MUSL_TARGET}/release/moments-worker" artifacts/moments-worker

      - uses: actions/upload-artifact@v3
        with: { name: moments-api, path: artifacts/moments-api, retention-days: 1 }
      - uses: actions/upload-artifact@v3
        with: { name: moments-worker, path: artifacts/moments-worker, retention-days: 1 }

  build-web:
    name: Build prerendered web
    # The prerender fetches VITE_API_BASE at build time, so this job bakes
    # whatever the api is serving when it runs. Without these dependencies it
    # raced the api/worker deploys and the crawler snapshot silently disagreed
    # with the live api for a whole day, until the nightly refresh: it won the
    # race once (a visibility fix reached both together) and lost it once (an
    # api field the snapshot then lacked). deploy-worker is named as well as
    # deploy-api even though deploy-api already implies it — stating it keeps
    # this job correct on its own terms if that chain is ever shortened.
    #
    # The cost is a serial deploy: binaries -> worker -> api -> web. Worth it;
    # the alternative is a snapshot whose correctness depends on which runner
    # finished first.
    needs: [deploy-api, deploy-worker]
    runs-on: fedora-44
    steps:
      - uses: actions/checkout@v4
      # The fedora-44 runner image bakes in node + pnpm (+ rsync) — no install.
      # pnpm 10 blocks dependency build scripts unless approved, and CI doesn't
      # reliably pick up the pnpm-workspace.yaml allowlist; so install with
      # --ignore-scripts (no approval gate) and explicitly rebuild the two native
      # deps vite needs (esbuild, @swc/core). Version-independent.
      - name: Build web (vite client + prerender)
        working-directory: ui
        run: |
          pnpm install --frozen-lockfile --ignore-scripts
          pnpm rebuild @swc/core esbuild
          pnpm run build
      - uses: actions/upload-artifact@v3
        with: { name: web-dist, path: ui/dist, retention-days: 1 }

  deploy-api:
    name: Deploy moments-api to nikola
    # The worker owns migrations (it connects as moments_rw; the api is
    # SELECT-only and cannot run them), so the schema has to be current before
    # a new api starts answering with it. These are separate hosts — nikola and
    # frootmig — so the systemd ordering the comment in moments-api/src/main.rs
    # refers to cannot reach across them; the dependency has to live here.
    # deploy-worker holds itself open until migrations have actually completed.
    needs: [build-binaries, deploy-worker]
    runs-on: fedora-44
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: moments-api, path: artifact }

      - name: SSH init
        run: |
          mkdir -p ~/.ssh
          echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
              gitea_ci@"${API_HOST}" 'hostname -f'

      - name: Render config + units from infra truth
        run: |
          mkdir -p rendered
          # HOSTNAME is the host's own fqdn (used for the postgres mTLS cert path).
          python3 - "$API_HOST" "$API_BIND" "$API_PORT" <<'PY'
          import sys
          host, bind, port = sys.argv[1:4]
          def render(src, dst, subs):
              t = open(src).read()
              for k, v in subs.items():
                  t = t.replace("{{%s}}" % k, v)
              open(dst, "w").write(t)
          render("asset/config/api.env.tmpl", "rendered/api.env",
                 {"HOSTNAME": host, "BIND": bind})
          render("asset/systemd/moments-api-cert.path", "rendered/moments-api-cert.path",
                 {"HOSTNAME": host})
          render("asset/firewalld/moments-api.xml.tmpl", "rendered/moments-api.xml",
                 {"API_PORT": port})
          PY

      - name: Provision service account + directories
        run: |
          # --mkpath: /etc/sysusers.d and /etc/firewalld/services don't exist by
          # default on Fedora (only the /usr/lib variants ship).
          rsync -az --mkpath --rsync-path='sudo rsync' \
            asset/systemd/moments.sysusers.conf \
            gitea_ci@"${API_HOST}":/etc/sysusers.d/moments.conf
          ssh gitea_ci@"${API_HOST}" '
            set -euo pipefail
            sudo /usr/bin/systemd-sysusers
            sudo /usr/bin/install -d -o root -g moments -m 0750 /etc/moments
            sudo /usr/bin/install -d -o moments -g moments -m 0750 /var/lib/moments'

      - name: Sync binary, env, units, firewalld service
        run: |
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0755 \
            artifact/moments-api gitea_ci@"${API_HOST}":/usr/local/bin/moments-api
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:moments --chmod=0640 \
            rendered/api.env gitea_ci@"${API_HOST}":/etc/moments/api.env
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            asset/systemd/moments-api.service \
            gitea_ci@"${API_HOST}":/etc/systemd/system/moments-api.service
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            rendered/moments-api-cert.path \
            gitea_ci@"${API_HOST}":/etc/systemd/system/moments-api-cert.path
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            asset/systemd/moments-api-cert-reload.service \
            gitea_ci@"${API_HOST}":/etc/systemd/system/moments-api-cert-reload.service
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            rendered/moments-api.xml \
            gitea_ci@"${API_HOST}":/etc/firewalld/services/moments-api.xml

      - name: Apply SELinux + firewalld + cert ACL (idempotent)
        run: |
          ssh gitea_ci@"${API_HOST}" '
            set -euo pipefail
            fqdn="$(hostname -f)"
            # Let the moments user read the host private key for postgres mTLS.
            sudo /usr/bin/setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem"
            sudo /usr/sbin/restorecon -R /usr/local/bin/moments-api /etc/moments /var/lib/moments
            if ! sudo /usr/sbin/semanage port -l | grep -E "^http_port_t" | grep -qw '"${API_PORT}"'; then
              sudo /usr/sbin/semanage port -a -t http_port_t -p tcp '"${API_PORT}"'
            fi
            # firewalld only learns a freshly-shipped custom service after reload.
            sudo /usr/bin/firewall-cmd --reload
            if ! sudo /usr/bin/firewall-cmd --query-service=moments-api; then
              sudo /usr/bin/firewall-cmd --add-service=moments-api --permanent
              sudo /usr/bin/firewall-cmd --reload
            fi'

      - name: Restart moments-api
        run: |
          ssh gitea_ci@"${API_HOST}" '
            set -euo pipefail
            sudo /usr/bin/systemctl daemon-reload
            sudo /usr/bin/systemctl enable --now moments-api-cert.path
            sudo /usr/bin/systemctl enable moments-api.service
            sudo /usr/bin/systemctl restart moments-api.service'

      - name: Health probe
        run: |
          for i in 1 2 3 4 5 6 7 8 9 10; do
            if ssh gitea_ci@"${API_HOST}" "curl -fsS http://127.0.0.1:${API_PORT}/v1/healthz"; then
              echo; echo "moments-api healthy"; exit 0
            fi
            sleep 2
          done
          echo "moments-api did not become healthy" >&2
          exit 1

      - name: Capture startup journal
        if: always()
        run: |
          sleep 3
          ssh gitea_ci@"${API_HOST}" 'journalctl --unit moments-api.service --no-pager -n 200'

  deploy-worker:
    name: Deploy moments-worker to frootmig
    needs: build-binaries
    runs-on: fedora-44
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: moments-worker, path: artifact }

      - name: SSH init
        run: |
          mkdir -p ~/.ssh
          echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
              gitea_ci@"${WORKER_HOST}" 'hostname -f'

      - name: Render worker config + cert.path from secrets
        env:
          # GITHUB_TOKEN / GITEA_TOKEN are reserved Actions secret names, so the
          # repo secrets are QUERY_*; the rendered worker.env still uses the
          # GITHUB_TOKEN / GITEA_TOKEN env vars the worker poller expects.
          GITHUB_TOKEN: ${{ secrets.QUERY_GITHUB_TOKEN }}
          GITEA_TOKEN: ${{ secrets.QUERY_GITEA_TOKEN }}
        run: |
          mkdir -p rendered
          # Literal substitution via python so tokens with shell-special chars
          # survive; secrets come from the environment, never a command line.
          python3 - "$WORKER_HOST" <<'PY'
          import os, sys
          host = sys.argv[1]
          t = open("asset/config/worker.env.tmpl").read()
          subs = {
              "HOSTNAME": host,
              "GITHUB_TOKEN": os.environ.get("GITHUB_TOKEN", ""),
              "GITEA_TOKEN": os.environ.get("GITEA_TOKEN", ""),
          }
          for k, v in subs.items():
              t = t.replace("{{%s}}" % k, v)
          open("rendered/worker.env", "w").write(t)
          c = open("asset/systemd/moments-worker-cert.path").read().replace("{{HOSTNAME}}", host)
          open("rendered/moments-worker-cert.path", "w").write(c)
          PY

      - name: Provision service account + directories
        run: |
          rsync -az --mkpath --rsync-path='sudo rsync' \
            asset/systemd/moments.sysusers.conf \
            gitea_ci@"${WORKER_HOST}":/etc/sysusers.d/moments.conf
          ssh gitea_ci@"${WORKER_HOST}" '
            set -euo pipefail
            sudo /usr/bin/systemd-sysusers
            sudo /usr/bin/install -d -o root -g moments -m 0750 /etc/moments
            sudo /usr/bin/install -d -o moments -g moments -m 0750 /var/lib/moments'

      - name: Sync binary, env, units
        run: |
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0755 \
            artifact/moments-worker gitea_ci@"${WORKER_HOST}":/usr/local/bin/moments-worker
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:moments --chmod=0640 \
            rendered/worker.env gitea_ci@"${WORKER_HOST}":/etc/moments/worker.env
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            asset/systemd/moments-worker.service \
            gitea_ci@"${WORKER_HOST}":/etc/systemd/system/moments-worker.service
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            rendered/moments-worker-cert.path \
            gitea_ci@"${WORKER_HOST}":/etc/systemd/system/moments-worker-cert.path
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            asset/systemd/moments-worker-cert-reload.service \
            gitea_ci@"${WORKER_HOST}":/etc/systemd/system/moments-worker-cert-reload.service

      - name: Apply cert ACL + SELinux, restart worker
        run: |
          ssh gitea_ci@"${WORKER_HOST}" '
            set -euo pipefail
            fqdn="$(hostname -f)"
            sudo /usr/bin/setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem"
            sudo /usr/sbin/restorecon -R /usr/local/bin/moments-worker /etc/moments /var/lib/moments
            sudo /usr/bin/systemctl daemon-reload
            sudo /usr/bin/systemctl enable --now moments-worker-cert.path
            sudo /usr/bin/systemctl enable moments-worker.service
            sudo /usr/bin/systemctl restart moments-worker.service
            sudo /usr/bin/systemctl is-active --quiet moments-worker.service'

      # The unit is Type=simple, so `restart` returns once the process is
      # exec'd and `is-active` is true immediately — neither says anything
      # about migrations. deploy-api waits on this job precisely so that the
      # schema is current before the new api answers a request, so the job
      # has to stay open until migrations are actually done.
      #
      # moments-worker awaits store.migrate() before it logs "worker started",
      # so that line is the signal. Scoping the search to the unit's current
      # InvocationID means a previous start's line can't satisfy the wait.
      - name: Wait for migrations to finish
        run: |
          ssh gitea_ci@"${WORKER_HOST}" '
            set -euo pipefail
            invocation="$(systemctl show --property=InvocationID --value moments-worker.service)"
            if [ -z "${invocation}" ]; then
              echo "no InvocationID for moments-worker.service" >&2
              exit 1
            fi
            for attempt in $(seq 1 60); do
              # Deliberately no `| grep -q`: a consumer that exits on first
              # match closes the pipe, journalctl dies of SIGPIPE, and under
              # `pipefail` the match reads as a failure. Let journalctl filter
              # itself and test whether anything came back. `|| true` because
              # --grep exits 1 when nothing matched yet.
              started="$(journalctl _SYSTEMD_INVOCATION_ID="${invocation}" \
                           --no-pager --grep "worker started" || true)"
              if [ -n "${started}" ]; then
                echo "migrations complete after ${attempt} check(s)"
                exit 0
              fi
              if ! systemctl is-active --quiet moments-worker.service; then
                echo "worker exited before reporting startup" >&2
                exit 1
              fi
              sleep 2
            done
            echo "timed out after 120s waiting for worker startup" >&2
            exit 1'

      - name: Capture startup journal
        if: always()
        run: |
          sleep 3
          ssh gitea_ci@"${WORKER_HOST}" 'journalctl --unit moments-worker.service --no-pager -n 200'

  deploy-web:
    name: Deploy web to oolon
    needs: build-web
    runs-on: fedora-44
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: web-dist, path: dist }

      - name: SSH init
        run: |
          mkdir -p ~/.ssh
          echo "${DEPLOY_KEY}" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
              gitea_ci@"${WEB_HOST}" 'hostname -f'

      - name: Render nginx vhost
        # Shared with refresh.yml so the two pipelines can't drift on what they
        # substitute; fails loudly if any {{PLACEHOLDER}} lacks an env value or
        # survives substitution, so a broken vhost never reaches oolon.
        run: python3 script/render-site-conf.py rendered/site.conf

      - name: Sync static site (prerendered)
        run: |
          ssh gitea_ci@"${WEB_HOST}" 'sudo /usr/bin/install -d -m 0755 '"${WEB_ROOT}"
          rsync -az --delete --mkpath --rsync-path='sudo rsync' \
            --chown=root:root --chmod=D755,F644 \
            dist/ gitea_ci@"${WEB_HOST}":"${WEB_ROOT}/"
          ssh gitea_ci@"${WEB_HOST}" 'sudo /usr/sbin/restorecon -R '"${WEB_ROOT}"

      - name: Sync nginx vhost + apply SELinux
        run: |
          rsync -az --mkpath --rsync-path='sudo rsync' --chown=root:root --chmod=0644 \
            rendered/site.conf \
            gitea_ci@"${WEB_HOST}":/etc/nginx/conf.d/"${SERVER_NAME}".conf
          ssh gitea_ci@"${WEB_HOST}" '
            set -euo pipefail
            # nginx proxies /api/ to the api host across the WG mesh.
            sudo /usr/sbin/setsebool -P httpd_can_network_connect on
            if ! sudo /usr/sbin/semanage port -l | grep -E "^http_port_t" | grep -qw '"${API_PORT}"'; then
              sudo /usr/sbin/semanage port -a -t http_port_t -p tcp '"${API_PORT}"'
            fi
            sudo /usr/sbin/restorecon -R /etc/nginx/conf.d/'"${SERVER_NAME}"'.conf
            sudo /usr/sbin/nginx -t
            # `nginx -t` cannot see a listener that collides across http{} and
            # stream{}, and `systemctl reload` is just SIGHUP — it exits 0 even
            # when nginx aborts the reconfiguration and keeps the old cycle. A
            # reload that really landed always spawns a fresh worker generation,
            # so assert that rather than trusting the exit code.
            master=$(cat /run/nginx.pid)
            before=$(pgrep -P "$master" | sort | tr "\n" " ")
            sudo /usr/bin/systemctl reload nginx
            for _ in $(seq 1 10); do
              sleep 1
              after=$(pgrep -P "$master" | sort | tr "\n" " ")
              [ "$before" = "$after" ] || break
            done
            if [ "$before" = "$after" ]; then
              echo "nginx reload did not take effect: worker generation unchanged" >&2
              sudo /usr/bin/tail -n 50 /var/log/nginx/error.log >&2 || true
              exit 1
            fi'