observer/.gitea/workflows/deploy.yaml

name: deploy

# The workflow is the source of infra truth (architecture/deployment-gitea-actions.md):
# hosts, ports, paths and component→host mapping live here and nowhere else.
# There is no separate deployment manifest.

on:
  push:
    branches: [main]
  # Every other branch runs ci.yaml, the same gate with no deploy behind it.
  workflow_dispatch:
    inputs:
      # `validate` runs the build and its gate, then checks what is *already*
      # deployed — the API's health from bob and from the edge proxy, and the
      # chain list through the public vhost — without shipping anything. It is
      # the post-deploy checks of deploy-api and deploy-web, run against the
      # live site. Nothing reads this input except the `if:` on the three jobs
      # below; until #28 nothing read it at all, and a "validate" dispatch
      # deployed.
      #
      # Deploying is main-only whatever the mode: a dispatch on any other ref
      # builds and gates, and stops there.
      mode:
        description: "deploy (apply then validate) or validate (gate, then check the live site; ships nothing)"
        type: choice
        options: [deploy, validate]
        default: deploy

concurrency:
  # Serialise deploys; never half-apply two at once.
  group: deploy
  cancel-in-progress: false

env:
  # --- infra truth -----------------------------------------------------------
  # The API runs beside quantus-node so it can read the node's loopback RPC.
  API_HOST: bob.hanzalova.internal
  API_PORT: "25864"
  # The site's edge proxy. It serves the built SPA and reverse-proxies /v1.
  # Cross-site on purpose: the API has to sit beside the node it reads, and the
  # public name is fronted from the DC.
  EDGE_HOST: oolon.kosherinata.internal
  WEBROOT: /var/www/blackbeard.observer
  PUBLIC_NAME: blackbeard.observer

  DEPLOY_KEY: |
    ${{ secrets.RSYNC_SSH_KEY }}

jobs:
  build:
    # `rust`, not `fedora-43`: the fedora images are node/shell runners with no
    # cargo at all (gitea-runners.md §4). `runner-rust` is built on
    # `runner-fedora-44`, so it carries node + pnpm too and one job can build
    # both halves.
    runs-on: rust
    outputs:
      rust: ${{ steps.changes.outputs.rust }}
    steps:
      - uses: actions/checkout@v4
        with:
          # The change detection below diffs against the previous head, which a
          # depth-1 clone does not contain.
          fetch-depth: 0

      - name: what changed
        id: changes
        # The Rust half is the expensive half — a cold musl release build plus
        # the gate is most of the wall clock — and a frontend commit cannot
        # change a byte of it. Deciding here rather than with `on.push.paths`
        # because that filter skips the *whole* workflow, and the site still
        # has to build and ship.
        #
        # Anything that is not clearly frontend-only counts as Rust. The cost of
        # a false positive is a slow deploy; the cost of a false negative is a
        # binary on bob that does not match this commit, which nothing would
        # report. Two entries are less obvious than they look:
        #
        # - `asset/` because deploy-api ships the systemd units, the firewalld
        #   service and the rendered config template from it.
        # - `web/src/api/generated/` because the drift gate below only runs
        #   when `cargo test` has regenerated those files. A hand-edit of them
        #   is exactly what that gate exists to catch, so it must not be able
        #   to arrive as a "frontend-only" change.
        #
        # `.gitea/` is in the list so that editing this file exercises the whole
        # path it describes.
        run: |
          set -euo pipefail
          base="${{ github.event.before }}"
          reason=""
          if [ "${{ github.event_name }}" != "push" ]; then
            reason="not a push"
          elif [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
            reason="no previous head — new branch or force push"
          elif ! git cat-file -e "${base}^{commit}" 2>/dev/null; then
            reason="previous head ${base} is not in this clone"
          fi

          if [ -n "$reason" ]; then
            echo "building everything: ${reason}"
            echo "rust=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Prefer the commit the API is *actually running* over the previous
          # push. They are usually the same and the difference only shows up
          # when it matters: if a run fails, its changes never ship, and the
          # next push diffs against a commit that was never deployed. That is
          # how a frontend asking for `window=3600` came to be served by an API
          # that only knew `six_hours` — the run that would have taught it
          # failed, and the next push was frontend-only, so nothing rebuilt the
          # API and nothing said so.
          #
          # Best effort: an unreachable API, an old binary with no stamp, or a
          # commit this clone does not have all fall back to `event.before`,
          # which is no worse than before. The gate only ever widens.
          live=$(curl -sS --max-time 10 "http://${API_HOST}:${API_PORT}/v1/healthz" 2>/dev/null \
                 | python3 -c 'import sys,json; print(json.load(sys.stdin).get("commit") or "")' 2>/dev/null || true)
          if [ -n "$live" ] && git cat-file -e "${live}^{commit}" 2>/dev/null; then
            if [ "$live" != "$base" ]; then
              echo "diffing against the running API at ${live}, not the previous push ${base}"
            fi
            base="$live"
          else
            echo "the running API did not name a commit we have; falling back to ${base}"
          fi

          changed=$(git diff --name-only "$base" "${{ github.sha }}")
          echo "changed since ${base}:"
          echo "$changed" | sed 's/^/  /'

          if echo "$changed" | grep -qE '^(crates/|asset/|script/|\.gitea/|\.cargo/|\.sqlx/|web/src/api/generated/|Cargo\.(toml|lock)$|rust-toolchain\.toml$|rustfmt\.toml$)'; then
            echo "rust=true" >> "$GITHUB_OUTPUT"
            echo "→ building and deploying the API"
          else
            echo "rust=false" >> "$GITHUB_OUTPUT"
            echo "→ frontend only; skipping the Rust build and the API deploy"
          fi

      - name: gate
        # Format, lint, tests, the ts-rs drift check and the web build, before
        # anything is built for a host, so a broken commit never reaches one.
        # Shared with ci.yaml, which runs it on every other branch: the steps
        # live in .gitea/actions/gate and nowhere else, so the two gates cannot
        # drift apart. The Rust half is skipped on a frontend-only push, as
        # before; the web half always runs.
        uses: ./.gitea/actions/gate
        with:
          rust: ${{ steps.changes.outputs.rust }}

      - name: build api and cli
        if: steps.changes.outputs.rust == 'true'
        env:
          SQLX_OFFLINE: "true"
        run: |
          set -euo pipefail
          # musl, statically linked. gitea-runners.md §4 notes a native build is
          # simpler when the runner matches the target fleet — it does not here:
          # `runner-rust` is Fedora 44 and the targets are Fedora 43, so a
          # glibc-linked binary built here would not load there.
          # Stamped so the API can answer "which commit am I?" — see the
          # deploy gate above, which asks exactly that.
          GIT_SHA="${{ github.sha }}" \
          cargo build --release --target x86_64-unknown-linux-musl \
            -p blackbeard-api -p blackbeard-cli
          file target/x86_64-unknown-linux-musl/release/blackbeard-api

      - uses: actions/upload-artifact@v3
        with:
          name: blackbeard
          # The binaries are absent on a frontend-only run. `warn` rather than
          # `error` for exactly that case — `deploy-api` is skipped alongside,
          # so nothing downstream goes looking for them.
          if-no-files-found: warn
          path: |
            target/x86_64-unknown-linux-musl/release/blackbeard-api
            target/x86_64-unknown-linux-musl/release/blackbeard
            web/dist
            asset

  deploy-api:
    needs: build
    # Skipped when nothing that reaches bob changed. The binary already there is
    # still the right one — this deploy is idempotent and its absence changes
    # nothing about what is running.
    #
    # Never from a ref other than main, and never on a `validate` dispatch. On
    # a push, `inputs.mode` is empty and so not 'validate'.
    if: >-
      needs.build.outputs.rust == 'true'
      && github.ref == 'refs/heads/main'
      && inputs.mode != 'validate'
    # `infra`, not `fedora-43`: the targets are mesh-only .internal names and
    # the fedora runners have no route to the WireGuard mesh. Same reason as
    # lair/quantus and lair/mail.
    runs-on: infra
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: blackbeard

      - name: ssh key and reachability
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s' "$DEPLOY_KEY" > ~/.ssh/id_deploy
          chmod 0600 ~/.ssh/id_deploy
          cat > ~/.ssh/config <<EOF
          Host *
            IdentityFile ~/.ssh/id_deploy
            StrictHostKeyChecking accept-new
            User gitea_ci
          EOF
          ssh "$API_HOST" hostname -f

      - name: preflight sudoers
        # Compare what infra-setup.sh grants against what the target actually
        # allows, and fail up front naming the missing paths — rather than
        # dying partway through an rsync with a bare "sudo: a password is
        # required" after some files have already landed.
        run: |
          set -euo pipefail
          granted=$(ssh "$API_HOST" sudo -n -l || true)
          missing=""
          for path in /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard \
                      /etc/blackbeard/config.toml /etc/sysusers.d/blackbeard.conf \
                      /etc/systemd/system/blackbeard-api.service \
                      /etc/firewalld/services/blackbeard-api.xml; do
            echo "$granted" | grep -qF "$path" || missing="$missing $path"
          done
          if [ -n "$missing" ]; then
            echo "gitea_ci on $API_HOST is not permitted to write:$missing" >&2
            echo "re-run script/infra-setup.sh against this host." >&2
            exit 1
          fi

      - name: render config
        # python3 .replace(), not sed or envsubst: a literal substitution
        # survives values containing shell-special characters.
        run: |
          set -euo pipefail
          fqdn=$(ssh "$API_HOST" hostname -f)
          # The API binds the host's mesh address, never 0.0.0.0 — the fleet has
          # one firewalld default zone, so a wildcard bind plus the named
          # service would publish this on every address the host carries.
          bind=$(ssh "$API_HOST" "ip -4 -o addr show | awk '/10\\.[0-9]+\\./ {print \$4}' | cut -d/ -f1 | head -1")
          if [ -z "$bind" ]; then
            echo "could not determine the mesh address of $API_HOST" >&2
            exit 1
          fi
          echo "binding $bind:$API_PORT on $fqdn"
          python3 - "$fqdn" "$bind" <<'PY'
          import sys, pathlib
          fqdn, bind = sys.argv[1], sys.argv[2]
          tmpl = pathlib.Path("asset/config/config.toml.tmpl").read_text()
          out = tmpl.replace("{{TARGET_FQDN}}", fqdn).replace("{{BIND_ADDRESS}}", bind)
          pathlib.Path("config.toml").write_text(out)
          PY

      - name: verify the rendered config
        # The binary validates its own configuration and exits without binding
        # anything. A bad config fails the deploy here rather than leaving a
        # daemon that will not start.
        run: |
          set -euo pipefail
          chmod +x target/x86_64-unknown-linux-musl/release/blackbeard-api
          # --check reads the certificate paths, which exist on the target and
          # not on the runner, so this runs on the target after the config
          # lands. Placeholder substitution is what is checked here.
          grep -q '{{' config.toml && { echo "config still has unsubstituted placeholders" >&2; exit 1; }
          echo "no placeholders remain"

      - name: ship
        run: |
          set -euo pipefail
          r() { rsync -a --checksum --mkpath --rsync-path='sudo rsync' "$@"; }
          # --checksum, not rsync's default size+mtime: the build produces a new
          # mtime every run, so the default heuristic reports a change on every
          # deploy and would restart a healthy daemon each time.
          r --chmod=0755 target/x86_64-unknown-linux-musl/release/blackbeard-api \
            "$API_HOST:/usr/local/bin/blackbeard-api"
          r --chmod=0755 target/x86_64-unknown-linux-musl/release/blackbeard \
            "$API_HOST:/usr/local/bin/blackbeard"
          r --chown=root:blackbeard --chmod=0640 config.toml \
            "$API_HOST:/etc/blackbeard/config.toml"
          r asset/systemd/blackbeard.sysusers.conf "$API_HOST:/etc/sysusers.d/blackbeard.conf"
          r asset/systemd/blackbeard-api.service "$API_HOST:/etc/systemd/system/blackbeard-api.service"
          r asset/systemd/blackbeard-api-cert.path "$API_HOST:/etc/systemd/system/blackbeard-api-cert.path"
          r asset/systemd/blackbeard-api-cert-reload.service \
            "$API_HOST:/etc/systemd/system/blackbeard-api-cert-reload.service"
          r asset/firewalld/blackbeard-api.xml "$API_HOST:/etc/firewalld/services/blackbeard-api.xml"

      - name: apply system state
        run: |
          set -euo pipefail
          ssh "$API_HOST" bash -euo pipefail <<'REMOTE'
            sudo systemd-sysusers
            sudo install -d -o root -g blackbeard -m 0750 /etc/blackbeard

            # The service account needs to read the host private key: it is the
            # credential for the mTLS Postgres connection and is not
            # world-readable (architecture/generic.md §11).
            sudo setfacl -m u:blackbeard:r "/etc/pki/tls/private/$(hostname -f).pem"

            sudo restorecon -R /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard /etc/blackbeard

            # SELinux must know about the port or the bind is denied. Guarded so
            # a re-run is a no-op; `semanage port -a` on an already-labelled
            # port reassigns it rather than failing cleanly.
            if ! sudo semanage port -l | grep -qE "^http_port_t.*\b25864\b"; then
              sudo semanage port -a -t http_port_t -p tcp 25864
            else
              echo "port 25864 already labelled http_port_t"
            fi

            # firewalld only learns a freshly-shipped custom service after a
            # reload; querying or adding it before reloading fails
            # INVALID_SERVICE.
            sudo firewall-cmd --reload
            zone=$(sudo firewall-cmd --get-default-zone)
            if ! sudo firewall-cmd --zone="$zone" --query-service=blackbeard-api; then
              sudo firewall-cmd --permanent --zone="$zone" --add-service=blackbeard-api
              sudo firewall-cmd --zone="$zone" --add-service=blackbeard-api
            else
              echo "blackbeard-api already open in zone $zone"
            fi

            sudo systemctl daemon-reload
            sudo systemctl enable --now blackbeard-api-cert.path

            # Validate the config on the target, where the certificates it names
            # actually exist.
            sudo -u blackbeard /usr/local/bin/blackbeard-api \
              --config /etc/blackbeard/config.toml --check

            sudo systemctl enable blackbeard-api.service
            sudo systemctl restart blackbeard-api.service
          REMOTE

      - name: health probe
        run: |
          set -euo pipefail
          for attempt in $(seq 1 30); do
            # NOT 127.0.0.1. The API binds the host's *mesh* address, never
            # loopback and never 0.0.0.0 — the fleet has one firewalld default
            # zone, so a wildcard bind plus the named service would publish it
            # on every address the host carries, and the edge proxy is at
            # another site so loopback-only would not serve it at all. Probing
            # loopback therefore gets connection-refused against a perfectly
            # healthy daemon. `hostname -f` resolves to the mesh address, so
            # this needs no value carried between steps.
            if ssh "$API_HOST" "curl -sf http://\$(hostname -f):$API_PORT/v1/healthz" > health.json; then
              cat health.json
              # `healthy` covers the database. Chain reachability is
              # deliberately not part of it: a chain configured before it
              # launches is not a deploy failure.
              python3 -c "import json,sys; sys.exit(0 if json.load(open('health.json'))['healthy'] else 1)"
              exit 0
            fi
            sleep 2
          done
          echo "blackbeard-api did not become healthy" >&2
          exit 1

      - name: reachable from the edge proxy
        # The health probe above runs on the API host over loopback and so
        # cannot see the hop that actually matters: the edge proxy is at a
        # different site, and if the firewalld service were scoped to this
        # host's own /16 the site would serve a static bundle with a dead /v1
        # and nothing would fail.
        run: |
          set -euo pipefail
          ssh "$EDGE_HOST" "curl -sf --max-time 10 http://$API_HOST:$API_PORT/v1/healthz -o /dev/null -w '%{http_code}\n'"

      - name: journal
        if: always()
        run: ssh "$API_HOST" journalctl -u blackbeard-api.service -n 80 --no-pager || true

  deploy-web:
    needs: build
    # The same guard as deploy-api, less the Rust gate: the bundle ships on
    # every push to main.
    if: github.ref == 'refs/heads/main' && inputs.mode != 'validate'
    runs-on: infra
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: blackbeard

      - name: ssh key and reachability
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s' "$DEPLOY_KEY" > ~/.ssh/id_deploy
          chmod 0600 ~/.ssh/id_deploy
          cat > ~/.ssh/config <<EOF
          Host *
            IdentityFile ~/.ssh/id_deploy
            StrictHostKeyChecking accept-new
            User gitea_ci
          EOF
          ssh "$EDGE_HOST" hostname -f

      - name: ship the bundle
        # --delete: hashed asset filenames accumulate forever otherwise. The
        # webroot holds only build output, so there is nothing else to lose.
        run: |
          set -euo pipefail
          rsync -a --checksum --delete --mkpath --rsync-path='sudo rsync' \
            web/dist/ "$EDGE_HOST:$WEBROOT/"

      - name: label and reload
        run: |
          set -euo pipefail
          ssh "$EDGE_HOST" bash -euo pipefail <<REMOTE
            # SELinux: an unlabelled webroot makes nginx return 403 for every
            # file, with nothing in the nginx error log to explain it.
            sudo restorecon -R "$WEBROOT"
            sudo nginx -t
            sudo systemctl reload nginx
          REMOTE

      - name: fetch the site
        # `nginx -t` parses without binding, so a passing test is not evidence
        # the reload landed. Fetching the page is — and fetching the chain list
        # through the vhost proves the whole path: nginx, the cross-site hop,
        # the API, and the telemetry directory behind it. A 200 with an empty
        # chain list is a broken deploy that a status code alone would pass.
        run: |
          set -euo pipefail
          # --resolve to loopback: from inside the mesh the public name resolves
          # to the site's WAN address and dead-ends on the OPNsense LAN
          # interface (reverse-proxies.md §2). Pinning it to 127.0.0.1 still
          # exercises the real path — the :443 stream router, SNI, the vhost,
          # the cross-site hop and the API — just without leaving the host.
          ssh "$EDGE_HOST" "curl -sf --max-time 20 --resolve $PUBLIC_NAME:443:127.0.0.1 https://$PUBLIC_NAME/v1/chains" > chains.json
          python3 - <<'PY'
          import json, sys
          chains = json.load(open("chains.json"))
          full = [c for c in chains if c["tracking"] == "full"]
          for c in chains:
              print(f"{c['node_count'] or 0:>4} nodes  {c['tracking']:<12} {c['display_name']}")
          if not full:
              print("no chain is tracked in full — the site would show an empty board", file=sys.stderr)
              sys.exit(1)
          PY

  validate:
    # What `mode: validate` means beyond the build: the read-only half of the two
    # deploy jobs, pointed at what is already running. Health from the API host,
    # reachability from the edge proxy, and the chain list through the public
    # vhost. It ships nothing, writes nothing on either host, and restarts
    # nothing.
    #
    # No `needs: build`: it checks the live site, which this ref's build has no
    # bearing on, and a red gate should not hide whether production is up.
    # main-only like the deploys, because it carries the deploy key.
    #
    # These checks are copies of the last steps of deploy-api and deploy-web and
    # must change with them. They are not shared through an action because the
    # deploy jobs never check out the repository — they run from the build
    # artifact — and changing what a deploy job has on disk to save twenty lines
    # is a change to the deploy path that only a production deploy would test.
    if: >-
      github.event_name == 'workflow_dispatch'
      && inputs.mode == 'validate'
      && github.ref == 'refs/heads/main'
    runs-on: infra
    steps:
      - name: ssh key and reachability
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s' "$DEPLOY_KEY" > ~/.ssh/id_deploy
          chmod 0600 ~/.ssh/id_deploy
          cat > ~/.ssh/config <<EOF
          Host *
            IdentityFile ~/.ssh/id_deploy
            StrictHostKeyChecking accept-new
            User gitea_ci
          EOF
          ssh "$API_HOST" hostname -f
          ssh "$EDGE_HOST" hostname -f

      - name: health probe
        # As in deploy-api, less the retry loop: nothing was just restarted, so
        # an API that is not healthy now is the finding, not a slow start. Mesh
        # address, never 127.0.0.1 — see deploy-api's probe.
        run: |
          set -euo pipefail
          ssh "$API_HOST" "curl -sf http://\$(hostname -f):$API_PORT/v1/healthz" > health.json
          cat health.json
          python3 -c "import json,sys; sys.exit(0 if json.load(open('health.json'))['healthy'] else 1)"

      - name: reachable from the edge proxy
        run: |
          set -euo pipefail
          ssh "$EDGE_HOST" "curl -sf --max-time 10 http://$API_HOST:$API_PORT/v1/healthz -o /dev/null -w '%{http_code}\n'"

      - name: fetch the site
        # As in deploy-web: a 200 with no chain tracked in full is a broken site
        # that a status code alone would pass.
        run: |
          set -euo pipefail
          ssh "$EDGE_HOST" "curl -sf --max-time 20 --resolve $PUBLIC_NAME:443:127.0.0.1 https://$PUBLIC_NAME/v1/chains" > chains.json
          python3 - <<'PY'
          import json, sys
          chains = json.load(open("chains.json"))
          full = [c for c in chains if c["tracking"] == "full"]
          for c in chains:
              print(f"{c['node_count'] or 0:>4} nodes  {c['tracking']:<12} {c['display_name']}")
          if not full:
              print("no chain is tracked in full — the site would show an empty board", file=sys.stderr)
              sys.exit(1)
          PY

      - name: journal
        if: always()
        run: ssh "$API_HOST" journalctl -u blackbeard-api.service -n 80 --no-pager || true