rustingface/.gitea/workflows/deploy.yml

name: deploy

on:
  push: { branches: [main] }
  workflow_dispatch:

concurrency:
  group: deploy
  cancel-in-progress: false

env:
  # --- infra truth: hosts, ports and paths live here, not in a manifest ---
  SERVICE_HOST: bob.hanzalova.internal
  # Bind the mesh address rather than a wildcard: the reverse proxy is on a
  # different host, so loopback will not do, and a wildcard bind on a host that
  # may later gain another interface would publish the registry there too.
  LISTEN_ADDR: 10.6.0.193:20482
  APP_PORT: "20482"
  S3_ENDPOINT: http://caveman.kosherinata.internal:9000
  S3_BUCKET: rustingface
  UPSTREAM_ENABLED: "true"
  AUTH_MODE: bearer
  AUTH_ANONYMOUS: catalog
  ALLOW_NEW_REPOS: "false"
  PROXY_HOST: hanzalova.internal
  WEBROOT: /var/www/rustingface

jobs:
  build:
    runs-on: rust
    steps:
      - uses: actions/checkout@v4

      # The gate runs before anything is built for deployment, so a broken
      # commit on main never reaches a host.
      - name: format
        run: cargo fmt --all -- --check

      - name: clippy
        run: cargo clippy --workspace --all-targets --all-features -- -D warnings

      - name: test
        run: cargo test --workspace --all-features

      # Static musl: the runner is Fedora 44 and the target is Fedora 43, so a
      # dynamically linked binary could reference a newer glibc than bob has.
      - name: build
        run: |
          rustup target add x86_64-unknown-linux-musl
          cargo build --release --target x86_64-unknown-linux-musl --bin rustingface

      - uses: actions/upload-artifact@v3
        with:
          name: rustingface
          path: target/x86_64-unknown-linux-musl/release/rustingface

  build-web:
    runs-on: fedora-43
    steps:
      - uses: actions/checkout@v4

      # The gate for the frontend, matching the Rust one: a type error or a
      # lint failure must not reach a host either.
      # The runner's pnpm is not necessarily the workstation's, and this
      # setting moved location between 10.x releases. Print what it actually
      # resolves to, so a failure here is diagnosable from the log rather than
      # by pushing another guess.
      - name: pnpm environment
        working-directory: web
        run: |
          pnpm --version
          pnpm config get --json onlyBuiltDependencies || true
          pnpm config list || true

      - name: install
        working-directory: web
        run: pnpm install --frozen-lockfile

      - name: lint
        working-directory: web
        run: pnpm lint

      - name: build
        working-directory: web
        # `pnpm build` runs `tsc -b` first, so type errors fail here.
        run: pnpm build

      - uses: actions/upload-artifact@v3
        with:
          name: rustingface-web
          path: web/dist

  deploy:
    needs: build
    runs-on: fedora-43
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: rustingface
          path: dist

      # Without this, a missing secret first surfaces as ssh failing to parse an
      # empty key, which names neither the secret nor the fix.
      - name: check the required secrets are set
        env:
          RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
          S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
          S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
          CLIENT_TOKENS: ${{ secrets.CLIENT_TOKENS }}
        run: |
          missing=""
          for name in RSYNC_SSH_KEY S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY CLIENT_TOKENS; do
            [ -n "${!name:-}" ] || missing="$missing $name"
          done
          if [ -n "$missing" ]; then
            echo "missing repo secret(s):$missing" >&2
            echo "Set them in Settings > Actions > Secrets. HF_TOKEN is optional and" >&2
            echo "only needed for gated repositories." >&2
            exit 1
          fi
          echo "all required secrets are present"

      - name: authenticate to the target
        id: auth
        run: |
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 600 ~/.ssh/id_gitea_ci
          cat > ~/.ssh/config <<CFG
          Host $SERVICE_HOST
            User gitea_ci
            IdentityFile ~/.ssh/id_gitea_ci
            IdentitiesOnly yes
            StrictHostKeyChecking accept-new
          CFG
          ssh "$SERVICE_HOST" hostname -f

      - name: render config
        env:
          S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
          S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          CLIENT_TOKENS: ${{ secrets.CLIENT_TOKENS }}
        run: |
          # Literal substitution, not a shell expansion: a secret containing
          # $, ` or \ must survive intact.
          python3 - <<'PY'
          import os, pathlib
          template = pathlib.Path("asset/config/config.toml.tmpl").read_text()
          for key in ("LISTEN_ADDR", "S3_ENDPOINT", "S3_BUCKET", "UPSTREAM_ENABLED",
                      "AUTH_MODE", "AUTH_ANONYMOUS", "ALLOW_NEW_REPOS"):
              template = template.replace("{{%s}}" % key, os.environ[key])
          pathlib.Path("dist/config.toml").write_text(template)
          for name, var in (("s3-access-key", "S3_ACCESS_KEY_ID"),
                            ("s3-secret-key", "S3_SECRET_ACCESS_KEY"),
                            ("hf-token", "HF_TOKEN"),
                            ("client-tokens", "CLIENT_TOKENS")):
              pathlib.Path("dist", name).write_text(os.environ.get(var, ""))
          PY
          # Fail loudly rather than shipping a config with an unrendered
          # placeholder that would only surface as a runtime parse error.
          # Matches the placeholder *shape* rather than a bare "{{", so the
          # template's own prose describing the syntax is not a false positive.
          if grep -nE '\{\{[A-Z_][A-Z0-9_]*\}\}' dist/config.toml; then
            echo "unrendered placeholder in the config" >&2
            exit 1
          fi
          if [ ! -s dist/s3-access-key ] || [ ! -s dist/s3-secret-key ]; then
            echo "S3 credentials are empty; set the S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY secrets" >&2
            exit 1
          fi
          # Bearer auth with no tokens refuses to start, so catch it here
          # rather than as a service that restart-loops after the files land.
          if [ "$AUTH_MODE" = bearer ] && [ ! -s dist/client-tokens ]; then
            echo "AUTH_MODE is bearer but the CLIENT_TOKENS secret is empty." >&2
            echo "Set it to one accepted token per line; '#' comments are allowed." >&2
            exit 1
          fi

      - name: create the service account and its config directory
        run: |
          rsync -az --rsync-path='sudo rsync' --mkpath \
            asset/systemd/rustingface.sysusers.conf \
            "$SERVICE_HOST:/etc/sysusers.d/rustingface.conf"
          ssh "$SERVICE_HOST" '
            set -euo pipefail
            sudo systemd-sysusers
            sudo install -d -o root -g rustingface -m 0750 /etc/rustingface'

      - name: ship the binary, config and credentials
        run: |
          set -euo pipefail
          rsync -az --rsync-path='sudo rsync' \
            dist/rustingface "$SERVICE_HOST:/usr/local/bin/rustingface"
          for f in config.toml s3-access-key s3-secret-key hf-token client-tokens; do
            rsync -az --rsync-path='sudo rsync' --mkpath \
              "dist/$f" "$SERVICE_HOST:/etc/rustingface/$f"
          done
          rsync -az --rsync-path='sudo rsync' --mkpath \
            asset/systemd/rustingface.service \
            "$SERVICE_HOST:/etc/systemd/system/rustingface.service"
          ssh "$SERVICE_HOST" '
            set -euo pipefail
            sudo chmod 0755 /usr/local/bin/rustingface
            sudo chown -R root:rustingface /etc/rustingface
            sudo chmod 0640 /etc/rustingface/config.toml /etc/rustingface/s3-access-key /etc/rustingface/s3-secret-key /etc/rustingface/hf-token /etc/rustingface/client-tokens
            sudo restorecon -R /usr/local/bin/rustingface /etc/rustingface'

      - name: firewalld
        run: |
          set -euo pipefail
          # rsync the XML first: firewalld only learns a freshly-shipped custom
          # service after --reload, and querying it before that fails.
          rsync -az --rsync-path='sudo rsync' --mkpath \
            asset/firewalld/rustingface.xml \
            "$SERVICE_HOST:/etc/firewalld/services/rustingface.xml"
          ssh "$SERVICE_HOST" '
            set -euo pipefail
            sudo firewall-cmd --reload
            zone=$(sudo firewall-cmd --get-default-zone)
            if sudo firewall-cmd --zone="$zone" --query-service=rustingface; then
              echo "rustingface already enabled in $zone"
            else
              sudo firewall-cmd --permanent --zone="$zone" --add-service=rustingface
              sudo firewall-cmd --zone="$zone" --add-service=rustingface
            fi'

      - name: restart
        run: |
          ssh "$SERVICE_HOST" '
            set -euo pipefail
            sudo systemctl daemon-reload
            sudo systemctl enable rustingface.service
            sudo systemctl restart rustingface.service'

      - name: health check
        run: |
          set -euo pipefail
          ssh "$SERVICE_HOST" 'sudo systemctl is-active rustingface.service'
          for attempt in $(seq 1 20); do
            if ssh "$SERVICE_HOST" "curl -fsS http://$LISTEN_ADDR/healthz"; then
              echo "healthy after $attempt attempt(s)"
              exit 0
            fi
            sleep 2
          done
          echo "rustingface did not answer /healthz" >&2
          exit 1

      - name: doctor
        run: |
          # Proves the deployment can actually reach and write to the bucket,
          # which a liveness probe deliberately does not.
          ssh "$SERVICE_HOST" \
            'sudo -u rustingface /usr/local/bin/rustingface --config /etc/rustingface/config.toml doctor'

      # Capture the unit's startup journal even when a later step failed --
      # that is the record worth having. Conditioned on the target actually
      # being reachable: without it, a job that failed before authenticating
      # ends with "Host key verification failed", which buries the real cause
      # under an error about something else entirely.
      - name: journal
        if: always() && steps.auth.outcome == 'success'
        run: ssh "$SERVICE_HOST" 'journalctl -u rustingface.service -n 200 --no-pager'

  deploy-web:
    # The frontend is only useful once the API it reads is live, and the vhost
    # it is served from proxies to that API.
    needs: [build-web, deploy]
    runs-on: fedora-43
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: rustingface-web
          path: dist

      - name: check the required secrets are set
        env:
          RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
        run: |
          [ -n "${RSYNC_SSH_KEY:-}" ] || { echo "missing repo secret: RSYNC_SSH_KEY" >&2; exit 1; }
          echo "all required secrets are present"

      - name: authenticate to the proxy
        id: auth
        run: |
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 600 ~/.ssh/id_gitea_ci
          cat > ~/.ssh/config <<CFG
          Host $PROXY_HOST
            User gitea_ci
            IdentityFile ~/.ssh/id_gitea_ci
            IdentitiesOnly yes
            StrictHostKeyChecking accept-new
          CFG
          ssh "$PROXY_HOST" hostname -f

      - name: ship the built frontend
        run: |
          set -euo pipefail
          [ -f dist/index.html ] || { echo "the build produced no index.html" >&2; exit 1; }
          # --delete so a rename leaves no orphaned asset behind; the webroot
          # holds nothing but this build's output.
          rsync -az --delete --rsync-path='sudo rsync' \
            dist/ "$PROXY_HOST:$WEBROOT/"
          # Webroots must be httpd_sys_content_t or nginx answers 403.
          ssh "$PROXY_HOST" "sudo restorecon -R $WEBROOT"

      - name: reload nginx
        run: |
          set -euo pipefail
          # `nginx -t` cannot catch a bind failure, but it does catch a config
          # that would break every other vhost on this shared proxy.
          ssh "$PROXY_HOST" 'sudo nginx -t'
          ssh "$PROXY_HOST" 'sudo systemctl reload nginx'

      - name: health check
        run: |
          set -euo pipefail
          # Probed from the proxy, not from this runner. The runner is a plain
          # Fedora container and does not carry the internal root CA, so a TLS
          # probe from here would fail on trust rather than on anything real
          # (verified: a fedora:43 container gets 000, the proxy gets 200).
          # The proxy resolves rf.internal to itself, and a mesh name does not
          # hit the public-name hairpin.
          ssh "$PROXY_HOST" 'set -euo pipefail
            # The app is served, and the API it reads is reachable through the
            # same vhost. Both, because a working page over a dead API looks
            # fine until somebody clicks Models.
            curl -fsS https://rf.internal/          | grep -q "id=.root."
            curl -fsS https://rf.internal/v1/status | grep -q bucket
            curl -fsS https://rf.internal/healthz   | grep -q ok
            # A client route must fall back to the app rather than 404.
            curl -fsS https://rf.internal/models    | grep -q "id=.root."
            # An API path must still reach the service, not the app. With
            # bearer auth on this is a 401, which is itself the proof that the
            # request reached the service rather than the static root.
            code=$(curl -s -o /dev/null -w "%{http_code}" https://rf.internal/api/whoami-v2)
            case "$code" in 200|401) ;; *) echo "api path returned $code" >&2; exit 1;; esac
            # The public vhost, if its certificate exists yet. Checked over
            # loopback with the SNI name, because the public name may not
            # resolve from inside the mesh until the DNS record is published.
            if sudo test -d /etc/letsencrypt/live/rustingface.com; then
              curl -fsS --resolve rustingface.com:443:127.0.0.1 \
                https://rustingface.com/ | grep -q "id=.root."
              # The catalogue is the public face and must answer anonymously.
              curl -fsS --resolve rustingface.com:443:127.0.0.1 \
                https://rustingface.com/v1/status | grep -q bucket
              # /metrics must not be public.
              m=$(curl -s -o /dev/null -w "%{http_code}" --resolve rustingface.com:443:127.0.0.1 \
                https://rustingface.com/metrics)
              [ "$m" = 404 ] || { echo "/metrics is exposed publicly ($m)" >&2; exit 1; }
              echo "public vhost answering, /metrics closed"
            else
              echo "rustingface.com has no certificate yet; skipping the public checks"
            fi'
          echo "frontend and API both answering through rf.internal"

      - name: nginx log
        if: always() && steps.auth.outcome == 'success'
        run: ssh "$PROXY_HOST" 'sudo tail -n 50 /var/log/nginx/rf.internal.error.log'