miner/.gitea/workflows/deploy.yaml

---
# Deploy — or validate — the miner on the fleet's mining hosts (quantus/miner#8).
#
# Every push to main builds the binary and lands it on each host in the matrix,
# validated, with rollback. This workflow is the source of infra truth for the
# miner: hosts and per-host settings live in the `deploy` matrix and nowhere
# else. The node it attaches to is deployed by quantus/chain; the fleet's
# monitoring, GPU power limits and nvidia metrics stay in lair/quantus.
#
# Build on cuda-13.0: it is Fedora 43 like the hosts (the `rust` image is
# Fedora 44 and its binaries fail on the hosts with GLIBC_2.43 not found), and
# it is the only runner with nvcc for the CUDA engine (#3). Deploy on
# fedora-43: ssh + rsync are on every runner (gitea-runners.md §3).
name: deploy

on:
  push:
    branches: [main]
    paths-ignore:
      - "**.md"
      - .gitea/workflows/ci.yml
      - .gitea/workflows/bench.yaml
      - .gitea/workflows/release.yaml
  workflow_dispatch:
    inputs:
      mode:
        description: "deploy (apply) or validate (check only, no changes)"
        required: false
        default: deploy
        type: choice
        options: [deploy, validate]
      chain:
        description: "chain id under the node's chains/ directory to take miner credentials from (overrides CHAIN); the node deploy's validate step prints it"
        required: false

# Never half-apply two deploys at once. (Not relied on for correctness of a
# single host: the restart is checksum-gated and the binary is rsynced atomically.)
concurrency:
  group: deploy-miner
  cancel-in-progress: false

env:
  # The chain spec: the miner credential path on the node host derives from it.
  # Must agree with lair/quantus's node deploy. Mainnet since 2026-09-09; the
  # node writes the credentials under chains/mainnet/.
  CHAIN: mainnet
  MINER_LINK_PORT: "9833"
  MINER_METRICS_PORT: "9900"
  # Fleet Prometheus host; the miner's exporter is unauthenticated and bound to
  # 0.0.0.0, so this is the only host allowed to reach it.
  METRICS_HOST: golgafrinchans.kosherinata.internal
  CARGO_TERM_COLOR: always

jobs:
  build:
    runs-on: cuda-13.0
    steps:
      - uses: actions/checkout@v4
      - name: build quantus-miner
        env:
          # Embedded by crates/miner-cli/build.rs into --version; validate below
          # asserts the host runs exactly this commit.
          MINER_BUILD_SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
          cargo build --release --locked -p miner-cli
          ./target/release/quantus-miner --version
      - uses: actions/upload-artifact@v3
        with:
          name: quantus-miner
          path: target/release/quantus-miner

  deploy:
    runs-on: fedora-43
    needs: build
    strategy:
      fail-fast: false        # one host's failure must not abort the other
      matrix:
        include:
          # `node` is the host whose QUIC control channel this miner attaches
          # to, and whose reward address therefore receives what it earns. The
          # node's deploy (quantus/chain) must list this host in its `miners`
          # so the firewalld rich rule admits it. lair/quantus's SCRAPE_MINERS
          # must list it so Prometheus scrapes it.
          #
          # Mainnet call 2026-09-09 (quantus/miner#1): all three hosts mine;
          # neuron (cortex inference) is disabled on each of them.
          # `kernel` is the kernel id validate expects on the per-device metric:
          # cuda (engine-cuda, #3) on every NVIDIA host once the build carries
          # the fat binary; u64 would mean the miner silently fell back to wgpu.
          - host: benjy.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "1"        # 1x RTX 4090 (sm_89); reference card for #2
            kernel: cuda
          - host: quadbrat.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "1"        # 1x RTX 3060 (sm_86)
            kernel: cuda
          - host: beast.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "2"        # 2x RTX 5090 (sm_120)
            kernel: cuda
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-miner, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: preflight — sudoers covers this deploy
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          # Every path deploy/infra-setup.sh grants must already be permitted on
          # the host. Compare up front, from the script itself so the two cannot
          # drift, and name the missing paths instead of failing forty lines into
          # an rsync with "sudo: a password is required".
          sed -n "/quantus-miner_gitea_ci.tmp/,/^SUDO$/p" deploy/infra-setup.sh \
            | grep '^gitea_ci ALL=' | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > expected-paths.txt
          ssh $SSHOPTS gitea_ci@${{ matrix.host }} 'sudo -n -l' \
            | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > permitted-paths.txt
          comm -23 expected-paths.txt permitted-paths.txt > missing-paths.txt
          if [ -s missing-paths.txt ]; then
            echo "the miner sudoers on ${{ matrix.host }} is out of date." >&2
            echo "not permitted, but this deploy needs them:" >&2
            sed 's/^/  /' missing-paths.txt >&2
            echo "" >&2
            echo "run: ./deploy/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub" >&2
            exit 1
          fi
          echo "sudoers covers all $(wc -l < expected-paths.txt) paths this deploy needs"

      - name: deploy miner
        id: deploy
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          GPU_DEVICES: ${{ matrix.gpu_devices }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          nrun() { ssh $SSHOPTS gitea_ci@"${{ matrix.node }}" "$@"; }
          run()  { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }

          # -c (checksum), not rsync's default size+mtime quick check: the
          # artifact is freshly built every run so mtimes always differ, and the
          # default heuristic would report a change on every deploy. Restart is
          # gated on an itemised content difference only.
          RESTART=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then
              RESTART=1
              printf '%s\n' "$out" | sed 's/^/    changed: /'
            fi
          }

          # 1. service account + dirs
          push --mkpath --chmod=F0644 \
            deploy/quantus-miner.sysusers.conf \
            gitea_ci@"${{ matrix.host }}":/etc/sysusers.d/quantus-miner.conf
          run sudo systemd-sysusers
          run sudo install -d -o root -g quantus-miner -m 0750 /etc/quantus-miner
          run sudo install -d -o quantus-miner -g quantus-miner -m 0750 /var/lib/quantus-miner

          # 2. The miner's credentials are GENERATED BY THE NODE on first start
          #    and regenerate if the node's base-path is ever wiped. Copying them
          #    on every deploy is what makes that self-healing instead of a
          #    silent auth failure. They pass through the runner in memory,
          #    never the workspace.
          umask 077
          tmp=$(mktemp -d)
          trap 'rm -rf "$tmp"' EXIT
          nrun sudo cat /var/lib/quantus-node/chains/${{ github.event.inputs.chain || env.CHAIN }}/miner-auth-token \
            > "$tmp/miner-auth-token"
          nrun sudo cat /var/lib/quantus-node/chains/${{ github.event.inputs.chain || env.CHAIN }}/miner-tls-cert-sha256 \
            > "$tmp/miner-tls-cert-sha256"
          test -s "$tmp/miner-auth-token"      || { echo "node auth token empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          test -s "$tmp/miner-tls-cert-sha256" || { echo "node TLS pin empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-auth-token" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-auth-token
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-tls-cert-sha256" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-tls-cert-sha256

          # 3. non-secret runtime config.
          #    --node-addr parses as a Rust SocketAddr: an IP and a port, with NO
          #    DNS resolution. Resolve on the MINER host — it is the one
          #    dialling — and keep the 10.x literal out of the repo.
          node_addrs=$(run "getent ahostsv4 ${{ matrix.node }}")
          node_ip=$(awk '{print $1; exit}' <<<"$node_addrs")
          case "$node_ip" in
            10.*) echo "node address: ${{ matrix.node }} -> ${node_ip}" ;;
            *) echo "refusing to point the miner at non-mesh address '${node_ip}'" >&2; exit 1 ;;
          esac
          export NODE_ADDR="${node_ip}:${{ env.MINER_LINK_PORT }}"

          python3 - <<'PY'
          import os, pathlib
          t = pathlib.Path("deploy/miner.env.tmpl").read_text()
          t = t.replace("{{QUANTUS_NODE_ADDR}}", os.environ["NODE_ADDR"])
          t = t.replace("{{QUANTUS_GPU_DEVICES}}", os.environ["GPU_DEVICES"])
          pathlib.Path("miner.env").write_text(t)
          PY
          push --chown=root:quantus-miner --chmod=F0640 \
            miner.env gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner.env

          # 4. binary + unit. Keep the running binary as .prev first so a failed
          #    validate can put it back (rollback step below).
          if run test -x /usr/local/bin/quantus-miner; then
            run sudo cp -p /usr/local/bin/quantus-miner /usr/local/bin/quantus-miner.prev
            echo "previous binary kept: $(run /usr/local/bin/quantus-miner.prev --version)"
          fi
          push --chmod=F0755 _bin/quantus-miner gitea_ci@"${{ matrix.host }}":/usr/local/bin/quantus-miner
          push --chmod=F0644 deploy/quantus-miner.service \
            gitea_ci@"${{ matrix.host }}":/etc/systemd/system/quantus-miner.service

          run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner

          # 5. firewalld for the exporter, scoped to the scrape host. The miner
          #    binds metrics on 0.0.0.0 unconditionally.
          rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic --mkpath --chmod=F0644 \
            deploy/quantus-miner-metrics.xml \
            gitea_ci@"${{ matrix.host }}":/etc/firewalld/services/quantus-miner-metrics.xml \
            | sed 's/^/    changed: /'
          run sudo firewall-cmd --reload
          zone=$(run sudo firewall-cmd --get-default-zone)
          metrics_addrs=$(run "getent ahostsv4 ${{ env.METRICS_HOST }}")
          metrics_ip=$(awk '{print $1; exit}' <<<"$metrics_addrs")
          case "$metrics_ip" in
            10.*) echo "scrape source: ${metrics_ip}" ;;
            *) echo "refusing to expose metrics to non-mesh address '${metrics_ip}'" >&2; exit 1 ;;
          esac
          mrich="rule family=ipv4 source address=${metrics_ip}/32 service name=quantus-miner-metrics accept"
          if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$mrich'"; then
            echo "firewalld: metrics rich rule already present in ${zone}"
          else
            run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$mrich'"
            run "sudo firewall-cmd --zone=$zone --add-rich-rule='$mrich'"
          fi

          run sudo systemctl enable quantus-miner.service   # idempotent
          if [ "$RESTART" = 1 ]; then
            echo "changes applied — restarting"
            run sudo systemctl daemon-reload
            run sudo systemctl restart quantus-miner.service
          elif run systemctl is-active --quiet quantus-miner.service; then
            echo "nothing changed and the miner is running — left alone"
          else
            echo "nothing changed but the miner is down — starting it"
            run sudo systemctl restart quantus-miner.service
          fi
          echo "restarted=$RESTART" >> "$GITHUB_OUTPUT"

      - name: validate miner
        id: validate
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }} -> ${{ matrix.node }}) ---"
          run systemctl is-active quantus-miner.service

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-miner --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ github.sha }}"*) echo "binary is this commit" ;;
            *)
              if [ "${{ github.event.inputs.mode }}" = validate ]; then
                echo "note: installed binary is not this commit (validate mode; not a failure)"
              else
                echo "installed binary does NOT carry commit ${{ github.sha }}" >&2; fail=1
              fi ;;
          esac

          echo "--- gpu ---"
          # An `active` miner that found no adapter still looks healthy to
          # systemd; assert the GPU is enumerated and that the miner sees the
          # number of devices the matrix says it should.
          run "nvidia-smi --query-gpu=name,power.draw,utilization.gpu --format=csv,noheader"

          echo "--- hashing ---"
          # The counter is the only honest evidence this process is doing work
          # rather than idling on a failed connection. After a restart the miner
          # exports miner_gpu_devices only once it has connected to the node and
          # miner_hashes_total only once it has hashed, so first wait for the
          # gauge to appear (readiness), then require the counter to advance.
          # Here-strings, not pipes: a pipe whose reader exits early SIGPIPEs the
          # writer and pipefail turns that into exit 141.
          scrape() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics" || true; }
          deadline=$((SECONDS + 90))
          devs=""
          while [ $SECONDS -lt $deadline ]; do
            m=$(scrape)
            devs=$(awk '/^miner_gpu_devices /{print $2; exit}' <<<"$m")
            [ -n "$devs" ] && break
            sleep 5
          done
          if [ -z "$devs" ]; then
            echo "  miner did not connect to ${{ matrix.node }} within 90s (no miner_gpu_devices exported)" >&2; fail=1
          elif [ "${devs%.*}" != "${{ matrix.gpu_devices }}" ]; then
            echo "  miner_gpu_devices is ${devs}, matrix says ${{ matrix.gpu_devices }}" >&2; fail=1
          else
            echo "  ok   connected; miner_gpu_devices ${devs%.*}"
          fi
          # The build-info gauge is what ties every other metric to a commit
          # (quantus/miner#9). In deploy mode it must carry this commit.
          if grep -q "^miner_build_info{.*commit=\"${{ github.sha }}\"" <<<"$m"; then
            echo "  ok   miner_build_info carries ${{ github.sha }}"
          elif [ "${{ github.event.inputs.mode }}" != validate ]; then
            echo "  miner_build_info does not carry commit ${{ github.sha }}" >&2; fail=1
          fi
          # The kernel actually running. A wgpu fallback on a host that should
          # run CUDA passes every other check at a fraction of the hashrate.
          if grep -q "^miner_device_hashes_total{.*kernel=\"${{ matrix.kernel }}\"" <<<"$m"; then
            echo "  ok   kernel ${{ matrix.kernel }} on device 0"
          else
            echo "  expected kernel ${{ matrix.kernel }}, exported series:" >&2
            grep "^miner_device_hashes_total" <<<"$m" >&2 || echo "  (none yet)" >&2
            if [ "${{ github.event.inputs.mode }}" != validate ]; then fail=1; fi
          fi

          deadline=$((SECONDS + 120))
          h1=""; h2=""; ok=0
          while [ $SECONDS -lt $deadline ]; do
            m=$(scrape)
            h=$(awk '/^miner_hashes_total /{print $2; exit}' <<<"$m")
            if [ -n "$h" ]; then
              if [ -z "$h1" ]; then
                h1="$h"
              elif [ "${h%.*}" -gt "${h1%.*}" ]; then
                h2="$h"; ok=1; break
              fi
            fi
            sleep 10
          done
          if [ "$ok" = 1 ]; then
            echo "  ok   miner_hashes_total ${h1%.*} -> ${h2%.*}"
          else
            echo "  miner_hashes_total did not advance within 120s (${h1:-none} -> ${h2:-none})" >&2
            fail=1
          fi

          exit $fail

      - name: rollback
        # Only when this run replaced the binary and validate then failed.
        if: ${{ failure() && steps.deploy.outputs.restarted == '1' }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          if run test -x /usr/local/bin/quantus-miner.prev; then
            echo "validate failed after a restart — restoring the previous binary"
            # install, not cp: cp writes in place and fails with "Text file busy"
            # on a running binary; install unlinks the destination first (the
            # same reason rsync's temp-file-and-rename works for the push).
            run sudo install -m 0755 /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner
            run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner
            run sudo systemctl restart quantus-miner.service
            echo "restored: $(run /usr/local/bin/quantus-miner --version)"
          else
            echo "no previous binary to restore" >&2
          fi

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-miner.service -n 60 --no-pager