quantus/.gitea/workflows/deploy.yaml

name: deploy
# Deploy — or validate — Quantus Planck nodes and their external GPU miners.
#
# Topology (the workflow is the source of infra truth,
# ~/git/architecture/deployment-gitea-actions.md — hosts belong here, and only
# here; see readme.md "Scope"):
#
#   quantus-node    consensus, P2P, rewards
#        ▲ QUIC/9833 (mesh only, rich-rule scoped to that node's miners)
#   quantus-miner   GPU search
#
# The node and the miner are separate processes on separate hosts by design: the
# node is disk+network bound and belongs on an always-on box, the miner is pure
# GPU compute. Setting --miner-listen-port DISABLES the node's built-in CPU
# mining, so the node authors nothing if the miner is absent — the validate mode
# below checks exactly that rather than trusting `is-active`.
#
# Runs on `infra`. gitea-runners.md §4 says a deploy needs only ssh+rsync and so
# fits `fedora-43`; that is about tooling, not routing. These targets are
# mesh-only `.internal` names and lair/mail's two working deploys both use
# `infra` for that reason. Deliberate deviation, noted per readme.md.

on:
  push:
    branches: [main]
    paths:
      - asset/**
      - .gitea/workflows/deploy.yaml
  workflow_dispatch:
    inputs:
      mode:
        description: "deploy (apply) or validate (check only, no changes)"
        required: false
        default: deploy
        type: choice
        options: [deploy, validate]
      node_version:
        description: quantus-node version (overrides the pin)
        required: false
      miner_version:
        description: quantus-miner version (overrides the pin)
        required: false

concurrency:              # never half-apply two deploys at once
  group: deploy
  cancel-in-progress: false

env:
  # The chain spec. Switching to mainnet is meant to be THIS line plus the
  # version pins — the unit takes it from config, and the miner credential path
  # (<base-path>/chains/<chain>/) already derives from it. Planck is a testnet;
  # mainnet is expected 2026-09-09.
  CHAIN: planck
  # Pinned; bump deliberately to upgrade. An unattended upgrade of a validator
  # is how you find out at 3am that a release changed a consensus rule.
  NODE_VERSION: "0.10.0"
  MINER_VERSION: "4.0.0"
  # Ports — port-allocations.md §5 registry. These are upstream protocol
  # defaults rather than derived numbers; see readme.md "Ports".
  P2P_PORT: "30333"
  MINER_LINK_PORT: "9833"
  NODE_METRICS_PORT: "9615"
  MINER_METRICS_PORT: "9900"
  # Fleet Prometheus/Grafana host. Exporters are unauthenticated, so this is the
  # only host allowed to reach them.
  METRICS_HOST: golgafrinchans.kosherinata.internal
  # Arena exporter: network hashrate, difficulty, authorship leaderboard.
  # Derived from the service name per port-allocations.md §3.
  ARENA_PORT: "25033"
  # Host-published port of the fleet Prometheus (port-allocations.md §5).
  PROM_PORT: "26559"
  GRAFANA_PORT: "28767"
  # Scrape targets, rendered into the Prometheus scrape config. These must agree
  # with the deploy-node / deploy-miner matrices below; `validate metrics`
  # asserts Prometheus actually has every target UP, so drift fails loudly rather
  # than silently monitoring nothing.
  SCRAPE_NODES: bob.hanzalova.internal
  SCRAPE_MINERS: benjy.hanzalova.internal quadbrat.hanzalova.internal
  # The arena exporter runs beside each node, reading its loopback RPC.
  SCRAPE_ARENA: bob.hanzalova.internal

jobs:
  fetch:
    runs-on: infra
    outputs:
      node_version: ${{ steps.v.outputs.node }}
      miner_version: ${{ steps.v.outputs.miner }}
    steps:
      - id: v
        run: |
          set -euo pipefail
          n="${{ github.event.inputs.node_version || env.NODE_VERSION }}"
          m="${{ github.event.inputs.miner_version || env.MINER_VERSION }}"
          echo "node=${n}"  >> "$GITHUB_OUTPUT"
          echo "miner=${m}" >> "$GITHUB_OUTPUT"
          echo "quantus-node ${n} / quantus-miner ${m}"

      - name: download quantus-node
        run: |
          set -euo pipefail
          v="${{ steps.v.outputs.node }}"
          curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
            -o node.tar.gz \
            "https://github.com/Quantus-Network/chain/releases/download/v${v}/quantus-node-v${v}-x86_64-unknown-linux-gnu.tar.gz"
          tar xzf node.tar.gz
          install -D -m 0755 "$(find . -name quantus-node -type f | head -1)" _bin/quantus-node
          _bin/quantus-node --version

      - name: download quantus-miner
        run: |
          set -euo pipefail
          v="${{ steps.v.outputs.miner }}"
          curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
            -o _bin/quantus-miner \
            "https://github.com/Quantus-Network/quantus-miner/releases/download/v${v}/quantus-miner-linux-x86_64"
          chmod 0755 _bin/quantus-miner
          _bin/quantus-miner --version

      - uses: actions/upload-artifact@v3
        with:
          name: quantus-bin
          path: _bin

  deploy-node:
    runs-on: infra
    needs: fetch
    strategy:
      fail-fast: false        # one site's failure must not abort the other
      matrix:
        include:
          # ONE inner_hash PER HOST — never share one across nodes.
          #
          # The reward preimage is published verbatim in the PreRuntime digest
          # of every block a node authors, and the payout address is
          # Poseidon2(inner_hash). It is therefore public, and it is STATIC —
          # derived per wallet, not per block. Two nodes sharing an inner_hash
          # are publicly and permanently identifiable as the same operator,
          # which silently collapses the independence that running nodes at
          # separate sites is meant to provide. See doc/wormhole-rewards.md §5.
          #
          # Not a secret, so it lives here with the rest of the infra truth.
          # Derive one per host, offline, from a DEDICATED mining wallet
          # (doc/wormhole-rewards.md §6):
          #   quantus-node key quantus --scheme wormhole --words < mnemonic.txt
          - host: bob.hanzalova.internal
            inner_hash: "0x134e73f06fa9bdb1dbfa909e149c563f5860ceb71a0e7307918f7033970edf59"
            # One node fans the same job out to every connected miner
            # (node/src/miner_server.rs: broadcast_job over a HashMap of
            # MinerHandle), and each miner picks its own random starting nonce,
            # so no coordination or range allocation is needed. A node per miner
            # would only be needed for a separate reward address.
            miners: benjy.hanzalova.internal quadbrat.hanzalova.internal
            # Complete flag, or "" for a node with no public ingress. Requires a
            # matching inbound TCP forward at the site edge — the flag only
            # advertises, it does not open anything.
            #
            # /dns4/ rather than a literal address: the site's WAN IP can change
            # without a redeploy, and this is the per-site indirection name the
            # fleet already uses (architecture public-dns.md).
            public_addr: "--public-addr /dns4/nh.thgttg.com/tcp/30333"
          # Second site, for decentralisation. Uncomment when provisioned, and
          # give it its OWN inner_hash derived from its OWN wallet — sharing
          # bob's would publicly tie the two sites to one operator and defeat
          # the point of running them separately (doc/wormhole-rewards.md §5).
          #
          # NOTE: a node with no miners authors nothing, because
          # --miner-listen-port disables built-in mining. A node intended purely
          # to relay and validate needs a unit WITHOUT
          # --validator/--miner-listen-port, not this one — and then it needs no
          # inner_hash at all.
          # - host: <second-node>.internal
          #   inner_hash: "0x..."
          #   miners: ""
          #   public_addr: ""
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-bin, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: render node config
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          QUANTUS_INNER_HASH: ${{ matrix.inner_hash }}
          QUANTUS_PUBLIC_ADDR: ${{ matrix.public_addr }}
          QUANTUS_CHAIN: ${{ env.CHAIN }}
        run: |
          set -euo pipefail
          case "$QUANTUS_INNER_HASH" in
            0x0000000000000000000000000000000000000000000000000000000000000000)
              echo "${{ matrix.host }}: inner_hash is still the placeholder." >&2
              echo "Set a real one for THIS HOST in the deploy-node matrix." >&2
              echo "Derive it from a dedicated mining wallet, offline:" >&2
              echo "  quantus-node key quantus --scheme wormhole --words < mnemonic.txt" >&2
              echo "See doc/wormhole-rewards.md §5-§6." >&2
              exit 1 ;;
          esac
          case "$QUANTUS_INNER_HASH" in
            0x*) ;;
            *) echo "inner_hash must be 0x-prefixed (the node rejects it otherwise)" >&2; exit 1 ;;
          esac
          # Literal substitution — never a shell/sed expansion, so a value with
          # regex or shell metacharacters survives intact.
          python3 - <<'PY'
          import os, pathlib
          tmpl = pathlib.Path("asset/config/node.env.tmpl").read_text()
          out = tmpl.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"])
          out = out.replace("{{QUANTUS_PUBLIC_ADDR}}", os.environ.get("QUANTUS_PUBLIC_ADDR", ""))
          out = out.replace("{{QUANTUS_CHAIN}}", os.environ["QUANTUS_CHAIN"])
          pathlib.Path("node.env").write_text(out)
          a = pathlib.Path("asset/config/arena.env.tmpl").read_text()
          a = a.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"])
          pathlib.Path("arena.env").write_text(a)
          PY

      - name: deploy node
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          HOST: ${{ matrix.host }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run()  { ssh $SSHOPTS gitea_ci@"$HOST" "$@"; }

          # -c (checksum), not rsync's default size+mtime quick check: `fetch`
          # re-downloads the binary every run, so its mtime is always new and the
          # default heuristic reports a change on every deploy. -c makes "changed"
          # mean "the content differs", which is what RESTART must key off.
          # -i itemizes, so an empty result is proof of no change.
          RESTART=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then
              RESTART=1
              printf '%s\n' "$out" | sed 's/^/    changed: /'
            fi
          }
          # firewalld definitions are picked up by --reload below and are never a
          # reason to bounce the daemon.
          pushfw() {
            rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@" | sed 's/^/    changed: /'
          }

          # 1. service account
          push --mkpath --chmod=F0644 \
            asset/systemd/quantus-node.sysusers.conf \
            gitea_ci@"$HOST":/etc/sysusers.d/quantus-node.conf
          run sudo systemd-sysusers

          # 2. directories. /etc is root-owned so the daemon can read but not
          #    rewrite its own config (generic.md §8).
          run sudo install -d -o root -g quantus-node -m 0750 /etc/quantus-node
          run sudo install -d -o quantus-node -g quantus-node -m 0750 /var/lib/quantus-node

          # 3. binary, unit, config, firewalld defs
          push --chmod=F0755 _bin/quantus-node gitea_ci@"$HOST":/usr/local/bin/quantus-node
          push --chmod=F0644 asset/systemd/quantus-node.service \
            gitea_ci@"$HOST":/etc/systemd/system/quantus-node.service
          push --chown=root:quantus-node --chmod=F0640 node.env \
            gitea_ci@"$HOST":/etc/quantus-node/node.env
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node.xml
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node-miner.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-miner.xml
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node-metrics.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-metrics.xml

          # 3b. Arena exporter — network-wide stats from the node's loopback RPC.
          push --mkpath --chmod=F0644 asset/systemd/quantus-arena.sysusers.conf \
            gitea_ci@"$HOST":/etc/sysusers.d/quantus-arena.conf
          run sudo systemd-sysusers
          run sudo install -d -o root -g quantus-arena -m 0750 /etc/quantus-arena
          run sudo install -d -o quantus-arena -g quantus-arena -m 0750 /var/lib/quantus-arena
          push --chmod=F0755 asset/arena/quantus-arena-exporter.py \
            gitea_ci@"$HOST":/usr/local/bin/quantus-arena-exporter.py
          push --chmod=F0644 asset/systemd/quantus-arena.service \
            gitea_ci@"$HOST":/etc/systemd/system/quantus-arena.service
          push --chown=root:quantus-arena --chmod=F0640 arena.env \
            gitea_ci@"$HOST":/etc/quantus-arena/arena.env
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-arena.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-arena.xml

          # 4. SELinux relabel. No `semanage port` needed: the unit runs under
          #    init_t, which may bind any port.
          run sudo restorecon -R /usr/local/bin/quantus-node /etc/quantus-node /var/lib/quantus-node
          run sudo restorecon -R /usr/local/bin/quantus-arena-exporter.py /etc/quantus-arena /var/lib/quantus-arena

          # 5. firewalld. Reload FIRST so the freshly-shipped definitions exist,
          #    or --query-service fails INVALID_SERVICE
          #    (deployment-gitea-actions.md §6).
          run sudo firewall-cmd --reload
          zone=$(run sudo firewall-cmd --get-default-zone)
          echo "default zone: ${zone}"

          # 5a. P2P: public, plain named service.
          if run sudo firewall-cmd --zone="$zone" --query-service=quantus-node; then
            echo "firewalld: quantus-node already enabled in ${zone}"
          else
            run sudo firewall-cmd --permanent --zone="$zone" --add-service=quantus-node
            run sudo firewall-cmd --zone="$zone" --add-service=quantus-node
          fi

          # 5b. Miner link: a rich rule scoped to this node's miner hosts, NOT a
          #     plain --add-service. With a single default zone (generic.md §9)
          #     adding the service outright would publish 9833/udp on every
          #     address the host carries. Resolve each miner's mesh address ON
          #     THE NODE — this repo carries no 10.x literals, and the node's own
          #     resolution is the address the miner will actually present.
          for m in ${{ matrix.miners }}; do
            miner_addrs=$(run "getent ahostsv4 $m")
            miner_ip=$(awk '{print $1; exit}' <<<"$miner_addrs")
            case "$miner_ip" in
              10.*) echo "miner source: ${m} -> ${miner_ip}" ;;
              *) echo "refusing to open ${{ env.MINER_LINK_PORT }}/udp to non-mesh address '${miner_ip}' for ${m}" >&2; exit 1 ;;
            esac
            rich="rule family=ipv4 source address=${miner_ip}/32 service name=quantus-node-miner accept"
            # Pass these as ONE pre-quoted string, not as separate run() args.
            # `run()` is `ssh ... "$@"`, and ssh concatenates its argument vector
            # with spaces for the REMOTE shell to re-split — so local quoting is
            # lost and a rich rule arrives as a dozen bare words
            # ("unrecognized arguments: family=ipv4 source address=..."). Every
            # other command here survives only because no other argument
            # contains a space. Keep the inner single quotes.
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$rich'"; then
              echo "firewalld: rich rule for ${m} already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$rich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$rich'"
            fi
          done

          # 5c. Metrics: scoped to the scrape host only. The exporter is
          #     unauthenticated and --prometheus-external binds every interface,
          #     so this rule is the whole access boundary.
          metrics_addrs=$(run "getent ahostsv4 ${{ env.METRICS_HOST }}")
          metrics_ip=$(awk '{print $1; exit}' <<<"$metrics_addrs")
          case "$metrics_ip" in
            10.*) echo "scrape source: ${metrics_ip}" ;;
            *) echo "refusing to expose metrics to non-mesh address '${metrics_ip}'" >&2; exit 1 ;;
          esac
          for svc in quantus-node-metrics quantus-arena; do
            mrich="rule family=ipv4 source address=${metrics_ip}/32 service name=${svc} accept"
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$mrich'"; then
              echo "firewalld: ${svc} rich rule already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$mrich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$mrich'"
            fi
          done

          # 6. (re)start — ONLY if something actually changed. Bouncing a
          #    syncing node costs it every peer and a RocksDB reopen, and this
          #    workflow also runs to validate an unchanged deployment.
          run sudo systemctl enable quantus-node.service    # idempotent, no service impact
          run sudo systemctl enable quantus-arena.service   # idempotent
          if [ "$RESTART" = 1 ]; then
            echo "changes applied — restarting"
            run sudo systemctl daemon-reload
            run sudo systemctl restart quantus-node.service
            run sudo systemctl restart quantus-arena.service
          elif run systemctl is-active --quiet quantus-node.service; then
            echo "nothing changed and the node is running — left alone"
          else
            echo "nothing changed but the node is down — starting it"
            run sudo systemctl restart quantus-node.service
          fi
          run systemctl is-active --quiet quantus-arena.service \
            || { echo "arena exporter down — starting it"; run sudo systemctl restart quantus-arena.service; }

      - name: validate node
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }}) ---"
          run systemctl is-active quantus-node.service

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-node --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ needs.fetch.outputs.node_version }}"*) echo "version matches pin" ;;
            *) echo "version does NOT match pin ${{ needs.fetch.outputs.node_version }}" >&2; fail=1 ;;
          esac

          echo "--- reward address ---"
          # The node logs the address it derived from this host's inner_hash.
          # Surfacing it makes a copy-pasted or shared inner_hash visible.
          run journalctl -u quantus-node.service -n 500 --no-pager \
            | grep -F "Rewards wormhole address" | tail -1 || {
              echo "  WARN no reward address in the recent journal" >&2; }

          echo "--- readiness ---"
          # A restarted node opens its RocksDB and initialises litep2p before it
          # binds anything; asserting immediately after `systemctl restart` reads
          # a node that is merely still starting as one that is broken. Wait for
          # the P2P socket, bounded, rather than sleeping a guessed constant.
          ready=0
          for i in $(seq 1 60); do
            if p=$(run "ss -Hln -t sport = :${{ env.P2P_PORT }}") && [ -n "$p" ]; then
              echo "  ready after ~$((i*2))s"; ready=1; break
            fi
            sleep 2
          done
          if [ "$ready" = 0 ]; then
            echo "  node did not open tcp/${{ env.P2P_PORT }} within 120s" >&2
            fail=1
          fi

          echo "--- listeners ---"
          # P2P must be listening; the miner link must be listening or the node
          # is authoring nothing (--miner-listen-port disables local mining).
          # `-t`/`-u` matter: without a protocol flag ss errors with
          # "RTNETLINK answers: Invalid argument" and reports nothing useful.
          # Capture, then test — piping into `grep -q` makes grep exit early,
          # SIGPIPEs ssh, and `set -o pipefail` turns that into a 141 that looks
          # exactly like "not listening".
          for spec in "${{ env.P2P_PORT }} tcp -t" "${{ env.MINER_LINK_PORT }} udp -u"; do
            set -- $spec
            if listeners=$(run "ss -Hln $3 sport = :$1"); then
              if [ -n "$listeners" ]; then
                echo "  ok   $2/$1 listening"
              else
                echo "  FAIL $2/$1 not listening" >&2; fail=1
              fi
            else
              echo "  FAIL could not query $2/$1 on the host" >&2; fail=1
            fi
          done

          echo "--- advertised address ---"
          if [ -n "${{ matrix.public_addr }}" ]; then
            echo "  advertising: ${{ matrix.public_addr }}"
          else
            echo "  none set — this node accepts no inbound peers and is a" >&2
            echo "  leech on the network rather than a contribution to it." >&2
          fi

          echo "--- sync ---"
          # Here-strings, not pipes: `awk ... exit` closes the pipe early, the
          # writer takes SIGPIPE, and pipefail turns a successful parse into 141.
          # The metrics body is large enough for that race to fire reliably.
          m=$(run "curl -fsS http://127.0.0.1:${{ env.NODE_METRICS_PORT }}/metrics")
          peers=$(awk '/^substrate_sub_libp2p_peers_count/{print $2; exit}' <<<"$m")
          best=$(awk '/^substrate_block_height\{status="best"/{print $2; exit}' <<<"$m")
          echo "  peers=${peers:-unknown} best_block=${best:-unknown}"
          if [ "${peers:-0}" = "0" ]; then echo "  WARN no peers" >&2; fi

          exit $fail

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-node.service -n 80 --no-pager

  deploy-miner:
    runs-on: infra
    needs: [fetch, deploy-node]
    strategy:
      fail-fast: false
      matrix:
        include:
          # `node` is the host whose QUIC control channel this miner attaches
          # to, and whose inner_hash therefore receives what it earns. A miner
          # holds no reward configuration of its own — the mining protocol
          # carries no payout address at all.
          - host: benjy.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "1"        # 1× RTX 4090, measured 183 MH/s @ ~450 W
          - host: quadbrat.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "1"        # 1× RTX 3060, hashrate not yet measured
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-bin, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: deploy miner
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          GPU_DEVICES: ${{ matrix.gpu_devices }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          nrun() { ssh $SSHOPTS gitea_ci@"${{ matrix.node }}" "$@"; }
          run()  { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }

          # See the node job for why -c rather than rsync's default quick check.
          # A rotated auth token or TLS pin shows up here as a content change and
          # correctly forces a restart.
          RESTART=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then
              RESTART=1
              printf '%s\n' "$out" | sed 's/^/    changed: /'
            fi
          }

          # 1. service account + dirs
          push --mkpath --chmod=F0644 \
            asset/systemd/quantus-miner.sysusers.conf \
            gitea_ci@"${{ matrix.host }}":/etc/sysusers.d/quantus-miner.conf
          run sudo systemd-sysusers
          run sudo install -d -o root -g quantus-miner -m 0750 /etc/quantus-miner
          run sudo install -d -o quantus-miner -g quantus-miner -m 0750 /var/lib/quantus-miner

          # 2. The miner's credentials are GENERATED BY THE NODE on first start
          #    and regenerate if the node's base-path is ever wiped. Copying them
          #    on every deploy — rather than once in infra-setup.sh — is what
          #    makes that self-healing instead of a silent auth failure.
          #    They pass through the runner in memory, never the workspace.
          umask 077
          tmp=$(mktemp -d)
          trap 'rm -rf "$tmp"' EXIT
          nrun sudo cat /var/lib/quantus-node/chains/${{ env.CHAIN }}/miner-auth-token \
            > "$tmp/miner-auth-token"
          nrun sudo cat /var/lib/quantus-node/chains/${{ env.CHAIN }}/miner-tls-cert-sha256 \
            > "$tmp/miner-tls-cert-sha256"
          test -s "$tmp/miner-auth-token"      || { echo "node auth token empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          test -s "$tmp/miner-tls-cert-sha256" || { echo "node TLS pin empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-auth-token" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-auth-token
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-tls-cert-sha256" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-tls-cert-sha256

          # 3. non-secret runtime config.
          #    --node-addr parses as a Rust SocketAddr: an IP and a port, with NO
          #    DNS resolution ("invalid socket address syntax" on a hostname).
          #    Resolve on the MINER host — it is the one dialling, so its own view
          #    of the node's address is the correct one — and keep the 10.x
          #    literal out of the repo, same as the firewall rich rule.
          node_addrs=$(run "getent ahostsv4 ${{ matrix.node }}")
          node_ip=$(awk '{print $1; exit}' <<<"$node_addrs")
          case "$node_ip" in
            10.*) echo "node address: ${{ matrix.node }} -> ${node_ip}" ;;
            *) echo "refusing to point the miner at non-mesh address '${node_ip}'" >&2; exit 1 ;;
          esac
          export NODE_ADDR="${node_ip}:${{ env.MINER_LINK_PORT }}"

          python3 - <<'PY'
          import os, pathlib
          t = pathlib.Path("asset/config/miner.env.tmpl").read_text()
          t = t.replace("{{QUANTUS_NODE_ADDR}}", os.environ["NODE_ADDR"])
          t = t.replace("{{QUANTUS_GPU_DEVICES}}", os.environ["GPU_DEVICES"])
          pathlib.Path("miner.env").write_text(t)
          PY
          push --chown=root:quantus-miner --chmod=F0640 \
            miner.env gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner.env

          # 4. binary + unit
          push --chmod=F0755 _bin/quantus-miner gitea_ci@"${{ matrix.host }}":/usr/local/bin/quantus-miner
          push --chmod=F0644 asset/systemd/quantus-miner.service \
            gitea_ci@"${{ matrix.host }}":/etc/systemd/system/quantus-miner.service

          run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner

          # 5. firewalld for the exporter. The miner binds metrics on 0.0.0.0
          #    unconditionally — there is no loopback option — so without this
          #    rule the port is simply closed, and with an unscoped one it would
          #    be open to the whole mesh.
          rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic --mkpath --chmod=F0644 \
            asset/firewalld/quantus-miner-metrics.xml \
            gitea_ci@"${{ matrix.host }}":/etc/firewalld/services/quantus-miner-metrics.xml \
            | sed 's/^/    changed: /'
          run sudo firewall-cmd --reload
          zone=$(run sudo firewall-cmd --get-default-zone)
          metrics_addrs=$(run "getent ahostsv4 ${{ env.METRICS_HOST }}")
          metrics_ip=$(awk '{print $1; exit}' <<<"$metrics_addrs")
          case "$metrics_ip" in
            10.*) echo "scrape source: ${metrics_ip}" ;;
            *) echo "refusing to expose metrics to non-mesh address '${metrics_ip}'" >&2; exit 1 ;;
          esac
          mrich="rule family=ipv4 source address=${metrics_ip}/32 service name=quantus-miner-metrics accept"
          if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$mrich'"; then
            echo "firewalld: metrics rich rule already present in ${zone}"
          else
            run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$mrich'"
            run "sudo firewall-cmd --zone=$zone --add-rich-rule='$mrich'"
          fi

          run sudo systemctl enable quantus-miner.service   # idempotent
          if [ "$RESTART" = 1 ]; then
            echo "changes applied — restarting"
            run sudo systemctl daemon-reload
            run sudo systemctl restart quantus-miner.service
          elif run systemctl is-active --quiet quantus-miner.service; then
            echo "nothing changed and the miner is running — left alone"
          else
            echo "nothing changed but the miner is down — starting it"
            run sudo systemctl restart quantus-miner.service
          fi

      - name: validate miner
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }} -> ${{ matrix.node }}) ---"
          run systemctl is-active quantus-miner.service

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-miner --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ needs.fetch.outputs.miner_version }}"*) echo "version matches pin" ;;
            *) echo "version does NOT match pin ${{ needs.fetch.outputs.miner_version }}" >&2; fail=1 ;;
          esac

          echo "--- gpu ---"
          # An `active` miner that found no adapter still looks healthy to
          # systemd; assert the GPU is actually enumerated and busy.
          run "nvidia-smi --query-gpu=name,power.draw,utilization.gpu --format=csv,noheader"

          echo "--- hashing ---"
          # The counter is the only honest evidence this process is doing work
          # rather than idling on a failed connection.
          # Here-strings, not pipes — see the node's sync check for why.
          m1=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics")
          h1=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m1")
          sleep 20
          m2=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics")
          h2=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m2")
          echo "  miner_hashes_total ${h1:-?} -> ${h2:-?}"
          if [ -n "${h1:-}" ] && [ -n "${h2:-}" ] && [ "${h2%.*}" -gt "${h1%.*}" ]; then
            echo "  ok   hash counter advancing"
          else
            echo "  WARN counter not advancing — ${{ matrix.node }} may still be syncing," >&2
            echo "       which is expected and not itself a deploy failure." >&2
          fi

          exit $fail

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-miner.service -n 80 --no-pager

  deploy-metrics:
    runs-on: infra
    needs: [deploy-node, deploy-miner]
    # Monitoring assets only. Deliberately last and deliberately separate: a
    # Grafana dashboard failing to provision must never be able to leave a
    # half-deployed validator behind it.
    steps:
      - uses: actions/checkout@v4

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ env.METRICS_HOST }} hostname -f

      - name: deploy scrape config and dashboard
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          HOST: ${{ env.METRICS_HOST }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run()  { ssh $SSHOPTS gitea_ci@"$HOST" "$@"; }

          RELOAD=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then RELOAD=1; printf '%s\n' "$out" | sed 's/^/    changed: /'; fi
          }

          # Render the scrape targets. Quoted, comma-separated, from the env
          # lists above — a space-separated host list becomes a YAML array.
          fmt() { local o=""; for h in $1; do o="${o:+$o, }\"${h}:${2}\""; done; printf '%s' "$o"; }
          NODE_TARGETS=$(fmt "${{ env.SCRAPE_NODES }}"  "${{ env.NODE_METRICS_PORT }}")
          MINER_TARGETS=$(fmt "${{ env.SCRAPE_MINERS }}" "${{ env.MINER_METRICS_PORT }}")
          ARENA_TARGETS=$(fmt "${{ env.SCRAPE_ARENA }}"  "${{ env.ARENA_PORT }}")
          echo "node targets:  ${NODE_TARGETS}"
          echo "miner targets: ${MINER_TARGETS}"
          echo "arena targets: ${ARENA_TARGETS}"

          NODE_TARGETS="$NODE_TARGETS" MINER_TARGETS="$MINER_TARGETS" ARENA_TARGETS="$ARENA_TARGETS" python3 - <<'PY'
          import os, pathlib
          t = pathlib.Path("asset/prometheus/quantus.yml.tmpl").read_text()
          t = t.replace("{{NODE_TARGETS}}", os.environ["NODE_TARGETS"])
          t = t.replace("{{MINER_TARGETS}}", os.environ["MINER_TARGETS"])
          t = t.replace("{{ARENA_TARGETS}}", os.environ["ARENA_TARGETS"])
          pathlib.Path("quantus-scrape.yml").write_text(t)
          PY
          python3 -c "import yaml,sys; yaml.safe_load(open('quantus-scrape.yml'))" \
            || { echo "rendered scrape config is not valid YAML" >&2; exit 1; }

          push --mkpath --chmod=F0644 quantus-scrape.yml \
            gitea_ci@"$HOST":/etc/prometheus/scrape_configs.d/quantus.yml
          push --mkpath --chmod=F0644 asset/grafana/quantus.json \
            gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus/quantus.json

          # The PROVIDER tells Grafana the dashboards directory exists at all;
          # without it the JSON above is just a file nobody reads. Tracked
          # separately from the dashboard because Grafana reads providers ONLY at
          # startup, while dashboards are re-read on updateIntervalSeconds — so a
          # provider change needs a restart and a dashboard change does not.
          PROVIDER=0
          pout=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic --mkpath --chmod=F0644 \
            asset/grafana/quantus-dashboards.yaml \
            gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus-dashboards.yaml)
          if [ -n "$pout" ]; then
            PROVIDER=1
            printf '%s\n' "$pout" | sed 's/^/    changed: /'
          fi

          if [ "$PROVIDER" = 1 ]; then
            echo "dashboard provider changed — restarting grafana (providers are read at startup only)"
            run sudo systemctl restart grafana.service
          fi

          if [ "$RELOAD" = 1 ]; then
            # Hot reload rather than a restart: Prometheus restarts drop nothing
            # permanent but do interrupt scraping, and --web.enable-lifecycle is
            # already set for exactly this.
            echo "config changed — reloading prometheus"
            run "curl -fsS -X POST http://127.0.0.1:${{ env.PROM_PORT }}/-/reload"
            echo "grafana picks the dashboard up on its own provisioning interval"
          else
            echo "nothing changed — no reload"
          fi

      - name: validate metrics
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ env.METRICS_HOST }}" "$@"; }
          fail=0

          echo "--- prometheus targets ---"
          # The point of this check: a target that drifted out of the matrices,
          # or a firewalld rule that never landed, shows up here as missing or
          # DOWN instead of an empty graph nobody looks at.
          #
          # Written as a quoted heredoc reading a file, NOT `python3 -c '...'`:
          # escaping quotes through the shell into an f-string produced
          # "SyntaxError: unexpected character after line continuation character",
          # because Python reads \" inside the expression as a continuation.
          # A target Prometheus has only just learned about reports health
          # "unknown" until its FIRST scrape completes. With a 15s scrape
          # interval and this step running seconds after the config reload, any
          # run that adds a target would otherwise fail on a target that is
          # perfectly healthy. Wait for the unknowns to resolve, bounded; a
          # genuinely dead target goes to "down", not "unknown", so this does
          # not paper over real failures.
          cat > count_unknown.py <<'PY'
          import json
          d = json.load(open("targets.json"))
          rows = [t for t in d["data"]["activeTargets"]
                  if t["labels"].get("job", "").startswith("quantus-")]
          print(sum(1 for t in rows if t["health"] == "unknown") if rows else 1)
          PY
          for i in $(seq 1 24); do
            run "curl -fsS 'http://127.0.0.1:${{ env.PROM_PORT }}/api/v1/targets?state=active'" > targets.json
            pending=$(python3 count_unknown.py)
            if [ "$pending" = "0" ]; then
              [ "$i" -gt 1 ] && echo "  settled after ~$((i*5))s"
              break
            fi
            [ "$i" = "24" ] && echo "  ${pending} target(s) still unscraped after 120s" >&2
            sleep 5
          done

          if python3 - <<'PY'
          import json, sys
          d = json.load(open("targets.json"))
          rows = [t for t in d["data"]["activeTargets"]
                  if t["labels"].get("job", "").startswith("quantus-")]
          if not rows:
              print("  FAIL prometheus has no quantus-* targets", file=sys.stderr)
              print("       is scrape_config_files wired into prometheus.yml?", file=sys.stderr)
              print("       one-time: script/infra-setup.sh --metrics-hosts", file=sys.stderr)
              sys.exit(1)
          bad = 0
          for t in sorted(rows, key=lambda x: (x["labels"].get("job", ""),
                                               x["labels"].get("instance", ""))):
              j = t["labels"].get("job")
              i = t["labels"].get("instance")
              h = t["health"]
              e = t.get("lastError", "")
              print(("  ok   " if h == "up" else "  FAIL ") + " ".join(filter(None, [j, i, h, e])))
              if h != "up":
                  bad += 1
          sys.exit(1 if bad else 0)
          PY
          then
            echo "  all quantus targets up"
          else
            echo "  FAIL not every quantus target is up" >&2; fail=1
          fi

          echo "--- grafana dashboard ---"
          # Grafana's /api/search requires auth (401 unauthenticated), and this
          # job holds no Grafana credentials by design — so assert provisioning
          # from the journal instead, which gitea_ci can read via systemd-journal.
          run "curl -fsS http://127.0.0.1:${{ env.GRAFANA_PORT }}/api/health" > /dev/null \
            && echo "  ok   grafana is healthy" \
            || { echo "  FAIL grafana health endpoint not responding" >&2; fail=1; }

          if run "test -s /etc/grafana/provisioning/dashboards/quantus/quantus.json" \
             && run "test -s /etc/grafana/provisioning/dashboards/quantus-dashboards.yaml"; then
            echo "  ok   dashboard and provider present on the host"
          else
            echo "  FAIL dashboard JSON or provider missing" >&2; fail=1
          fi

          # A dashboard that fails to parse is logged by the provisioner and then
          # silently absent from the UI, so treat any provisioning error as fatal.
          if run "journalctl -u grafana.service --since '-10min' --no-pager" \
               | grep -E 'logger=provisioning\.dashboard.*level=error'; then
            echo "  FAIL grafana reported a dashboard provisioning error" >&2; fail=1
          else
            echo "  ok   no dashboard provisioning errors in the last 10 minutes"
          fi

          exit $fail