quantus/.gitea/workflows/deploy.yaml

name: deploy
# Deploy — or validate — Quantus Planck nodes, and the fleet monitoring for
# nodes and miners. The miners themselves deploy from quantus/miner.
#
# Topology (the workflow is the source of infra truth,
# ~/git/architecture/deployment-gitea-actions.md — hosts belong here, and only
# here; see readme.md "Scope"):
#
#   quantus-node    consensus, P2P, rewards            (deployed here)
#        ▲ QUIC/9833 (mesh only, rich-rule scoped to that node's miners)
#   quantus-miner   GPU search                         (deployed by quantus/miner)
#
# The node and the miner are separate processes on separate hosts by design: the
# node is disk+network bound and belongs on an always-on box, the miner is pure
# GPU compute. Setting --miner-listen-port DISABLES the node's built-in CPU
# mining, so the node authors nothing if the miner is absent — the validate mode
# below checks exactly that rather than trusting `is-active`.
#
# Runs on `infra`. gitea-runners.md §4 says a deploy needs only ssh+rsync and so
# fits `fedora-43`; that is about tooling, not routing. These targets are
# mesh-only `.internal` names and lair/mail's two working deploys both use
# `infra` for that reason. Deliberate deviation, noted per readme.md.

on:
  push:
    branches: [main]
    paths:
      - asset/**
      - .gitea/workflows/deploy.yaml
  workflow_dispatch:
    inputs:
      mode:
        description: "deploy (apply) or validate (check only, no changes)"
        required: false
        default: deploy
        type: choice
        options: [deploy, validate]
      node_version:
        description: quantus-node version (overrides the pin)
        required: false

concurrency:              # never half-apply two deploys at once
  group: deploy
  cancel-in-progress: false

env:
  # The chain spec. Switching to mainnet is meant to be THIS line plus the
  # version pins — the unit takes it from config, and the miner credential path
  # (<base-path>/chains/<chain>/) already derives from it. Planck is a testnet;
  # mainnet is expected 2026-09-09.
  CHAIN: planck
  # Pinned; bump deliberately to upgrade. An unattended upgrade of a validator
  # is how you find out at 3am that a release changed a consensus rule.
  NODE_VERSION: "0.11.1"
  # The miner is deployed by quantus/miner's own workflow, following its main
  # branch (quantus/miner#8, lair/quantus#4). This repo no longer ships it;
  # SCRAPE_MINERS below must still agree with that repo's deploy matrix.
  # Ports — port-allocations.md §5 registry. These are upstream protocol
  # defaults rather than derived numbers; see readme.md "Ports".
  P2P_PORT: "30333"
  MINER_LINK_PORT: "9833"
  NODE_METRICS_PORT: "9615"
  MINER_METRICS_PORT: "9900"
  # Fleet Prometheus/Grafana host. Exporters are unauthenticated, so this is the
  # only host allowed to reach them.
  METRICS_HOST: golgafrinchans.kosherinata.internal
  # substrate-telemetry feed for this chain. Found in /tmp/env-config.js on the
  # telemetry site — NOT the /feed path on the main host, which 404s.
  TELEMETRY_URL: wss://feed-telemetry.quantus.cat/feed
  # Arena exporter: network hashrate, difficulty, authorship leaderboard.
  # Derived from the service name per port-allocations.md §3.
  ARENA_PORT: "25033"
  # Host-published port of the fleet Prometheus (port-allocations.md §5).
  PROM_PORT: "26559"
  GRAFANA_PORT: "28767"
  # Scrape targets, rendered into the Prometheus scrape config. These must agree
  # with the deploy-node matrix below and quantus/miner's deploy matrix; `validate metrics`
  # asserts Prometheus actually has every target UP, so drift fails loudly rather
  # than silently monitoring nothing.
  SCRAPE_NODES: bob.hanzalova.internal
  SCRAPE_MINERS: benjy.hanzalova.internal quadbrat.hanzalova.internal
  # The arena exporter runs beside each node, reading its loopback RPC.
  SCRAPE_ARENA: bob.hanzalova.internal

jobs:
  fetch:
    runs-on: infra
    outputs:
      node_version: ${{ steps.v.outputs.node }}
    steps:
      - id: v
        run: |
          set -euo pipefail
          n="${{ github.event.inputs.node_version || env.NODE_VERSION }}"
          echo "node=${n}"  >> "$GITHUB_OUTPUT"
          echo "quantus-node ${n}"

      - name: download quantus-node
        run: |
          set -euo pipefail
          v="${{ steps.v.outputs.node }}"
          curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
            -o node.tar.gz \
            "https://github.com/Quantus-Network/chain/releases/download/v${v}/quantus-node-v${v}-x86_64-unknown-linux-gnu.tar.gz"
          tar xzf node.tar.gz
          install -D -m 0755 "$(find . -name quantus-node -type f | head -1)" _bin/quantus-node
          _bin/quantus-node --version

      - uses: actions/upload-artifact@v3
        with:
          name: quantus-bin
          path: _bin

  deploy-node:
    runs-on: infra
    needs: fetch
    strategy:
      fail-fast: false        # one site's failure must not abort the other
      matrix:
        include:
          # ONE inner_hash PER HOST — never share one across nodes.
          #
          # The reward preimage is published verbatim in the PreRuntime digest
          # of every block a node authors, and the payout address is
          # Poseidon2(inner_hash). It is therefore public, and it is STATIC —
          # derived per wallet, not per block. Two nodes sharing an inner_hash
          # are publicly and permanently identifiable as the same operator,
          # which silently collapses the independence that running nodes at
          # separate sites is meant to provide. See doc/wormhole-rewards.md §5.
          #
          # Not a secret, so it lives here with the rest of the infra truth.
          # Derive one per host, offline, from a DEDICATED mining wallet
          # (doc/wormhole-rewards.md §6):
          #   quantus-node key quantus --scheme wormhole --words < mnemonic.txt
          - host: bob.hanzalova.internal
            inner_hash: "0x134e73f06fa9bdb1dbfa909e149c563f5860ceb71a0e7307918f7033970edf59"
            # One node fans the same job out to every connected miner
            # (node/src/miner_server.rs: broadcast_job over a HashMap of
            # MinerHandle), and each miner picks its own random starting nonce,
            # so no coordination or range allocation is needed. A node per miner
            # would only be needed for a separate reward address.
            miners: benjy.hanzalova.internal quadbrat.hanzalova.internal
            # Complete flag, or "" for a node with no public ingress. Requires a
            # matching inbound TCP forward at the site edge — the flag only
            # advertises, it does not open anything.
            #
            # /dns4/ rather than a literal address: the site's WAN IP can change
            # without a redeploy, and this is the per-site indirection name the
            # fleet already uses (architecture public-dns.md).
            public_addr: "--public-addr /dns4/nh.thgttg.com/tcp/30333"
            # JSON-RPC for other machines on the site LAN. Empty = loopback
            # only. `safe` is not optional here: it is what keeps
            # author_rotateKeys and friends off an exposed port. Loopback is
            # rate-limit-whitelisted so the arena exporter beside the node is not
            # throttled by its own polling.
            # --unsafe-rpc-external, NOT --rpc-external: Substrate REFUSES
            # --rpc-external on a validator outright and exits, whatever
            # --rpc-methods says. The two flags bind identically ("Same as
            # `--rpc-external`" per --help); the unsafe- name is purely a
            # forced acknowledgement. What actually keeps this safe is
            # --rpc-methods safe, which stays.
            rpc_expose: "--unsafe-rpc-external --rpc-methods safe --rpc-cors all --rpc-rate-limit 300 --rpc-rate-limit-whitelisted-ips 127.0.0.1/32"
          # Second site, for decentralisation. Uncomment when provisioned, and
          # give it its OWN inner_hash derived from its OWN wallet — sharing
          # bob's would publicly tie the two sites to one operator and defeat
          # the point of running them separately (doc/wormhole-rewards.md §5).
          #
          # NOTE: a node with no miners authors nothing, because
          # --miner-listen-port disables built-in mining. A node intended purely
          # to relay and validate needs a unit WITHOUT
          # --validator/--miner-listen-port, not this one — and then it needs no
          # inner_hash at all.
          # - host: <second-node>.internal
          #   inner_hash: "0x..."
          #   miners: ""
          #   public_addr: ""
          #   rpc_expose: ""
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-bin, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: preflight — sudoers covers this deploy
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          # Every path infra-setup.sh grants for this role must already be
          # permitted on the host. Adding an asset here without re-running
          # infra-setup produces a bare "sudo: a password is required" forty
          # lines into the deploy, after some files have already landed. Compare
          # up front instead, from infra-setup.sh itself so the two cannot drift.
          sed -n "/quantus-node_gitea_ci.tmp/,/^SUDO$/p" script/infra-setup.sh \
            | grep '^gitea_ci ALL=' | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > expected-paths.txt
          ssh $SSHOPTS gitea_ci@${{ matrix.host }} 'sudo -n -l' \
            | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > permitted-paths.txt
          comm -23 expected-paths.txt permitted-paths.txt > missing-paths.txt
          if [ -s missing-paths.txt ]; then
            echo "the node sudoers on ${{ matrix.host }} is out of date." >&2
            echo "not permitted, but this deploy needs them:" >&2
            sed 's/^/  /' missing-paths.txt >&2
            echo "" >&2
            echo "run: ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub" >&2
            exit 1
          fi
          echo "sudoers covers all $(wc -l < expected-paths.txt) paths this deploy needs"
      - name: render node config
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          QUANTUS_INNER_HASH: ${{ matrix.inner_hash }}
          QUANTUS_PUBLIC_ADDR: ${{ matrix.public_addr }}
          QUANTUS_CHAIN: ${{ env.CHAIN }}
          QUANTUS_RPC_EXPOSE: ${{ matrix.rpc_expose }}
          QUANTUS_TELEMETRY_URL: ${{ env.TELEMETRY_URL }}
        run: |
          set -euo pipefail
          case "$QUANTUS_INNER_HASH" in
            0x0000000000000000000000000000000000000000000000000000000000000000)
              echo "${{ matrix.host }}: inner_hash is still the placeholder." >&2
              echo "Set a real one for THIS HOST in the deploy-node matrix." >&2
              echo "Derive it from a dedicated mining wallet, offline:" >&2
              echo "  quantus-node key quantus --scheme wormhole --words < mnemonic.txt" >&2
              echo "See doc/wormhole-rewards.md §5-§6." >&2
              exit 1 ;;
          esac
          case "$QUANTUS_INNER_HASH" in
            0x*) ;;
            *) echo "inner_hash must be 0x-prefixed (the node rejects it otherwise)" >&2; exit 1 ;;
          esac
          # Literal substitution — never a shell/sed expansion, so a value with
          # regex or shell metacharacters survives intact.
          python3 - <<'PY'
          import os, pathlib
          tmpl = pathlib.Path("asset/config/node.env.tmpl").read_text()
          out = tmpl.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"])
          out = out.replace("{{QUANTUS_PUBLIC_ADDR}}", os.environ.get("QUANTUS_PUBLIC_ADDR", ""))
          out = out.replace("{{QUANTUS_CHAIN}}", os.environ["QUANTUS_CHAIN"])
          out = out.replace("{{QUANTUS_RPC_EXPOSE}}", os.environ.get("QUANTUS_RPC_EXPOSE", ""))
          pathlib.Path("node.env").write_text(out)
          a = pathlib.Path("asset/config/arena.env.tmpl").read_text()
          a = a.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"])
          a = a.replace("{{QUANTUS_TELEMETRY_URL}}", os.environ.get("QUANTUS_TELEMETRY_URL", ""))
          pathlib.Path("arena.env.partial").write_text(a)
          PY

      - name: deploy node
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          HOST: ${{ matrix.host }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run()  { ssh $SSHOPTS gitea_ci@"$HOST" "$@"; }

          # -c (checksum), not rsync's default size+mtime quick check: `fetch`
          # re-downloads the binary every run, so its mtime is always new and the
          # default heuristic reports a change on every deploy. -c makes "changed"
          # mean "the content differs", which is what RESTART must key off.
          # -i itemizes, so an empty result is proof of no change.
          RESTART=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then
              RESTART=1
              printf '%s\n' "$out" | sed 's/^/    changed: /'
            fi
          }
          # firewalld definitions are picked up by --reload below and are never a
          # reason to bounce the daemon.
          pushfw() {
            rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@" | sed 's/^/    changed: /'
          }

          # 1. service account
          push --mkpath --chmod=F0644 \
            asset/systemd/quantus-node.sysusers.conf \
            gitea_ci@"$HOST":/etc/sysusers.d/quantus-node.conf
          run sudo systemd-sysusers

          # 2. directories. /etc is root-owned so the daemon can read but not
          #    rewrite its own config (generic.md §8).
          run sudo install -d -o root -g quantus-node -m 0750 /etc/quantus-node
          run sudo install -d -o quantus-node -g quantus-node -m 0750 /var/lib/quantus-node

          # 3. binary, unit, config, firewalld defs
          push --chmod=F0755 _bin/quantus-node gitea_ci@"$HOST":/usr/local/bin/quantus-node
          push --chmod=F0644 asset/systemd/quantus-node.service \
            gitea_ci@"$HOST":/etc/systemd/system/quantus-node.service
          push --chown=root:quantus-node --chmod=F0640 node.env \
            gitea_ci@"$HOST":/etc/quantus-node/node.env
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node.xml
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node-miner.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-miner.xml
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node-metrics.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-metrics.xml
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-node-rpc.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-rpc.xml

          # 3b. Arena exporter — network-wide stats from the node's loopback RPC.
          push --mkpath --chmod=F0644 asset/systemd/quantus-arena.sysusers.conf \
            gitea_ci@"$HOST":/etc/sysusers.d/quantus-arena.conf
          run sudo systemd-sysusers
          run sudo install -d -o root -g quantus-arena -m 0750 /etc/quantus-arena
          run sudo install -d -o quantus-arena -g quantus-arena -m 0750 /var/lib/quantus-arena
          push --chmod=F0755 asset/arena/quantus-arena-exporter.py \
            gitea_ci@"$HOST":/usr/local/bin/quantus-arena-exporter.py
          push --chmod=F0644 asset/systemd/quantus-arena.service \
            gitea_ci@"$HOST":/etc/systemd/system/quantus-arena.service
          # The reward address is Poseidon2(inner_hash); rather than reimplement
          # that, take it from the node's own startup log, which prints it. Empty
          # is tolerated — on a genuinely first deploy the node has not logged it
          # yet, the exporter omits balance metrics, and the next deploy fills it.
          run "journalctl -u quantus-node.service --no-pager" > node-journal.txt
          if grep -qF "Rewards wormhole address" node-journal.txt; then
            reward_addr=$(grep -F "Rewards wormhole address" node-journal.txt \
                          | tail -1 | sed 's/.*address: //' | tr -d '[:space:]')
            echo "reward address: ${reward_addr}"
          else
            reward_addr=""
            echo "WARN no reward address in the node journal yet — balance metrics will be omitted" >&2
          fi
          REWARD_ADDR="$reward_addr" python3 - <<'PY'
          import os, pathlib
          a = pathlib.Path("arena.env.partial").read_text()
          a = a.replace("{{QUANTUS_REWARD_ADDRESS}}", os.environ["REWARD_ADDR"])
          pathlib.Path("arena.env").write_text(a)
          PY
          push --chown=root:quantus-arena --chmod=F0640 arena.env \
            gitea_ci@"$HOST":/etc/quantus-arena/arena.env
          pushfw --mkpath --chmod=F0644 asset/firewalld/quantus-arena.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-arena.xml

          # 4. SELinux relabel. No `semanage port` needed: the unit runs under
          #    init_t, which may bind any port.
          run sudo restorecon -R /usr/local/bin/quantus-node /etc/quantus-node /var/lib/quantus-node
          run sudo restorecon -R /usr/local/bin/quantus-arena-exporter.py /etc/quantus-arena /var/lib/quantus-arena

          # 5. firewalld. Reload FIRST so the freshly-shipped definitions exist,
          #    or --query-service fails INVALID_SERVICE
          #    (deployment-gitea-actions.md §6).
          run sudo firewall-cmd --reload
          zone=$(run sudo firewall-cmd --get-default-zone)
          echo "default zone: ${zone}"

          # 5a. P2P: public, plain named service.
          if run sudo firewall-cmd --zone="$zone" --query-service=quantus-node; then
            echo "firewalld: quantus-node already enabled in ${zone}"
          else
            run sudo firewall-cmd --permanent --zone="$zone" --add-service=quantus-node
            run sudo firewall-cmd --zone="$zone" --add-service=quantus-node
          fi

          # 5b. Miner link: a rich rule scoped to this node's miner hosts, NOT a
          #     plain --add-service. With a single default zone (generic.md §9)
          #     adding the service outright would publish 9833/udp on every
          #     address the host carries. Resolve each miner's mesh address ON
          #     THE NODE — this repo carries no 10.x literals, and the node's own
          #     resolution is the address the miner will actually present.
          for m in ${{ matrix.miners }}; do
            miner_addrs=$(run "getent ahostsv4 $m")
            miner_ip=$(awk '{print $1; exit}' <<<"$miner_addrs")
            case "$miner_ip" in
              10.*) echo "miner source: ${m} -> ${miner_ip}" ;;
              *) echo "refusing to open ${{ env.MINER_LINK_PORT }}/udp to non-mesh address '${miner_ip}' for ${m}" >&2; exit 1 ;;
            esac
            rich="rule family=ipv4 source address=${miner_ip}/32 service name=quantus-node-miner accept"
            # Pass these as ONE pre-quoted string, not as separate run() args.
            # `run()` is `ssh ... "$@"`, and ssh concatenates its argument vector
            # with spaces for the REMOTE shell to re-split — so local quoting is
            # lost and a rich rule arrives as a dozen bare words
            # ("unrecognized arguments: family=ipv4 source address=..."). Every
            # other command here survives only because no other argument
            # contains a space. Keep the inner single quotes.
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$rich'"; then
              echo "firewalld: rich rule for ${m} already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$rich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$rich'"
            fi
          done

          # 5c. Metrics: scoped to the scrape host only. The exporter is
          #     unauthenticated and --prometheus-external binds every interface,
          #     so this rule is the whole access boundary.
          metrics_addrs=$(run "getent ahostsv4 ${{ env.METRICS_HOST }}")
          metrics_ip=$(awk '{print $1; exit}' <<<"$metrics_addrs")
          case "$metrics_ip" in
            10.*) echo "scrape source: ${metrics_ip}" ;;
            *) echo "refusing to expose metrics to non-mesh address '${metrics_ip}'" >&2; exit 1 ;;
          esac
          for svc in quantus-node-metrics quantus-arena; do
            mrich="rule family=ipv4 source address=${metrics_ip}/32 service name=${svc} accept"
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$mrich'"; then
              echo "firewalld: ${svc} rich rule already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$mrich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$mrich'"
            fi
          done

          # 5d. JSON-RPC, if exposed. Scoped to the node's own site subnet,
          #     derived from its own address so this repo carries no 10.x
          #     literal and the rule follows the host to whatever site it is in.
          if [ -n "${{ matrix.rpc_expose }}" ]; then
            self_addrs=$(run "getent ahostsv4 $HOST")
            self_ip=$(awk '{print $1; exit}' <<<"$self_addrs")
            case "$self_ip" in
              10.*) ;;
              *) echo "refusing to derive an RPC subnet from non-mesh address '${self_ip}'" >&2; exit 1 ;;
            esac
            subnet="$(cut -d. -f1,2 <<<"$self_ip").0.0/16"
            echo "rpc allowed from: ${subnet}"
            rrich="rule family=ipv4 source address=${subnet} service name=quantus-node-rpc accept"
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$rrich'"; then
              echo "firewalld: rpc rich rule already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$rrich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$rrich'"
            fi
          else
            echo "rpc: loopback only (rpc_expose empty)"
          fi

          # 6. (re)start — ONLY if something actually changed. Bouncing a
          #    syncing node costs it every peer and a RocksDB reopen, and this
          #    workflow also runs to validate an unchanged deployment.
          run sudo systemctl enable quantus-node.service    # idempotent, no service impact
          run sudo systemctl enable quantus-arena.service   # idempotent
          if [ "$RESTART" = 1 ]; then
            echo "changes applied — restarting"
            run sudo systemctl daemon-reload
            run sudo systemctl restart quantus-node.service
            run sudo systemctl restart quantus-arena.service
          elif run systemctl is-active --quiet quantus-node.service; then
            echo "nothing changed and the node is running — left alone"
          else
            echo "nothing changed but the node is down — starting it"
            run sudo systemctl restart quantus-node.service
          fi
          run systemctl is-active --quiet quantus-arena.service \
            || { echo "arena exporter down — starting it"; run sudo systemctl restart quantus-arena.service; }

      - name: validate node
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }}) ---"
          # NOT a bare `systemctl is-active`: that exits 3 the instant the unit
          # is "activating", which under set -e aborts the whole step before any
          # diagnostic runs. A crash-looping unit sits in activating/auto-restart
          # indefinitely, so this waits, then reports WHY rather than an exit
          # code — that is how a validator refusing its own flags looked like a
          # bare "exit code 3".
          state=""
          for i in $(seq 1 30); do
            state=$(run "systemctl show quantus-node.service -p ActiveState --value")
            [ "$state" = "active" ] && break
            [ "$state" = "failed" ] && break
            sleep 4
          done
          if [ "$state" = "active" ]; then
            echo "  active"
          else
            restarts=$(run "systemctl show quantus-node.service -p NRestarts --value")
            echo "  FAIL unit is '${state}' after 120s (NRestarts=${restarts}) — crash-looping" >&2
            jr=$(run "journalctl -u quantus-node.service --no-pager -n 40")
            printf '%s\n' "$jr" | grep -iE "error|panic|refus" | tail -5 | sed 's/^/    /' >&2
            exit 1
          fi

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-node --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ needs.fetch.outputs.node_version }}"*) echo "version matches pin" ;;
            *) echo "version does NOT match pin ${{ needs.fetch.outputs.node_version }}" >&2; fail=1 ;;
          esac

          echo "--- reward address ---"
          # The node logs the address it derived from this host's inner_hash.
          # Surfacing it makes a copy-pasted or shared inner_hash visible.
          run journalctl -u quantus-node.service -n 500 --no-pager \
            | grep -F "Rewards wormhole address" | tail -1 || {
              echo "  WARN no reward address in the recent journal" >&2; }

          echo "--- readiness ---"
          # A restarted node opens its RocksDB and initialises litep2p before it
          # binds anything; asserting immediately after `systemctl restart` reads
          # a node that is merely still starting as one that is broken. Wait for
          # the P2P socket, bounded, rather than sleeping a guessed constant.
          ready=0
          for i in $(seq 1 60); do
            if p=$(run "ss -Hln -t sport = :${{ env.P2P_PORT }}") && [ -n "$p" ]; then
              echo "  ready after ~$((i*2))s"; ready=1; break
            fi
            sleep 2
          done
          if [ "$ready" = 0 ]; then
            echo "  node did not open tcp/${{ env.P2P_PORT }} within 120s" >&2
            fail=1
          fi

          echo "--- listeners ---"
          # P2P must be listening; the miner link must be listening or the node
          # is authoring nothing (--miner-listen-port disables local mining).
          # `-t`/`-u` matter: without a protocol flag ss errors with
          # "RTNETLINK answers: Invalid argument" and reports nothing useful.
          # Capture, then test — piping into `grep -q` makes grep exit early,
          # SIGPIPEs ssh, and `set -o pipefail` turns that into a 141 that looks
          # exactly like "not listening".
          for spec in "${{ env.P2P_PORT }} tcp -t" "${{ env.MINER_LINK_PORT }} udp -u"; do
            set -- $spec
            if listeners=$(run "ss -Hln $3 sport = :$1"); then
              if [ -n "$listeners" ]; then
                echo "  ok   $2/$1 listening"
              else
                echo "  FAIL $2/$1 not listening" >&2; fail=1
              fi
            else
              echo "  FAIL could not query $2/$1 on the host" >&2; fail=1
            fi
          done

          echo "--- json-rpc ---"
          if [ -n "${{ matrix.rpc_expose }}" ]; then
            if rpcl=$(run "ss -Hln -t sport = :9944") && [ -n "$rpcl" ]; then
              echo "  ok   rpc listening"
              printf '%s\n' "$rpcl" | sed 's/^/       /'
            else
              echo "  FAIL rpc_expose is set but nothing is listening on 9944" >&2; fail=1
            fi
          else
            echo "  loopback only by configuration"
          fi

          echo "--- advertised address ---"
          if [ -n "${{ matrix.public_addr }}" ]; then
            echo "  advertising: ${{ matrix.public_addr }}"
          else
            echo "  none set — this node accepts no inbound peers and is a" >&2
            echo "  leech on the network rather than a contribution to it." >&2
          fi

          echo "--- sync ---"
          # Here-strings, not pipes: `awk ... exit` closes the pipe early, the
          # writer takes SIGPIPE, and pipefail turns a successful parse into 141.
          # The metrics body is large enough for that race to fire reliably.
          m=$(run "curl -fsS http://127.0.0.1:${{ env.NODE_METRICS_PORT }}/metrics")
          peers=$(awk '/^substrate_sub_libp2p_peers_count/{print $2; exit}' <<<"$m")
          best=$(awk '/^substrate_block_height\{status="best"/{print $2; exit}' <<<"$m")
          echo "  peers=${peers:-unknown} best_block=${best:-unknown}"
          if [ "${peers:-0}" = "0" ]; then echo "  WARN no peers" >&2; fi

          exit $fail

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-node.service -n 80 --no-pager

  deploy-metrics:
    runs-on: infra
    needs: [deploy-node]
    # Monitoring assets only. Deliberately last and deliberately separate: a
    # Grafana dashboard failing to provision must never be able to leave a
    # half-deployed validator behind it.
    steps:
      - uses: actions/checkout@v4

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ env.METRICS_HOST }} hostname -f

      - name: preflight — sudoers covers this deploy
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          # Every path infra-setup.sh grants for this role must already be
          # permitted on the host. Adding an asset here without re-running
          # infra-setup produces a bare "sudo: a password is required" forty
          # lines into the deploy, after some files have already landed. Compare
          # up front instead, from infra-setup.sh itself so the two cannot drift.
          sed -n "/quantus-metrics_gitea_ci.tmp/,/^SUDO$/p" script/infra-setup.sh \
            | grep '^gitea_ci ALL=' | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > expected-paths.txt
          ssh $SSHOPTS gitea_ci@${{ env.METRICS_HOST }} 'sudo -n -l' \
            | grep -oE '(/etc|/usr|/var)/[^ ]*' | sort -u > permitted-paths.txt
          comm -23 expected-paths.txt permitted-paths.txt > missing-paths.txt
          if [ -s missing-paths.txt ]; then
            echo "the metrics sudoers on ${{ env.METRICS_HOST }} is out of date." >&2
            echo "not permitted, but this deploy needs them:" >&2
            sed 's/^/  /' missing-paths.txt >&2
            echo "" >&2
            echo "run: ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub" >&2
            exit 1
          fi
          echo "sudoers covers all $(wc -l < expected-paths.txt) paths this deploy needs"
      - name: deploy scrape config and dashboard
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          HOST: ${{ env.METRICS_HOST }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run()  { ssh $SSHOPTS gitea_ci@"$HOST" "$@"; }

          RELOAD=0
          push() {
            local out
            out=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic "$@")
            if [ -n "$out" ]; then RELOAD=1; printf '%s\n' "$out" | sed 's/^/    changed: /'; fi
          }

          # Render the scrape targets. Quoted, comma-separated, from the env
          # lists above — a space-separated host list becomes a YAML array.
          fmt() { local o=""; for h in $1; do o="${o:+$o, }\"${h}:${2}\""; done; printf '%s' "$o"; }
          NODE_TARGETS=$(fmt "${{ env.SCRAPE_NODES }}"  "${{ env.NODE_METRICS_PORT }}")
          MINER_TARGETS=$(fmt "${{ env.SCRAPE_MINERS }}" "${{ env.MINER_METRICS_PORT }}")
          ARENA_TARGETS=$(fmt "${{ env.SCRAPE_ARENA }}"  "${{ env.ARENA_PORT }}")
          echo "node targets:  ${NODE_TARGETS}"
          echo "miner targets: ${MINER_TARGETS}"
          echo "arena targets: ${ARENA_TARGETS}"

          NODE_TARGETS="$NODE_TARGETS" MINER_TARGETS="$MINER_TARGETS" ARENA_TARGETS="$ARENA_TARGETS" python3 - <<'PY'
          import os, pathlib
          t = pathlib.Path("asset/prometheus/quantus.yml.tmpl").read_text()
          t = t.replace("{{NODE_TARGETS}}", os.environ["NODE_TARGETS"])
          t = t.replace("{{MINER_TARGETS}}", os.environ["MINER_TARGETS"])
          t = t.replace("{{ARENA_TARGETS}}", os.environ["ARENA_TARGETS"])
          pathlib.Path("quantus-scrape.yml").write_text(t)
          PY
          python3 -c "import yaml,sys; yaml.safe_load(open('quantus-scrape.yml'))" \
            || { echo "rendered scrape config is not valid YAML" >&2; exit 1; }

          push --mkpath --chmod=F0644 quantus-scrape.yml \
            gitea_ci@"$HOST":/etc/prometheus/scrape_configs.d/quantus.yml
          python3 -c "import yaml,sys; yaml.safe_load(open('asset/prometheus/quantus-rules.yml'))" \
            || { echo "recording rules are not valid YAML" >&2; exit 1; }
          push --mkpath --chmod=F0644 asset/prometheus/quantus-rules.yml \
            gitea_ci@"$HOST":/etc/prometheus/rules.d/quantus.yml
          push --mkpath --chmod=F0644 asset/grafana/quantus.json \
            gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus/quantus.json
          push --mkpath --chmod=F0644 asset/grafana/gpu.json \
            gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus/gpu.json

          # The PROVIDER tells Grafana the dashboards directory exists at all;
          # without it the JSON above is just a file nobody reads. Tracked
          # separately from the dashboard because Grafana reads providers ONLY at
          # startup, while dashboards are re-read on updateIntervalSeconds — so a
          # provider change needs a restart and a dashboard change does not.
          PROVIDER=0
          pout=$(rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' -ic --mkpath --chmod=F0644 \
            asset/grafana/quantus-dashboards.yaml \
            gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus-dashboards.yaml)
          if [ -n "$pout" ]; then
            PROVIDER=1
            printf '%s\n' "$pout" | sed 's/^/    changed: /'
          fi

          if [ "$PROVIDER" = 1 ]; then
            echo "dashboard provider changed — restarting grafana (providers are read at startup only)"
            run sudo systemctl restart grafana.service
          fi

          if [ "$RELOAD" = 1 ]; then
            # Hot reload rather than a restart: Prometheus restarts drop nothing
            # permanent but do interrupt scraping, and --web.enable-lifecycle is
            # already set for exactly this.
            echo "config changed — reloading prometheus"
            run "curl -fsS -X POST http://127.0.0.1:${{ env.PROM_PORT }}/-/reload"
            echo "grafana picks the dashboard up on its own provisioning interval"
          else
            echo "nothing changed — no reload"
          fi

      - name: validate metrics
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ env.METRICS_HOST }}" "$@"; }
          fail=0

          echo "--- prometheus rules ---"
          r=$(run "curl -fsS http://127.0.0.1:${{ env.PROM_PORT }}/api/v1/rules")
          if grep -q '"name":"quantus-miner"' <<<"$r"; then
            echo "  ok   rule group quantus-miner loaded"
          else
            echo "  FAIL rule group quantus-miner not loaded" >&2
            echo "       is rule_files wired into prometheus.yml? (infra-setup.sh --metrics-hosts)" >&2
            fail=1
          fi

          echo "--- prometheus targets ---"
          # The point of this check: a target that drifted out of the matrices,
          # or a firewalld rule that never landed, shows up here as missing or
          # DOWN instead of an empty graph nobody looks at.
          #
          # Written as a quoted heredoc reading a file, NOT `python3 -c '...'`:
          # escaping quotes through the shell into an f-string produced
          # "SyntaxError: unexpected character after line continuation character",
          # because Python reads \" inside the expression as a continuation.
          # A target Prometheus has only just learned about reports health
          # "unknown" until its FIRST scrape completes. With a 15s scrape
          # interval and this step running seconds after the config reload, any
          # run that adds a target would otherwise fail on a target that is
          # perfectly healthy. Wait for the unknowns to resolve, bounded; a
          # genuinely dead target goes to "down", not "unknown", so this does
          # not paper over real failures.
          cat > count_unknown.py <<'PY'
          import json
          d = json.load(open("targets.json"))
          rows = [t for t in d["data"]["activeTargets"]
                  if t["labels"].get("job", "").startswith("quantus-")]
          print(sum(1 for t in rows if t["health"] == "unknown") if rows else 1)
          PY
          for i in $(seq 1 24); do
            run "curl -fsS 'http://127.0.0.1:${{ env.PROM_PORT }}/api/v1/targets?state=active'" > targets.json
            pending=$(python3 count_unknown.py)
            if [ "$pending" = "0" ]; then
              [ "$i" -gt 1 ] && echo "  settled after ~$((i*5))s"
              break
            fi
            [ "$i" = "24" ] && echo "  ${pending} target(s) still unscraped after 120s" >&2
            sleep 5
          done

          if python3 - <<'PY'
          import json, sys
          d = json.load(open("targets.json"))
          rows = [t for t in d["data"]["activeTargets"]
                  if t["labels"].get("job", "").startswith("quantus-")]
          if not rows:
              print("  FAIL prometheus has no quantus-* targets", file=sys.stderr)
              print("       is scrape_config_files wired into prometheus.yml?", file=sys.stderr)
              print("       one-time: script/infra-setup.sh --metrics-hosts", file=sys.stderr)
              sys.exit(1)
          bad = 0
          for t in sorted(rows, key=lambda x: (x["labels"].get("job", ""),
                                               x["labels"].get("instance", ""))):
              j = t["labels"].get("job")
              i = t["labels"].get("instance")
              h = t["health"]
              e = t.get("lastError", "")
              print(("  ok   " if h == "up" else "  FAIL ") + " ".join(filter(None, [j, i, h, e])))
              if h != "up":
                  bad += 1
          sys.exit(1 if bad else 0)
          PY
          then
            echo "  all quantus targets up"
          else
            echo "  FAIL not every quantus target is up" >&2; fail=1
          fi

          echo "--- grafana dashboard ---"
          # Grafana's /api/search requires auth (401 unauthenticated), and this
          # job holds no Grafana credentials by design — so assert provisioning
          # from the journal instead, which gitea_ci can read via systemd-journal.
          run "curl -fsS http://127.0.0.1:${{ env.GRAFANA_PORT }}/api/health" > /dev/null \
            && echo "  ok   grafana is healthy" \
            || { echo "  FAIL grafana health endpoint not responding" >&2; fail=1; }

          if run "test -s /etc/grafana/provisioning/dashboards/quantus/quantus.json" \
             && run "test -s /etc/grafana/provisioning/dashboards/quantus-dashboards.yaml"; then
            echo "  ok   dashboard and provider present on the host"
          else
            echo "  FAIL dashboard JSON or provider missing" >&2; fail=1
          fi

          # A dashboard that fails to parse is logged by the provisioner and then
          # silently absent from the UI, so treat any provisioning error as fatal.
          if run "journalctl -u grafana.service --since '-10min' --no-pager" \
               | grep -E 'logger=provisioning\.dashboard.*level=error'; then
            echo "  FAIL grafana reported a dashboard provisioning error" >&2; fail=1
          else
            echo "  ok   no dashboard provisioning errors in the last 10 minutes"
          fi

          exit $fail