quantus/.gitea/workflows/deploy.yaml

name: deploy
# Deploy — or validate — Quantus Planck nodes and their external GPU miners.
#
# Topology (the workflow is the source of infra truth,
# ~/git/architecture/deployment-gitea-actions.md — hosts belong here, and only
# here; see readme.md "Scope"):
#
#   quantus-node    consensus, P2P, rewards
#        ▲ QUIC/9833 (mesh only, rich-rule scoped to that node's miners)
#   quantus-miner   GPU search
#
# The node and the miner are separate processes on separate hosts by design: the
# node is disk+network bound and belongs on an always-on box, the miner is pure
# GPU compute. Setting --miner-listen-port DISABLES the node's built-in CPU
# mining, so the node authors nothing if the miner is absent — the validate mode
# below checks exactly that rather than trusting `is-active`.
#
# Runs on `infra`. gitea-runners.md §4 says a deploy needs only ssh+rsync and so
# fits `fedora-43`; that is about tooling, not routing. These targets are
# mesh-only `.internal` names and lair/mail's two working deploys both use
# `infra` for that reason. Deliberate deviation, noted per readme.md.

on:
  push:
    branches: [main]
    paths:
      - asset/**
      - .gitea/workflows/deploy.yaml
  workflow_dispatch:
    inputs:
      mode:
        description: "deploy (apply) or validate (check only, no changes)"
        required: false
        default: deploy
        type: choice
        options: [deploy, validate]
      node_version:
        description: quantus-node version (overrides the pin)
        required: false
      miner_version:
        description: quantus-miner version (overrides the pin)
        required: false

concurrency:              # never half-apply two deploys at once
  group: deploy
  cancel-in-progress: false

env:
  CHAIN: planck
  # Pinned; bump deliberately to upgrade. An unattended upgrade of a validator
  # is how you find out at 3am that a release changed a consensus rule.
  NODE_VERSION: "0.10.0"
  MINER_VERSION: "4.0.0"
  # Ports — port-allocations.md §5 registry. These are upstream protocol
  # defaults rather than derived numbers; see readme.md "Ports".
  P2P_PORT: "30333"
  MINER_LINK_PORT: "9833"
  NODE_METRICS_PORT: "9615"
  MINER_METRICS_PORT: "9900"

jobs:
  fetch:
    runs-on: infra
    outputs:
      node_version: ${{ steps.v.outputs.node }}
      miner_version: ${{ steps.v.outputs.miner }}
    steps:
      - id: v
        run: |
          set -euo pipefail
          n="${{ github.event.inputs.node_version || env.NODE_VERSION }}"
          m="${{ github.event.inputs.miner_version || env.MINER_VERSION }}"
          echo "node=${n}"  >> "$GITHUB_OUTPUT"
          echo "miner=${m}" >> "$GITHUB_OUTPUT"
          echo "quantus-node ${n} / quantus-miner ${m}"

      - name: download quantus-node
        run: |
          set -euo pipefail
          v="${{ steps.v.outputs.node }}"
          curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
            -o node.tar.gz \
            "https://github.com/Quantus-Network/chain/releases/download/v${v}/quantus-node-v${v}-x86_64-unknown-linux-gnu.tar.gz"
          tar xzf node.tar.gz
          install -D -m 0755 "$(find . -name quantus-node -type f | head -1)" _bin/quantus-node
          _bin/quantus-node --version

      - name: download quantus-miner
        run: |
          set -euo pipefail
          v="${{ steps.v.outputs.miner }}"
          curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
            -o _bin/quantus-miner \
            "https://github.com/Quantus-Network/quantus-miner/releases/download/v${v}/quantus-miner-linux-x86_64"
          chmod 0755 _bin/quantus-miner
          _bin/quantus-miner --version

      - uses: actions/upload-artifact@v3
        with:
          name: quantus-bin
          path: _bin

  deploy-node:
    runs-on: infra
    needs: fetch
    strategy:
      fail-fast: false        # one site's failure must not abort the other
      matrix:
        include:
          # ONE inner_hash PER HOST — never share one across nodes.
          #
          # The reward preimage is published verbatim in the PreRuntime digest
          # of every block a node authors, and the payout address is
          # Poseidon2(inner_hash). It is therefore public, and it is STATIC —
          # derived per wallet, not per block. Two nodes sharing an inner_hash
          # are publicly and permanently identifiable as the same operator,
          # which silently collapses the independence that running nodes at
          # separate sites is meant to provide. See doc/wormhole-rewards.md §5.
          #
          # Not a secret, so it lives here with the rest of the infra truth.
          # Derive one per host, offline, from a DEDICATED mining wallet
          # (doc/wormhole-rewards.md §6):
          #   quantus-node key quantus --scheme wormhole --words < mnemonic.txt
          - host: bob.hanzalova.internal
            inner_hash: "0x134e73f06fa9bdb1dbfa909e149c563f5860ceb71a0e7307918f7033970edf59"
            miners: benjy.hanzalova.internal   # space-separated if more than one
          # Second site, for decentralisation. Uncomment when provisioned, and
          # give it its OWN inner_hash derived from its OWN wallet — sharing
          # bob's would publicly tie the two sites to one operator and defeat
          # the point of running them separately (doc/wormhole-rewards.md §5).
          #
          # NOTE: a node with no miners authors nothing, because
          # --miner-listen-port disables built-in mining. A node intended purely
          # to relay and validate needs a unit WITHOUT
          # --validator/--miner-listen-port, not this one — and then it needs no
          # inner_hash at all.
          # - host: <second-node>.internal
          #   inner_hash: "0x..."
          #   miners: ""
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-bin, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: render node config
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          QUANTUS_INNER_HASH: ${{ matrix.inner_hash }}
        run: |
          set -euo pipefail
          case "$QUANTUS_INNER_HASH" in
            0x0000000000000000000000000000000000000000000000000000000000000000)
              echo "${{ matrix.host }}: inner_hash is still the placeholder." >&2
              echo "Set a real one for THIS HOST in the deploy-node matrix." >&2
              echo "Derive it from a dedicated mining wallet, offline:" >&2
              echo "  quantus-node key quantus --scheme wormhole --words < mnemonic.txt" >&2
              echo "See doc/wormhole-rewards.md §5-§6." >&2
              exit 1 ;;
          esac
          case "$QUANTUS_INNER_HASH" in
            0x*) ;;
            *) echo "inner_hash must be 0x-prefixed (the node rejects it otherwise)" >&2; exit 1 ;;
          esac
          # Literal substitution — never a shell/sed expansion, so a value with
          # regex or shell metacharacters survives intact.
          python3 - <<'PY'
          import os, pathlib
          tmpl = pathlib.Path("asset/config/node.env.tmpl").read_text()
          out = tmpl.replace("{{QUANTUS_INNER_HASH}}", os.environ["QUANTUS_INNER_HASH"])
          pathlib.Path("node.env").write_text(out)
          PY

      - name: deploy node
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          HOST: ${{ matrix.host }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run()  { ssh $SSHOPTS gitea_ci@"$HOST" "$@"; }
          push() { rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' "$@"; }

          # 1. service account
          push --mkpath --chmod=F0644 \
            asset/systemd/quantus-node.sysusers.conf \
            gitea_ci@"$HOST":/etc/sysusers.d/quantus-node.conf
          run sudo systemd-sysusers

          # 2. directories. /etc is root-owned so the daemon can read but not
          #    rewrite its own config (generic.md §8).
          run sudo install -d -o root -g quantus-node -m 0750 /etc/quantus-node
          run sudo install -d -o quantus-node -g quantus-node -m 0750 /var/lib/quantus-node

          # 3. binary, unit, config, firewalld defs
          push --chmod=F0755 _bin/quantus-node gitea_ci@"$HOST":/usr/local/bin/quantus-node
          push --chmod=F0644 asset/systemd/quantus-node.service \
            gitea_ci@"$HOST":/etc/systemd/system/quantus-node.service
          push --chown=root:quantus-node --chmod=F0640 node.env \
            gitea_ci@"$HOST":/etc/quantus-node/node.env
          push --mkpath --chmod=F0644 asset/firewalld/quantus-node.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node.xml
          push --mkpath --chmod=F0644 asset/firewalld/quantus-node-miner.xml \
            gitea_ci@"$HOST":/etc/firewalld/services/quantus-node-miner.xml

          # 4. SELinux relabel. No `semanage port` needed: the unit runs under
          #    init_t, which may bind any port.
          run sudo restorecon -R /usr/local/bin/quantus-node /etc/quantus-node /var/lib/quantus-node

          # 5. firewalld. Reload FIRST so the freshly-shipped definitions exist,
          #    or --query-service fails INVALID_SERVICE
          #    (deployment-gitea-actions.md §6).
          run sudo firewall-cmd --reload
          zone=$(run sudo firewall-cmd --get-default-zone)
          echo "default zone: ${zone}"

          # 5a. P2P: public, plain named service.
          if run sudo firewall-cmd --zone="$zone" --query-service=quantus-node; then
            echo "firewalld: quantus-node already enabled in ${zone}"
          else
            run sudo firewall-cmd --permanent --zone="$zone" --add-service=quantus-node
            run sudo firewall-cmd --zone="$zone" --add-service=quantus-node
          fi

          # 5b. Miner link: a rich rule scoped to this node's miner hosts, NOT a
          #     plain --add-service. With a single default zone (generic.md §9)
          #     adding the service outright would publish 9833/udp on every
          #     address the host carries. Resolve each miner's mesh address ON
          #     THE NODE — this repo carries no 10.x literals, and the node's own
          #     resolution is the address the miner will actually present.
          for m in ${{ matrix.miners }}; do
            miner_ip=$(run "getent ahostsv4 $m" | awk '{print $1; exit}')
            case "$miner_ip" in
              10.*) echo "miner source: ${m} -> ${miner_ip}" ;;
              *) echo "refusing to open ${{ env.MINER_LINK_PORT }}/udp to non-mesh address '${miner_ip}' for ${m}" >&2; exit 1 ;;
            esac
            rich="rule family=ipv4 source address=${miner_ip}/32 service name=quantus-node-miner accept"
            # Pass these as ONE pre-quoted string, not as separate run() args.
            # `run()` is `ssh ... "$@"`, and ssh concatenates its argument vector
            # with spaces for the REMOTE shell to re-split — so local quoting is
            # lost and a rich rule arrives as a dozen bare words
            # ("unrecognized arguments: family=ipv4 source address=..."). Every
            # other command here survives only because no other argument
            # contains a space. Keep the inner single quotes.
            if run "sudo firewall-cmd --zone=$zone --query-rich-rule='$rich'"; then
              echo "firewalld: rich rule for ${m} already present in ${zone}"
            else
              run "sudo firewall-cmd --permanent --zone=$zone --add-rich-rule='$rich'"
              run "sudo firewall-cmd --zone=$zone --add-rich-rule='$rich'"
            fi
          done

          # 6. (re)start
          run sudo systemctl daemon-reload
          run sudo systemctl enable quantus-node.service
          run sudo systemctl restart quantus-node.service

      - name: validate node
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }}) ---"
          run systemctl is-active quantus-node.service

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-node --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ needs.fetch.outputs.node_version }}"*) echo "version matches pin" ;;
            *) echo "version does NOT match pin ${{ needs.fetch.outputs.node_version }}" >&2; fail=1 ;;
          esac

          echo "--- reward address ---"
          # The node logs the address it derived from this host's inner_hash.
          # Surfacing it makes a copy-pasted or shared inner_hash visible.
          run journalctl -u quantus-node.service -n 500 --no-pager \
            | grep -F "Rewards wormhole address" | tail -1 || {
              echo "  WARN no reward address in the recent journal" >&2; }

          echo "--- listeners ---"
          # P2P must be listening; the miner link must be listening or the node
          # is authoring nothing (--miner-listen-port disables local mining).
          for spec in "${{ env.P2P_PORT }} tcp" "${{ env.MINER_LINK_PORT }} udp"; do
            set -- $spec
            if run "ss -Hln sport = :$1" | grep -q .; then
              echo "  ok   $2/$1 listening"
            else
              echo "  FAIL $2/$1 not listening" >&2; fail=1
            fi
          done

          echo "--- sync ---"
          m=$(run "curl -fsS http://127.0.0.1:${{ env.NODE_METRICS_PORT }}/metrics")
          peers=$(printf '%s' "$m" | awk '/^substrate_sub_libp2p_peers_count/{print $2; exit}')
          best=$(printf '%s' "$m" | awk '/^substrate_block_height\{status="best"/{print $2; exit}')
          echo "  peers=${peers:-unknown} best_block=${best:-unknown}"
          if [ "${peers:-0}" = "0" ]; then echo "  WARN no peers" >&2; fi

          exit $fail

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-node.service -n 80 --no-pager

  deploy-miner:
    runs-on: infra
    needs: [fetch, deploy-node]
    strategy:
      fail-fast: false
      matrix:
        include:
          # `node` is the host whose QUIC control channel this miner attaches
          # to, and whose inner_hash therefore receives what it earns. A miner
          # holds no reward configuration of its own — the mining protocol
          # carries no payout address at all.
          - host: benjy.hanzalova.internal
            node: bob.hanzalova.internal
            gpu_devices: "1"        # 1× RTX 4090, measured 183 MH/s @ ~450 W
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with: { name: quantus-bin, path: _bin }

      - name: write ssh key
        run: |
          set -euo pipefail
          install -d -m 0700 ~/.ssh
          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
          chmod 0600 ~/.ssh/id_gitea_ci

      - name: reachability
        run: |
          set -euo pipefail
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} hostname -f

      - name: deploy miner
        if: ${{ github.event.inputs.mode != 'validate' }}
        env:
          NODE_ADDR: ${{ matrix.node }}:${{ env.MINER_LINK_PORT }}
          GPU_DEVICES: ${{ matrix.gpu_devices }}
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          nrun() { ssh $SSHOPTS gitea_ci@"${{ matrix.node }}" "$@"; }
          run()  { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          push() { rsync -e "ssh $SSHOPTS" --rsync-path='sudo rsync' "$@"; }

          # 1. service account + dirs
          push --mkpath --chmod=F0644 \
            asset/systemd/quantus-miner.sysusers.conf \
            gitea_ci@"${{ matrix.host }}":/etc/sysusers.d/quantus-miner.conf
          run sudo systemd-sysusers
          run sudo install -d -o root -g quantus-miner -m 0750 /etc/quantus-miner
          run sudo install -d -o quantus-miner -g quantus-miner -m 0750 /var/lib/quantus-miner

          # 2. The miner's credentials are GENERATED BY THE NODE on first start
          #    and regenerate if the node's base-path is ever wiped. Copying them
          #    on every deploy — rather than once in infra-setup.sh — is what
          #    makes that self-healing instead of a silent auth failure.
          #    They pass through the runner in memory, never the workspace.
          umask 077
          tmp=$(mktemp -d)
          trap 'rm -rf "$tmp"' EXIT
          nrun sudo cat /var/lib/quantus-node/chains/${{ env.CHAIN }}/miner-auth-token \
            > "$tmp/miner-auth-token"
          nrun sudo cat /var/lib/quantus-node/chains/${{ env.CHAIN }}/miner-tls-cert-sha256 \
            > "$tmp/miner-tls-cert-sha256"
          test -s "$tmp/miner-auth-token"      || { echo "node auth token empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          test -s "$tmp/miner-tls-cert-sha256" || { echo "node TLS pin empty — has ${{ matrix.node }} started?" >&2; exit 1; }
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-auth-token" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-auth-token
          push --chown=root:quantus-miner --chmod=F0640 \
            "$tmp/miner-tls-cert-sha256" gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner-tls-cert-sha256

          # 3. non-secret runtime config
          python3 - <<'PY'
          import os, pathlib
          t = pathlib.Path("asset/config/miner.env.tmpl").read_text()
          t = t.replace("{{QUANTUS_NODE_ADDR}}", os.environ["NODE_ADDR"])
          t = t.replace("{{QUANTUS_GPU_DEVICES}}", os.environ["GPU_DEVICES"])
          pathlib.Path("miner.env").write_text(t)
          PY
          push --chown=root:quantus-miner --chmod=F0640 \
            miner.env gitea_ci@"${{ matrix.host }}":/etc/quantus-miner/miner.env

          # 4. binary + unit
          push --chmod=F0755 _bin/quantus-miner gitea_ci@"${{ matrix.host }}":/usr/local/bin/quantus-miner
          push --chmod=F0644 asset/systemd/quantus-miner.service \
            gitea_ci@"${{ matrix.host }}":/etc/systemd/system/quantus-miner.service

          run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner

          run sudo systemctl daemon-reload
          run sudo systemctl enable quantus-miner.service
          run sudo systemctl restart quantus-miner.service

      - name: validate miner
        run: |
          set -euo pipefail
          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
          run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
          fail=0

          echo "--- unit (${{ matrix.host }} -> ${{ matrix.node }}) ---"
          run systemctl is-active quantus-miner.service

          echo "--- version ---"
          got=$(run /usr/local/bin/quantus-miner --version)
          echo "installed: ${got}"
          case "$got" in
            *"${{ needs.fetch.outputs.miner_version }}"*) echo "version matches pin" ;;
            *) echo "version does NOT match pin ${{ needs.fetch.outputs.miner_version }}" >&2; fail=1 ;;
          esac

          echo "--- gpu ---"
          # An `active` miner that found no adapter still looks healthy to
          # systemd; assert the GPU is actually enumerated and busy.
          run "nvidia-smi --query-gpu=name,power.draw,utilization.gpu --format=csv,noheader"

          echo "--- hashing ---"
          # The counter is the only honest evidence this process is doing work
          # rather than idling on a failed connection.
          h1=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics" \
                | awk '/^miner_hashes_total/{print $2; exit}')
          sleep 20
          h2=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics" \
                | awk '/^miner_hashes_total/{print $2; exit}')
          echo "  miner_hashes_total ${h1:-?} -> ${h2:-?}"
          if [ -n "${h1:-}" ] && [ -n "${h2:-}" ] && [ "${h2%.*}" -gt "${h1%.*}" ]; then
            echo "  ok   hash counter advancing"
          else
            echo "  WARN counter not advancing — ${{ matrix.node }} may still be syncing," >&2
            echo "       which is expected and not itself a deploy failure." >&2
          fi

          exit $fail

      - name: journal
        if: always()
        run: |
          ssh -i ~/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new \
            gitea_ci@${{ matrix.host }} journalctl -u quantus-miner.service -n 80 --no-pager