.gitea/workflows/deploy.yaml

@@ -286,30 +286,46 @@ jobs:
           echo "--- hashing ---"
           # The counter is the only honest evidence this process is doing work
           # rather than idling on a failed connection. After a restart the miner
-          # must reconnect and receive a job, so poll rather than sample twice.
+          # exports miner_gpu_devices only once it has connected to the node and
+          # miner_hashes_total only once it has hashed, so first wait for the
+          # gauge to appear (readiness), then require the counter to advance.
           # Here-strings, not pipes: a pipe whose reader exits early SIGPIPEs the
           # writer and pipefail turns that into exit 141.
-          deadline=$((SECONDS + 120))
-          m=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics")
-          h1=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m")
-          devs=$(awk '/^miner_gpu_devices/{print $2; exit}' <<<"$m")
-          if [ "${devs%.*}" != "${{ matrix.gpu_devices }}" ]; then
-            echo "miner_gpu_devices is ${devs:-?}, matrix says ${{ matrix.gpu_devices }}" >&2; fail=1
-          else
-            echo "  ok   miner_gpu_devices ${devs%.*}"
-          fi
-          ok=0
+          scrape() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics" || true; }
+          deadline=$((SECONDS + 90))
+          devs=""
           while [ $SECONDS -lt $deadline ]; do
-            sleep 10
-            m=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics")
-            h2=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m")
-            if [ -n "${h1:-}" ] && [ -n "${h2:-}" ] && [ "${h2%.*}" -gt "${h1%.*}" ]; then
-              echo "  ok   miner_hashes_total ${h1%.*} -> ${h2%.*}"
-              ok=1; break
-            fi
+            m=$(scrape)
+            devs=$(awk '/^miner_gpu_devices /{print $2; exit}' <<<"$m")
+            [ -n "$devs" ] && break
+            sleep 5
           done
-          if [ "$ok" != 1 ]; then
-            echo "  miner_hashes_total did not advance within 120s (${h1:-?} -> ${h2:-?})" >&2
+          if [ -z "$devs" ]; then
+            echo "  miner did not connect to ${{ matrix.node }} within 90s (no miner_gpu_devices exported)" >&2; fail=1
+          elif [ "${devs%.*}" != "${{ matrix.gpu_devices }}" ]; then
+            echo "  miner_gpu_devices is ${devs}, matrix says ${{ matrix.gpu_devices }}" >&2; fail=1
+          else
+            echo "  ok   connected; miner_gpu_devices ${devs%.*}"
+          fi
+
+          deadline=$((SECONDS + 120))
+          h1=""; h2=""; ok=0
+          while [ $SECONDS -lt $deadline ]; do
+            m=$(scrape)
+            h=$(awk '/^miner_hashes_total /{print $2; exit}' <<<"$m")
+            if [ -n "$h" ]; then
+              if [ -z "$h1" ]; then
+                h1="$h"
+              elif [ "${h%.*}" -gt "${h1%.*}" ]; then
+                h2="$h"; ok=1; break
+              fi
+            fi
+            sleep 10
+          done
+          if [ "$ok" = 1 ]; then
+            echo "  ok   miner_hashes_total ${h1%.*} -> ${h2%.*}"
+          else
+            echo "  miner_hashes_total did not advance within 120s (${h1:-none} -> ${h2:-none})" >&2
             fail=1
           fi
 
@@ -324,7 +340,10 @@ jobs:
           run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; }
           if run test -x /usr/local/bin/quantus-miner.prev; then
             echo "validate failed after a restart — restoring the previous binary"
-            run sudo cp -p /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner
+            # install, not cp: cp writes in place and fails with "Text file busy"
+            # on a running binary; install unlinks the destination first (the
+            # same reason rsync's temp-file-and-rename works for the push).
+            run sudo install -m 0755 /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner
             run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner
             run sudo systemctl restart quantus-miner.service
             echo "restored: $(run /usr/local/bin/quantus-miner --version)"

deploy/infra-setup.sh

@@ -110,11 +110,12 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-miner.service
 # mining for a measurement window and resume it afterwards.
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop quantus-miner.service
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start quantus-miner.service
-# Rollback: the deploy keeps the previous binary as .prev and restores it when
-# validate fails. Deploying from main means a bad commit reaches production;
+# Rollback: the deploy keeps the previous binary as .prev (cp) and restores it
+# with install when validate fails; cp back would hit "Text file busy" on the
+# running binary, install unlinks the destination first. Deploying from main means a bad commit reaches production;
 # this is what makes that survivable.
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/cp -p /usr/local/bin/quantus-miner /usr/local/bin/quantus-miner.prev
-gitea_ci ALL=(root) NOPASSWD: /usr/bin/cp -p /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner
+gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -m 0755 /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner
 SUDO
     chmod 0440 "$tmp"
     visudo -cf "$tmp"