bench: serialise measurements per host with a flock, per-run file names

Gitea's concurrency group did not serialise two runs on benjy: the second
run's scp failed with ETXTBSY on the binary the first was executing (safe,
it happened before the miner was touched, but the run was lost). The pause,
lock, measure and resume sequence now lives in bench-on-host.sh, executed
over one ssh call under flock on the host, with per-run binary and record
names so staging never collides.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5

.gitea/workflows/bench.yaml

@@ -80,6 +80,7 @@ jobs:
     runs-on: fedora-43
     needs: build
     steps:
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with: { name: quantus-bench, path: _bin }
 
@@ -96,53 +97,25 @@ jobs:
           SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
           run() { ssh $SSHOPTS gitea_ci@"$BENCH_HOST" "$@"; }
 
-          echo "--- host ---"
-          run hostname -f
-          # The enforced power limit is the largest confound; record it and
-          # hand it to the harness so a drifted limit refuses the run.
-          limit=$(run "nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits" | awk 'NR==1{printf "%d", $1}')
-          echo "power limit: ${limit} W"
+          # Per-run file names: two runs may overlap on the host until the
+          # host-side flock serialises them, and scp over a running binary
+          # fails with ETXTBSY.
+          id="${{ github.run_id }}"
+          dir=/var/lib/gitea_ci/bench
+          run "install -d -m 0750 $dir"
+          scp $SSHOPTS -q _bin/quantus-bench gitea_ci@"$BENCH_HOST":$dir/quantus-bench.$id
+          scp $SSHOPTS -q crates/bench-harness/bench-on-host.sh gitea_ci@"$BENCH_HOST":$dir/bench-on-host.$id.sh
+          run "chmod 0755 $dir/quantus-bench.$id"
+          cleanup() { ssh $SSHOPTS gitea_ci@"$BENCH_HOST" "rm -f $dir/quantus-bench.$id $dir/bench-on-host.$id.sh $dir/record.$id.json"; }
+          trap cleanup EXIT
 
-          echo "--- stage binary ---"
-          run "install -d -m 0750 /var/lib/gitea_ci/bench"
-          scp $SSHOPTS -q _bin/quantus-bench gitea_ci@"$BENCH_HOST":/var/lib/gitea_ci/bench/quantus-bench
-          run "chmod 0755 /var/lib/gitea_ci/bench/quantus-bench && /var/lib/gitea_ci/bench/quantus-bench --version"
-
-          echo "--- pause miner ---"
-          was_active=0
-          if run systemctl is-active --quiet quantus-miner.service; then
-            was_active=1
-            run sudo systemctl stop quantus-miner.service
-          else
-            echo "quantus-miner.service was not active; nothing to pause"
-          fi
-          # Whatever happens below, the miner comes back if it was running.
-          resume() {
-            if [ "$was_active" = 1 ]; then
-              echo "--- resume miner ---"
-              ssh $SSHOPTS gitea_ci@"$BENCH_HOST" sudo systemctl start quantus-miner.service
-              ssh $SSHOPTS gitea_ci@"$BENCH_HOST" systemctl is-active quantus-miner.service
-            fi
-          }
-          trap resume EXIT
-
-          # Refuse to measure a card something else is using (on beast that
-          # would be inference). A few seconds for the miner to release it.
-          sleep 3
-          util=$(run "nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits" | awk 'NR==1{printf "%d", $1}')
-          echo "utilisation before measuring: ${util}%"
-          if [ "$util" -gt 5 ]; then
-            echo "GPU is busy (${util}%) with the miner stopped; refusing to measure" >&2
-            exit 1
-          fi
-
-          echo "--- measure ---"
           label="${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}"
-          run "cd /var/lib/gitea_ci/bench && RUST_LOG=info ./quantus-bench \
-                --duration-secs $DURATION --runs $RUNS --batch-size $BATCH --workers $WORKERS \
-                --expect-power-limit $limit --label $label --json record.json" | tee bench.md
-
-          scp $SSHOPTS -q gitea_ci@"$BENCH_HOST":/var/lib/gitea_ci/bench/record.json record.json
+          # The host script pauses the miner, holds the per-host lock, measures,
+          # and resumes the miner under its own trap.
+          run "bash $dir/bench-on-host.$id.sh $dir/quantus-bench.$id $DURATION $RUNS $BATCH $WORKERS $label $dir/record.$id.json" | tee bench.log
+          # The summary is everything from the harness's markdown header on.
+          sed -n '/^## quantus-bench/,$p' bench.log > bench.md
+          scp $SSHOPTS -q gitea_ci@"$BENCH_HOST":$dir/record.$id.json record.json
           { echo; cat bench.md; } >> "$GITHUB_STEP_SUMMARY"
 
       - uses: actions/upload-artifact@v3

crates/bench-harness/bench-on-host.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# lair: run quantus-bench on a mining host, as gitea_ci, with the miner paused.
+#
+# Executed over ssh by .gitea/workflows/bench.yaml. Everything that must be
+# serialised per host lives here under one flock: Gitea's workflow concurrency
+# group did NOT serialise two runs on the same host (observed: the second run's
+# scp hit ETXTBSY on the binary the first was executing), so the host is the
+# arbiter, not the forge.
+#
+# usage: bench-on-host.sh <binary> <duration_secs> <runs> <batch_size> <workers> <label> <out_json>
+set -euo pipefail
+
+BIN="$1"; DURATION="$2"; RUNS="$3"; BATCH="$4"; WORKERS="$5"; LABEL="$6"; OUT="$7"
+DIR=/var/lib/gitea_ci/bench
+LOCK="$DIR/.lock"
+
+exec 9>"$LOCK"
+if ! flock -w 1800 9; then
+  echo "another measurement has held $LOCK for 30 minutes; giving up" >&2
+  exit 1
+fi
+
+echo "--- host ---"
+hostname -f
+limit=$(nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits | awk 'NR==1{printf "%d", $1}')
+echo "power limit: ${limit} W"
+"$BIN" --version
+
+echo "--- pause miner ---"
+was_active=0
+if systemctl is-active --quiet quantus-miner.service; then
+  was_active=1
+  sudo systemctl stop quantus-miner.service
+else
+  echo "quantus-miner.service was not active; nothing to pause"
+fi
+resume() {
+  if [ "$was_active" = 1 ]; then
+    echo "--- resume miner ---"
+    sudo systemctl start quantus-miner.service
+    systemctl is-active quantus-miner.service
+  fi
+}
+trap resume EXIT
+
+# Refuse to measure a card something else is using (on beast that would be
+# inference). A few seconds for the miner to release it.
+sleep 3
+util=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | awk 'NR==1{printf "%d", $1}')
+echo "utilisation before measuring: ${util}%"
+if [ "$util" -gt 5 ]; then
+  echo "GPU is busy (${util}%) with the miner stopped; refusing to measure" >&2
+  exit 1
+fi
+
+echo "--- measure ---"
+cd "$DIR"
+RUST_LOG=info "$BIN" \
+  --duration-secs "$DURATION" --runs "$RUNS" --batch-size "$BATCH" --workers "$WORKERS" \
+  --expect-power-limit "$limit" --label "$LABEL" --json "$OUT"