bench: quantus-bench harness and a workflow that measures on a mining host

quantus/miner#2. The measurement gate every kernel and submission change
passes before merge.

crates/bench-harness (lair-owned, never touched by origin merges):
- drives GpuEngine through MinerEngine, one worker thread per card, with
  fixed timed windows after a warm-up; reports per-worker median MH/s and
  spread and refuses (exit 5) when spread exceeds 2%
- captures nvidia-smi state (driver, power limit, draw, clock, temperature)
  before and after, and refuses to run (exit 3) when the enforced power limit
  is not the expected one, since that is the largest confound on these cards
- GPU/CPU parity on random jobs, same shape as engine-gpu's gpu_cpu_parity
  example, exit 4 on any mismatch
- one JSON record per run (schema 1) plus a markdown summary on stdout
- build.rs embeds the commit like miner-cli's

.gitea/workflows/bench.yaml:
- runner containers on the GPU hosts have no device passthrough, so the
  binary is built on cuda-13.0 (Fedora 43, matching the hosts; the rust
  image is Fedora 44 and its binaries need a newer glibc) and executed on
  the host over ssh as gitea_ci
- stops quantus-miner.service for the window and starts it again under a
  trap; refuses to measure a card that is busy with the miner stopped
- benjy (4090, dedicated) by default; host, duration, runs, batch size and
  workers are dispatch inputs; PRs touching engine or service crates trigger
  it and get the summary as a comment
- one measurement per host at a time (concurrency group)

The stop/start sudoers grants were added to lair/quantus infra-setup.sh
(lair/quantus#5) and applied to benjy and quadbrat.

Origin coupling: one line in the workspace members list, marked.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBgs2nSi4H2mdh8kD8vMX5

.gitea/workflows/bench.yaml

@@ -0,0 +1,169 @@
+---
+# GPU hashrate and parity measurement on a fleet mining host (quantus/miner#2).
+#
+# Runner containers on the GPU hosts have no device passthrough (gongfoo gives
+# them `devices: None`), so the benchmark cannot run inside the runner. The
+# binary is built on a runner and executed on the host over ssh as gitea_ci,
+# which has the GPU device nodes (0666) and the Vulkan ICD, exactly like the
+# miner. The host's quantus-miner.service is stopped for the window and started
+# again afterwards, under a trap, so a failed run cannot leave the host idle.
+#
+# Benjy (4090, dedicated) is the reference card; beast (2x 5090) also serves
+# inference and is only benchmarked by manual dispatch.
+name: bench
+
+on:
+  workflow_dispatch:
+    inputs:
+      host:
+        description: mining host to measure
+        default: benjy.hanzalova.internal
+        type: choice
+        options:
+          - benjy.hanzalova.internal
+          - quadbrat.hanzalova.internal
+          - beast.hanzalova.internal
+      duration_secs:
+        description: seconds per timed window
+        default: "30"
+      runs:
+        description: timed windows (median reported)
+        default: "5"
+      batch_size:
+        description: nonces per GPU batch
+        default: "1000000"
+      workers:
+        description: worker threads (one per card)
+        default: "1"
+  pull_request:
+    paths:
+      - crates/engine-gpu/**
+      - crates/engine-cpu/**
+      - crates/pow-core/**
+      - crates/miner-service/**
+      - crates/bench-harness/**
+      - Cargo.lock
+      - .gitea/workflows/bench.yaml
+
+env:
+  CARGO_TERM_COLOR: always
+  BENCH_HOST: ${{ github.event.inputs.host || 'benjy.hanzalova.internal' }}
+  DURATION: ${{ github.event.inputs.duration_secs || '30' }}
+  RUNS: ${{ github.event.inputs.runs || '5' }}
+  BATCH: ${{ github.event.inputs.batch_size || '1000000' }}
+  WORKERS: ${{ github.event.inputs.workers || '1' }}
+
+# One measurement per host at a time; a second one would share the card.
+concurrency:
+  group: bench-${{ github.event.inputs.host || 'benjy.hanzalova.internal' }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    # cuda-13.0, not rust: the rust image is Fedora 44 and the mining hosts are
+    # Fedora 43, so a binary built there needs a glibc the hosts do not have
+    # (observed: "GLIBC_2.43 not found"). cuda-13.0 is Fedora 43 based, matches
+    # the hosts, and is what the deploy in #8 builds on anyway.
+    runs-on: cuda-13.0
+    steps:
+      - uses: actions/checkout@v4
+      - name: build quantus-bench
+        env:
+          MINER_BUILD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: cargo build --release --locked -p bench-harness
+      - uses: actions/upload-artifact@v3
+        with:
+          name: quantus-bench
+          path: target/release/quantus-bench
+
+  measure:
+    runs-on: fedora-43
+    needs: build
+    steps:
+      - uses: actions/download-artifact@v3
+        with: { name: quantus-bench, path: _bin }
+
+      - name: write ssh key
+        run: |
+          set -euo pipefail
+          install -d -m 0700 ~/.ssh
+          printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
+          chmod 0600 ~/.ssh/id_gitea_ci
+
+      - name: measure on ${{ env.BENCH_HOST }}
+        run: |
+          set -euo pipefail
+          SSHOPTS="-i $HOME/.ssh/id_gitea_ci -o StrictHostKeyChecking=accept-new"
+          run() { ssh $SSHOPTS gitea_ci@"$BENCH_HOST" "$@"; }
+
+          echo "--- host ---"
+          run hostname -f
+          # The enforced power limit is the largest confound; record it and
+          # hand it to the harness so a drifted limit refuses the run.
+          limit=$(run "nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits" | awk 'NR==1{printf "%d", $1}')
+          echo "power limit: ${limit} W"
+
+          echo "--- stage binary ---"
+          run "install -d -m 0750 /var/lib/gitea_ci/bench"
+          scp $SSHOPTS -q _bin/quantus-bench gitea_ci@"$BENCH_HOST":/var/lib/gitea_ci/bench/quantus-bench
+          run "chmod 0755 /var/lib/gitea_ci/bench/quantus-bench && /var/lib/gitea_ci/bench/quantus-bench --version"
+
+          echo "--- pause miner ---"
+          was_active=0
+          if run systemctl is-active --quiet quantus-miner.service; then
+            was_active=1
+            run sudo systemctl stop quantus-miner.service
+          else
+            echo "quantus-miner.service was not active; nothing to pause"
+          fi
+          # Whatever happens below, the miner comes back if it was running.
+          resume() {
+            if [ "$was_active" = 1 ]; then
+              echo "--- resume miner ---"
+              ssh $SSHOPTS gitea_ci@"$BENCH_HOST" sudo systemctl start quantus-miner.service
+              ssh $SSHOPTS gitea_ci@"$BENCH_HOST" systemctl is-active quantus-miner.service
+            fi
+          }
+          trap resume EXIT
+
+          # Refuse to measure a card something else is using (on beast that
+          # would be inference). A few seconds for the miner to release it.
+          sleep 3
+          util=$(run "nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits" | awk 'NR==1{printf "%d", $1}')
+          echo "utilisation before measuring: ${util}%"
+          if [ "$util" -gt 5 ]; then
+            echo "GPU is busy (${util}%) with the miner stopped; refusing to measure" >&2
+            exit 1
+          fi
+
+          echo "--- measure ---"
+          label="${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}"
+          run "cd /var/lib/gitea_ci/bench && RUST_LOG=info ./quantus-bench \
+                --duration-secs $DURATION --runs $RUNS --batch-size $BATCH --workers $WORKERS \
+                --expect-power-limit $limit --label $label --json record.json" | tee bench.md
+
+          scp $SSHOPTS -q gitea_ci@"$BENCH_HOST":/var/lib/gitea_ci/bench/record.json record.json
+          { echo; cat bench.md; } >> "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: bench-record-${{ env.BENCH_HOST }}
+          path: |
+            record.json
+            bench.md
+
+      - name: comment on the pull request
+        if: github.event_name == 'pull_request'
+        run: |
+          set -euo pipefail
+          body=$(python3 - <<'PY'
+          import json, pathlib
+          print(json.dumps({"body": pathlib.Path("bench.md").read_text()}))
+          PY
+          )
+          curl -fsS -X POST \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${{ github.event.pull_request.number }}/comments" \
+            -d "$body" > /dev/null

Cargo.lock

@@ -122,6 +122,22 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "bench-harness"
+version = "4.0.2"
+dependencies = [
+ "clap",
+ "engine-cpu",
+ "engine-gpu",
+ "env_logger",
+ "log",
+ "pow-core",
+ "primitive-types 0.13.1",
+ "rand 0.9.2",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+	"crates/bench-harness", # lair: quantus/miner#2
 	"crates/engine-cpu",
 	"crates/engine-gpu",
 	"crates/metrics",

crates/bench-harness/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "bench-harness"
+version.workspace = true
+edition.workspace = true
+publish = false
+description = "lair: reproducible GPU hashrate and parity measurement for the Quantus miner (quantus/miner#2)"
+
+[[bin]]
+name = "quantus-bench"
+path = "src/main.rs"
+
+[dependencies]
+engine-cpu = { path = "../engine-cpu" }
+engine-gpu = { path = "../engine-gpu" }
+pow-core = { path = "../pow-core" }
+primitive-types = { workspace = true }
+clap = { workspace = true, features = ["derive", "env"] }
+serde = { workspace = true }
+serde_json = { workspace = true, features = ["std"] }
+rand = { workspace = true }
+env_logger = { workspace = true }
+log = { workspace = true }

crates/bench-harness/build.rs

@@ -0,0 +1,51 @@
+// lair: embed the git commit in the binary so `--version` and the build-info
+// metric identify a deployed build by commit, not by the workspace semver
+// (which does not change between commits on a branch that deploys on push).
+//
+// Resolution order:
+//   1. MINER_BUILD_SHA in the environment (CI sets it from the checked-out ref)
+//   2. `git rev-parse HEAD` of the workspace, with "-dirty" if the tree differs
+//   3. "unknown"
+use std::process::Command;
+
+fn git(args: &[&str]) -> Option<String> {
+    let out = Command::new("git").args(args).output().ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let s = String::from_utf8(out.stdout).ok()?;
+    let s = s.trim();
+    if s.is_empty() {
+        None
+    } else {
+        Some(s.to_string())
+    }
+}
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=MINER_BUILD_SHA");
+
+    let sha = match std::env::var("MINER_BUILD_SHA") {
+        Ok(s) if !s.trim().is_empty() => s.trim().to_string(),
+        _ => match git(&["rev-parse", "--short=12", "HEAD"]) {
+            Some(head) => {
+                // Re-run when HEAD moves so a rebuild after a commit picks it up.
+                if let Some(dir) = git(&["rev-parse", "--git-dir"]) {
+                    println!("cargo:rerun-if-changed={dir}/HEAD");
+                    println!("cargo:rerun-if-changed={dir}/refs/heads");
+                }
+                let dirty = git(&["status", "--porcelain", "--untracked-files=no"])
+                    .map(|s| !s.is_empty())
+                    .unwrap_or(false);
+                if dirty {
+                    format!("{head}-dirty")
+                } else {
+                    head
+                }
+            }
+            None => "unknown".to_string(),
+        },
+    };
+
+    println!("cargo:rustc-env=MINER_BUILD_SHA={sha}");
+}

crates/bench-harness/src/main.rs

@@ -0,0 +1,452 @@
+//! lair: `quantus-bench`, the measurement gate for quantus/miner#2.
+//!
+//! Drives a GPU engine through the same `MinerEngine` trait the miner uses,
+//! records hashrate over fixed windows with the GPU's power and clock state
+//! captured before and after, verifies GPU/CPU parity on random jobs, and
+//! writes one JSON record so runs are comparable across commits.
+//!
+//! It deliberately measures one card per worker thread, with no node, no
+//! QUIC and no job churn: this is the kernel-plus-submission number. The
+//! production number, with all of that included, is quantus/miner#9.
+
+use clap::Parser;
+use engine_cpu::{AtomicBoolCancelCheck, EngineStatus, MinerEngine, Range};
+use engine_gpu::GpuEngine;
+use primitive_types::U512;
+use rand::RngCore;
+use serde::Serialize;
+use std::process::Command;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+/// Reproducible GPU hashrate and parity measurement.
+#[derive(Parser, Debug)]
+#[command(version = VERSION, about)]
+struct Args {
+    /// Seconds per timed window.
+    #[arg(long, default_value_t = 30)]
+    duration_secs: u64,
+
+    /// Number of timed windows (median and spread are reported).
+    #[arg(long, default_value_t = 5)]
+    runs: usize,
+
+    /// Seconds of untimed warm-up before the first window.
+    #[arg(long, default_value_t = 5)]
+    warmup_secs: u64,
+
+    /// Nonces per GPU batch (the miner's --gpu-batch-size).
+    #[arg(long, default_value_t = 1_000_000)]
+    batch_size: u32,
+
+    /// Worker threads; each is assigned its own device by the engine, so on a
+    /// multi-card host N threads measure N cards. Extra threads beyond the
+    /// card count are refused.
+    #[arg(long, default_value_t = 1)]
+    workers: usize,
+
+    /// Random jobs for the GPU/CPU parity check (0 to skip).
+    #[arg(long, default_value_t = 25)]
+    parity_jobs: usize,
+
+    /// Refuse to run unless every GPU reports exactly this enforced power
+    /// limit in watts. Power limit is the largest confound on these cards.
+    #[arg(long)]
+    expect_power_limit: Option<u32>,
+
+    /// Commit the binary under test was built from.
+    #[arg(long, default_value = BUILD_SHA)]
+    commit: String,
+
+    /// Free-form label stored in the record (e.g. the PR number).
+    #[arg(long, default_value = "")]
+    label: String,
+
+    /// Write the JSON record here.
+    #[arg(long)]
+    json: Option<std::path::PathBuf>,
+}
+
+// Set by build.rs (same resolution as miner-cli's).
+const BUILD_SHA: &str = env!("MINER_BUILD_SHA");
+const VERSION: &str = concat!(
+    env!("CARGO_PKG_VERSION"),
+    " (",
+    env!("MINER_BUILD_SHA"),
+    ")"
+);
+
+#[derive(Serialize, Debug, Clone)]
+struct GpuState {
+    index: u32,
+    name: String,
+    driver: String,
+    power_limit_w: f64,
+    power_draw_w: f64,
+    sm_clock_mhz: f64,
+    temperature_c: f64,
+}
+
+#[derive(Serialize, Debug)]
+struct WorkerResult {
+    worker: usize,
+    /// MH/s per timed window, in order.
+    windows_mhs: Vec<f64>,
+    median_mhs: f64,
+    /// (max - min) / median over the windows.
+    spread: f64,
+}
+
+#[derive(Serialize, Debug)]
+struct Parity {
+    jobs: usize,
+    found: usize,
+    ok: bool,
+}
+
+#[derive(Serialize, Debug)]
+struct Record {
+    schema: u32,
+    timestamp_unix: u64,
+    host: String,
+    commit: String,
+    label: String,
+    engine: String,
+    batch_size: u32,
+    duration_secs: u64,
+    runs: usize,
+    warmup_secs: u64,
+    workers: usize,
+    gpu_before: Vec<GpuState>,
+    gpu_after: Vec<GpuState>,
+    results: Vec<WorkerResult>,
+    /// Sum of worker medians: the host's number.
+    total_median_mhs: f64,
+    parity: Option<Parity>,
+}
+
+fn nvidia_smi() -> Vec<GpuState> {
+    let out = Command::new("nvidia-smi")
+        .args([
+            "--query-gpu=index,name,driver_version,power.limit,power.draw,clocks.sm,temperature.gpu",
+            "--format=csv,noheader,nounits",
+        ])
+        .output();
+    let Ok(out) = out else {
+        log::warn!("nvidia-smi not available; GPU state not recorded");
+        return Vec::new();
+    };
+    if !out.status.success() {
+        log::warn!("nvidia-smi failed; GPU state not recorded");
+        return Vec::new();
+    }
+    String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter_map(|l| {
+            let f: Vec<&str> = l.split(',').map(str::trim).collect();
+            if f.len() != 7 {
+                return None;
+            }
+            let num = |s: &str| s.parse::<f64>().unwrap_or(f64::NAN);
+            Some(GpuState {
+                index: f[0].parse().ok()?,
+                name: f[1].to_string(),
+                driver: f[2].to_string(),
+                power_limit_w: num(f[3]),
+                power_draw_w: num(f[4]),
+                sm_clock_mhz: num(f[5]),
+                temperature_c: num(f[6]),
+            })
+        })
+        .collect()
+}
+
+fn hostname() -> String {
+    Command::new("hostname")
+        .arg("-f")
+        .output()
+        .ok()
+        .filter(|o| o.status.success())
+        .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
+        .unwrap_or_else(|| "unknown".into())
+}
+
+fn median(xs: &[f64]) -> f64 {
+    let mut v = xs.to_vec();
+    v.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    let n = v.len();
+    if n == 0 {
+        return f64::NAN;
+    }
+    if n % 2 == 1 {
+        v[n / 2]
+    } else {
+        (v[n / 2 - 1] + v[n / 2]) / 2.0
+    }
+}
+
+/// One search window: search from a random start until `cancel` fires,
+/// returning hashes and elapsed seconds. The difficulty is high enough that
+/// a solution is effectively impossible, matching production where the second
+/// squeeze is almost never taken.
+fn window(engine: &GpuEngine, secs: u64, rng: &mut impl RngCore) -> (u64, f64) {
+    let mut header = [0u8; 32];
+    rng.fill_bytes(&mut header);
+    let ctx = engine.prepare_context(header, U512::from(1u64) << 200);
+
+    let mut start_bytes = [0u8; 64];
+    rng.fill_bytes(&mut start_bytes);
+    start_bytes[0] = 0; // keep clear of the top so the range cannot wrap
+    let start = U512::from_big_endian(&start_bytes);
+    let range = Range {
+        start,
+        end: start + (U512::from(1u64) << 200),
+    };
+
+    let flag = Arc::new(AtomicBool::new(false));
+    let timer_flag = flag.clone();
+    let timer = std::thread::spawn(move || {
+        std::thread::sleep(Duration::from_secs(secs));
+        timer_flag.store(true, Ordering::Relaxed);
+    });
+
+    let t0 = Instant::now();
+    let status = engine.search_range(&ctx, range, &AtomicBoolCancelCheck(&flag));
+    let elapsed = t0.elapsed().as_secs_f64();
+    timer.join().expect("timer thread");
+
+    let hashes = match status {
+        EngineStatus::Cancelled { hash_count } | EngineStatus::Exhausted { hash_count } => {
+            hash_count
+        }
+        EngineStatus::Found { hash_count, .. } => {
+            log::warn!("window found a solution at 2^-200 odds; counting hashes anyway");
+            hash_count
+        }
+        other => panic!("unexpected engine status: {other:?}"),
+    };
+    (hashes, elapsed)
+}
+
+fn parity(engine: &GpuEngine, jobs: usize) -> Parity {
+    // Same shape as engine-gpu's gpu_cpu_parity example, kept in lockstep with it.
+    let mut rng = rand::rng();
+    let cancel_flag = AtomicBool::new(false);
+    let cancel = AtomicBoolCancelCheck(&cancel_flag);
+    let mut found = 0usize;
+    let mut ok = true;
+    for job in 0..jobs {
+        let mut header = [0u8; 32];
+        rng.fill_bytes(&mut header);
+        let ctx = engine.prepare_context(header, U512::from(100_000u64));
+        let start = if job == 0 {
+            // Cross a 2^256 boundary to exercise the midstate batch clamp.
+            (U512::from(3u64) << 256) - U512::from(1_000u64)
+        } else {
+            let mut b = [0u8; 64];
+            rng.fill_bytes(&mut b);
+            b[0] = 0;
+            U512::from_big_endian(&b)
+        };
+        let range = Range {
+            start,
+            end: start + U512::from(10_000_000u64),
+        };
+        match engine.search_range(&ctx, range.clone(), &cancel) {
+            EngineStatus::Found { candidate, .. } => {
+                let cpu = pow_core::hash_from_nonce(&ctx, candidate.nonce);
+                let in_range = candidate.nonce >= range.start && candidate.nonce <= range.end;
+                if cpu != candidate.hash || cpu >= ctx.target || !in_range {
+                    log::error!(
+                        "parity job {job}: nonce {} gpu_hash {} cpu_hash {} in_range {in_range}",
+                        candidate.nonce,
+                        candidate.hash,
+                        cpu
+                    );
+                    ok = false;
+                }
+                found += 1;
+            }
+            EngineStatus::Exhausted { .. } => {}
+            other => {
+                log::error!("parity job {job}: unexpected status {other:?}");
+                ok = false;
+            }
+        }
+    }
+    if found == 0 {
+        log::error!("parity: no solutions found across {jobs} jobs");
+        ok = false;
+    }
+    Parity { jobs, found, ok }
+}
+
+fn main() {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+    let args = Args::parse();
+    if args.runs == 0 || args.workers == 0 {
+        eprintln!("--runs and --workers must be at least 1");
+        std::process::exit(2);
+    }
+
+    let gpu_before = nvidia_smi();
+    if let Some(want) = args.expect_power_limit {
+        for g in &gpu_before {
+            if (g.power_limit_w - want as f64).abs() > 0.5 {
+                eprintln!(
+                    "gpu {} ({}) power limit is {:.0} W, expected {want} W; refusing to measure",
+                    g.index, g.name, g.power_limit_w
+                );
+                std::process::exit(3);
+            }
+        }
+    }
+
+    let engine =
+        Arc::new(GpuEngine::try_new(args.batch_size, 0, false).expect("GPU engine init failed"));
+    let devices = engine.device_count();
+    if args.workers > devices {
+        eprintln!(
+            "--workers {} exceeds the {devices} GPU device(s) present",
+            args.workers
+        );
+        std::process::exit(2);
+    }
+    log::info!(
+        "engine {} with {devices} device(s); measuring {} worker(s), {} x {}s windows after {}s warm-up, batch {}",
+        engine.name(),
+        args.workers,
+        args.runs,
+        args.duration_secs,
+        args.warmup_secs,
+        args.batch_size
+    );
+
+    let mut handles = Vec::new();
+    for w in 0..args.workers {
+        let engine = engine.clone();
+        let (runs, dur, warm) = (args.runs, args.duration_secs, args.warmup_secs);
+        handles.push(std::thread::spawn(move || {
+            let mut rng = rand::rng();
+            if warm > 0 {
+                let _ = window(&engine, warm, &mut rng);
+            }
+            let mut windows = Vec::with_capacity(runs);
+            for i in 0..runs {
+                let (h, s) = window(&engine, dur, &mut rng);
+                let mhs = h as f64 / s / 1e6;
+                log::info!("worker {w} window {i}: {h} hashes in {s:.2}s = {mhs:.2} MH/s");
+                windows.push(mhs);
+            }
+            let med = median(&windows);
+            let (mn, mx) = windows
+                .iter()
+                .fold((f64::INFINITY, f64::NEG_INFINITY), |(a, b), &x| {
+                    (a.min(x), b.max(x))
+                });
+            WorkerResult {
+                worker: w,
+                windows_mhs: windows,
+                median_mhs: med,
+                spread: if med > 0.0 { (mx - mn) / med } else { f64::NAN },
+            }
+        }));
+    }
+    let results: Vec<WorkerResult> = handles
+        .into_iter()
+        .map(|h| h.join().expect("worker thread"))
+        .collect();
+    let gpu_after = nvidia_smi();
+
+    let parity_result = if args.parity_jobs > 0 {
+        Some(parity(&engine, args.parity_jobs))
+    } else {
+        None
+    };
+
+    let record = Record {
+        schema: 1,
+        timestamp_unix: SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_secs())
+            .unwrap_or(0),
+        host: hostname(),
+        commit: args.commit.clone(),
+        label: args.label.clone(),
+        engine: engine.name().to_string(),
+        batch_size: args.batch_size,
+        duration_secs: args.duration_secs,
+        runs: args.runs,
+        warmup_secs: args.warmup_secs,
+        workers: args.workers,
+        gpu_before,
+        gpu_after,
+        total_median_mhs: results.iter().map(|r| r.median_mhs).sum(),
+        results,
+        parity: parity_result,
+    };
+
+    // Human summary on stdout (the workflow appends it to the job summary).
+    println!("## quantus-bench {} on {}", record.commit, record.host);
+    println!();
+    println!("| worker | median MH/s | spread | windows |");
+    println!("| --- | --- | --- | --- |");
+    for r in &record.results {
+        let w: Vec<String> = r.windows_mhs.iter().map(|x| format!("{x:.1}")).collect();
+        println!(
+            "| {} | {:.2} | {:.1}% | {} |",
+            r.worker,
+            r.median_mhs,
+            r.spread * 100.0,
+            w.join(", ")
+        );
+    }
+    println!();
+    println!(
+        "total median: **{:.2} MH/s** (batch {}, {} x {}s, engine {})",
+        record.total_median_mhs,
+        record.batch_size,
+        record.runs,
+        record.duration_secs,
+        record.engine
+    );
+    for g in &record.gpu_after {
+        println!(
+            "- gpu {} {}: driver {}, limit {:.0} W, draw {:.0} W, sm {:.0} MHz, {:.0} C",
+            g.index,
+            g.name,
+            g.driver,
+            g.power_limit_w,
+            g.power_draw_w,
+            g.sm_clock_mhz,
+            g.temperature_c
+        );
+    }
+    if let Some(p) = &record.parity {
+        println!(
+            "- parity: {} ({}/{} jobs found solutions, all verified against CPU)",
+            if p.ok { "OK" } else { "FAILED" },
+            p.found,
+            p.jobs
+        );
+    }
+
+    if let Some(path) = &args.json {
+        let s = serde_json::to_string_pretty(&record).expect("serialise record");
+        std::fs::write(path, s).expect("write json record");
+        log::info!("record written to {}", path.display());
+    }
+
+    let spread_bad = record.results.iter().any(|r| r.spread > 0.02);
+    if spread_bad {
+        eprintln!("spread above 2% on at least one worker; run is not stable enough to compare");
+    }
+    if record.parity.as_ref().is_some_and(|p| !p.ok) {
+        eprintln!("PARITY FAILED");
+        std::process::exit(4);
+    }
+    if spread_bad {
+        std::process::exit(5);
+    }
+}