diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 6908ab9..e70a1a0 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -88,8 +88,9 @@ jobs: # so the firewalld rich rule admits it. lair/quantus's SCRAPE_MINERS # must list it so Prometheus scrapes it. # - # Mainnet call 2026-09-09 (quantus/miner#1): all three hosts mine; - # neuron (cortex inference) is disabled on each of them. + # Mainnet call 2026-09-09 (quantus/miner#1): all three hosts mine + # when neuron (helexa inference) is not enabled on them. A host with + # neuron enabled keeps its miner files current but is not started. # `kernel` is the kernel id validate expects on the per-device metric: # cuda (engine-cuda, #3) on every NVIDIA host once the build carries # the fat binary; u64 would mean the miner silently fell back to wgpu. @@ -263,6 +264,20 @@ jobs: run "sudo firewall-cmd --zone=$zone --add-rich-rule='$mrich'" fi + # A host whose neuron.service is enabled is serving LLM inference. + # neuron conflicts with the miner (lair/quantus drop-in), so enabling + # or starting the miner here would stop inference mid-request and put + # the host back to mining behind the operator's back. Ship the files, + # reload so the next deliberate start sees them, and leave it there. + # Switching a host is `systemctl disable --now neuron && systemctl + # enable --now quantus-miner`, done on purpose. + if run systemctl is-enabled --quiet neuron.service; then + run sudo systemctl daemon-reload + echo "neuron.service is enabled on ${{ matrix.host }}: inference host, miner left stopped" + echo "restarted=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + run sudo systemctl enable quantus-miner.service # idempotent if [ "$RESTART" = 1 ]; then echo "changes applied — restarting" @@ -284,6 +299,14 @@ jobs: run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; } fail=0 + # Deploy leaves the miner stopped on an inference host (see above); + # there is nothing to validate there, and failing would make every + # miner deploy red for as long as the host serves. + if run systemctl is-enabled --quiet neuron.service; then + echo "neuron.service is enabled on ${{ matrix.host }}: inference host, miner not expected to run" + exit 0 + fi + echo "--- unit (${{ matrix.host }} -> ${{ matrix.node }}) ---" run systemctl is-active quantus-miner.service diff --git a/crates/bench-harness/bench-on-host.sh b/crates/bench-harness/bench-on-host.sh index fb777be..35808f2 100755 --- a/crates/bench-harness/bench-on-host.sh +++ b/crates/bench-harness/bench-on-host.sh @@ -22,10 +22,15 @@ fi echo "--- host ---" hostname -f -limit=$(nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits | awk 'NR==1{printf "%d", $1}') -echo "power limit: ${limit} W" "$BIN" --version +# A host serving inference is not a bench host, even when idle: the window +# locks the memory clock, which would cripple any request that lands in it. +if systemctl is-active --quiet neuron.service; then + echo "neuron.service is active on this host; refusing to measure" >&2 + exit 1 +fi + echo "--- pause miner ---" was_active=0 if systemctl is-active --quiet quantus-miner.service; then @@ -37,8 +42,12 @@ fi resume() { if [ "$was_active" = 1 ]; then echo "--- resume miner ---" + # Its ExecStartPre applies the miner profile again. sudo systemctl start quantus-miner.service systemctl is-active quantus-miner.service + else + echo "--- restore default GPU profile ---" + sudo /usr/local/bin/nvidia-power-limit.sh default fi } trap resume EXIT @@ -53,6 +62,14 @@ if [ "$util" -gt 5 ]; then exit 1 fi +# Stopping the miner reset the card to its default profile (ExecStopPost). +# Measure under the profile the miner actually runs with, and read the limit +# the harness asserts only after applying it. +echo "--- apply miner GPU profile ---" +sudo /usr/local/bin/nvidia-power-limit.sh miner +limit=$(nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits | awk 'NR==1{printf "%d", $1}') +echo "power limit: ${limit} W" + echo "--- measure ---" cd "$DIR" RUST_LOG=info "$BIN" \ diff --git a/deploy/README.md b/deploy/README.md index 1edeac3..6b0d91f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -37,6 +37,18 @@ deploy on every run. Nothing secret lives here. advances and `miner_gpu_devices` matches the matrix. - **Binaries must be built on `cuda-13.0`** (Fedora 43, like the hosts). The `rust` runner is Fedora 44; its binaries fail with `GLIBC_2.43 not found`. +- **The GPU profile lives and dies with the unit.** `quantus-miner.service` + applies the power limit and 810 MHz memory lock in `ExecStartPre` and resets + to driver defaults in `ExecStopPost`, via `nvidia-power-limit.sh` from + lair/quantus. The lock cuts LLM inference to a sixth of its speed with nothing + reporting as throttled, so it must not outlive the miner. Stopping the unit + therefore also resets the clocks — the benchmark re-applies the profile for + its window. +- **An inference host is left alone.** When `neuron.service` is enabled on a + host, the deploy ships the miner's files but does not enable or start it, and + validate passes without checking hashing. neuron conflicts with the miner, so + starting it would stop inference. Switch hosts deliberately: + `systemctl disable --now neuron && systemctl enable --now quantus-miner`. - **A benchmark (`bench.yaml`) stops the miner on its host.** A deploy landing on the same host during a measurement would find the unit down and start it mid-window. Both are serialised only by not running them together. diff --git a/deploy/infra-setup.sh b/deploy/infra-setup.sh index bddad2c..a767b4e 100755 --- a/deploy/infra-setup.sh +++ b/deploy/infra-setup.sh @@ -110,6 +110,11 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-miner.service # mining for a measurement window and resume it afterwards. gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop quantus-miner.service gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start quantus-miner.service +# The benchmark measures under the miner's GPU profile, which stopping the unit +# resets (its ExecStopPost). bench-on-host.sh re-applies it for the window and +# restores whichever profile the host was in afterwards. +gitea_ci ALL=(root) NOPASSWD: /usr/local/bin/nvidia-power-limit.sh miner +gitea_ci ALL=(root) NOPASSWD: /usr/local/bin/nvidia-power-limit.sh default # Rollback: the deploy keeps the previous binary as .prev (cp) and restores it # with install when validate fails; cp back would hit "Text file busy" on the # running binary, install unlinks the destination first. Deploying from main means a bad commit reaches production; diff --git a/deploy/quantus-miner.service b/deploy/quantus-miner.service index 141e641..67be972 100644 --- a/deploy/quantus-miner.service +++ b/deploy/quantus-miner.service @@ -9,8 +9,13 @@ # finds no adapter, silently falling back to nothing. # # Poseidon2-over-Goldilocks is compute-bound in registers, not memory-bound, so -# there is no tuning knob here worth more than the GPU power limit — which is a -# host-level concern (nvidia-smi -pl), not a unit-file one. +# the tuning that matters is the GPU profile: a lower power limit and memory +# locked at 810 MHz (lair/quantus readme, "GPU memory clock"). That profile is +# applied here, for the lifetime of this unit, and reset when it stops — it must +# never outlive the miner, because the same lock cuts LLM inference on the host +# to a sixth of its speed. The script and its per-host config +# (/etc/default/nvidia-power-limit) are installed by lair/quantus's +# infra-setup.sh; neuron.service on the same host conflicts with this unit. # # --cpu-workers 0 is deliberate: a current-gen desktop CPU contributes ~4 MH/s # next to a 4090's ~183 MH/s, for a couple hundred watts. Measured, not assumed. @@ -37,6 +42,12 @@ ExecStart=/usr/local/bin/quantus-miner serve \ --metrics-port 9900 EnvironmentFile=/etc/quantus-miner/miner.env +# '+' runs these as root; the miner itself stays unprivileged. ExecStopPost runs +# however the service ends — stop, crash, failed start — so the lock cannot be +# left behind by a miner that is no longer running. +ExecStartPre=+/usr/local/bin/nvidia-power-limit.sh miner +ExecStopPost=+/usr/local/bin/nvidia-power-limit.sh default + # The node may be down, still syncing, or mid-redeploy; reconnecting is normal # operation, not an error condition. Restart=always