gpu: apply the miner's clock profile only while the miner runs #34
@@ -1,85 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# Apply a persistent NVIDIA power limit at boot.
|
||||
# Apply one of two GPU profiles: the miner's tuned power limit and memory clock,
|
||||
# or the driver defaults.
|
||||
#
|
||||
# nvidia-smi -pl does NOT survive a reboot, and without persistence mode it is
|
||||
# also lost whenever the last GPU client detaches and the driver unloads. Both
|
||||
# are silent: the card simply returns to its default limit and nobody notices
|
||||
# until a connector runs warm. Hence a boot-time unit rather than a one-off.
|
||||
# nvidia-power-limit.sh miner # while quantus-miner.service runs
|
||||
# nvidia-power-limit.sh default # at boot, when the miner stops, and before
|
||||
# # any other GPU workload (e.g. neuron) starts
|
||||
#
|
||||
# Lower is not a compromise here. Measured on a 4090 mining Poseidon2:
|
||||
# The miner profile is a property of the miner, not of the host. It is applied
|
||||
# by quantus-miner.service's ExecStartPre and undone by its ExecStopPost, so it
|
||||
# exists only while the miner is running. A host that switches to LLM inference
|
||||
# must not keep it: inference is memory-bandwidth bound, and a memory clock
|
||||
# locked at 810 MHz on a 5090 cut decode from ~22 to ~4 tok/s with nothing
|
||||
# reporting as throttled (the lock is not a "clocks event reason").
|
||||
#
|
||||
# Why the power limit is lower in the miner profile, measured on a 4090 mining
|
||||
# Poseidon2:
|
||||
# 450 W -> 185.5 MH/s, 70 C, 0.413 MH/s/W
|
||||
# 300 W -> 183.2 MH/s, 61 C, 0.612 MH/s/W
|
||||
# a third less power and 9 C cooler for 1.2% less hashrate, because this
|
||||
# workload is register-resident integer math and does not need the top of the
|
||||
# clock curve. It also cuts current through the 12VHPWR connector, which is the
|
||||
# one component on 4090/5090 cards with a documented melting failure mode.
|
||||
#
|
||||
# nvidia-smi -pl and -lmc survive neither a reboot nor, without persistence
|
||||
# mode, the last GPU client detaching. Both modes enable persistence mode so a
|
||||
# profile holds for as long as it is meant to and no longer.
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:-}"
|
||||
case "$MODE" in
|
||||
miner|default) ;;
|
||||
*) echo "usage: $0 miner|default" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# Persistence mode keeps the driver resident so a limit is not lost when the
|
||||
# last client detaches.
|
||||
if ! nvidia-smi -pm 1 >/dev/null; then
|
||||
echo "warning: could not enable persistence mode" >&2
|
||||
fi
|
||||
|
||||
if [ "$MODE" = default ]; then
|
||||
# Each card's own default limit, not a literal: the fleet mixes 3060, 4090
|
||||
# and 5090, whose defaults differ.
|
||||
nvidia-smi --query-gpu=index,name,power.default_limit --format=csv,noheader,nounits |
|
||||
while IFS=',' read -r idx name defw; do
|
||||
idx="${idx// /}"; defw="${defw%%.*}"; defw="${defw// /}"
|
||||
if nvidia-smi -i "$idx" -pl "$defw" >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): power limit reset to default ${defw}W"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to reset power limit to ${defw}W" >&2
|
||||
exit 1
|
||||
fi
|
||||
if nvidia-smi -i "$idx" -rmc >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): memory clock reset to driver default"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to reset memory clock" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── miner profile ──────────────────────────────────────────────────────────
|
||||
CONF="${CONF:-/etc/default/nvidia-power-limit}"
|
||||
[ -r "$CONF" ] && . "$CONF"
|
||||
WANT="${NVIDIA_POWER_LIMIT_WATTS:-}"
|
||||
# Locked memory clock. The miner never touches VRAM, so memory at full clock is
|
||||
# power taken from the SMs inside the same cap; 810 MHz measured +13% on the
|
||||
# 4090 at 250 W. Empty means "driver default" and resets any previous lock.
|
||||
# 4090 at 250 W. Empty means "driver default".
|
||||
# 405 MHz drops the card into a low-power state and is refused below.
|
||||
WANT_MEM="${NVIDIA_MEMORY_CLOCK_MHZ:-}"
|
||||
|
||||
if [ -z "$WANT" ] && [ -z "$WANT_MEM" ]; then
|
||||
echo "no NVIDIA_POWER_LIMIT_WATTS or NVIDIA_MEMORY_CLOCK_MHZ set in ${CONF} — leaving the cards at default" >&2
|
||||
exit 0
|
||||
echo "no NVIDIA_POWER_LIMIT_WATTS or NVIDIA_MEMORY_CLOCK_MHZ set in ${CONF} — miner runs at driver defaults"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Persistence mode keeps the driver resident so the limit is not lost when the
|
||||
# last client detaches. It does NOT survive a reboot on its own, which is what
|
||||
# this unit is for.
|
||||
nvidia-smi -pm 1 >/dev/null || echo "warning: could not enable persistence mode" >&2
|
||||
|
||||
[ -n "$WANT" ] && nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \
|
||||
--format=csv,noheader,nounits |
|
||||
while IFS=',' read -r idx name minw maxw; do
|
||||
idx="${idx// /}"; minw="${minw%%.*}"; maxw="${maxw%%.*}"
|
||||
minw="${minw// /}"; maxw="${maxw// /}"
|
||||
target="$WANT"
|
||||
idx="${idx// /}"; minw="${minw%%.*}"; maxw="${maxw%%.*}"
|
||||
minw="${minw// /}"; maxw="${maxw// /}"
|
||||
target="$WANT"
|
||||
|
||||
# Clamp rather than fail: a request below the card's floor is a
|
||||
# configuration mistake, not a reason to leave the GPU at its default —
|
||||
# which would be the opposite of what was intended.
|
||||
if [ "$target" -lt "$minw" ]; then
|
||||
echo "gpu ${idx} (${name# }): requested ${WANT}W is below the ${minw}W floor — clamping to ${minw}W" >&2
|
||||
target="$minw"
|
||||
elif [ "$target" -gt "$maxw" ]; then
|
||||
echo "gpu ${idx} (${name# }): requested ${WANT}W exceeds the ${maxw}W ceiling — clamping to ${maxw}W" >&2
|
||||
target="$maxw"
|
||||
fi
|
||||
# Clamp rather than fail: a request below the card's floor is a
|
||||
# configuration mistake, not a reason to leave the GPU at its default —
|
||||
# which would be the opposite of what was intended.
|
||||
if [ "$target" -lt "$minw" ]; then
|
||||
echo "gpu ${idx} (${name# }): requested ${WANT}W is below the ${minw}W floor — clamping to ${minw}W" >&2
|
||||
target="$minw"
|
||||
elif [ "$target" -gt "$maxw" ]; then
|
||||
echo "gpu ${idx} (${name# }): requested ${WANT}W exceeds the ${maxw}W ceiling — clamping to ${maxw}W" >&2
|
||||
target="$maxw"
|
||||
fi
|
||||
|
||||
if nvidia-smi -i "$idx" -pl "$target" >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): power limit set to ${target}W"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to set power limit to ${target}W" >&2
|
||||
exit 1
|
||||
fi
|
||||
if nvidia-smi -i "$idx" -pl "$target" >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): power limit set to ${target}W"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to set power limit to ${target}W" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Memory clock, after the power limit so the freed watts are already available.
|
||||
nvidia-smi --query-gpu=index,name --format=csv,noheader,nounits |
|
||||
while IFS=',' read -r idx name; do
|
||||
idx="${idx// /}"
|
||||
if [ -z "$WANT_MEM" ]; then
|
||||
nvidia-smi -i "$idx" -rmc >/dev/null && echo "gpu ${idx} (${name# }): memory clock reset to driver default"
|
||||
continue
|
||||
fi
|
||||
if [ "$WANT_MEM" -le 405 ]; then
|
||||
echo "gpu ${idx} (${name# }): refusing memory clock ${WANT_MEM} MHz — 405 puts the card in a low-power state" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -qE "Memory\s+: ${WANT_MEM} MHz"; then
|
||||
echo "gpu ${idx} (${name# }): memory clock ${WANT_MEM} MHz is not a supported state; supported: $(nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -E '^\s+Memory\s+:' | grep -oE '[0-9]+ MHz' | sort -un | tr '\n' ' ')" >&2
|
||||
exit 1
|
||||
fi
|
||||
if nvidia-smi -i "$idx" -lmc "$WANT_MEM,$WANT_MEM" >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): memory clock locked at ${WANT_MEM} MHz"
|
||||
idx="${idx// /}"
|
||||
if [ -z "$WANT_MEM" ]; then
|
||||
if nvidia-smi -i "$idx" -rmc >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): memory clock reset to driver default"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to lock memory clock at ${WANT_MEM} MHz" >&2
|
||||
exit 1
|
||||
echo "gpu ${idx} (${name# }): FAILED to reset memory clock" >&2
|
||||
exit 1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$WANT_MEM" -le 405 ]; then
|
||||
echo "gpu ${idx} (${name# }): refusing memory clock ${WANT_MEM} MHz — 405 puts the card in a low-power state" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -qE "Memory\s+: ${WANT_MEM} MHz"; then
|
||||
echo "gpu ${idx} (${name# }): memory clock ${WANT_MEM} MHz is not a supported state; supported: $(nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -E '^\s+Memory\s+:' | grep -oE '[0-9]+ MHz' | sort -un | tr '\n' ' ')" >&2
|
||||
exit 1
|
||||
fi
|
||||
if nvidia-smi -i "$idx" -lmc "$WANT_MEM,$WANT_MEM" >/dev/null; then
|
||||
echo "gpu ${idx} (${name# }): memory clock locked at ${WANT_MEM} MHz"
|
||||
else
|
||||
echo "gpu ${idx} (${name# }): FAILED to lock memory clock at ${WANT_MEM} MHz" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Drop-in for helexa's neuron.service on a host that can also mine.
|
||||
# Lands at /etc/systemd/system/neuron.service.d/50-gpu-exclusive.conf.
|
||||
#
|
||||
# neuron (LLM inference) and quantus-miner want the same GPUs with opposite
|
||||
# tuning: the miner runs with memory locked at 810 MHz, inference is
|
||||
# memory-bandwidth bound and collapses under that lock. So they never run
|
||||
# together, and whichever starts second takes the host:
|
||||
#
|
||||
# systemctl start neuron # stops the miner, resets clocks, serves
|
||||
# systemctl start quantus-miner # stops neuron (drain + unload), mines
|
||||
#
|
||||
# Conflicts= works in both directions from this one file, so the miner's own
|
||||
# unit needs no knowledge of neuron. Keep only one of the two enabled at boot;
|
||||
# with both enabled, which one wins is decided by job ordering, not intent.
|
||||
[Unit]
|
||||
Conflicts=quantus-miner.service
|
||||
# Ordered after the miner so that when starting neuron stops the miner, the
|
||||
# miner's ExecStopPost (reset to the default profile) has finished before
|
||||
# neuron touches the GPU. When the miner is started instead, systemd orders the
|
||||
# stop of neuron before the start of the miner regardless of direction.
|
||||
After=quantus-miner.service
|
||||
|
||||
[Service]
|
||||
# Reset anyway: a lock left by anything other than the miner's own lifecycle
|
||||
# (a hand-run nvidia-smi -lmc, a benchmark) must not reach inference. '+' runs it as root; neuron itself stays unprivileged.
|
||||
ExecStartPre=+/usr/local/bin/nvidia-power-limit.sh default
|
||||
@@ -1,18 +1,20 @@
|
||||
# Reapply the NVIDIA power limit at boot. See asset/gpu/nvidia-power-limit.sh
|
||||
# for why this cannot be a one-off nvidia-smi invocation.
|
||||
# Put the GPUs in the driver-default profile at boot. The miner's tuned power
|
||||
# limit and memory clock are applied by quantus-miner.service itself
|
||||
# (ExecStartPre) and undone when it stops (ExecStopPost), so they exist only
|
||||
# while the miner runs. See asset/gpu/nvidia-power-limit.sh.
|
||||
[Unit]
|
||||
Description=Apply NVIDIA GPU power limit
|
||||
Description=Reset NVIDIA GPUs to the default profile
|
||||
Documentation=https://git.lair.cafe/lair/quantus
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
After=basic.target
|
||||
# Ordered before the miner so the GPU is never briefly at its default limit
|
||||
# while under full mining load. Harmless on hosts with no miner.
|
||||
Before=quantus-miner.service
|
||||
# Before both GPU workloads, so neither can start and then have its profile
|
||||
# overwritten by this unit.
|
||||
Before=quantus-miner.service neuron.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/nvidia-power-limit.sh
|
||||
ExecStart=/usr/local/bin/nvidia-power-limit.sh default
|
||||
# Needs root and the nvidia device nodes; holds no credentials and reads
|
||||
# nothing sensitive.
|
||||
NoNewPrivileges=true
|
||||
|
||||
@@ -82,10 +82,13 @@ explicit that this port must never face the internet.
|
||||
|
||||
## GPU power limits
|
||||
|
||||
Applied at boot by `nvidia-power-limit.service`, configured per host in
|
||||
`infra-setup.sh`'s `GPU_POWER_LIMITS` as `host=watts`, and **clamped to each
|
||||
card's own floor and ceiling** — a request the hardware refuses is corrected with
|
||||
a log line rather than silently ignored.
|
||||
Part of the **miner profile**, configured per host in `infra-setup.sh`'s
|
||||
`GPU_POWER_LIMITS` as `host=watts` and **clamped to each card's own floor and
|
||||
ceiling** — a request the hardware refuses is corrected with a log line rather
|
||||
than silently ignored. The profile is applied by `quantus-miner.service` itself
|
||||
(`ExecStartPre=+nvidia-power-limit.sh miner`) and undone when it stops
|
||||
(`ExecStopPost=+nvidia-power-limit.sh default`), so it exists only while the
|
||||
miner runs. See [GPU profiles and inference](#gpu-profiles-and-inference).
|
||||
|
||||
This is not a compromise. Measured on a 4090 mining Poseidon2:
|
||||
|
||||
@@ -101,16 +104,15 @@ the last few hundred MHz were very nearly pure heat. It also cuts current throug
|
||||
the 12VHPWR connector, the one component on 4090/5090 cards with a documented
|
||||
melting failure mode.
|
||||
|
||||
A boot-time unit is required because `nvidia-smi -pl` survives neither a reboot
|
||||
nor a driver unload. The unit also enables persistence mode, which stops the
|
||||
limit being lost when the last GPU client detaches — a silent reversion that
|
||||
would otherwise show up only as a warm connector. The dashboard's **GPU Power**
|
||||
gauge reads draw against the *enforced* limit, so a reverted cap appears as the
|
||||
gauge dropping from ~100% to ~60% rather than as nothing at all.
|
||||
`nvidia-smi -pl` survives neither a reboot nor a driver unload, so the script
|
||||
also enables persistence mode, which stops the limit being lost when the last
|
||||
GPU client detaches. The dashboard's **GPU Power** gauge reads draw against the
|
||||
*enforced* limit, so a reverted cap appears as the gauge dropping from ~100% to
|
||||
~60% rather than as nothing at all.
|
||||
|
||||
## GPU memory clock
|
||||
|
||||
Same unit, same config file: `GPU_MEMORY_CLOCKS` in `infra-setup.sh` as
|
||||
Same profile, same config file: `GPU_MEMORY_CLOCKS` in `infra-setup.sh` as
|
||||
`host=MHz`, applied with `nvidia-smi -lmc` after the power limit. The miner never
|
||||
touches VRAM, so memory running at full clock is power spent inside the same cap
|
||||
for nothing. Measured with the CUDA engine (quantus/miner#3), miner paused, the
|
||||
@@ -128,9 +130,43 @@ card at its usual limit:
|
||||
**+13% on the 4090 for the same watts**, the freed power going straight into SM
|
||||
clock. The 405 MHz state is not "lower still": it moves the card into a
|
||||
low-power performance state that also caps the core, and the script refuses it.
|
||||
Hosts that serve inference are deliberately not listed: LLM inference is
|
||||
memory-bandwidth bound and would be crippled. A host with no entry has its
|
||||
memory clock reset to the driver default, so the config is authoritative both ways.
|
||||
The lock is invisible to the usual checks: `nvidia-smi` shows no active "clocks
|
||||
event reason" for it, utilisation reads ~98%, and only the P5 state, the
|
||||
810 MHz memory clock and the low power draw give it away. That is why it must
|
||||
never outlive the miner — see below.
|
||||
|
||||
## GPU profiles and inference
|
||||
|
||||
The miner profile is a property of the miner, not of the host. Three pieces
|
||||
keep it that way:
|
||||
|
||||
- `nvidia-power-limit.service` runs `nvidia-power-limit.sh default` at boot:
|
||||
driver-default power limit and memory clock.
|
||||
- `quantus-miner.service` applies the miner profile in `ExecStartPre` and resets
|
||||
to default in `ExecStopPost`, which runs however the miner stops (clean stop,
|
||||
crash, restart). The unit lives in quantus/miner; the script it calls is
|
||||
installed by this repo's `infra-setup.sh`.
|
||||
- On every GPU host, `infra-setup.sh` installs a drop-in for helexa's
|
||||
`neuron.service` (`asset/systemd/neuron-gpu-exclusive.conf`) that declares
|
||||
`Conflicts=quantus-miner.service` and resets to default in `ExecStartPre`.
|
||||
|
||||
So switching a host between mining and inference is one command, and the clocks
|
||||
follow:
|
||||
|
||||
```sh
|
||||
sudo systemctl disable --now quantus-miner && sudo systemctl enable --now neuron # serve
|
||||
sudo systemctl disable --now neuron && sudo systemctl enable --now quantus-miner # mine
|
||||
```
|
||||
|
||||
Keep only one of the two enabled at boot. `systemctl start` of either one stops
|
||||
the other, but with both enabled, which one wins at boot is decided by job
|
||||
ordering, not intent. The miner deploy (quantus/miner) will not enable or start
|
||||
the miner on a host where neuron is enabled.
|
||||
|
||||
Why it matters: when this was a boot-time host setting, a host switched back to
|
||||
inference kept the miner's 810 MHz lock with the miner stopped. A 27B model on
|
||||
2× RTX 5090 decoded at 3.8 tok/s on the same build that had measured 21.9 tok/s
|
||||
— a 5.8× slowdown with no error anywhere.
|
||||
|
||||
## Monitoring
|
||||
|
||||
@@ -389,21 +425,16 @@ auto-update on a validator on purpose. The miner follows quantus/miner `main`.
|
||||
|
||||
## Known conflict: GPU contention with helexa ⚠️
|
||||
|
||||
A miner host may already be running helexa `neuron` against the same GPU
|
||||
(`architecture/generic.md` §11, `inference.md`). They contend directly: the miner
|
||||
drives the GPU to ~95% utilisation and near its power limit continuously, which
|
||||
makes inference latency on that host by turns awful and unpredictable.
|
||||
A miner host may also be a helexa `neuron` host (`architecture/generic.md` §11,
|
||||
`inference.md`). Running both at once is not supported: the miner drives the GPU
|
||||
to ~95% utilisation continuously, and its memory-clock lock cripples inference.
|
||||
The two units therefore conflict (see
|
||||
[GPU profiles and inference](#gpu-profiles-and-inference)); a host does one or
|
||||
the other, and switching is a `systemctl` operation.
|
||||
|
||||
Nothing here resolves that, and the deploy does not try to arbitrate. Options,
|
||||
in rough order of preference:
|
||||
|
||||
1. Put mining on a host that is not serving inference.
|
||||
2. Make the miner yield — `--gpu-throttle-ms`, or an `ExecCondition` that
|
||||
refuses to start while a model is loaded.
|
||||
3. Accept the contention and treat inference on that host as best-effort.
|
||||
|
||||
Check what a candidate miner host is already serving before enabling
|
||||
`quantus-miner.service` on it, not after.
|
||||
Mining in neuron's idle time — yielding the GPU when a model is requested — is
|
||||
not implemented. It would need neuron to unload its models and release VRAM on
|
||||
demand, which is a scheduling feature on the helexa side, not a unit-file one.
|
||||
|
||||
## Deviations from house convention
|
||||
|
||||
|
||||
+42
-28
@@ -55,22 +55,24 @@ EDGE_HOSTS="${EDGE_HOSTS-hanzalova.internal}"
|
||||
# quantus-specific — it lives here because there is no monitoring repo yet, and
|
||||
# should move to one if that changes.
|
||||
GPU_HOSTS="${GPU_HOSTS-beast.hanzalova.internal benjy.hanzalova.internal quadbrat.hanzalova.internal}"
|
||||
# Per-host GPU power limits, as `host=watts`. Applied at boot by
|
||||
# nvidia-power-limit.service and clamped to each card's own floor/ceiling, so a
|
||||
# value the hardware refuses is corrected loudly rather than silently ignored.
|
||||
# Conservative on purpose: a 4090 measured 183.2 MH/s at 300W against 185.5 at
|
||||
# 450W, so the hashrate cost is ~1% and the thermal and connector-current
|
||||
# margin is large. A 5090's floor is 400W; requests below it are clamped.
|
||||
# Per-host MINER profile: power limit as `host=watts`, memory clock as
|
||||
# `host=MHz`. Written to /etc/default/nvidia-power-limit and applied only while
|
||||
# quantus-miner.service runs (its ExecStartPre; ExecStopPost resets to driver
|
||||
# defaults), so listing a host that also serves LLM inference is safe: neuron
|
||||
# and the miner conflict, and neuron starts on a reset GPU.
|
||||
#
|
||||
# Power: clamped to each card's own floor/ceiling, so a value the hardware
|
||||
# refuses is corrected loudly rather than silently ignored. Conservative on
|
||||
# purpose: a 4090 measured 183.2 MH/s at 300W against 185.5 at 450W, so the
|
||||
# hashrate cost is ~1% and the thermal and connector-current margin is large. A
|
||||
# 5090's floor is 400W; requests below it are clamped.
|
||||
GPU_POWER_LIMITS="${GPU_POWER_LIMITS-beast.hanzalova.internal=300 benjy.hanzalova.internal=250 quadbrat.hanzalova.internal=130}"
|
||||
# Per-host locked memory clock, as `host=MHz`, applied by the same unit. The
|
||||
# miner never touches VRAM, so memory at full clock is watts taken from the
|
||||
# SMs inside the same power cap: locking it at 810 MHz measured +13% on the
|
||||
# 4090 at 250 W (410 -> 463 MH/s) and +2% on the 3060 at 130 W. 405 MHz drops
|
||||
# the card into a low-power state and must not be used. Hosts serving
|
||||
# beast joined 2026-09-09 once inference was switched off there (+2% on each
|
||||
# 5090 at 400 W, 555 -> 567 MH/s, SM clock 2250 -> 2370 MHz); a host serving
|
||||
# LLM inference must NOT be listed, inference is memory-bound. A host with no
|
||||
# entry gets its memory clock reset to the driver default.
|
||||
# Memory: the miner never touches VRAM, so memory at full clock is watts taken
|
||||
# from the SMs inside the same power cap: locking it at 810 MHz measured +13% on
|
||||
# the 4090 at 250 W (410 -> 463 MH/s), +2% on the 3060 at 130 W and +2% on each
|
||||
# 5090 at 400 W (555 -> 567 MH/s, SM clock 2250 -> 2370 MHz). 405 MHz drops the
|
||||
# card into a low-power state and must not be used. A host with no entry mines
|
||||
# at the driver-default memory clock.
|
||||
GPU_MEMORY_CLOCKS="${GPU_MEMORY_CLOCKS-benjy.hanzalova.internal=810 quadbrat.hanzalova.internal=810 beast.hanzalova.internal=810}"
|
||||
# Vanity name served by the edge proxy. Dot-free short name: the step@ renewal
|
||||
# unit is a template instance and %i cannot carry dots (internal-tls.md §1).
|
||||
@@ -351,8 +353,16 @@ provision_gpu() {
|
||||
asset/systemd/nvidia-textfile.service asset/systemd/nvidia-textfile.timer \
|
||||
asset/systemd/nvidia-power-limit.service \
|
||||
"${ADMIN_USER}@${host}:/etc/systemd/system/"
|
||||
# neuron and the miner are mutually exclusive on the GPU (see the drop-in).
|
||||
# Installed on every GPU host whether or not neuron is: a drop-in for an
|
||||
# absent unit is inert.
|
||||
rsync -az --mkpath --rsync-path='sudo rsync' --chmod=F0644 \
|
||||
asset/systemd/neuron-gpu-exclusive.conf \
|
||||
"${ADMIN_USER}@${host}:/etc/systemd/system/neuron.service.d/50-gpu-exclusive.conf"
|
||||
|
||||
# Per-host power limit and memory clock, if configured for this host.
|
||||
# Per-host miner profile. Always written, empty values included, so removing
|
||||
# a host from the lists above actually removes its profile instead of leaving
|
||||
# a stale file the miner would keep applying.
|
||||
local watts="" memclk=""
|
||||
for pair in $GPU_POWER_LIMITS; do
|
||||
case "$pair" in "${host}="*) watts="${pair#*=}" ;; esac
|
||||
@@ -360,25 +370,29 @@ provision_gpu() {
|
||||
for pair in $GPU_MEMORY_CLOCKS; do
|
||||
case "$pair" in "${host}="*) memclk="${pair#*=}" ;; esac
|
||||
done
|
||||
if [ -n "$watts" ] || [ -n "$memclk" ]; then
|
||||
printf 'NVIDIA_POWER_LIMIT_WATTS=%s\nNVIDIA_MEMORY_CLOCK_MHZ=%s\n' "$watts" "$memclk" \
|
||||
| ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null"
|
||||
echo " = requested power limit ${watts:-default}W, memory clock ${memclk:-default} MHz"
|
||||
else
|
||||
echo " = no power limit or memory clock configured for this host — cards left at default"
|
||||
fi
|
||||
printf 'NVIDIA_POWER_LIMIT_WATTS=%s\nNVIDIA_MEMORY_CLOCK_MHZ=%s\n' "$watts" "$memclk" \
|
||||
| ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null"
|
||||
echo " = miner profile: power limit ${watts:-default}W, memory clock ${memclk:-default} MHz"
|
||||
|
||||
ssh "${ADMIN_USER}@${host}" "
|
||||
sudo install -d -m 0755 /var/lib/prometheus/node-exporter
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now nvidia-textfile.timer
|
||||
sudo systemctl start nvidia-textfile.service"
|
||||
if [ -n "$watts" ] || [ -n "$memclk" ]; then
|
||||
ssh "${ADMIN_USER}@${host}" "sudo systemctl enable nvidia-power-limit.service \
|
||||
&& sudo systemctl restart nvidia-power-limit.service"
|
||||
ssh "${ADMIN_USER}@${host}" \
|
||||
"nvidia-smi --query-gpu=index,power.limit,clocks.mem --format=csv,noheader" | sed "s/^/ now: /"
|
||||
# The boot unit resets to defaults; the miner applies its own profile. Enable
|
||||
# it but do not restart it here — that would reset a running miner. Instead
|
||||
# apply whichever profile matches what is running now, so a config change
|
||||
# takes effect without bouncing the workload.
|
||||
ssh "${ADMIN_USER}@${host}" "sudo systemctl daemon-reload \
|
||||
&& sudo systemctl enable nvidia-power-limit.service"
|
||||
local profile=default
|
||||
if ssh "${ADMIN_USER}@${host}" "systemctl is-active --quiet quantus-miner.service"; then
|
||||
profile=miner
|
||||
fi
|
||||
echo " = applying the ${profile} profile to match the running workload"
|
||||
ssh "${ADMIN_USER}@${host}" "sudo /usr/local/bin/nvidia-power-limit.sh ${profile}" | sed "s/^/ /"
|
||||
ssh "${ADMIN_USER}@${host}" \
|
||||
"nvidia-smi --query-gpu=index,power.limit,clocks.mem --format=csv,noheader" | sed "s/^/ now: /"
|
||||
|
||||
local n
|
||||
n=$(ssh "${ADMIN_USER}@${host}" \
|
||||
|
||||
Reference in New Issue
Block a user