From e178bcbdf97e67e7c6a486df5267d9faa8300aed Mon Sep 17 00:00:00 2001 From: Rob Thijssen Date: Tue, 1 Sep 2026 06:42:32 +0300 Subject: [PATCH] feat: boot-time GPU power limits, clamped to hardware floors nvidia-smi -pl survives neither a reboot nor a driver unload, and both failures are silent -- the card returns to its default limit and nobody notices until a connector runs warm. Hence a boot-time unit, which also enables persistence mode so the limit is not lost when the last GPU client detaches. Per-host limits configured as host=watts in infra-setup.sh, clamped to each card's own min/max. Verified: a 300W request on a 5090 whose floor is 400W is corrected with an explicit log line rather than failing or being ignored. Measured on a 4090 mining Poseidon2: 450W (stock) 185.5 MH/s 70C 0.413 MH/s/W 300W 183.2 MH/s 61C 0.611 250W 180.3 MH/s 58C 0.721 44% less power and 12C cooler for 2.8% less hashrate -- this workload is register-resident integer math and does not need the top of the clock curve. It also cuts current through the 12VHPWR connector, which is the one component on these cards with a documented melting failure mode. Ordered Before=quantus-miner.service so a GPU is never briefly at its default limit under full mining load. --- asset/gpu/nvidia-power-limit.sh | 56 ++++++++++++++++++++++++ asset/systemd/nvidia-power-limit.service | 25 +++++++++++ readme.md | 28 ++++++++++++ script/infra-setup.sh | 30 +++++++++++++ 4 files changed, 139 insertions(+) create mode 100755 asset/gpu/nvidia-power-limit.sh create mode 100644 asset/systemd/nvidia-power-limit.service diff --git a/asset/gpu/nvidia-power-limit.sh b/asset/gpu/nvidia-power-limit.sh new file mode 100755 index 0000000..3794bf6 --- /dev/null +++ b/asset/gpu/nvidia-power-limit.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Apply a persistent NVIDIA power limit at boot. +# +# nvidia-smi -pl does NOT survive a reboot, and without persistence mode it is +# also lost whenever the last GPU client detaches and the driver unloads. Both +# are silent: the card simply returns to its default limit and nobody notices +# until a connector runs warm. Hence a boot-time unit rather than a one-off. +# +# Lower is not a compromise here. Measured on a 4090 mining Poseidon2: +# 450 W -> 185.5 MH/s, 70 C, 0.413 MH/s/W +# 300 W -> 183.2 MH/s, 61 C, 0.612 MH/s/W +# a third less power and 9 C cooler for 1.2% less hashrate, because this +# workload is register-resident integer math and does not need the top of the +# clock curve. It also cuts current through the 12VHPWR connector, which is the +# one component on 4090/5090 cards with a documented melting failure mode. +set -euo pipefail + +CONF="${CONF:-/etc/default/nvidia-power-limit}" +[ -r "$CONF" ] && . "$CONF" +WANT="${NVIDIA_POWER_LIMIT_WATTS:-}" + +if [ -z "$WANT" ]; then + echo "no NVIDIA_POWER_LIMIT_WATTS set in ${CONF} — leaving limits at default" >&2 + exit 0 +fi + +# Persistence mode keeps the driver resident so the limit is not lost when the +# last client detaches. It does NOT survive a reboot on its own, which is what +# this unit is for. +nvidia-smi -pm 1 >/dev/null || echo "warning: could not enable persistence mode" >&2 + +nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \ + --format=csv,noheader,nounits | +while IFS=',' read -r idx name minw maxw; do + idx="${idx// /}"; minw="${minw%%.*}"; maxw="${maxw%%.*}" + minw="${minw// /}"; maxw="${maxw// /}" + target="$WANT" + + # Clamp rather than fail: a request below the card's floor is a + # configuration mistake, not a reason to leave the GPU at its default — + # which would be the opposite of what was intended. + if [ "$target" -lt "$minw" ]; then + echo "gpu ${idx} (${name# }): requested ${WANT}W is below the ${minw}W floor — clamping to ${minw}W" >&2 + target="$minw" + elif [ "$target" -gt "$maxw" ]; then + echo "gpu ${idx} (${name# }): requested ${WANT}W exceeds the ${maxw}W ceiling — clamping to ${maxw}W" >&2 + target="$maxw" + fi + + if nvidia-smi -i "$idx" -pl "$target" >/dev/null; then + echo "gpu ${idx} (${name# }): power limit set to ${target}W" + else + echo "gpu ${idx} (${name# }): FAILED to set power limit to ${target}W" >&2 + exit 1 + fi +done diff --git a/asset/systemd/nvidia-power-limit.service b/asset/systemd/nvidia-power-limit.service new file mode 100644 index 0000000..46d5314 --- /dev/null +++ b/asset/systemd/nvidia-power-limit.service @@ -0,0 +1,25 @@ +# Reapply the NVIDIA power limit at boot. See asset/gpu/nvidia-power-limit.sh +# for why this cannot be a one-off nvidia-smi invocation. +[Unit] +Description=Apply NVIDIA GPU power limit +Documentation=https://git.lair.cafe/lair/quantus +ConditionPathExists=/usr/bin/nvidia-smi +After=basic.target +# Ordered before the miner so the GPU is never briefly at its default limit +# while under full mining load. Harmless on hosts with no miner. +Before=quantus-miner.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/nvidia-power-limit.sh +# Needs root and the nvidia device nodes; holds no credentials and reads +# nothing sensitive. +NoNewPrivileges=true +ProtectHome=true +ProtectControlGroups=true +RestrictSUIDSGID=true +LockPersonality=true + +[Install] +WantedBy=multi-user.target diff --git a/readme.md b/readme.md index 3343fd8..5803ceb 100644 --- a/readme.md +++ b/readme.md @@ -78,6 +78,34 @@ fleet uses a single default zone (`generic.md` §9): adding the service outright would publish `9833/udp` on every address the host carries, and upstream is explicit that this port must never face the internet. +## GPU power limits + +Applied at boot by `nvidia-power-limit.service`, configured per host in +`infra-setup.sh`'s `GPU_POWER_LIMITS` as `host=watts`, and **clamped to each +card's own floor and ceiling** — a request the hardware refuses is corrected with +a log line rather than silently ignored. + +This is not a compromise. Measured on a 4090 mining Poseidon2: + +| limit | hashrate | temp | efficiency | +| --- | --- | --- | --- | +| 450 W (stock) | 185.5 MH/s | 70 °C | 0.413 MH/s/W | +| 300 W | 183.2 MH/s | 61 °C | 0.611 MH/s/W | +| 250 W | 180.3 MH/s | 58 °C | **0.721 MH/s/W** | + +**44% less power and 12 °C cooler for 2.8% less hashrate.** The workload is +register-resident integer math and does not need the top of the clock curve, so +the last few hundred MHz were very nearly pure heat. It also cuts current through +the 12VHPWR connector, the one component on 4090/5090 cards with a documented +melting failure mode. + +A boot-time unit is required because `nvidia-smi -pl` survives neither a reboot +nor a driver unload. The unit also enables persistence mode, which stops the +limit being lost when the last GPU client detaches — a silent reversion that +would otherwise show up only as a warm connector. The dashboard's **GPU Power** +gauge reads draw against the *enforced* limit, so a reverted cap appears as the +gauge dropping from ~100% to ~60% rather than as nothing at all. + ## Monitoring Scraped by the fleet Prometheus and visualised in Grafana (folder **Quantus**). diff --git a/script/infra-setup.sh b/script/infra-setup.sh index bba9d93..6b43116 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -56,6 +56,13 @@ EDGE_HOSTS="${EDGE_HOSTS-hanzalova.internal}" # quantus-specific — it lives here because there is no monitoring repo yet, and # should move to one if that changes. GPU_HOSTS="${GPU_HOSTS-beast.hanzalova.internal benjy.hanzalova.internal quadbrat.hanzalova.internal}" +# Per-host GPU power limits, as `host=watts`. Applied at boot by +# nvidia-power-limit.service and clamped to each card's own floor/ceiling, so a +# value the hardware refuses is corrected loudly rather than silently ignored. +# Conservative on purpose: a 4090 measured 183.2 MH/s at 300W against 185.5 at +# 450W, so the hashrate cost is ~1% and the thermal and connector-current +# margin is large. A 5090's floor is 400W; requests below it are clamped. +GPU_POWER_LIMITS="${GPU_POWER_LIMITS-beast.hanzalova.internal=300 benjy.hanzalova.internal=250 quadbrat.hanzalova.internal=150}" # Vanity name served by the edge proxy. Dot-free short name: the step@ renewal # unit is a template instance and %i cannot carry dots (internal-tls.md §1). EDGE_NAME="${EDGE_NAME:-quantus}" @@ -74,6 +81,7 @@ while [[ $# -gt 0 ]]; do --metrics-hosts) METRICS_HOSTS="$2"; shift 2 ;; --edge-hosts) EDGE_HOSTS="$2"; shift 2 ;; --gpu-hosts) GPU_HOSTS="$2"; shift 2 ;; + --gpu-power-limits) GPU_POWER_LIMITS="$2"; shift 2 ;; --admin) ADMIN_USER="$2"; shift 2 ;; --node-version) NODE_VERSION="$2"; shift 2 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; @@ -383,15 +391,37 @@ provision_gpu() { rsync -az --rsync-path='sudo rsync' --chmod=F0755 \ asset/gpu/nvidia-textfile.sh "${ADMIN_USER}@${host}:/usr/local/bin/nvidia-textfile.sh" + rsync -az --rsync-path='sudo rsync' --chmod=F0755 \ + asset/gpu/nvidia-power-limit.sh "${ADMIN_USER}@${host}:/usr/local/bin/nvidia-power-limit.sh" rsync -az --rsync-path='sudo rsync' --chmod=F0644 \ asset/systemd/nvidia-textfile.service asset/systemd/nvidia-textfile.timer \ + asset/systemd/nvidia-power-limit.service \ "${ADMIN_USER}@${host}:/etc/systemd/system/" + # Per-host power limit, if one is configured for this host. + local watts="" + for pair in $GPU_POWER_LIMITS; do + case "$pair" in "${host}="*) watts="${pair#*=}" ;; esac + done + if [ -n "$watts" ]; then + printf 'NVIDIA_POWER_LIMIT_WATTS=%s\n' "$watts" \ + | ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null" + echo " = requested power limit ${watts}W" + else + echo " = no power limit configured for this host — cards left at default" + fi + ssh "${ADMIN_USER}@${host}" " sudo install -d -m 0755 /var/lib/prometheus/node-exporter sudo systemctl daemon-reload sudo systemctl enable --now nvidia-textfile.timer sudo systemctl start nvidia-textfile.service" + if [ -n "$watts" ]; then + ssh "${ADMIN_USER}@${host}" "sudo systemctl enable nvidia-power-limit.service \ + && sudo systemctl restart nvidia-power-limit.service" + ssh "${ADMIN_USER}@${host}" \ + "nvidia-smi --query-gpu=index,power.limit --format=csv,noheader" | sed "s/^/ now: /" + fi local n n=$(ssh "${ADMIN_USER}@${host}" \