diff --git a/asset/gpu/nvidia-power-limit.sh b/asset/gpu/nvidia-power-limit.sh index 3794bf6..589bc41 100755 --- a/asset/gpu/nvidia-power-limit.sh +++ b/asset/gpu/nvidia-power-limit.sh @@ -18,9 +18,14 @@ set -euo pipefail CONF="${CONF:-/etc/default/nvidia-power-limit}" [ -r "$CONF" ] && . "$CONF" WANT="${NVIDIA_POWER_LIMIT_WATTS:-}" +# Locked memory clock. The miner never touches VRAM, so memory at full clock is +# power taken from the SMs inside the same cap; 810 MHz measured +13% on the +# 4090 at 250 W. Empty means "driver default" and resets any previous lock. +# 405 MHz drops the card into a low-power state and is refused below. +WANT_MEM="${NVIDIA_MEMORY_CLOCK_MHZ:-}" -if [ -z "$WANT" ]; then - echo "no NVIDIA_POWER_LIMIT_WATTS set in ${CONF} — leaving limits at default" >&2 +if [ -z "$WANT" ] && [ -z "$WANT_MEM" ]; then + echo "no NVIDIA_POWER_LIMIT_WATTS or NVIDIA_MEMORY_CLOCK_MHZ set in ${CONF} — leaving the cards at default" >&2 exit 0 fi @@ -29,7 +34,7 @@ fi # this unit is for. nvidia-smi -pm 1 >/dev/null || echo "warning: could not enable persistence mode" >&2 -nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \ +[ -n "$WANT" ] && nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \ --format=csv,noheader,nounits | while IFS=',' read -r idx name minw maxw; do idx="${idx// /}"; minw="${minw%%.*}"; maxw="${maxw%%.*}" @@ -54,3 +59,27 @@ while IFS=',' read -r idx name minw maxw; do exit 1 fi done + +# Memory clock, after the power limit so the freed watts are already available. +nvidia-smi --query-gpu=index,name --format=csv,noheader,nounits | +while IFS=',' read -r idx name; do + idx="${idx// /}" + if [ -z "$WANT_MEM" ]; then + nvidia-smi -i "$idx" -rmc >/dev/null && echo "gpu ${idx} (${name# }): memory clock reset to driver default" + continue + fi + if [ "$WANT_MEM" -le 405 ]; then + echo "gpu ${idx} (${name# }): refusing memory clock ${WANT_MEM} MHz — 405 puts the card in a low-power state" >&2 + exit 1 + fi + if ! nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -qE "Memory\s+: ${WANT_MEM} MHz"; then + echo "gpu ${idx} (${name# }): memory clock ${WANT_MEM} MHz is not a supported state; supported: $(nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -E '^\s+Memory\s+:' | grep -oE '[0-9]+ MHz' | sort -un | tr '\n' ' ')" >&2 + exit 1 + fi + if nvidia-smi -i "$idx" -lmc "$WANT_MEM,$WANT_MEM" >/dev/null; then + echo "gpu ${idx} (${name# }): memory clock locked at ${WANT_MEM} MHz" + else + echo "gpu ${idx} (${name# }): FAILED to lock memory clock at ${WANT_MEM} MHz" >&2 + exit 1 + fi +done diff --git a/readme.md b/readme.md index d874d3a..41db992 100644 --- a/readme.md +++ b/readme.md @@ -108,6 +108,30 @@ would otherwise show up only as a warm connector. The dashboard's **GPU Power** gauge reads draw against the *enforced* limit, so a reverted cap appears as the gauge dropping from ~100% to ~60% rather than as nothing at all. +## GPU memory clock + +Same unit, same config file: `GPU_MEMORY_CLOCKS` in `infra-setup.sh` as +`host=MHz`, applied with `nvidia-smi -lmc` after the power limit. The miner never +touches VRAM, so memory running at full clock is power spent inside the same cap +for nothing. Measured with the CUDA engine (quantus/miner#3), miner paused, the +card at its usual limit: + +| card | memory clock | hashrate | SM clock | +| --- | --- | --- | --- | +| RTX 4090, 250 W | 10,251 MHz (default under load) | 410 MH/s | 1,950 MHz | +| RTX 4090, 250 W | 5,001 MHz | 435 to 447 MH/s | 2,040 to 2,175 MHz | +| RTX 4090, 250 W | **810 MHz** | **463 MH/s** | 2,190 MHz | +| RTX 4090, 250 W | 405 MHz | 155 MH/s | 645 MHz, low-power state | +| RTX 3060, 130 W | 7,301 MHz (default) | 81 to 82 MH/s | 1,725 to 1,800 MHz | +| RTX 3060, 130 W | **810 MHz** | 83 to 84 MH/s | 1,790 MHz | + +**+13% on the 4090 for the same watts**, the freed power going straight into SM +clock. The 405 MHz state is not "lower still": it moves the card into a +low-power performance state that also caps the core, and the script refuses it. +Hosts that serve inference are deliberately not listed: LLM inference is +memory-bandwidth bound and would be crippled. A host with no entry has its +memory clock reset to the driver default, so the config is authoritative both ways. + ## Monitoring Scraped by the fleet Prometheus and visualised in Grafana (folder **Quantus**). diff --git a/script/infra-setup.sh b/script/infra-setup.sh index 722e0d1..ac8d5c8 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -62,6 +62,14 @@ GPU_HOSTS="${GPU_HOSTS-beast.hanzalova.internal benjy.hanzalova.internal quadbra # 450W, so the hashrate cost is ~1% and the thermal and connector-current # margin is large. A 5090's floor is 400W; requests below it are clamped. GPU_POWER_LIMITS="${GPU_POWER_LIMITS-beast.hanzalova.internal=300 benjy.hanzalova.internal=250 quadbrat.hanzalova.internal=130}" +# Per-host locked memory clock, as `host=MHz`, applied by the same unit. The +# miner never touches VRAM, so memory at full clock is watts taken from the +# SMs inside the same power cap: locking it at 810 MHz measured +13% on the +# 4090 at 250 W (410 -> 463 MH/s) and +2% on the 3060 at 130 W. 405 MHz drops +# the card into a low-power state and must not be used. Hosts serving +# inference (beast) are NOT listed: LLM inference is memory-bound. A host with +# no entry gets its memory clock reset to the driver default. +GPU_MEMORY_CLOCKS="${GPU_MEMORY_CLOCKS-benjy.hanzalova.internal=810 quadbrat.hanzalova.internal=810}" # Vanity name served by the edge proxy. Dot-free short name: the step@ renewal # unit is a template instance and %i cannot carry dots (internal-tls.md §1). EDGE_NAME="${EDGE_NAME:-quantus}" @@ -80,6 +88,7 @@ while [[ $# -gt 0 ]]; do --edge-hosts) EDGE_HOSTS="$2"; shift 2 ;; --gpu-hosts) GPU_HOSTS="$2"; shift 2 ;; --gpu-power-limits) GPU_POWER_LIMITS="$2"; shift 2 ;; + --gpu-memory-clocks) GPU_MEMORY_CLOCKS="$2"; shift 2 ;; --admin) ADMIN_USER="$2"; shift 2 ;; --node-version) NODE_VERSION="$2"; shift 2 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; @@ -340,17 +349,20 @@ provision_gpu() { asset/systemd/nvidia-power-limit.service \ "${ADMIN_USER}@${host}:/etc/systemd/system/" - # Per-host power limit, if one is configured for this host. - local watts="" + # Per-host power limit and memory clock, if configured for this host. + local watts="" memclk="" for pair in $GPU_POWER_LIMITS; do case "$pair" in "${host}="*) watts="${pair#*=}" ;; esac done - if [ -n "$watts" ]; then - printf 'NVIDIA_POWER_LIMIT_WATTS=%s\n' "$watts" \ + for pair in $GPU_MEMORY_CLOCKS; do + case "$pair" in "${host}="*) memclk="${pair#*=}" ;; esac + done + if [ -n "$watts" ] || [ -n "$memclk" ]; then + printf 'NVIDIA_POWER_LIMIT_WATTS=%s\nNVIDIA_MEMORY_CLOCK_MHZ=%s\n' "$watts" "$memclk" \ | ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null" - echo " = requested power limit ${watts}W" + echo " = requested power limit ${watts:-default}W, memory clock ${memclk:-default} MHz" else - echo " = no power limit configured for this host — cards left at default" + echo " = no power limit or memory clock configured for this host — cards left at default" fi ssh "${ADMIN_USER}@${host}" " @@ -358,11 +370,11 @@ provision_gpu() { sudo systemctl daemon-reload sudo systemctl enable --now nvidia-textfile.timer sudo systemctl start nvidia-textfile.service" - if [ -n "$watts" ]; then + if [ -n "$watts" ] || [ -n "$memclk" ]; then ssh "${ADMIN_USER}@${host}" "sudo systemctl enable nvidia-power-limit.service \ - && sudo systemctl restart nvidia-power-limit.service" + && sudo systemctl restart nvidia-power-limit.service" ssh "${ADMIN_USER}@${host}" \ - "nvidia-smi --query-gpu=index,power.limit --format=csv,noheader" | sed "s/^/ now: /" + "nvidia-smi --query-gpu=index,power.limit,clocks.mem --format=csv,noheader" | sed "s/^/ now: /" fi local n