asset/gpu/nvidia-power-limit.sh

@@ -18,9 +18,14 @@ set -euo pipefail
 CONF="${CONF:-/etc/default/nvidia-power-limit}"
 [ -r "$CONF" ] && . "$CONF"
 WANT="${NVIDIA_POWER_LIMIT_WATTS:-}"
+# Locked memory clock. The miner never touches VRAM, so memory at full clock is
+# power taken from the SMs inside the same cap; 810 MHz measured +13% on the
+# 4090 at 250 W. Empty means "driver default" and resets any previous lock.
+# 405 MHz drops the card into a low-power state and is refused below.
+WANT_MEM="${NVIDIA_MEMORY_CLOCK_MHZ:-}"
 
-if [ -z "$WANT" ]; then
-    echo "no NVIDIA_POWER_LIMIT_WATTS set in ${CONF} — leaving limits at default" >&2
+if [ -z "$WANT" ] && [ -z "$WANT_MEM" ]; then
+    echo "no NVIDIA_POWER_LIMIT_WATTS or NVIDIA_MEMORY_CLOCK_MHZ set in ${CONF} — leaving the cards at default" >&2
     exit 0
 fi
 
@@ -29,7 +34,7 @@ fi
 # this unit is for.
 nvidia-smi -pm 1 >/dev/null || echo "warning: could not enable persistence mode" >&2
 
-nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \
+[ -n "$WANT" ] && nvidia-smi --query-gpu=index,name,power.min_limit,power.max_limit \
            --format=csv,noheader,nounits |
 while IFS=',' read -r idx name minw maxw; do
     idx="${idx// /}"; minw="${minw%%.*}"; maxw="${maxw%%.*}"
@@ -54,3 +59,27 @@ while IFS=',' read -r idx name minw maxw; do
         exit 1
     fi
 done
+
+# Memory clock, after the power limit so the freed watts are already available.
+nvidia-smi --query-gpu=index,name --format=csv,noheader,nounits |
+while IFS=',' read -r idx name; do
+    idx="${idx// /}"
+    if [ -z "$WANT_MEM" ]; then
+        nvidia-smi -i "$idx" -rmc >/dev/null && echo "gpu ${idx} (${name# }): memory clock reset to driver default"
+        continue
+    fi
+    if [ "$WANT_MEM" -le 405 ]; then
+        echo "gpu ${idx} (${name# }): refusing memory clock ${WANT_MEM} MHz — 405 puts the card in a low-power state" >&2
+        exit 1
+    fi
+    if ! nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -qE "Memory\s+: ${WANT_MEM} MHz"; then
+        echo "gpu ${idx} (${name# }): memory clock ${WANT_MEM} MHz is not a supported state; supported: $(nvidia-smi -i "$idx" -q -d SUPPORTED_CLOCKS | grep -E '^\s+Memory\s+:' | grep -oE '[0-9]+ MHz' | sort -un | tr '\n' ' ')" >&2
+        exit 1
+    fi
+    if nvidia-smi -i "$idx" -lmc "$WANT_MEM,$WANT_MEM" >/dev/null; then
+        echo "gpu ${idx} (${name# }): memory clock locked at ${WANT_MEM} MHz"
+    else
+        echo "gpu ${idx} (${name# }): FAILED to lock memory clock at ${WANT_MEM} MHz" >&2
+        exit 1
+    fi
+done

readme.md

@@ -108,6 +108,30 @@ would otherwise show up only as a warm connector. The dashboard's **GPU Power**
 gauge reads draw against the *enforced* limit, so a reverted cap appears as the
 gauge dropping from ~100% to ~60% rather than as nothing at all.
 
+## GPU memory clock
+
+Same unit, same config file: `GPU_MEMORY_CLOCKS` in `infra-setup.sh` as
+`host=MHz`, applied with `nvidia-smi -lmc` after the power limit. The miner never
+touches VRAM, so memory running at full clock is power spent inside the same cap
+for nothing. Measured with the CUDA engine (quantus/miner#3), miner paused, the
+card at its usual limit:
+
+| card | memory clock | hashrate | SM clock |
+| --- | --- | --- | --- |
+| RTX 4090, 250 W | 10,251 MHz (default under load) | 410 MH/s | 1,950 MHz |
+| RTX 4090, 250 W | 5,001 MHz | 435 to 447 MH/s | 2,040 to 2,175 MHz |
+| RTX 4090, 250 W | **810 MHz** | **463 MH/s** | 2,190 MHz |
+| RTX 4090, 250 W | 405 MHz | 155 MH/s | 645 MHz, low-power state |
+| RTX 3060, 130 W | 7,301 MHz (default) | 81 to 82 MH/s | 1,725 to 1,800 MHz |
+| RTX 3060, 130 W | **810 MHz** | 83 to 84 MH/s | 1,790 MHz |
+
+**+13% on the 4090 for the same watts**, the freed power going straight into SM
+clock. The 405 MHz state is not "lower still": it moves the card into a
+low-power performance state that also caps the core, and the script refuses it.
+Hosts that serve inference are deliberately not listed: LLM inference is
+memory-bandwidth bound and would be crippled. A host with no entry has its
+memory clock reset to the driver default, so the config is authoritative both ways.
+
 ## Monitoring
 
 Scraped by the fleet Prometheus and visualised in Grafana (folder **Quantus**).

script/infra-setup.sh

@@ -62,6 +62,14 @@ GPU_HOSTS="${GPU_HOSTS-beast.hanzalova.internal benjy.hanzalova.internal quadbra
 # 450W, so the hashrate cost is ~1% and the thermal and connector-current
 # margin is large. A 5090's floor is 400W; requests below it are clamped.
 GPU_POWER_LIMITS="${GPU_POWER_LIMITS-beast.hanzalova.internal=300 benjy.hanzalova.internal=250 quadbrat.hanzalova.internal=130}"
+# Per-host locked memory clock, as `host=MHz`, applied by the same unit. The
+# miner never touches VRAM, so memory at full clock is watts taken from the
+# SMs inside the same power cap: locking it at 810 MHz measured +13% on the
+# 4090 at 250 W (410 -> 463 MH/s) and +2% on the 3060 at 130 W. 405 MHz drops
+# the card into a low-power state and must not be used. Hosts serving
+# inference (beast) are NOT listed: LLM inference is memory-bound. A host with
+# no entry gets its memory clock reset to the driver default.
+GPU_MEMORY_CLOCKS="${GPU_MEMORY_CLOCKS-benjy.hanzalova.internal=810 quadbrat.hanzalova.internal=810}"
 # Vanity name served by the edge proxy. Dot-free short name: the step@ renewal
 # unit is a template instance and %i cannot carry dots (internal-tls.md §1).
 EDGE_NAME="${EDGE_NAME:-quantus}"
@@ -80,6 +88,7 @@ while [[ $# -gt 0 ]]; do
     --edge-hosts)       EDGE_HOSTS="$2"; shift 2 ;;
     --gpu-hosts)        GPU_HOSTS="$2"; shift 2 ;;
     --gpu-power-limits) GPU_POWER_LIMITS="$2"; shift 2 ;;
+    --gpu-memory-clocks) GPU_MEMORY_CLOCKS="$2"; shift 2 ;;
     --admin)            ADMIN_USER="$2"; shift 2 ;;
     --node-version)     NODE_VERSION="$2"; shift 2 ;;
     *) echo "unknown arg: $1" >&2; exit 2 ;;
@@ -340,17 +349,20 @@ provision_gpu() {
     asset/systemd/nvidia-power-limit.service \
     "${ADMIN_USER}@${host}:/etc/systemd/system/"
 
-  # Per-host power limit, if one is configured for this host.
-  local watts=""
+  # Per-host power limit and memory clock, if configured for this host.
+  local watts="" memclk=""
   for pair in $GPU_POWER_LIMITS; do
     case "$pair" in "${host}="*) watts="${pair#*=}" ;; esac
   done
-  if [ -n "$watts" ]; then
-    printf 'NVIDIA_POWER_LIMIT_WATTS=%s\n' "$watts" \
+  for pair in $GPU_MEMORY_CLOCKS; do
+    case "$pair" in "${host}="*) memclk="${pair#*=}" ;; esac
+  done
+  if [ -n "$watts" ] || [ -n "$memclk" ]; then
+    printf 'NVIDIA_POWER_LIMIT_WATTS=%s\nNVIDIA_MEMORY_CLOCK_MHZ=%s\n' "$watts" "$memclk" \
       | ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null"
-    echo "  = requested power limit ${watts}W"
+    echo "  = requested power limit ${watts:-default}W, memory clock ${memclk:-default} MHz"
   else
-    echo "  = no power limit configured for this host — cards left at default"
+    echo "  = no power limit or memory clock configured for this host — cards left at default"
   fi
 
   ssh "${ADMIN_USER}@${host}" "
@@ -358,11 +370,11 @@ provision_gpu() {
     sudo systemctl daemon-reload
     sudo systemctl enable --now nvidia-textfile.timer
     sudo systemctl start nvidia-textfile.service"
-  if [ -n "$watts" ]; then
+  if [ -n "$watts" ] || [ -n "$memclk" ]; then
     ssh "${ADMIN_USER}@${host}" "sudo systemctl enable nvidia-power-limit.service \
-      && sudo systemctl restart nvidia-power-limit.service" 
+      && sudo systemctl restart nvidia-power-limit.service"
     ssh "${ADMIN_USER}@${host}" \
-      "nvidia-smi --query-gpu=index,power.limit --format=csv,noheader" | sed "s/^/    now: /"
+      "nvidia-smi --query-gpu=index,power.limit,clocks.mem --format=csv,noheader" | sed "s/^/    now: /"
   fi
 
   local n