.gitea/workflows/deploy.yaml

@@ -660,6 +660,10 @@ jobs:
 
           push --mkpath --chmod=F0644 quantus-scrape.yml \
             gitea_ci@"$HOST":/etc/prometheus/scrape_configs.d/quantus.yml
+          python3 -c "import yaml,sys; yaml.safe_load(open('asset/prometheus/quantus-rules.yml'))" \
+            || { echo "recording rules are not valid YAML" >&2; exit 1; }
+          push --mkpath --chmod=F0644 asset/prometheus/quantus-rules.yml \
+            gitea_ci@"$HOST":/etc/prometheus/rules.d/quantus.yml
           push --mkpath --chmod=F0644 asset/grafana/quantus.json \
             gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus/quantus.json
           push --mkpath --chmod=F0644 asset/grafana/gpu.json \
@@ -702,6 +706,16 @@ jobs:
           run() { ssh $SSHOPTS gitea_ci@"${{ env.METRICS_HOST }}" "$@"; }
           fail=0
 
+          echo "--- prometheus rules ---"
+          r=$(run "curl -fsS http://127.0.0.1:${{ env.PROM_PORT }}/api/v1/rules")
+          if grep -q '"name":"quantus-miner"' <<<"$r"; then
+            echo "  ok   rule group quantus-miner loaded"
+          else
+            echo "  FAIL rule group quantus-miner not loaded" >&2
+            echo "       is rule_files wired into prometheus.yml? (infra-setup.sh --metrics-hosts)" >&2
+            fail=1
+          fi
+
           echo "--- prometheus targets ---"
           # The point of this check: a target that drifted out of the matrices,
           # or a firewalld rule that never landed, shows up here as missing or

asset/gpu/nvidia-textfile.sh

@@ -20,7 +20,9 @@ TMP="$(mktemp "${OUT}.XXXXXX")"
 trap 'rm -f "$TMP"' EXIT
 
 QUERY=index,uuid,name,temperature.gpu,utilization.gpu,utilization.memory,\
-memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed
+memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed,\
+clocks_event_reasons.sw_power_cap,clocks_event_reasons.hw_slowdown,clocks_event_reasons.sw_thermal_slowdown,\
+clocks_event_reasons.hw_thermal_slowdown,clocks_event_reasons.hw_power_brake_slowdown
 
 {
   echo "# HELP nvidia_gpu_info Static GPU identity, always 1."
@@ -45,6 +47,8 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current
   echo "# TYPE nvidia_gpu_clock_memory_hertz gauge"
   echo "# HELP nvidia_gpu_fan_speed_ratio Fan duty cycle, 0-1."
   echo "# TYPE nvidia_gpu_fan_speed_ratio gauge"
+  echo "# HELP nvidia_gpu_event_reason_active 1 while the clock is being held down for this reason (power cap, thermal, brake), else 0."
+  echo "# TYPE nvidia_gpu_event_reason_active gauge"
   echo "# HELP nvidia_gpu_scrape_ok 1 when nvidia-smi was queried successfully."
   echo "# TYPE nvidia_gpu_scrape_ok gauge"
 
@@ -60,6 +64,7 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current
   while IFS= read -r line; do
     [ -n "$line" ] || continue
     IFS=',' read -r idx uuid name temp util umem mused mtotal pdraw plimit csm cmem fan \
+      r_pcap r_hwslow r_swtherm r_hwtherm r_brake \
       <<< "$(printf '%s' "$line" | sed 's/, */,/g')"
     # A label value may contain characters Prometheus treats specially.
     esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; }
@@ -78,6 +83,14 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current
     num "$csm"    >/dev/null && awk -v l="$L" -v v="$csm"  'BEGIN{printf "nvidia_gpu_clock_sm_hertz{%s} %.0f\n", l, v*1000000}'
     num "$cmem"   >/dev/null && awk -v l="$L" -v v="$cmem" 'BEGIN{printf "nvidia_gpu_clock_memory_hertz{%s} %.0f\n", l, v*1000000}'
     num "$fan"    >/dev/null && awk -v l="$L" -v v="$fan"  'BEGIN{printf "nvidia_gpu_fan_speed_ratio{%s} %.4f\n", l, v/100}'
+    # Why a card is below its boost clock. A hashrate regression that
+    # coincides with a thermal or power-brake reason is not a code regression.
+    reason() { case "$2" in Active) echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 1" ;; "Not Active") echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 0" ;; esac; }
+    reason sw_power_cap "$r_pcap"
+    reason hw_slowdown "$r_hwslow"
+    reason sw_thermal_slowdown "$r_swtherm"
+    reason hw_thermal_slowdown "$r_hwtherm"
+    reason hw_power_brake_slowdown "$r_brake"
   done <<< "$smi"
 } > "$TMP"
 

asset/grafana/quantus.json

@@ -9,7 +9,7 @@
   ],
   "timezone": "browser",
   "schemaVersion": 39,
-  "version": 10,
+  "version": 11,
   "refresh": "30s",
   "time": {
     "from": "now-6h",
@@ -2764,6 +2764,520 @@
           "values": false
         }
       }
+    },
+    {
+      "type": "row",
+      "title": "Performance by build (quantus/miner#9)",
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 116
+      },
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Hashrate by device and commit",
+      "description": "Per-card throughput carrying the build that produced it. A new series starts at every deploy; the legend is the experiment key.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 117
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "MHs",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:device_hashrate_by_commit:rate5m / 1e6",
+          "legendFormat": "{{host}} dev{{device}} {{kernel}} @{{commit}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Hashrate per watt",
+      "description": "Hashes per joule per card. The honest comparison when a power limit changes between builds.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 117
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "none",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:device_hashrate_per_watt:rate5m / 1e6",
+          "legendFormat": "{{host}} dev{{device}} {{name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Batch time off the GPU",
+      "description": "Share of each batch spent on the host (submit gaps, readback) rather than executing on the card. Pipelining and batch sizing lower this.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 125
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percentunit",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "max": 1,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:gpu_batch_host_fraction:rate5m",
+          "legendFormat": "{{instance}} dev{{device}} {{kernel}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Stale work",
+      "description": "Fraction of hashes done for a job after a newer one had been issued. Grows with batch size and with node job churn.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 125
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percentunit",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "max": 1,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:stale_hash_fraction:rate1h",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Seal latency (p95)",
+      "description": "Solution found on the card to the result sent to the node. Grows with batch size; costs blocks on a fast chain.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 133
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "s",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:seal_latency_seconds:p95_5m",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Idle waiting for the node",
+      "description": "Fraction of wall-clock between sending a result and receiving the next job. Node-attributable; see quantus/chain#1.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 133
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percentunit",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "max": 1,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "quantus:job_idle_fraction:rate5m",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Jobs per minute",
+      "description": "Job cadence from the node. Every job restarts every worker's search.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 141
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "rate(miner_jobs_received_total[5m]) * 60",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Clock held down",
+      "description": "Why a card is below its boost clock (nvidia-smi clock event reasons). A regression that coincides with thermal or brake is not a code regression; power cap is the normal state under a limit.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 141
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "none",
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 8,
+            "spanNulls": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "nvidia_gpu_event_reason_active == 1",
+          "legendFormat": "{{host}} gpu{{gpu}} {{reason}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
     }
   ],
   "annotations": {
@@ -2807,6 +3321,19 @@
         "step": "1m",
         "titleFormat": "miner restart",
         "textFormat": "counter reset \u2014 hashrate readings near this line are unreliable"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "enable": true,
+        "iconColor": "blue",
+        "name": "miner deploy",
+        "expr": "changes(miner_build_info[2m]) > 0 or (miner_build_info unless miner_build_info offset 2m)",
+        "titleFormat": "deploy {{instance}}",
+        "textFormat": "{{version}} {{commit}}",
+        "step": "1m"
       }
     ]
   }

asset/prometheus/quantus-rules.yml

@@ -0,0 +1,53 @@
+# Recording rules for the Quantus miners, deployed to
+# /etc/prometheus/rules.d/quantus.yml on the metrics host (quantus/miner#9).
+#
+# The miner's series (job quantus-miner, instance host:9900) and the GPU
+# series from node-exporter's textfile collector (job node, `host` label) are
+# joined on a derived `host` and on `device` (the miner's engine index) =
+# `gpu` (nvidia-smi index). On single-card hosts both are 0; on a multi-card
+# host the engine orders adapters discrete-first as Vulkan enumerates them,
+# which is expected to match nvidia-smi's order but is not guaranteed.
+groups:
+  - name: quantus-miner
+    interval: 30s
+    rules:
+      # Per-device throughput, keyed so it can be joined with the GPU series.
+      - record: quantus:device_hashrate:rate5m
+        expr: label_replace(rate(miner_device_hashes_total[5m]), "host", "$1", "instance", "([^:]+):.*")
+
+      # The same, carrying the build that produced it. The series set changes
+      # on every deploy, which is the annotation the dashboard draws.
+      - record: quantus:device_hashrate_by_commit:rate5m
+        expr: quantus:device_hashrate:rate5m * on(instance) group_left(commit, version) miner_build_info
+
+      # GPU draw keyed like the miner series.
+      - record: quantus:gpu_power_watts
+        expr: label_replace(nvidia_gpu_power_watts, "device", "$1", "gpu", "(.*)")
+
+      # Hashes per joule: the honest number when a power limit changes.
+      - record: quantus:device_hashrate_per_watt:rate5m
+        expr: quantus:device_hashrate:rate5m / on(host, device) group_left(name) quantus:gpu_power_watts
+
+      # Share of each batch spent off the GPU (submit gaps, readback). This is
+      # what pipelining and batch sizing reduce.
+      - record: quantus:gpu_batch_host_fraction:rate5m
+        expr: >
+          sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum{phase="host"}[5m]))
+          /
+          sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum[5m]))
+
+      # Work done for jobs that had already been superseded.
+      - record: quantus:stale_hash_fraction:rate1h
+        expr: >
+          sum by (instance) (rate(miner_stale_hashes_total[1h]))
+          /
+          sum by (instance) (rate(miner_hashes_total[1h]))
+
+      # Found to sent, 95th percentile.
+      - record: quantus:seal_latency_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (instance, le) (rate(miner_seal_latency_seconds_bucket[5m])))
+
+      # Fraction of wall-clock spent waiting for the node to issue the next
+      # job after a result. Node-attributable, not miner-attributable.
+      - record: quantus:job_idle_fraction:rate5m
+        expr: rate(miner_job_idle_seconds_total[5m])

script/infra-setup.sh

@@ -239,6 +239,7 @@ provision_metrics() {
     usermod -aG systemd-journal gitea_ci
 
     install -d -m 0755 /etc/prometheus/scrape_configs.d
+    install -d -m 0755 /etc/prometheus/rules.d
     install -d -m 0755 /etc/grafana/provisioning/dashboards/quantus
 
     # 1. include drop-in scrape configs
@@ -258,6 +259,23 @@ PROM
       curl -fsS -X POST http://127.0.0.1:26559/-/reload && echo "  = prometheus reloaded"
     fi
 
+    # 1b. include drop-in recording rules (quantus/miner#9), same shape.
+    if grep -q '^rule_files:' /etc/prometheus/prometheus.yml; then
+      echo "  = prometheus.yml already includes rules.d"
+    else
+      cp -a /etc/prometheus/prometheus.yml \
+            "/etc/prometheus/prometheus.yml.bak.$(date +%s)"
+      cat >> /etc/prometheus/prometheus.yml <<'PROM'
+
+# Per-project recording and alerting rules. Added by lair/quantus
+# script/infra-setup.sh.
+rule_files:
+  - /etc/prometheus/rules.d/*.yml
+PROM
+      echo "  + prometheus.yml now includes rules.d"
+      curl -fsS -X POST http://127.0.0.1:26559/-/reload && echo "  = prometheus reloaded"
+    fi
+
     # 2. give Grafana somewhere to read provisioned dashboards from
     q=/etc/containers/systemd/grafana.container
     if grep -q 'provisioning/dashboards:' "$q"; then
@@ -275,6 +293,7 @@ PROM
     tmp=/etc/sudoers.d/.quantus-metrics_gitea_ci.tmp
     cat > "$tmp" <<'SUDO'
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/prometheus/scrape_configs.d/quantus.yml
+gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/prometheus/rules.d/quantus.yml
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/quantus.json
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/gpu.json
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus-dashboards.yaml