diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 7296a8c..00f3823 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -660,6 +660,10 @@ jobs: push --mkpath --chmod=F0644 quantus-scrape.yml \ gitea_ci@"$HOST":/etc/prometheus/scrape_configs.d/quantus.yml + python3 -c "import yaml,sys; yaml.safe_load(open('asset/prometheus/quantus-rules.yml'))" \ + || { echo "recording rules are not valid YAML" >&2; exit 1; } + push --mkpath --chmod=F0644 asset/prometheus/quantus-rules.yml \ + gitea_ci@"$HOST":/etc/prometheus/rules.d/quantus.yml push --mkpath --chmod=F0644 asset/grafana/quantus.json \ gitea_ci@"$HOST":/etc/grafana/provisioning/dashboards/quantus/quantus.json push --mkpath --chmod=F0644 asset/grafana/gpu.json \ @@ -702,6 +706,16 @@ jobs: run() { ssh $SSHOPTS gitea_ci@"${{ env.METRICS_HOST }}" "$@"; } fail=0 + echo "--- prometheus rules ---" + r=$(run "curl -fsS http://127.0.0.1:${{ env.PROM_PORT }}/api/v1/rules") + if grep -q '"name":"quantus-miner"' <<<"$r"; then + echo " ok rule group quantus-miner loaded" + else + echo " FAIL rule group quantus-miner not loaded" >&2 + echo " is rule_files wired into prometheus.yml? (infra-setup.sh --metrics-hosts)" >&2 + fail=1 + fi + echo "--- prometheus targets ---" # The point of this check: a target that drifted out of the matrices, # or a firewalld rule that never landed, shows up here as missing or diff --git a/asset/gpu/nvidia-textfile.sh b/asset/gpu/nvidia-textfile.sh index ddc234f..cb6f17b 100755 --- a/asset/gpu/nvidia-textfile.sh +++ b/asset/gpu/nvidia-textfile.sh @@ -20,7 +20,9 @@ TMP="$(mktemp "${OUT}.XXXXXX")" trap 'rm -f "$TMP"' EXIT QUERY=index,uuid,name,temperature.gpu,utilization.gpu,utilization.memory,\ -memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed +memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current.memory,fan.speed,\ +clocks_event_reasons.sw_power_cap,clocks_event_reasons.hw_slowdown,clocks_event_reasons.sw_thermal_slowdown,\ +clocks_event_reasons.hw_thermal_slowdown,clocks_event_reasons.hw_power_brake_slowdown { echo "# HELP nvidia_gpu_info Static GPU identity, always 1." @@ -45,6 +47,8 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current echo "# TYPE nvidia_gpu_clock_memory_hertz gauge" echo "# HELP nvidia_gpu_fan_speed_ratio Fan duty cycle, 0-1." echo "# TYPE nvidia_gpu_fan_speed_ratio gauge" + echo "# HELP nvidia_gpu_event_reason_active 1 while the clock is being held down for this reason (power cap, thermal, brake), else 0." + echo "# TYPE nvidia_gpu_event_reason_active gauge" echo "# HELP nvidia_gpu_scrape_ok 1 when nvidia-smi was queried successfully." echo "# TYPE nvidia_gpu_scrape_ok gauge" @@ -60,6 +64,7 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current while IFS= read -r line; do [ -n "$line" ] || continue IFS=',' read -r idx uuid name temp util umem mused mtotal pdraw plimit csm cmem fan \ + r_pcap r_hwslow r_swtherm r_hwtherm r_brake \ <<< "$(printf '%s' "$line" | sed 's/, */,/g')" # A label value may contain characters Prometheus treats specially. esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } @@ -78,6 +83,14 @@ memory.used,memory.total,power.draw,power.limit,clocks.current.sm,clocks.current num "$csm" >/dev/null && awk -v l="$L" -v v="$csm" 'BEGIN{printf "nvidia_gpu_clock_sm_hertz{%s} %.0f\n", l, v*1000000}' num "$cmem" >/dev/null && awk -v l="$L" -v v="$cmem" 'BEGIN{printf "nvidia_gpu_clock_memory_hertz{%s} %.0f\n", l, v*1000000}' num "$fan" >/dev/null && awk -v l="$L" -v v="$fan" 'BEGIN{printf "nvidia_gpu_fan_speed_ratio{%s} %.4f\n", l, v/100}' + # Why a card is below its boost clock. A hashrate regression that + # coincides with a thermal or power-brake reason is not a code regression. + reason() { case "$2" in Active) echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 1" ;; "Not Active") echo "nvidia_gpu_event_reason_active{${L},reason=\"$1\"} 0" ;; esac; } + reason sw_power_cap "$r_pcap" + reason hw_slowdown "$r_hwslow" + reason sw_thermal_slowdown "$r_swtherm" + reason hw_thermal_slowdown "$r_hwtherm" + reason hw_power_brake_slowdown "$r_brake" done <<< "$smi" } > "$TMP" diff --git a/asset/grafana/quantus.json b/asset/grafana/quantus.json index fc7be7f..27409c8 100644 --- a/asset/grafana/quantus.json +++ b/asset/grafana/quantus.json @@ -9,7 +9,7 @@ ], "timezone": "browser", "schemaVersion": 39, - "version": 10, + "version": 11, "refresh": "30s", "time": { "from": "now-6h", @@ -2764,6 +2764,520 @@ "values": false } } + }, + { + "type": "row", + "title": "Performance by build (quantus/miner#9)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 116 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Hashrate by device and commit", + "description": "Per-card throughput carrying the build that produced it. A new series starts at every deploy; the legend is the experiment key.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 117 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "MHs", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:device_hashrate_by_commit:rate5m / 1e6", + "legendFormat": "{{host}} dev{{device}} {{kernel}} @{{commit}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Hashrate per watt", + "description": "Hashes per joule per card. The honest comparison when a power limit changes between builds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 117 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:device_hashrate_per_watt:rate5m / 1e6", + "legendFormat": "{{host}} dev{{device}} {{name}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Batch time off the GPU", + "description": "Share of each batch spent on the host (submit gaps, readback) rather than executing on the card. Pipelining and batch sizing lower this.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 125 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "max": 1, + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:gpu_batch_host_fraction:rate5m", + "legendFormat": "{{instance}} dev{{device}} {{kernel}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Stale work", + "description": "Fraction of hashes done for a job after a newer one had been issued. Grows with batch size and with node job churn.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 125 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "max": 1, + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:stale_hash_fraction:rate1h", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Seal latency (p95)", + "description": "Solution found on the card to the result sent to the node. Grows with batch size; costs blocks on a fast chain.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 133 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:seal_latency_seconds:p95_5m", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Idle waiting for the node", + "description": "Fraction of wall-clock between sending a result and receiving the next job. Node-attributable; see quantus/chain#1.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 133 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "max": 1, + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantus:job_idle_fraction:rate5m", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Jobs per minute", + "description": "Job cadence from the node. Every job restarts every worker's search.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 141 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(miner_jobs_received_total[5m]) * 60", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Clock held down", + "description": "Why a card is below its boost clock (nvidia-smi clock event reasons). A regression that coincides with thermal or brake is not a code regression; power cap is the normal state under a limit.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 141 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none", + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_gpu_event_reason_active == 1", + "legendFormat": "{{host}} gpu{{gpu}} {{reason}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ], "annotations": { @@ -2807,6 +3321,19 @@ "step": "1m", "titleFormat": "miner restart", "textFormat": "counter reset \u2014 hashrate readings near this line are unreliable" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "iconColor": "blue", + "name": "miner deploy", + "expr": "changes(miner_build_info[2m]) > 0 or (miner_build_info unless miner_build_info offset 2m)", + "titleFormat": "deploy {{instance}}", + "textFormat": "{{version}} {{commit}}", + "step": "1m" } ] } diff --git a/asset/prometheus/quantus-rules.yml b/asset/prometheus/quantus-rules.yml new file mode 100644 index 0000000..f71c40d --- /dev/null +++ b/asset/prometheus/quantus-rules.yml @@ -0,0 +1,53 @@ +# Recording rules for the Quantus miners, deployed to +# /etc/prometheus/rules.d/quantus.yml on the metrics host (quantus/miner#9). +# +# The miner's series (job quantus-miner, instance host:9900) and the GPU +# series from node-exporter's textfile collector (job node, `host` label) are +# joined on a derived `host` and on `device` (the miner's engine index) = +# `gpu` (nvidia-smi index). On single-card hosts both are 0; on a multi-card +# host the engine orders adapters discrete-first as Vulkan enumerates them, +# which is expected to match nvidia-smi's order but is not guaranteed. +groups: + - name: quantus-miner + interval: 30s + rules: + # Per-device throughput, keyed so it can be joined with the GPU series. + - record: quantus:device_hashrate:rate5m + expr: label_replace(rate(miner_device_hashes_total[5m]), "host", "$1", "instance", "([^:]+):.*") + + # The same, carrying the build that produced it. The series set changes + # on every deploy, which is the annotation the dashboard draws. + - record: quantus:device_hashrate_by_commit:rate5m + expr: quantus:device_hashrate:rate5m * on(instance) group_left(commit, version) miner_build_info + + # GPU draw keyed like the miner series. + - record: quantus:gpu_power_watts + expr: label_replace(nvidia_gpu_power_watts, "device", "$1", "gpu", "(.*)") + + # Hashes per joule: the honest number when a power limit changes. + - record: quantus:device_hashrate_per_watt:rate5m + expr: quantus:device_hashrate:rate5m / on(host, device) group_left(name) quantus:gpu_power_watts + + # Share of each batch spent off the GPU (submit gaps, readback). This is + # what pipelining and batch sizing reduce. + - record: quantus:gpu_batch_host_fraction:rate5m + expr: > + sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum{phase="host"}[5m])) + / + sum by (instance, device, kernel) (rate(miner_gpu_batch_seconds_sum[5m])) + + # Work done for jobs that had already been superseded. + - record: quantus:stale_hash_fraction:rate1h + expr: > + sum by (instance) (rate(miner_stale_hashes_total[1h])) + / + sum by (instance) (rate(miner_hashes_total[1h])) + + # Found to sent, 95th percentile. + - record: quantus:seal_latency_seconds:p95_5m + expr: histogram_quantile(0.95, sum by (instance, le) (rate(miner_seal_latency_seconds_bucket[5m]))) + + # Fraction of wall-clock spent waiting for the node to issue the next + # job after a result. Node-attributable, not miner-attributable. + - record: quantus:job_idle_fraction:rate5m + expr: rate(miner_job_idle_seconds_total[5m]) diff --git a/script/infra-setup.sh b/script/infra-setup.sh index f501a92..722e0d1 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -239,6 +239,7 @@ provision_metrics() { usermod -aG systemd-journal gitea_ci install -d -m 0755 /etc/prometheus/scrape_configs.d + install -d -m 0755 /etc/prometheus/rules.d install -d -m 0755 /etc/grafana/provisioning/dashboards/quantus # 1. include drop-in scrape configs @@ -258,6 +259,23 @@ PROM curl -fsS -X POST http://127.0.0.1:26559/-/reload && echo " = prometheus reloaded" fi + # 1b. include drop-in recording rules (quantus/miner#9), same shape. + if grep -q '^rule_files:' /etc/prometheus/prometheus.yml; then + echo " = prometheus.yml already includes rules.d" + else + cp -a /etc/prometheus/prometheus.yml \ + "/etc/prometheus/prometheus.yml.bak.$(date +%s)" + cat >> /etc/prometheus/prometheus.yml <<'PROM' + +# Per-project recording and alerting rules. Added by lair/quantus +# script/infra-setup.sh. +rule_files: + - /etc/prometheus/rules.d/*.yml +PROM + echo " + prometheus.yml now includes rules.d" + curl -fsS -X POST http://127.0.0.1:26559/-/reload && echo " = prometheus reloaded" + fi + # 2. give Grafana somewhere to read provisioned dashboards from q=/etc/containers/systemd/grafana.container if grep -q 'provisioning/dashboards:' "$q"; then @@ -275,6 +293,7 @@ PROM tmp=/etc/sudoers.d/.quantus-metrics_gitea_ci.tmp cat > "$tmp" <<'SUDO' gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/prometheus/scrape_configs.d/quantus.yml +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/prometheus/rules.d/quantus.yml gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/quantus.json gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/gpu.json gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus-dashboards.yaml