metrics: stale work is the batch in flight at cancellation, not the whole search #15

Merged
grenade merged 1 commits from metrics/stale-semantics into main 2026-09-03 11:28:44 +00:00
3 changed files with 24 additions and 22 deletions

View File

@@ -618,6 +618,9 @@ impl MinerEngine for GpuEngine {
let mut total_hashes: u64 = 0;
let mut current_start = range.start;
let mut batch_num = 0u64;
// lair: hashes of the most recent batch; if the job turns out to have
// been superseded while it ran, that batch was wasted work.
let mut last_batch_hashes: u64 = 0;
log::info!(
target: "gpu_engine",
@@ -632,6 +635,7 @@ impl MinerEngine for GpuEngine {
while current_start <= range.end {
// Check for cancellation at host level BEFORE starting each batch
if cancel.is_cancelled() {
gpu_ctx.metrics.record_stale_hashes(last_batch_hashes); // lair
let elapsed = search_start.elapsed();
let hash_rate = total_hashes as f64 / elapsed.as_secs_f64();
log::info!(
@@ -699,6 +703,7 @@ impl MinerEngine for GpuEngine {
}
BatchResult::NotFound { hash_count } => {
total_hashes += hash_count;
last_batch_hashes = hash_count; // lair
}
BatchResult::DeviceLost => {
gpu_ctx.metrics.record_device_lost(); // lair

View File

@@ -116,6 +116,18 @@ static DEVICE_LOST: Lazy<IntCounterVec> = Lazy::new(|| {
.expect("miner_device_lost_total"))
});
static DEVICE_STALE_HASHES: Lazy<IntCounterVec> = Lazy::new(|| {
reg(IntCounterVec::new(
Opts::new(
"miner_stale_hashes_total",
"Hashes computed after the job they were for had been superseded: the batch in \
flight when a new job arrived. The wasted-work cost of batch size.",
),
&["device", "kernel"],
)
.expect("miner_stale_hashes_total"))
});
static GPU_BATCH_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
reg(HistogramVec::new(
HistogramOpts::new(
@@ -138,6 +150,7 @@ pub struct DeviceMetrics {
hashes: IntCounter,
solutions: IntCounter,
lost: IntCounter,
stale: IntCounter,
batch_gpu: Histogram,
batch_host: Histogram,
}
@@ -149,6 +162,7 @@ impl DeviceMetrics {
hashes: DEVICE_HASHES.with_label_values(&[&d, kernel]),
solutions: DEVICE_SOLUTIONS.with_label_values(&[&d, kernel]),
lost: DEVICE_LOST.with_label_values(&[&d, kernel]),
stale: DEVICE_STALE_HASHES.with_label_values(&[&d, kernel]),
batch_gpu: GPU_BATCH_SECONDS.with_label_values(&[&d, kernel, "gpu"]),
batch_host: GPU_BATCH_SECONDS.with_label_values(&[&d, kernel, "host"]),
}
@@ -166,6 +180,11 @@ impl DeviceMetrics {
self.lost.inc();
}
/// The batch that completed after its job was superseded.
pub fn record_stale_hashes(&self, n: u64) {
self.stale.inc_by(n);
}
/// `gpu` is submit-to-completion on the device; `host` is everything else
/// in the batch (buffer writes, encoding, readback, bookkeeping).
pub fn observe_batch(&self, gpu: Duration, host: Duration) {
@@ -185,17 +204,6 @@ static JOBS_RECEIVED: Lazy<IntCounter> = Lazy::new(|| {
)
});
static STALE_HASHES: Lazy<IntCounterVec> = Lazy::new(|| {
reg(IntCounterVec::new(
Opts::new(
"miner_stale_hashes_total",
"Hashes reported for a job after a newer job had been issued (wasted work)",
),
&["engine"],
)
.expect("miner_stale_hashes_total"))
});
static JOB_PICKUP_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
reg(HistogramVec::new(
HistogramOpts::new(
@@ -252,10 +260,6 @@ pub fn record_job_received() {
JOBS_RECEIVED.inc();
}
pub fn record_stale_hashes(engine: &str, n: u64) {
STALE_HASHES.with_label_values(&[engine]).inc_by(n);
}
pub fn observe_job_pickup(engine: &str, d: Duration) {
JOB_PICKUP_SECONDS
.with_label_values(&[engine])

View File

@@ -186,13 +186,6 @@ async fn handle_connection(
// Check if this result is for the current job (not stale)
if worker_result.job_id != internal_job_id {
metrics::record_stale_hashes(
match worker_result.engine_type {
EngineType::Cpu => "CPU",
EngineType::Gpu => "GPU",
},
worker_result.hash_count,
); // lair
log::debug!(
"⏰ Discarding stale result from worker {} (result job_id {} != current {})",
worker_result.thread_id,