metrics: stale work is the batch in flight at cancellation, not the whole search #15
@@ -618,6 +618,9 @@ impl MinerEngine for GpuEngine {
|
||||
let mut total_hashes: u64 = 0;
|
||||
let mut current_start = range.start;
|
||||
let mut batch_num = 0u64;
|
||||
// lair: hashes of the most recent batch; if the job turns out to have
|
||||
// been superseded while it ran, that batch was wasted work.
|
||||
let mut last_batch_hashes: u64 = 0;
|
||||
|
||||
log::info!(
|
||||
target: "gpu_engine",
|
||||
@@ -632,6 +635,7 @@ impl MinerEngine for GpuEngine {
|
||||
while current_start <= range.end {
|
||||
// Check for cancellation at host level BEFORE starting each batch
|
||||
if cancel.is_cancelled() {
|
||||
gpu_ctx.metrics.record_stale_hashes(last_batch_hashes); // lair
|
||||
let elapsed = search_start.elapsed();
|
||||
let hash_rate = total_hashes as f64 / elapsed.as_secs_f64();
|
||||
log::info!(
|
||||
@@ -699,6 +703,7 @@ impl MinerEngine for GpuEngine {
|
||||
}
|
||||
BatchResult::NotFound { hash_count } => {
|
||||
total_hashes += hash_count;
|
||||
last_batch_hashes = hash_count; // lair
|
||||
}
|
||||
BatchResult::DeviceLost => {
|
||||
gpu_ctx.metrics.record_device_lost(); // lair
|
||||
|
||||
@@ -116,6 +116,18 @@ static DEVICE_LOST: Lazy<IntCounterVec> = Lazy::new(|| {
|
||||
.expect("miner_device_lost_total"))
|
||||
});
|
||||
|
||||
static DEVICE_STALE_HASHES: Lazy<IntCounterVec> = Lazy::new(|| {
|
||||
reg(IntCounterVec::new(
|
||||
Opts::new(
|
||||
"miner_stale_hashes_total",
|
||||
"Hashes computed after the job they were for had been superseded: the batch in \
|
||||
flight when a new job arrived. The wasted-work cost of batch size.",
|
||||
),
|
||||
&["device", "kernel"],
|
||||
)
|
||||
.expect("miner_stale_hashes_total"))
|
||||
});
|
||||
|
||||
static GPU_BATCH_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
|
||||
reg(HistogramVec::new(
|
||||
HistogramOpts::new(
|
||||
@@ -138,6 +150,7 @@ pub struct DeviceMetrics {
|
||||
hashes: IntCounter,
|
||||
solutions: IntCounter,
|
||||
lost: IntCounter,
|
||||
stale: IntCounter,
|
||||
batch_gpu: Histogram,
|
||||
batch_host: Histogram,
|
||||
}
|
||||
@@ -149,6 +162,7 @@ impl DeviceMetrics {
|
||||
hashes: DEVICE_HASHES.with_label_values(&[&d, kernel]),
|
||||
solutions: DEVICE_SOLUTIONS.with_label_values(&[&d, kernel]),
|
||||
lost: DEVICE_LOST.with_label_values(&[&d, kernel]),
|
||||
stale: DEVICE_STALE_HASHES.with_label_values(&[&d, kernel]),
|
||||
batch_gpu: GPU_BATCH_SECONDS.with_label_values(&[&d, kernel, "gpu"]),
|
||||
batch_host: GPU_BATCH_SECONDS.with_label_values(&[&d, kernel, "host"]),
|
||||
}
|
||||
@@ -166,6 +180,11 @@ impl DeviceMetrics {
|
||||
self.lost.inc();
|
||||
}
|
||||
|
||||
/// The batch that completed after its job was superseded.
|
||||
pub fn record_stale_hashes(&self, n: u64) {
|
||||
self.stale.inc_by(n);
|
||||
}
|
||||
|
||||
/// `gpu` is submit-to-completion on the device; `host` is everything else
|
||||
/// in the batch (buffer writes, encoding, readback, bookkeeping).
|
||||
pub fn observe_batch(&self, gpu: Duration, host: Duration) {
|
||||
@@ -185,17 +204,6 @@ static JOBS_RECEIVED: Lazy<IntCounter> = Lazy::new(|| {
|
||||
)
|
||||
});
|
||||
|
||||
static STALE_HASHES: Lazy<IntCounterVec> = Lazy::new(|| {
|
||||
reg(IntCounterVec::new(
|
||||
Opts::new(
|
||||
"miner_stale_hashes_total",
|
||||
"Hashes reported for a job after a newer job had been issued (wasted work)",
|
||||
),
|
||||
&["engine"],
|
||||
)
|
||||
.expect("miner_stale_hashes_total"))
|
||||
});
|
||||
|
||||
static JOB_PICKUP_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
|
||||
reg(HistogramVec::new(
|
||||
HistogramOpts::new(
|
||||
@@ -252,10 +260,6 @@ pub fn record_job_received() {
|
||||
JOBS_RECEIVED.inc();
|
||||
}
|
||||
|
||||
pub fn record_stale_hashes(engine: &str, n: u64) {
|
||||
STALE_HASHES.with_label_values(&[engine]).inc_by(n);
|
||||
}
|
||||
|
||||
pub fn observe_job_pickup(engine: &str, d: Duration) {
|
||||
JOB_PICKUP_SECONDS
|
||||
.with_label_values(&[engine])
|
||||
|
||||
@@ -186,13 +186,6 @@ async fn handle_connection(
|
||||
|
||||
// Check if this result is for the current job (not stale)
|
||||
if worker_result.job_id != internal_job_id {
|
||||
metrics::record_stale_hashes(
|
||||
match worker_result.engine_type {
|
||||
EngineType::Cpu => "CPU",
|
||||
EngineType::Gpu => "GPU",
|
||||
},
|
||||
worker_result.hash_count,
|
||||
); // lair
|
||||
log::debug!(
|
||||
"⏰ Discarding stale result from worker {} (result job_id {} != current {})",
|
||||
worker_result.thread_id,
|
||||
|
||||
Reference in New Issue
Block a user