diff --git a/crates/metrics/src/lib.rs b/crates/metrics/src/lib.rs index e21f2a7..fb5d694 100644 --- a/crates/metrics/src/lib.rs +++ b/crates/metrics/src/lib.rs @@ -19,10 +19,15 @@ //! - The HTTP exporter is spawned as a background task and does not block. //! - If the `http-exporter` feature is disabled, `start_http_exporter` becomes //! a no-op that returns immediately. +//! - When jobs/threads end, prefer removing gauge label children (series) rather than +//! writing a zero; this avoids scrape-timing artifacts and produces cleaner rollups. +//! - Service-level observability: expose an `active_jobs` gauge and a `mine_requests_total` +//! counter (labeled by result) to disambiguate "idle because no jobs" vs "actively mining". use once_cell::sync::Lazy; use prometheus::{ - opts, Encoder, Gauge, GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry, TextEncoder, + opts, Encoder, Gauge, GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry, + TextEncoder, }; #[cfg(feature = "http-exporter")] @@ -42,9 +47,43 @@ use anyhow::Result; static REGISTRY: Lazy = Lazy::new(Registry::new); +// ------------------------------------------------------------------------------------- +// High-level service metrics +// ------------------------------------------------------------------------------------- + +static ACTIVE_JOBS: Lazy = Lazy::new(|| { + let g = IntGauge::new( + "miner_active_jobs", + "Number of currently running mining jobs", + ) + .expect("create miner_active_jobs"); + REGISTRY + .register(Box::new(g.clone())) + .expect("register miner_active_jobs"); + g +}); + +static MINE_REQUESTS_TOTAL: Lazy = Lazy::new(|| { + let c = IntCounterVec::new( + opts!( + "miner_mine_requests_total", + "Count of /mine requests by result" + ), + &["result"], // accepted, duplicate, invalid, error + ) + .expect("create miner_mine_requests_total"); + REGISTRY + .register(Box::new(c.clone())) + .expect("register miner_mine_requests_total"); + c +}); + static JOBS_TOTAL: Lazy = Lazy::new(|| { - let c = IntCounterVec::new(opts!("miner_jobs_total", "Number of jobs by status"), &["status"]) - .expect("create miner_jobs_total"); + let c = IntCounterVec::new( + opts!("miner_jobs_total", "Number of jobs by status"), + &["status"], + ) + .expect("create miner_jobs_total"); REGISTRY .register(Box::new(c.clone())) .expect("register miner_jobs_total"); @@ -187,6 +226,15 @@ pub fn default_registry() -> &'static Registry { /// Increment the total hashes counter by `n` (number of nonces tested). // Labeled helpers for engine/job/thread metrics +// +// Service-level helpers +pub fn set_active_jobs(n: i64) { + ACTIVE_JOBS.set(n); +} + +pub fn inc_mine_requests(result: &str) { + MINE_REQUESTS_TOTAL.with_label_values(&[result]).inc(); +} pub fn inc_job_hashes(engine: &str, job_id: &str, n: u64) { JOB_HASHES_TOTAL @@ -195,9 +243,7 @@ pub fn inc_job_hashes(engine: &str, job_id: &str, n: u64) { } pub fn set_job_hash_rate(engine: &str, job_id: &str, rate: f64) { - JOB_HASH_RATE - .with_label_values(&[engine, job_id]) - .set(rate); + JOB_HASH_RATE.with_label_values(&[engine, job_id]).set(rate); } pub fn inc_thread_hashes(engine: &str, job_id: &str, thread_id: &str, n: u64) { @@ -245,6 +291,20 @@ pub fn inc_http_request(endpoint: &str, code: u16) { .inc(); } +// ------------------------------------------------------------------------------------- +// Removal helpers for end-of-life series +// ------------------------------------------------------------------------------------- + +/// Remove the per-job hash rate series for a finished job. +pub fn remove_job_hash_rate(engine: &str, job_id: &str) { + let _ = JOB_HASH_RATE.remove_label_values(&[engine, job_id]); +} + +/// Remove the per-thread hash rate series for a finished thread. +pub fn remove_thread_hash_rate(engine: &str, job_id: &str, thread_id: &str) { + let _ = THREAD_HASH_RATE.remove_label_values(&[engine, job_id, thread_id]); +} + // ------------------------------------------------------------------------------------- // HTTP Exporter (feature: http-exporter) // ------------------------------------------------------------------------------------- @@ -270,7 +330,9 @@ pub async fn start_http_exporter(port: u16) -> Result<()> { let encoder = TextEncoder::new(); let metric_families = REGISTRY.gather(); let mut buffer = Vec::with_capacity(16 * 1024); - encoder.encode(&metric_families, &mut buffer).unwrap_or_default(); + encoder + .encode(&metric_families, &mut buffer) + .unwrap_or_default(); Response::builder() .header("Content-Type", encoder.format_type()) diff --git a/crates/miner-cli/src/main.rs b/crates/miner-cli/src/main.rs index 9f416a1..875c595 100644 --- a/crates/miner-cli/src/main.rs +++ b/crates/miner-cli/src/main.rs @@ -18,6 +18,11 @@ struct Args { #[arg(long, env = "MINER_METRICS_PORT")] metrics_port: Option, + /// Target milliseconds for per-thread progress updates (chunking). + /// Smaller values increase metrics freshness but add a bit of overhead. + #[arg(long = "progress-chunk-ms", env = "MINER_PROGRESS_CHUNK_MS")] + progress_chunk_ms: Option, + /// Mining engine to use (default: cpu-fast). Options: cpu-baseline, cpu-fast #[arg(long, env = "MINER_ENGINE", value_enum, default_value_t = EngineCli::CpuFast)] engine: EngineCli, @@ -60,7 +65,9 @@ async fn main() { log::info!("API listening port: {}", args.port); match args.num_cores { Some(n) if n > 0 => log::info!("Using specified number of cores: {}", n), - Some(_) => log::warn!("Number of cores must be positive. Defaulting to all available cores."), + Some(_) => { + log::warn!("Number of cores must be positive. Defaulting to all available cores.") + } None => log::info!("Using all available cores (auto-detected)."), } match args.metrics_port { @@ -73,6 +80,7 @@ async fn main() { port: args.port, num_cores: args.num_cores, metrics_port: args.metrics_port, + progress_chunk_ms: args.progress_chunk_ms, engine: args.engine.into(), }; diff --git a/crates/miner-service/src/lib.rs b/crates/miner-service/src/lib.rs index cf00500..eae6fae 100644 --- a/crates/miner-service/src/lib.rs +++ b/crates/miner-service/src/lib.rs @@ -24,6 +24,8 @@ pub struct ServiceConfig { pub num_cores: Option, /// Optional metrics port. When Some, metrics endpoint starts; when None, metrics are disabled. pub metrics_port: Option, + /// Target milliseconds for per-thread progress updates (chunking). If None, defaults to 2000ms. + pub progress_chunk_ms: Option, /// Engine selection (future use). For now, CPU baseline/fast engines are supported. pub engine: EngineSelection, } @@ -43,6 +45,7 @@ impl Default for ServiceConfig { port: 9833, num_cores: None, metrics_port: None, + progress_chunk_ms: None, engine: EngineSelection::CpuBaseline, } } @@ -54,14 +57,20 @@ pub struct MiningService { pub jobs: Arc>>, pub num_cores: usize, pub engine: Arc, + /// Target milliseconds for per-thread progress updates (chunking). + pub progress_chunk_ms: u64, + /// Gauge of currently running jobs (for metrics) + pub active_jobs_gauge: Arc>, } impl MiningService { - pub fn new(num_cores: usize, engine: Arc) -> Self { + pub fn new(num_cores: usize, engine: Arc, progress_chunk_ms: u64) -> Self { Self { jobs: Arc::new(Mutex::new(HashMap::new())), num_cores, engine, + progress_chunk_ms, + active_jobs_gauge: Arc::new(tokio::sync::Mutex::new(0)), } } @@ -80,8 +89,14 @@ impl MiningService { metrics::set_job_status_gauge(self.engine.name(), &job_id, "completed", 0); metrics::set_job_status_gauge(self.engine.name(), &job_id, "failed", 0); metrics::set_job_status_gauge(self.engine.name(), &job_id, "cancelled", 0); + // increment active jobs + { + let mut g = self.active_jobs_gauge.lock().await; + *g += 1; + metrics::set_active_jobs(*g); + } } - job.start_mining(self.engine.clone(), self.num_cores); + job.start_mining(self.engine.clone(), self.num_cores, self.progress_chunk_ms); jobs.insert(job_id, job); Ok(()) } @@ -96,6 +111,14 @@ impl MiningService { if let Some(mut job) = jobs.remove(job_id) { log::debug!(target: "miner", "Removing job: {}", job_id); job.cancel(); + #[cfg(feature = "metrics")] + { + let mut g = self.active_jobs_gauge.lock().await; + if *g > 0 { + *g -= 1; + } + metrics::set_active_jobs(*g); + } Some(job) } else { None @@ -106,6 +129,14 @@ impl MiningService { let mut jobs = self.jobs.lock().await; if let Some(job) = jobs.get_mut(job_id) { job.cancel(); + #[cfg(feature = "metrics")] + { + let mut g = self.active_jobs_gauge.lock().await; + if *g > 0 { + *g -= 1; + } + metrics::set_active_jobs(*g); + } true } else { false @@ -144,15 +175,12 @@ impl MiningService { #[cfg(feature = "metrics")] { // Aggregate and per-job hash-rate estimate across running jobs (nonces/sec) + // Aggregate hash-rate across running jobs from last recorded per-job rates let mut total_rate = 0.0; for (job_id, job) in jobs_guard.iter() { if job.status == JobStatus::Running { - let elapsed = job.start_time.elapsed().as_secs_f64(); - if elapsed > 0.0 { - let rate = job.total_hash_count as f64 / elapsed; - total_rate += rate; - metrics::set_job_hash_rate(job.engine_name, job_id, rate); - } + total_rate += job.last_hash_rate; + metrics::set_job_hash_rate(job.engine_name, job_id, job.last_hash_rate); } } metrics::set_hash_rate(total_rate); @@ -192,6 +220,7 @@ pub struct MiningJob { pub status: JobStatus, pub start_time: Instant, pub total_hash_count: u64, + pub last_hash_rate: f64, pub best_result: Option, pub engine_name: &'static str, @@ -215,6 +244,7 @@ impl Clone for MiningJob { status: self.status.clone(), start_time: self.start_time, total_hash_count: self.total_hash_count, + last_hash_rate: self.last_hash_rate, best_result: self.best_result.clone(), engine_name: self.engine_name, job_id: self.job_id.clone(), @@ -253,6 +283,7 @@ impl MiningJob { status: JobStatus::Running, start_time: Instant::now(), total_hash_count: 0, + last_hash_rate: 0.0, best_result: None, engine_name: "unknown", job_id: None, @@ -265,7 +296,12 @@ impl MiningJob { } } - pub fn start_mining(&mut self, engine: Arc, num_cores: usize) { + pub fn start_mining( + &mut self, + engine: Arc, + num_cores: usize, + progress_chunk_ms: u64, + ) { let (sender, receiver) = bounded(num_cores * 2); self.result_receiver = Some(receiver); self.engine_name = engine.name(); @@ -300,6 +336,7 @@ impl MiningJob { let sender = sender.clone(); let ctx = ctx.clone(); let engine = engine.clone(); + let progress_chunk_ms = progress_chunk_ms; let handle = thread::spawn(move || { mine_range_with_engine( @@ -309,6 +346,7 @@ impl MiningJob { EngineRange { start, end }, cancel_flag, sender, + progress_chunk_ms, ); }); @@ -329,6 +367,15 @@ impl MiningJob { metrics::set_job_status_gauge(self.engine_name, job_id, "completed", 0); metrics::set_job_status_gauge(self.engine_name, job_id, "failed", 0); metrics::set_job_status_gauge(self.engine_name, job_id, "cancelled", 1); + // Remove job hash rate series on cancellation (avoid scrape-timing artifacts) + self.last_hash_rate = 0.0; + metrics::remove_job_hash_rate(self.engine_name, job_id); + // Remove all per-thread hash rate series on cancellation and clear tracking + for (tid, _) in self.thread_rate_ema.iter() { + metrics::remove_thread_hash_rate(self.engine_name, job_id, &tid.to_string()); + } + self.thread_last_update.clear(); + self.thread_rate_ema.clear(); } } @@ -372,7 +419,8 @@ impl MiningJob { .get(&thread_result.thread_id) .copied() .unwrap_or(instant_rate); - let ema = THREAD_RATE_EMA_ALPHA * instant_rate + (1.0 - THREAD_RATE_EMA_ALPHA) * prev; + let ema = THREAD_RATE_EMA_ALPHA * instant_rate + + (1.0 - THREAD_RATE_EMA_ALPHA) * prev; metrics::set_thread_hash_rate( self.engine_name, job_id, @@ -380,12 +428,15 @@ impl MiningJob { ema, ); // Store updated EMA for next delta - self.thread_rate_ema - .insert(thread_result.thread_id, ema); + self.thread_rate_ema.insert(thread_result.thread_id, ema); } // Update last-seen timestamp for this thread - self.thread_last_update - .insert(thread_result.thread_id, now); + self.thread_last_update.insert(thread_result.thread_id, now); + + // Update per-job hash rate as the sum of per-thread EMAs and publish + let job_rate: f64 = self.thread_rate_ema.values().copied().sum(); + self.last_hash_rate = job_rate; + metrics::set_job_hash_rate(self.engine_name, job_id, job_rate); } } @@ -393,6 +444,14 @@ impl MiningJob { self.completed_threads += 1; #[cfg(feature = "metrics")] { + if let Some(job_id) = &self.job_id { + // Remove this thread's hash rate series on completion + metrics::remove_thread_hash_rate( + self.engine_name, + job_id, + &thread_result.thread_id.to_string(), + ); + } // Cleanup per-thread tracking on completion self.thread_last_update.remove(&thread_result.thread_id); self.thread_rate_ema.remove(&thread_result.thread_id); @@ -430,6 +489,18 @@ impl MiningJob { metrics::set_job_status_gauge(self.engine_name, job_id, "completed", 1); metrics::set_job_status_gauge(self.engine_name, job_id, "failed", 0); metrics::set_job_status_gauge(self.engine_name, job_id, "cancelled", 0); + // Remove job hash rate on completion and clear per-thread series + self.last_hash_rate = 0.0; + metrics::remove_job_hash_rate(self.engine_name, job_id); + for (tid, _) in self.thread_rate_ema.iter() { + metrics::remove_thread_hash_rate( + self.engine_name, + job_id, + &tid.to_string(), + ); + } + self.thread_last_update.clear(); + self.thread_rate_ema.clear(); } } } else if self.completed_threads >= self.thread_handles.len() @@ -445,6 +516,18 @@ impl MiningJob { metrics::set_job_status_gauge(self.engine_name, job_id, "completed", 0); metrics::set_job_status_gauge(self.engine_name, job_id, "failed", 1); metrics::set_job_status_gauge(self.engine_name, job_id, "cancelled", 0); + // Remove job hash rate on failure and clear per-thread series + self.last_hash_rate = 0.0; + metrics::remove_job_hash_rate(self.engine_name, job_id); + for (tid, _) in self.thread_rate_ema.iter() { + metrics::remove_thread_hash_rate( + self.engine_name, + job_id, + &tid.to_string(), + ); + } + self.thread_last_update.clear(); + self.thread_rate_ema.clear(); } } } @@ -461,6 +544,7 @@ fn mine_range_with_engine( range: EngineRange, cancel_flag: Arc, sender: Sender, + progress_chunk_ms: u64, ) { log::debug!( "Thread {} mining range {} to {} (inclusive)", @@ -469,44 +553,111 @@ fn mine_range_with_engine( range.end ); - let status = engine.search_range(&ctx, range.clone(), &cancel_flag); + // Chunk the range into subranges to emit periodic progress updates for metrics + let mut current_start = range.start; + let end = range.end; + // Derive a rough chunk size from target milliseconds and an estimate of hashes/sec. + // Start with a conservative default of 100k ops/sec per thread and scale by target ms. + let target_ms = progress_chunk_ms; + let est_ops_per_sec: u64 = 100_000; + let derived_chunk: u64 = ((est_ops_per_sec.saturating_mul(target_ms)) / 1000).max(5_000); + let chunk_size = U512::from(derived_chunk); + let mut done = false; - let mut final_result = ThreadResult { - thread_id, - result: None, - hash_count: 0, - completed: true, - }; + while current_start <= end && !cancel_flag.load(Ordering::Relaxed) { + let mut current_end = current_start + .saturating_add(chunk_size) + .saturating_sub(U512::from(1u64)); + if current_end > end { + current_end = end; + } - match status { - engine_cpu::EngineStatus::Found { - candidate: EngineCandidate { - nonce, - work, - distance, - }, - hash_count, - } => { - final_result.result = Some(MiningJobResult { - nonce, - work, - distance, - }); - final_result.hash_count = hash_count; + let sub_range = EngineRange { + start: current_start, + end: current_end, + }; + + let status = engine.search_range(&ctx, sub_range.clone(), &cancel_flag); + + match status { + engine_cpu::EngineStatus::Found { + candidate: + EngineCandidate { + nonce, + work, + distance, + }, + hash_count, + } => { + // Send final result with found candidate and the hashes covered in this subrange + let final_result = ThreadResult { + thread_id, + result: Some(MiningJobResult { + nonce, + work, + distance, + }), + hash_count, + completed: true, + }; + if sender.send(final_result).is_err() { + log::warn!("Thread {} failed to send final result", thread_id); + } + done = true; + break; + } + engine_cpu::EngineStatus::Exhausted { hash_count } => { + // Send intermediate progress update for this chunk + let update = ThreadResult { + thread_id, + result: None, + hash_count, + completed: false, + }; + if sender.send(update).is_err() { + log::warn!("Thread {} failed to send progress update", thread_id); + break; + } + } + engine_cpu::EngineStatus::Cancelled { hash_count } => { + // Send last progress update and stop + let update = ThreadResult { + thread_id, + result: None, + hash_count, + completed: false, + }; + if sender.send(update).is_err() { + log::warn!("Thread {} failed to send cancel update", thread_id); + } + done = true; + break; + } + engine_cpu::EngineStatus::Running { .. } => { + // Not expected from a synchronous engine chunk call; ignore + } } - engine_cpu::EngineStatus::Exhausted { hash_count } => { - final_result.hash_count = hash_count; - } - engine_cpu::EngineStatus::Cancelled { hash_count } => { - final_result.hash_count = hash_count; - } - engine_cpu::EngineStatus::Running { .. } => { - // Should not happen for synchronous search_range; treat as completed without result. + + if current_end == end { + break; } + current_start = current_end.saturating_add(U512::from(1u64)); } - if sender.send(final_result).is_err() { - log::warn!("Thread {} failed to send completion status", thread_id); + if !done { + // Signal thread completion with no final candidate + let final_result = ThreadResult { + thread_id, + result: None, + hash_count: 0, + completed: true, + }; + if sender.send(final_result).is_err() { + log::warn!( + "Thread {} failed to send completion status after chunked search", + thread_id + ); + } } log::debug!("Thread {} completed.", thread_id); @@ -556,6 +707,10 @@ pub async fn handle_mine_request( log::debug!("Received mine request: {:?}", request); if let Err(e) = validate_mining_request(&request) { log::warn!("Invalid mine request ({}): {}", request.job_id, e); + #[cfg(feature = "metrics")] + { + metrics::inc_mine_requests("invalid"); + } return Ok(warp::reply::with_status( warp::reply::json(&MiningResponse { status: ApiResponseStatus::Error, @@ -579,6 +734,10 @@ pub async fn handle_mine_request( match state.add_job(request.job_id.clone(), job).await { Ok(_) => { log::debug!(target: "miner", "Accepted mine request for job ID: {}", request.job_id); + #[cfg(feature = "metrics")] + { + metrics::inc_mine_requests("accepted"); + } Ok(warp::reply::with_status( warp::reply::json(&MiningResponse { status: ApiResponseStatus::Accepted, @@ -590,6 +749,15 @@ pub async fn handle_mine_request( } Err(e) => { log::error!("Failed to add job {}: {}", request.job_id, e); + #[cfg(feature = "metrics")] + { + let result = if e.contains("already") { + "duplicate" + } else { + "error" + }; + metrics::inc_mine_requests(result); + } Ok(warp::reply::with_status( warp::reply::json(&MiningResponse { status: ApiResponseStatus::Error, @@ -627,7 +795,7 @@ pub async fn handle_result_request( } }; - let api_status = match job.status { + let status = match job.status { JobStatus::Running => ApiResponseStatus::Running, JobStatus::Completed => ApiResponseStatus::Completed, JobStatus::Failed => ApiResponseStatus::Failed, @@ -646,7 +814,7 @@ pub async fn handle_result_request( Ok(warp::reply::with_status( warp::reply::json(&resonance_miner_api::MiningResult { - status: api_status, + status: status, job_id, nonce, work, @@ -738,7 +906,8 @@ pub async fn run(config: ServiceConfig) -> anyhow::Result<()> { }; log::info!("Using engine: {}", engine.name()); - let service = MiningService::new(num_cores, engine.clone()); + let progress_chunk_ms = config.progress_chunk_ms.unwrap_or(2000); + let service = MiningService::new(num_cores, engine.clone(), progress_chunk_ms); // Start mining loop service.start_mining_loop().await; diff --git a/docs/grafana/engines-comparison-dashboard.json b/docs/grafana/engines-comparison-dashboard.json index 3603649..5af7e71 100644 --- a/docs/grafana/engines-comparison-dashboard.json +++ b/docs/grafana/engines-comparison-dashboard.json @@ -32,7 +32,11 @@ } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "orientation": "auto", "colorMode": "value", "graphMode": "none", @@ -61,7 +65,11 @@ } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "orientation": "horizontal", "colorMode": "value", "graphMode": "none", @@ -86,7 +94,11 @@ } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "orientation": "auto", "colorMode": "value", "graphMode": "area", @@ -107,7 +119,7 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine) (miner_job_hash_rate{engine=~\"$engine\"})", + "expr": "sum by (engine) (max_over_time(miner_job_hash_rate{engine=~\"$engine\"}[1m]))", "legendFormat": "{{engine}}" } ], @@ -128,7 +140,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine, status) (miner_jobs_by_engine_total{engine=~\"$engine\"})" + "expr": "sum by (engine, status) (increase(miner_jobs_by_engine_total{engine=~\"$engine\"}[1m]))", + "legendFormat": "{{engine}}, {{status}}" } ], "fieldConfig": { @@ -138,7 +151,11 @@ "options": { "displayMode": "gradient", "orientation": "horizontal", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "showUnfilled": true } }, @@ -151,7 +168,7 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine) (increase(miner_job_hashes_total{engine=~\"$engine\"}[$__rate_interval]))", + "expr": "sum by (engine) (increase(miner_job_hashes_total{engine=~\"$engine\"}[1m]))", "legendFormat": "{{engine}}" } ], @@ -194,7 +211,8 @@ "targets": [ { "refId": "A", - "expr": "topk(10, miner_job_hash_rate{engine=~\"$engine\"})" + "expr": "topk(10, miner_job_hash_rate{engine=~\"$engine\"})", + "legendFormat": "{{engine}}, {{job_id}}" } ], "fieldConfig": { @@ -219,7 +237,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine) (increase(miner_job_hashes_total{engine=~\"$engine\"}[5m]))" + "expr": "sum by (engine) (increase(miner_job_hashes_total{engine=~\"$engine\"}[5m]))", + "legendFormat": "{{engine}}" } ], "fieldConfig": { @@ -245,8 +264,8 @@ "targets": [ { "refId": "A", - "expr": "miner_job_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\"}", - "legendFormat": "{{engine}} | {{job_id}}" + "expr": "max_over_time(miner_job_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\"}[1m])", + "legendFormat": "{{engine}}, {{job_id}}" } ], "fieldConfig": { @@ -267,7 +286,8 @@ "targets": [ { "refId": "A", - "expr": "miner_thread_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\", thread_id=~\"$thread_id\"}" + "expr": "max_over_time(miner_thread_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\", thread_id=~\"$thread_id\"}[1m])", + "legendFormat": "{{engine}}, {{job_id}}, {{thread_id}}" } ], "fieldConfig": { diff --git a/docs/grafana/instances-comparison-dashboard.json b/docs/grafana/instances-comparison-dashboard.json new file mode 100644 index 0000000..e9c72ed --- /dev/null +++ b/docs/grafana/instances-comparison-dashboard.json @@ -0,0 +1,416 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "stat", + "title": "24h Peak Hash Rate (selected instances)", + "description": "Sum of 24h max per-instance job hash rate across selected instances (trend-friendly)", + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 0 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\"}[24h:1m]))" + } + ], + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { "unit": "ops" }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Global Hash Rate by Instance", + "description": "sum of per-job hash rate grouped by instance (1m max window for trend)", + "gridPos": { "h": 8, "w": 16, "x": 8, "y": 0 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\"}[1m]))", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops" }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "table", + "title": "Per-Instance Hash Rate (current vs 1h peak)", + "description": "Current (1m max) and 1h peak per instance (filter by engine)", + "gridPos": { "h": 9, "w": 8, "x": 0, "y": 5 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "current" + }, + { + "refId": "B", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1h:1m]))", + "legendFormat": "peak_1h" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": {}, + "aggregations": [] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true + }, + "indexByName": { + "instance": 0, + "A": 1, + "B": 2 + }, + "renameByName": { + "A": "current_ops", + "B": "peak_1h_ops" + } + } + }, + { + "id": "calculateField", + "options": { + "mode": "binary", + "binary": { + "reducer": "last", + "left": "A", + "operator": "/", + "right": "B", + "replaceFields": false + }, + "reduce": { + "reducer": "last" + }, + "alias": "delta_current_over_peak" + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "auto" }, + "unit": "ops", + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 0.5 }, + { "color": "red", "value": 0.8 } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { "show": false }, + "frameIndex": 0 + } + }, + { + "type": "timeseries", + "title": "Per-Instance Job Hash Rate (over time)", + "description": "sum(max_over_time(miner_job_hash_rate[1m])) grouped by instance (filter by engine) for trend", + "gridPos": { "h": 8, "w": 16, "x": 8, "y": 8 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops" }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "timeseries", + "title": "Average Per-Thread Hash Rate by Instance", + "description": "avg(max_over_time(miner_thread_hash_rate[1m])) grouped by instance (EMA-smoothed; filter by engine)", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops" }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "table", + "title": "Top 15 Jobs by Instance (filter by engine)", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 24 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "topk(15, max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "{{instance}}, {{engine}}, {{job_id}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "auto" }, + "unit": "ops", + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { "show": false }, + "frameIndex": 0 + } + }, + { + "type": "table", + "title": "Per-Instance Total Hashes (1m increase; filter by engine)", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 24 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (increase(miner_job_hashes_total{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "auto" }, + "unit": "ops", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { "show": false }, + "frameIndex": 0 + } + }, + { + "type": "timeseries", + "title": "Per-Instance Per-Engine Job Hash Rate", + "description": "sum(max_over_time(miner_job_hash_rate[1m])) grouped by instance and engine", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 33 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance, engine) (max_over_time(miner_job_hash_rate{job=~\"$job\", instance=~\"$instance\", engine=~\"$engine\"}[1m]))", + "legendFormat": "{{instance}}, {{engine}}" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops" }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "none" } + } + }, + { + "type": "stat", + "title": "Active Jobs (by instance)", + "description": "Number of currently running mining jobs on each instance", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 42 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance) (miner_active_jobs{job=~\"$job\", instance=~\"$instance\"})" + } + ], + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { "unit": "short" }, + "overrides": [] + } + }, + { + "type": "table", + "title": "Mine Requests (5m increase; by instance & result)", + "description": "Accepted, duplicate, invalid, error counts", + "gridPos": { "h": 9, "w": 18, "x": 6, "y": 42 }, + "datasource": "$DS_PROMETHEUS", + "targets": [ + { + "refId": "A", + "expr": "sum by (instance, result) (increase(miner_mine_requests_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", + "legendFormat": "{{instance}}, {{result}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "auto" }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { "show": false }, + "frameIndex": 0 + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["quantus", "miner", "prometheus", "instances"], + "templating": { + "list": [ + { + "type": "datasource", + "name": "DS_PROMETHEUS", + "hide": 0, + "query": "prometheus", + "current": {}, + "options": [] + }, + { + "type": "query", + "name": "instance", + "label": "Instance(s)", + "hide": 0, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(miner_job_hash_rate, instance)", + "includeAll": true, + "multi": true, + "refresh": 2, + "sort": 1, + "query": "label_values(miner_job_hash_rate, instance)", + "regex": "", + "current": { + "selected": false, + "text": ["All"], + "value": ["$__all"] + } + }, + { + "type": "query", + "name": "engine", + "label": "Engine(s)", + "hide": 0, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(miner_job_hash_rate, engine)", + "includeAll": true, + "multi": true, + "refresh": 2, + "sort": 1, + "query": "label_values(miner_job_hash_rate, engine)", + "regex": "", + "current": { + "selected": false, + "text": ["All"], + "value": ["$__all"] + } + }, + { + "type": "query", + "name": "job", + "label": "Job(s)", + "hide": 0, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(miner_job_hash_rate, job)", + "includeAll": true, + "multi": true, + "refresh": 2, + "sort": 1, + "query": "label_values(miner_job_hash_rate, job)", + "regex": "", + "current": { + "selected": false, + "text": ["All"], + "value": ["$__all"] + } + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Quantus Miner - Instances Comparison", + "version": 1, + "uid": null +} diff --git a/docs/grafana/miner-dashboard.json b/docs/grafana/miner-dashboard.json index 41d4354..f5d8ae7 100644 --- a/docs/grafana/miner-dashboard.json +++ b/docs/grafana/miner-dashboard.json @@ -32,7 +32,11 @@ } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "orientation": "auto", "colorMode": "value", "graphMode": "none", @@ -52,7 +56,7 @@ "targets": [ { "refId": "A", - "expr": "sum(miner_hash_rate)", + "expr": "sum(max_over_time(miner_job_hash_rate[1m]))", "legendFormat": "global" } ], @@ -73,7 +77,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (status) (miner_jobs_total)" + "expr": "sum by (status) (miner_jobs_total)", + "legendFormat": "{{status}}" } ], "fieldConfig": { @@ -83,7 +88,11 @@ "options": { "displayMode": "gradient", "orientation": "horizontal", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "showUnfilled": true } }, @@ -95,7 +104,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine, status) (miner_jobs_by_engine_total)" + "expr": "sum by (engine, status) (increase(miner_jobs_by_engine_total[1m]))", + "legendFormat": "{{engine}}, {{status}}" } ], "fieldConfig": { @@ -105,7 +115,11 @@ "options": { "displayMode": "gradient", "orientation": "horizontal", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "showUnfilled": true } }, @@ -139,8 +153,8 @@ "targets": [ { "refId": "A", - "expr": "miner_thread_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\", thread_id=~\"$thread_id\"}", - "legendFormat": "{{engine}} | job={{job_id}} | th={{thread_id}}" + "expr": "max_over_time(miner_thread_hash_rate{engine=~\"$engine\", job_id=~\"$job_id\", thread_id=~\"$thread_id\"}[1m])", + "legendFormat": "{{engine}}, {{job_id}}, {{thread_id}}" } ], "fieldConfig": { @@ -160,11 +174,15 @@ "targets": [ { "refId": "A", - "expr": "increase(miner_hashes_total[5m])" + "expr": "increase(miner_hashes_total[1m])" } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, "orientation": "auto", "colorMode": "value", "graphMode": "none", @@ -184,7 +202,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (engine, job_id) (increase(miner_job_hashes_total{engine=~\"$engine\"}[5m]))" + "expr": "sum by (engine, job_id) (increase(miner_job_hashes_total{engine=~\"$engine\"}[1m]))", + "legendFormat": "{{engine}}, {{job_id}}" } ], "fieldConfig": { @@ -206,7 +225,7 @@ "targets": [ { "refId": "A", - "expr": "miner_job_status{engine=~\"$engine\", job_id=~\"$job_id\"}", + "expr": "max_over_time(miner_job_status{engine=~\"$engine\", job_id=~\"$job_id\"}[1m])", "legendFormat": "{{job_id}}:{{status}}" } ], diff --git a/docs/implementation/04-prompt-metrics-chunking-and-dashboards.md b/docs/implementation/04-prompt-metrics-chunking-and-dashboards.md new file mode 100644 index 0000000..e3ebda5 --- /dev/null +++ b/docs/implementation/04-prompt-metrics-chunking-and-dashboards.md @@ -0,0 +1,169 @@ +# 04 — Metrics Freshness, Chunking Knob, Remove-on-End, and Dashboards + +This document summarizes the prompts and responses since the last update (03), focused on making the miner’s metrics accurate, timely, and useful in Grafana for engine comparability and per-instance analysis. + +Scope covered: +- Fixing miner hash-rate metrics (job/global) so they reflect active work and show trend meaningfully +- Emitting periodic progress updates (chunking) and adding a time-based chunking knob +- Switching to remove-on-end for rate gauges to avoid scrape-timing artifacts +- Extending dashboards with cleaner labels, trend queries, 24h/1h peaks, current vs peak visuals, and a job filter +- Adding service-level metrics to distinguish “idle because no jobs” vs “actively mining” +- Prometheus scrape interval guidance + +--- + +## 1) What we observed + +- On resonance (busy), job/global hash rates appeared; on heisenberg (quiet), they often showed 0. +- miner_thread_hash_rate was present (thanks to per-thread EMA), but miner_job_hash_rate/miner_hash_rate often flatlined at 0. +- “Instances Comparison” dashboard had labels rendered as Prometheus label objects (e.g., `{instance="a1.i.res.fm"}`) and some panels were not informative. +- We needed trend-oriented panels for engine comparability, not just instantaneous gauges. + +Root causes: +- Engines reported results only at the end of large ranges, so totals/elapsed were stale most of the time. +- Job/global rates were computed as totals/elapsed and reset aggressively on job completion, leading to zeros or absent data. +- Thread EMA rates weren’t cleared on completion — this made thread charts look alive even when jobs weren’t running. +- Some dashboards summed the “instant” miner_hash_rate instead of a trend-friendly aggregate. + +--- + +## 2) Decisions + +- Emit periodic sub-range progress from each worker (chunking) for real-time metrics. +- Compute per-thread delta-rate (hash_count / Δt) and smooth with EMA; derive per-job and global rates from the sum of thread EMAs. +- Switch from “write zero on end” to “remove-on-end” for rate series to avoid scrape-timing artifacts and produce clean rollups. +- Add a time-based knob for progress chunking: `--progress-chunk-ms` (`MINER_PROGRESS_CHUNK_MS`). +- Adopt trend-oriented queries in Grafana (max_over_time/increase) to show recent performance (current vs. 1h peak, etc.). +- Add service-level metrics: + - `miner_active_jobs` (gauge) + - `miner_mine_requests_total{result}` (counter: accepted, duplicate, invalid, error) +- Add a job filter to dashboards to select a specific scrape job per chain (e.g., heisenberg vs. resonance). + +--- + +## 3) Implementation highlights + +A) Periodic progress updates (chunking) +- The mining loop (`mine_range_with_engine`) now splits the assigned range into sub-ranges (“chunks”). +- After each chunk, a `ThreadResult` is emitted with `hash_count` even if no solution is found. +- Default chunk size is derived from `--progress-chunk-ms`: + - `derived_chunk = max(5_000, est_ops_per_sec * progress_chunk_ms / 1000)`, with `est_ops_per_sec = 100_000` as a conservative starting point. +- This gives a steady cadence of per-thread updates and fresher job/global rates. + +B) Rates derived from thread EMAs +- For each `ThreadResult`, compute instantaneous rate = `hash_count / Δt`, then smooth with EMA (`THREAD_RATE_EMA_ALPHA = 0.2`). +- Publish `miner_thread_hash_rate{engine,job_id,thread_id}` = EMA. +- Per-job rate = sum of thread EMAs per job → `miner_job_hash_rate{engine,job_id}`. +- Global rate = sum of per-job rates across running jobs → `miner_hash_rate`. + +C) Remove-on-end semantics +- When a thread ends or a job transitions to Completed/Failed/Cancelled: + - Remove thread rate series: `remove_thread_hash_rate(engine, job_id, thread_id)`. + - Remove job rate series: `remove_job_hash_rate(engine, job_id)`. +- Rationale: this avoids scrape-race zero spikes and produces cleaner trend rollups. + +D) Metrics added +- `miner_active_jobs` (gauge) + - Increment on job accept, decrement on job cancel/remove. Reflects #running jobs “now”. +- `miner_mine_requests_total{result}` (counter) + - Increment per /mine request outcome: `accepted`, `duplicate`, `invalid`, `error`. +- Effect: makes it obvious when a miner is idle because the node isn’t issuing /mine (active_jobs = 0, requests not increasing) vs actively mining. + +E) CLI knob for chunking +- `--progress-chunk-ms` (env: `MINER_PROGRESS_CHUNK_MS`) sets target milliseconds for per-thread progress updates. +- Default: 2000 ms (if not provided). +- Use ~1000–2000 ms with `scrape_interval: 2s` for responsive charts. + +--- + +## 4) Prometheus/Grafana updates + +A) Scrape interval guidance +- Global: keep 15s. +- Miner jobs: override to 2s (or 1s for short experiments; 5s to reduce Prometheus load). +- Example per-job override: + ``` + scrape_configs: + - job_name: 'quantus-miner-resonance' + scrape_interval: 2s + scrape_timeout: 1.5s + scheme: https + metrics_path: /metrics/miner + static_configs: + - targets: [a1.i.res.fm, a2.i.res.fm, a3.i.res.fm] + - job_name: 'quantus-miner-heisenberg' + scrape_interval: 2s + scrape_timeout: 1.5s + scheme: https + metrics_path: /metrics/miner + static_configs: + - targets: [a1.t.res.fm, a2.t.res.fm, a3.t.res.fm] + ``` + +B) Dashboards +- Three JSONs under `docs/grafana/`: + - `miner-dashboard.json` (operator overview) + - `engines-comparison-dashboard.json` (baseline vs fast comparison) + - `instances-comparison-dashboard.json` (per-instance analysis) +- Label display cleanup: + - Legends use `{{engine}}, {{job_id}}, {{thread_id}}`, etc. (no `{label="value"}` surfacing). +- Trend-oriented queries: + - Use `max_over_time(...[1m])` for rates and `increase(...[1m])` for counters so panels show recent performance rather than just instantaneous snapshots. +- Peak vs Current visuals: + - Instances comparison now includes: + - A 24h peak stat (computed via PromQL subqueries: `[24h:1m]`) for trend-friendly peak potential. + - A per-instance table (current vs 1h peak + delta), with filters for job/instance/engine: + - Current: `sum by (instance) (max_over_time(miner_job_hash_rate{...}[1m]))` + - Peak 1h: `sum by (instance) (max_over_time(miner_job_hash_rate{...}[1h:1m]))` + - Delta column: `current / peak_1h` +- New panels: + - Active Jobs by instance: `miner_active_jobs` + - Mine Requests 5m increase by instance & result: `increase(miner_mine_requests_total{...}[5m])` + +--- + +## 5) Operational guidance + +- Scrape: + - Per-job: 2s scrape interval (1s for deep dives). +- Chunking: + - Start with `--progress-chunk-ms 2000` (or 1000 for more frequent updates). +- Engines: + - Compare `cpu-baseline` vs `cpu-fast` by setting different instances to different engines and using the Engines Comparison dashboard. +- Trend windows: + - Dashboards use `[1m]` windows for charts to reduce “zero gaps” while preserving short-time responsiveness. + - Peak windows can be adjusted (e.g., `[6h:1m]` or `[24h:1m]`) depending on what “recent performance” means for your use case. + +--- + +## 6) Next steps + +- Auto-tune chunk size: + - Estimate per-thread throughput in the first few chunks and adapt `derived_chunk` dynamically rather than relying on the static `est_ops_per_sec`. +- Better per-thread trend smoothing: + - Expose alpha as a CLI/env knob if operators want to tune EMA smoothing. +- Dashboard polish: + - Add a percent unit & color thresholds for the delta column; optionally include sparkline mini-charts. +- Alerts/recording rules: + - Record per-engine/per-instance hash rate aggregates and set basic alerts (e.g., “no jobs for X minutes” or “hash rate below expected baseline for Y minutes”). + +--- + +## 7) Quick reference + +- CLI: + - `--engine cpu-baseline|cpu-fast` + - `--progress-chunk-ms ` + - `--metrics-port ` + - `--cores ` (alias: `--num-cores`) +- Service metrics added: + - `miner_active_jobs` (gauge) + - `miner_mine_requests_total{result}` (counter) +- Remove-on-end: + - Thread/job rate gauges removed (series deleted) when completed/failed/cancelled. +- Grafana JSONs: + - `docs/grafana/miner-dashboard.json` + - `docs/grafana/engines-comparison-dashboard.json` + - `docs/grafana/instances-comparison-dashboard.json` + +These changes make the miner’s metrics fresher, more accurate, and easy to compare across engines and instances, while avoiding scrape artifacts and providing the context needed to explain “0 ops/s” situations on quiet networks. \ No newline at end of file diff --git a/docs/implementation/04-response-metrics-chunking-and-dashboards.md b/docs/implementation/04-response-metrics-chunking-and-dashboards.md new file mode 100644 index 0000000..c837e91 --- /dev/null +++ b/docs/implementation/04-response-metrics-chunking-and-dashboards.md @@ -0,0 +1,165 @@ +# 04 — Response Summary: Metrics Freshness, Chunking Knob, Remove-on-End, and Dashboards + +This document summarises what was implemented since iteration 03 to improve metric accuracy, responsiveness, and usability for engine/instance comparisons. + +## Overview + +We addressed four main areas: +1. Metrics freshness: emit periodic progress updates from workers so job/global hash rates update in near‑real time. +2. Accuracy and artifact reduction: switch to remove‑on‑end for rate series (instead of writing zeros) to avoid scrape-timing spikes. +3. Operator control: add a time‑based chunking knob to tune the heartbeat cadence without code changes. +4. Observability and UX: extend Prometheus metrics and update Grafana dashboards to show meaningful trends and to disambiguate “idle because no jobs” vs “actively mining”. + +--- + +## Code Changes + +### 1) Periodic progress updates (chunking) + +- Mining worker now splits its assigned nonce range into time‑targeted “chunks.” +- After each chunk completes, the worker sends a `ThreadResult` with `hash_count` even if it didn’t find a solution. +- Default chunk size derived from a time target: + - `derived_chunk ≈ max(5_000, est_ops_per_sec * progress_chunk_ms / 1000)` + - Using a conservative `est_ops_per_sec = 100_000` as an initial constant. +- Effect: Prometheus/Grafana receive frequent progress updates so per‑job/global rates are responsive while work is ongoing. + +### 2) Per-thread delta-rate with EMA smoothing, per‑job/global derived from threads + +- On each `ThreadResult`, compute per‑thread instantaneous rate: + - `instant_rate = hash_count / Δt`, then smooth: + - `ema = alpha * instant_rate + (1 - alpha) * prev_ema` with `THREAD_RATE_EMA_ALPHA = 0.2`. +- Publish `miner_thread_hash_rate{engine,job_id,thread_id} = ema` (gauge). +- Per‑job rate = sum of thread EMAs → `miner_job_hash_rate{engine,job_id}` (gauge). +- Global rate = sum of per‑job rates across running jobs → `miner_hash_rate` (gauge). + +### 3) Remove-on-end for rate series + +- When a thread completes or a job transitions to Completed/Failed/Cancelled: + - Remove the corresponding gauge series instead of writing zeros: + - `remove_thread_hash_rate(engine, job_id, thread_id)` + - `remove_job_hash_rate(engine, job_id)` +- Rationale: avoids scrape timing artifacts (spurious zeros) and yields cleaner trend rollups with range functions. + +### 4) Time‑based chunking knob + +- CLI/env: + - `--progress-chunk-ms ` (env: `MINER_PROGRESS_CHUNK_MS`) + - Default (if omitted): 2000ms +- Use ~1000–2000ms at a 2s Prometheus scrape interval for smooth charts and low overhead. + +### 5) Service-level metrics + +New Prometheus series: +- `miner_active_jobs` (gauge): number of currently running jobs in the miner. +- `miner_mine_requests_total{result}` (counter): counts `/mine` requests by `result`: + - `accepted`, `duplicate`, `invalid`, `error`. + +Wiring: +- On job accept: `active_jobs + 1`, `mine_requests_total{result="accepted"} + 1` +- On job cancel/remove: `active_jobs - 1` +- On invalid request: `mine_requests_total{result="invalid"} + 1` +- On duplicate or other add-job error: `result="duplicate"` or `result="error"` accordingly. + +--- + +## Dashboard Updates (Grafana JSONs in `docs/grafana/`) + +We maintained a clean, value‑only label style (e.g., `a1.i.res.fm`, `cpu-baseline`, `job_id`, `thread_id`), and adopted short range window functions to show trends rather than purely instantaneous views. + +### 1) Miner Overview — `miner-dashboard.json` + +- Global Hash Rate (over time): + - `sum(max_over_time(miner_job_hash_rate[1m]))` +- Jobs by Engine and Status: + - `sum by (engine, status) (increase(miner_jobs_by_engine_total[1m]))` +- Thread Hash Rate: + - `max_over_time(miner_thread_hash_rate[1m])` +- Total Hashes / Per-Job Hashes: + - `increase(...[1m])` +- Job Status: + - `max_over_time(miner_job_status[1m])` + +### 2) Engines Comparison — `engines-comparison-dashboard.json` + +- Hash Rate by Engine (trend): + - `sum by (engine) (max_over_time(miner_job_hash_rate{engine=~"$engine"}[1m]))` +- Jobs by Engine and Status: + - `sum by (engine, status) (increase(miner_jobs_by_engine_total{engine=~"$engine"}[1m]))` +- Per-Engine Total Hashes: + - `sum by (engine) (increase(miner_job_hashes_total{engine=~"$engine"}[1m]))` +- Per-Job / Per-Thread (trend): + - `max_over_time(miner_job_hash_rate{...}[1m])` + - `max_over_time(miner_thread_hash_rate{...}[1m])` + +### 3) Instances Comparison — `instances-comparison-dashboard.json` + +- Added a `job` filter variable (multi‑select) alongside `instance` and `engine`, and applied `job=~"$job"` across queries to filter by scrape job (e.g., chain). +- Global Hash Rate by Instance (trend): + - `sum by (instance) (max_over_time(miner_job_hash_rate{job=~"$job", instance=~"$instance"}[1m]))` +- Per‑Instance Job Hash Rate (trend): + - `sum by (instance) (max_over_time(miner_job_hash_rate{job=~"$job", instance=~"$instance", engine=~"$engine"}[1m]))` +- Current vs 1h Peak (per instance) — converted to a table with delta: + - Current (1m max): + - `sum by (instance) (max_over_time(miner_job_hash_rate{job=~"$job", instance=~"$instance", engine=~"$engine"}[1m]))` + - Peak 1h (1m resolution subquery): + - `sum by (instance) (max_over_time(miner_job_hash_rate{job=~"$job", instance=~"$instance", engine=~"$engine"}[1h:1m]))` + - Delta: + - Calculated as `current / peak_1h` via a table transformation (`calculateField`). + - Columns: `instance | current_ops | peak_1h_ops | delta_current_over_peak` +- 24h Peak Hash Rate (selected instances): + - `sum by (instance) (max_over_time(miner_job_hash_rate{job=~"$job", instance=~"$instance"}[24h:1m]))` +- New panels to disambiguate idle vs active: + - Active Jobs (by instance): + - `sum by (instance) (miner_active_jobs{job=~"$job", instance=~"$instance"})` + - Mine Requests (5m increase; by instance & result): + - `sum by (instance, result) (increase(miner_mine_requests_total{job=~"$job", instance=~"$instance"}[5m]))` + +--- + +## Prometheus Settings + +- Keep global defaults (e.g., `scrape_interval: 15s`), but override miner jobs to 2s: + - `scrape_interval: 2s`, `scrape_timeout: 1.5s` +- For short-lived performance experiments, consider 1s; for load-sensitive prod, 5s works with chunking still providing reasonable freshness. +- Align `--progress-chunk-ms` with your scrape interval (e.g., chunk ~1000–2000ms for a 2s scrape). + +--- + +## Operational Results + +- Job/global rates are now responsive while work is active and disappear cleanly after completion. +- Trend panels (using `max_over_time` / `increase`) provide a meaningful “recent performance” view for engine/instance comparisons. +- New service-level metrics (`miner_active_jobs`, `miner_mine_requests_total`) make it obvious when a miner is idle because no jobs are being issued (e.g., on a quiet chain). +- Label formatting is consistent and readable across dashboards. + +--- + +## Next Actions (Optional Enhancements) + +- Auto‑tune chunk size based on observed per‑thread throughput in the first chunks. +- Expose EMA alpha (smoothing) as a knob for advanced tuning. +- Add field overrides to render the delta column as a percentage with thresholds. +- Provide baseline alerts/recording rules, e.g.: + - “No active_jobs for X minutes” (node not issuing /mine). + - “Hash rate below expected baseline for Y minutes.” + +--- + +## Quick Reference + +- CLI: + - `--engine cpu-baseline|cpu-fast` + - `--progress-chunk-ms ` (env: `MINER_PROGRESS_CHUNK_MS`) + - `--metrics-port ` + - `--cores ` (alias: `--num-cores`) +- New metrics: + - `miner_active_jobs` (gauge) + - `miner_mine_requests_total{result}` (counter) +- Rate series handling: + - Per-thread and per-job rate series are removed on end, avoiding zero artifacts. +- Dashboards: + - `miner-dashboard.json` — Operator overview + - `engines-comparison-dashboard.json` — Engine comparability + - `instances-comparison-dashboard.json` — Per-instance analysis with `job` filter + +These changes yield accurate, timely, and comparative metrics for evaluating engine performance and instance health across your testnets and (eventually) production. \ No newline at end of file