From 39c29c3bb620f91e84a3bb266bb496dfb697c101 Mon Sep 17 00:00:00 2001 From: jif Date: Thu, 19 Nov 2026 13:06:34 +0000 Subject: [PATCH] Add Statsig counters for `exec_command` latency (#62507) ## Why Dividing total handler time by the call count gives a per-call mean without averaging histogram export batches. ## What changed Record `codex.exec_command.duration_total_ms` and `codex.exec_command.calls` for built-in `exec_command` handler runs, including failures. Measure from handler entry through command exit or the first yield, including shell setup and startup. Exclude calls rejected before the handler runs and retain tags such as `shell_snapshot_mode`, `success`, and `product_sku` when available. ## Testing Cover Statsig export of both counters and check that their counts and duration totals match the tool duration histogram for eligible calls within each tag set. GitOrigin-RevId: 4bf6dc628bca5758999bf0ed5f67051f870a61e8 --- codex-rs/otel/src/events/session_telemetry.rs | 16 ++ codex-rs/otel/src/metrics/names.rs | 5 + codex-rs/otel/src/provider.rs | 13 +- codex-rs/otel/tests/suite/manager_metrics.rs | 142 ++++++++++++++++++ 4 files changed, 175 insertions(+), 1 deletion(-) diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index 61d6248a6e..14ad832dc6 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -5,6 +5,8 @@ use crate::events::shared::log_event; use crate::events::shared::trace_event; use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_DURATION_METRIC; +use crate::metrics::EXEC_COMMAND_COUNT_METRIC; +use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_FAILURE_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC; use crate::metrics::MetricsClient; @@ -1334,6 +1336,20 @@ impl SessionTelemetry { Err(error) => (error.to_string(), false), }; + if tool_name.is_default_namespace() && tool_name.name == "exec_command" { + // Use the full handler interval: shell setup/startup and command work until + // exit or the first yield. Direct tool_result_with_tags calls also report + // pre-handler rejections, which must not become zero-latency samples here. + let mut tags = extra_tags.to_vec(); + tags.push(("success", if success { "true" } else { "false" })); + if let Some(product_sku) = self.metadata.product_sku { + tags.push(("product_sku", product_sku)); + } + let duration_ms = duration.as_millis().min(i64::MAX as u128) as i64; + self.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, duration_ms, &tags); + self.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &tags); + } + self.tool_result_with_tags( tool_name, call_id, diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs index f8d9374f69..1ca5d1f425 100644 --- a/codex-rs/otel/src/metrics/names.rs +++ b/codex-rs/otel/src/metrics/names.rs @@ -2,6 +2,11 @@ use std::sync::LazyLock; pub const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call"; pub const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms"; +/// Paired Statsig-exported counters for exec-command handler latency, including +/// shell startup through exit or the first yield. Divide total milliseconds by +/// calls to compute a mean without averaging histogram export batches. +pub const EXEC_COMMAND_DURATION_TOTAL_METRIC: &str = "codex.exec_command.duration_total_ms"; +pub const EXEC_COMMAND_COUNT_METRIC: &str = "codex.exec_command.calls"; pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec"; pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure"; pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str = diff --git a/codex-rs/otel/src/provider.rs b/codex-rs/otel/src/provider.rs index 1b4ecffdc7..b0f81b0ba8 100644 --- a/codex-rs/otel/src/provider.rs +++ b/codex-rs/otel/src/provider.rs @@ -705,6 +705,8 @@ mod tests { use super::*; use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_DURATION_METRIC; + use crate::metrics::EXEC_COMMAND_COUNT_METRIC; + use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC; use crate::metrics::EXEC_SERVER_CLIENT_REQUEST_COUNT_METRIC; use crate::metrics::MetricsExporter; use crate::metrics::RESPONSES_API_ENGINE_IAPI_TTFT_DURATION_METRIC; @@ -895,6 +897,8 @@ mod tests { )?; metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?; + metrics.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, /*inc*/ 25, &[])?; + metrics.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?; metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?; metrics.counter("codex.turns", /*inc*/ 1, &[])?; @@ -909,7 +913,14 @@ mod tests { .collect(); names.sort_unstable(); names.dedup(); - assert_eq!(names, vec!["codex.turns"]); + assert_eq!( + names, + vec![ + EXEC_COMMAND_COUNT_METRIC, + EXEC_COMMAND_DURATION_TOTAL_METRIC, + "codex.turns" + ], + ); Ok(()) } diff --git a/codex-rs/otel/tests/suite/manager_metrics.rs b/codex-rs/otel/tests/suite/manager_metrics.rs index 653ed5be36..d0e9818ee2 100644 --- a/codex-rs/otel/tests/suite/manager_metrics.rs +++ b/codex-rs/otel/tests/suite/manager_metrics.rs @@ -3,6 +3,8 @@ use crate::harness::build_metrics_with_defaults; use crate::harness::find_metric; use crate::harness::histogram_data; use crate::harness::latest_metrics; +use codex_otel::EXEC_COMMAND_COUNT_METRIC; +use codex_otel::EXEC_COMMAND_DURATION_TOTAL_METRIC; use codex_otel::MULTI_AGENT_SPAWN_FAILURE_METRIC; use codex_otel::PLUGIN_INSTALL_ELICITATION_SENT_METRIC; use codex_otel::PLUGIN_INSTALL_SUGGESTION_METRIC; @@ -22,6 +24,146 @@ use pretty_assertions::assert_eq; use std::collections::BTreeMap; use std::time::Duration; +#[tokio::test] +async fn exec_command_latency_keeps_snapshot_cohorts_and_the_full_tool_interval() -> Result<()> { + let (metrics, exporter) = build_metrics_with_defaults(&[])?; + let manager = SessionTelemetry::new( + ThreadId::new(), + "gpt-5.1", + "gpt-5.1", + /*account_id*/ None, + /*account_email*/ None, + /*auth_mode*/ None, + "test_originator".to_string(), + /*log_user_prompts*/ false, + "tty".to_string(), + SessionSource::Cli, + ) + .with_metrics_without_metadata_tags(metrics) + .with_product_sku(Some("codex")); + + for (tool, mode, result) in [ + (ToolName::plain("exec_command"), "v1", Ok(true)), + ( + ToolName::namespaced("functions", "exec_command"), + "v1", + Ok(true), + ), + (ToolName::plain("exec_command"), "v2", Ok(false)), + (ToolName::plain("exec_command"), "v2", Err("launch failed")), + (ToolName::plain("write_stdin"), "v1", Ok(true)), + ( + ToolName::namespaced("mcp__example", "exec_command"), + "v2", + Ok(true), + ), + ] { + assert_eq!( + manager + .log_tool_result_with_tags( + &tool, + "call-1", + "{}", + &[("shell_snapshot_mode", mode)], + &[], + || async { + // Include awaited handler work so a zero-duration regression fails. + tokio::time::sleep(Duration::from_millis(5)).await; + result + }, + |success| (String::new(), *success), + ) + .await, + result, + ); + } + manager.tool_result_with_tags( + &ToolName::plain("exec_command"), + "rejected-before-handler", + "{}", + Duration::ZERO, + /*success*/ false, + "rejected", + &[("shell_snapshot_mode", "v1")], + &[], + ); + manager.shutdown_metrics()?; + let resource_metrics = latest_metrics(&exporter); + let histogram_metric = + find_metric(&resource_metrics, TOOL_CALL_DURATION_METRIC).expect("tool duration metric"); + let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = histogram_metric.data() else { + panic!("expected duration histogram"); + }; + let expected = histogram + .data_points() + .filter_map(|point| { + let mut tags = attributes_to_map(point.attributes()); + if tags.remove("tool").as_deref() != Some("exec_command") + || (tags["shell_snapshot_mode"] == "v1" && tags["success"] == "false") + { + return None; + } + Some((tags, (point.count(), point.sum() as u64))) + }) + .collect::>(); + let mut snapshots = BTreeMap::new(); + for name in [ + EXEC_COMMAND_COUNT_METRIC, + EXEC_COMMAND_DURATION_TOTAL_METRIC, + ] { + let metric = find_metric(&resource_metrics, name).expect("exec counter"); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { + panic!("expected counter"); + }; + let points = sum + .data_points() + .map(|point| (attributes_to_map(point.attributes()), point.value())) + .collect::>(); + snapshots.insert(name, points); + } + assert_eq!( + snapshots[EXEC_COMMAND_COUNT_METRIC], + expected + .iter() + .map(|(tags, (count, _))| (tags.clone(), *count)) + .collect(), + ); + assert_eq!( + snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC], + expected + .iter() + .map(|(tags, (_, total_ms))| (tags.clone(), *total_ms)) + .collect(), + ); + assert!( + snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC] + .values() + .all(|total_ms| *total_ms > 0) + ); + assert_eq!( + snapshots[EXEC_COMMAND_COUNT_METRIC], + BTreeMap::from([ + ( + BTreeMap::from([ + ("product_sku".to_string(), "codex".to_string()), + ("shell_snapshot_mode".to_string(), "v1".to_string()), + ("success".to_string(), "true".to_string()), + ]), + 2 + ), + ( + BTreeMap::from([ + ("product_sku".to_string(), "codex".to_string()), + ("shell_snapshot_mode".to_string(), "v2".to_string()), + ("success".to_string(), "false".to_string()), + ]), + 2 + ), + ]), + ); + Ok(()) +} + #[test] fn tool_metrics_keep_product_skus_separate_on_a_shared_client() -> Result<()> { let (metrics, exporter) = build_metrics_with_defaults(&[])?;