diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index 61d6248a6e..14ad832dc6 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -5,6 +5,8 @@ use crate::events::shared::log_event; use crate::events::shared::trace_event; use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_DURATION_METRIC; +use crate::metrics::EXEC_COMMAND_COUNT_METRIC; +use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_FAILURE_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC; use crate::metrics::MetricsClient; @@ -1334,6 +1336,20 @@ impl SessionTelemetry { Err(error) => (error.to_string(), false), }; + if tool_name.is_default_namespace() && tool_name.name == "exec_command" { + // Use the full handler interval: shell setup/startup and command work until + // exit or the first yield. Direct tool_result_with_tags calls also report + // pre-handler rejections, which must not become zero-latency samples here. + let mut tags = extra_tags.to_vec(); + tags.push(("success", if success { "true" } else { "false" })); + if let Some(product_sku) = self.metadata.product_sku { + tags.push(("product_sku", product_sku)); + } + let duration_ms = duration.as_millis().min(i64::MAX as u128) as i64; + self.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, duration_ms, &tags); + self.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &tags); + } + self.tool_result_with_tags( tool_name, call_id, diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs index f8d9374f69..1ca5d1f425 100644 --- a/codex-rs/otel/src/metrics/names.rs +++ b/codex-rs/otel/src/metrics/names.rs @@ -2,6 +2,11 @@ use std::sync::LazyLock; pub const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call"; pub const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms"; +/// Paired Statsig-exported counters for exec-command handler latency, including +/// shell startup through exit or the first yield. Divide total milliseconds by +/// calls to compute a mean without averaging histogram export batches. +pub const EXEC_COMMAND_DURATION_TOTAL_METRIC: &str = "codex.exec_command.duration_total_ms"; +pub const EXEC_COMMAND_COUNT_METRIC: &str = "codex.exec_command.calls"; pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec"; pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure"; pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str = diff --git a/codex-rs/otel/src/provider.rs b/codex-rs/otel/src/provider.rs index 1b4ecffdc7..b0f81b0ba8 100644 --- a/codex-rs/otel/src/provider.rs +++ b/codex-rs/otel/src/provider.rs @@ -705,6 +705,8 @@ mod tests { use super::*; use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_DURATION_METRIC; + use crate::metrics::EXEC_COMMAND_COUNT_METRIC; + use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC; use crate::metrics::EXEC_SERVER_CLIENT_REQUEST_COUNT_METRIC; use crate::metrics::MetricsExporter; use crate::metrics::RESPONSES_API_ENGINE_IAPI_TTFT_DURATION_METRIC; @@ -895,6 +897,8 @@ mod tests { )?; metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?; + metrics.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, /*inc*/ 25, &[])?; + metrics.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?; metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?; metrics.counter("codex.turns", /*inc*/ 1, &[])?; @@ -909,7 +913,14 @@ mod tests { .collect(); names.sort_unstable(); names.dedup(); - assert_eq!(names, vec!["codex.turns"]); + assert_eq!( + names, + vec![ + EXEC_COMMAND_COUNT_METRIC, + EXEC_COMMAND_DURATION_TOTAL_METRIC, + "codex.turns" + ], + ); Ok(()) } diff --git a/codex-rs/otel/tests/suite/manager_metrics.rs b/codex-rs/otel/tests/suite/manager_metrics.rs index 653ed5be36..d0e9818ee2 100644 --- a/codex-rs/otel/tests/suite/manager_metrics.rs +++ b/codex-rs/otel/tests/suite/manager_metrics.rs @@ -3,6 +3,8 @@ use crate::harness::build_metrics_with_defaults; use crate::harness::find_metric; use crate::harness::histogram_data; use crate::harness::latest_metrics; +use codex_otel::EXEC_COMMAND_COUNT_METRIC; +use codex_otel::EXEC_COMMAND_DURATION_TOTAL_METRIC; use codex_otel::MULTI_AGENT_SPAWN_FAILURE_METRIC; use codex_otel::PLUGIN_INSTALL_ELICITATION_SENT_METRIC; use codex_otel::PLUGIN_INSTALL_SUGGESTION_METRIC; @@ -22,6 +24,146 @@ use pretty_assertions::assert_eq; use std::collections::BTreeMap; use std::time::Duration; +#[tokio::test] +async fn exec_command_latency_keeps_snapshot_cohorts_and_the_full_tool_interval() -> Result<()> { + let (metrics, exporter) = build_metrics_with_defaults(&[])?; + let manager = SessionTelemetry::new( + ThreadId::new(), + "gpt-5.1", + "gpt-5.1", + /*account_id*/ None, + /*account_email*/ None, + /*auth_mode*/ None, + "test_originator".to_string(), + /*log_user_prompts*/ false, + "tty".to_string(), + SessionSource::Cli, + ) + .with_metrics_without_metadata_tags(metrics) + .with_product_sku(Some("codex")); + + for (tool, mode, result) in [ + (ToolName::plain("exec_command"), "v1", Ok(true)), + ( + ToolName::namespaced("functions", "exec_command"), + "v1", + Ok(true), + ), + (ToolName::plain("exec_command"), "v2", Ok(false)), + (ToolName::plain("exec_command"), "v2", Err("launch failed")), + (ToolName::plain("write_stdin"), "v1", Ok(true)), + ( + ToolName::namespaced("mcp__example", "exec_command"), + "v2", + Ok(true), + ), + ] { + assert_eq!( + manager + .log_tool_result_with_tags( + &tool, + "call-1", + "{}", + &[("shell_snapshot_mode", mode)], + &[], + || async { + // Include awaited handler work so a zero-duration regression fails. + tokio::time::sleep(Duration::from_millis(5)).await; + result + }, + |success| (String::new(), *success), + ) + .await, + result, + ); + } + manager.tool_result_with_tags( + &ToolName::plain("exec_command"), + "rejected-before-handler", + "{}", + Duration::ZERO, + /*success*/ false, + "rejected", + &[("shell_snapshot_mode", "v1")], + &[], + ); + manager.shutdown_metrics()?; + let resource_metrics = latest_metrics(&exporter); + let histogram_metric = + find_metric(&resource_metrics, TOOL_CALL_DURATION_METRIC).expect("tool duration metric"); + let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = histogram_metric.data() else { + panic!("expected duration histogram"); + }; + let expected = histogram + .data_points() + .filter_map(|point| { + let mut tags = attributes_to_map(point.attributes()); + if tags.remove("tool").as_deref() != Some("exec_command") + || (tags["shell_snapshot_mode"] == "v1" && tags["success"] == "false") + { + return None; + } + Some((tags, (point.count(), point.sum() as u64))) + }) + .collect::>(); + let mut snapshots = BTreeMap::new(); + for name in [ + EXEC_COMMAND_COUNT_METRIC, + EXEC_COMMAND_DURATION_TOTAL_METRIC, + ] { + let metric = find_metric(&resource_metrics, name).expect("exec counter"); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { + panic!("expected counter"); + }; + let points = sum + .data_points() + .map(|point| (attributes_to_map(point.attributes()), point.value())) + .collect::>(); + snapshots.insert(name, points); + } + assert_eq!( + snapshots[EXEC_COMMAND_COUNT_METRIC], + expected + .iter() + .map(|(tags, (count, _))| (tags.clone(), *count)) + .collect(), + ); + assert_eq!( + snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC], + expected + .iter() + .map(|(tags, (_, total_ms))| (tags.clone(), *total_ms)) + .collect(), + ); + assert!( + snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC] + .values() + .all(|total_ms| *total_ms > 0) + ); + assert_eq!( + snapshots[EXEC_COMMAND_COUNT_METRIC], + BTreeMap::from([ + ( + BTreeMap::from([ + ("product_sku".to_string(), "codex".to_string()), + ("shell_snapshot_mode".to_string(), "v1".to_string()), + ("success".to_string(), "true".to_string()), + ]), + 2 + ), + ( + BTreeMap::from([ + ("product_sku".to_string(), "codex".to_string()), + ("shell_snapshot_mode".to_string(), "v2".to_string()), + ("success".to_string(), "false".to_string()), + ]), + 2 + ), + ]), + ); + Ok(()) +} + #[test] fn tool_metrics_keep_product_skus_separate_on_a_shared_client() -> Result<()> { let (metrics, exporter) = build_metrics_with_defaults(&[])?;