Add Statsig counters for exec_command latency (#62507)

## Why

Dividing total handler time by the call count gives a per-call mean
without averaging histogram export batches.

## What changed

Record `codex.exec_command.duration_total_ms` and
`codex.exec_command.calls` for built-in `exec_command` handler runs,
including failures. Measure from handler entry through command exit or
the first yield, including shell setup and startup. Exclude calls rejected
before the handler runs and retain tags such as `shell_snapshot_mode`,
`success`, and `product_sku` when available.

## Testing

Cover Statsig export of both counters and check that their counts and
duration totals match the tool duration histogram for eligible calls
within each tag set.

GitOrigin-RevId: 4bf6dc628bca5758999bf0ed5f67051f870a61e8
This commit is contained in:
jif
2026-11-19 13:18:44 +00:00
committed by copyberry
parent 1826d80eca
commit 39c29c3bb6
4 changed files with 175 additions and 1 deletions
@@ -5,6 +5,8 @@ use crate::events::shared::log_event;
use crate::events::shared::trace_event; use crate::events::shared::trace_event;
use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_COUNT_METRIC;
use crate::metrics::API_CALL_DURATION_METRIC; use crate::metrics::API_CALL_DURATION_METRIC;
use crate::metrics::EXEC_COMMAND_COUNT_METRIC;
use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC;
use crate::metrics::MULTI_AGENT_SPAWN_FAILURE_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_FAILURE_METRIC;
use crate::metrics::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC; use crate::metrics::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC;
use crate::metrics::MetricsClient; use crate::metrics::MetricsClient;
@@ -1334,6 +1336,20 @@ impl SessionTelemetry {
Err(error) => (error.to_string(), false), Err(error) => (error.to_string(), false),
}; };
if tool_name.is_default_namespace() && tool_name.name == "exec_command" {
// Use the full handler interval: shell setup/startup and command work until
// exit or the first yield. Direct tool_result_with_tags calls also report
// pre-handler rejections, which must not become zero-latency samples here.
let mut tags = extra_tags.to_vec();
tags.push(("success", if success { "true" } else { "false" }));
if let Some(product_sku) = self.metadata.product_sku {
tags.push(("product_sku", product_sku));
}
let duration_ms = duration.as_millis().min(i64::MAX as u128) as i64;
self.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, duration_ms, &tags);
self.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &tags);
}
self.tool_result_with_tags( self.tool_result_with_tags(
tool_name, tool_name,
call_id, call_id,
+5
View File
@@ -2,6 +2,11 @@ use std::sync::LazyLock;
pub const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call"; pub const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call";
pub const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms"; pub const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms";
/// Paired Statsig-exported counters for exec-command handler latency, including
/// shell startup through exit or the first yield. Divide total milliseconds by
/// calls to compute a mean without averaging histogram export batches.
pub const EXEC_COMMAND_DURATION_TOTAL_METRIC: &str = "codex.exec_command.duration_total_ms";
pub const EXEC_COMMAND_COUNT_METRIC: &str = "codex.exec_command.calls";
pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec"; pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec";
pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure"; pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure";
pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str = pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str =
+12 -1
View File
@@ -705,6 +705,8 @@ mod tests {
use super::*; use super::*;
use crate::metrics::API_CALL_COUNT_METRIC; use crate::metrics::API_CALL_COUNT_METRIC;
use crate::metrics::API_CALL_DURATION_METRIC; use crate::metrics::API_CALL_DURATION_METRIC;
use crate::metrics::EXEC_COMMAND_COUNT_METRIC;
use crate::metrics::EXEC_COMMAND_DURATION_TOTAL_METRIC;
use crate::metrics::EXEC_SERVER_CLIENT_REQUEST_COUNT_METRIC; use crate::metrics::EXEC_SERVER_CLIENT_REQUEST_COUNT_METRIC;
use crate::metrics::MetricsExporter; use crate::metrics::MetricsExporter;
use crate::metrics::RESPONSES_API_ENGINE_IAPI_TTFT_DURATION_METRIC; use crate::metrics::RESPONSES_API_ENGINE_IAPI_TTFT_DURATION_METRIC;
@@ -895,6 +897,8 @@ mod tests {
)?; )?;
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?; metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
metrics.counter(EXEC_COMMAND_DURATION_TOTAL_METRIC, /*inc*/ 25, &[])?;
metrics.counter(EXEC_COMMAND_COUNT_METRIC, /*inc*/ 1, &[])?;
metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?; metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?;
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?; metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
metrics.counter("codex.turns", /*inc*/ 1, &[])?; metrics.counter("codex.turns", /*inc*/ 1, &[])?;
@@ -909,7 +913,14 @@ mod tests {
.collect(); .collect();
names.sort_unstable(); names.sort_unstable();
names.dedup(); names.dedup();
assert_eq!(names, vec!["codex.turns"]); assert_eq!(
names,
vec![
EXEC_COMMAND_COUNT_METRIC,
EXEC_COMMAND_DURATION_TOTAL_METRIC,
"codex.turns"
],
);
Ok(()) Ok(())
} }
@@ -3,6 +3,8 @@ use crate::harness::build_metrics_with_defaults;
use crate::harness::find_metric; use crate::harness::find_metric;
use crate::harness::histogram_data; use crate::harness::histogram_data;
use crate::harness::latest_metrics; use crate::harness::latest_metrics;
use codex_otel::EXEC_COMMAND_COUNT_METRIC;
use codex_otel::EXEC_COMMAND_DURATION_TOTAL_METRIC;
use codex_otel::MULTI_AGENT_SPAWN_FAILURE_METRIC; use codex_otel::MULTI_AGENT_SPAWN_FAILURE_METRIC;
use codex_otel::PLUGIN_INSTALL_ELICITATION_SENT_METRIC; use codex_otel::PLUGIN_INSTALL_ELICITATION_SENT_METRIC;
use codex_otel::PLUGIN_INSTALL_SUGGESTION_METRIC; use codex_otel::PLUGIN_INSTALL_SUGGESTION_METRIC;
@@ -22,6 +24,146 @@ use pretty_assertions::assert_eq;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::Duration; use std::time::Duration;
#[tokio::test]
async fn exec_command_latency_keeps_snapshot_cohorts_and_the_full_tool_interval() -> Result<()> {
let (metrics, exporter) = build_metrics_with_defaults(&[])?;
let manager = SessionTelemetry::new(
ThreadId::new(),
"gpt-5.1",
"gpt-5.1",
/*account_id*/ None,
/*account_email*/ None,
/*auth_mode*/ None,
"test_originator".to_string(),
/*log_user_prompts*/ false,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics_without_metadata_tags(metrics)
.with_product_sku(Some("codex"));
for (tool, mode, result) in [
(ToolName::plain("exec_command"), "v1", Ok(true)),
(
ToolName::namespaced("functions", "exec_command"),
"v1",
Ok(true),
),
(ToolName::plain("exec_command"), "v2", Ok(false)),
(ToolName::plain("exec_command"), "v2", Err("launch failed")),
(ToolName::plain("write_stdin"), "v1", Ok(true)),
(
ToolName::namespaced("mcp__example", "exec_command"),
"v2",
Ok(true),
),
] {
assert_eq!(
manager
.log_tool_result_with_tags(
&tool,
"call-1",
"{}",
&[("shell_snapshot_mode", mode)],
&[],
|| async {
// Include awaited handler work so a zero-duration regression fails.
tokio::time::sleep(Duration::from_millis(5)).await;
result
},
|success| (String::new(), *success),
)
.await,
result,
);
}
manager.tool_result_with_tags(
&ToolName::plain("exec_command"),
"rejected-before-handler",
"{}",
Duration::ZERO,
/*success*/ false,
"rejected",
&[("shell_snapshot_mode", "v1")],
&[],
);
manager.shutdown_metrics()?;
let resource_metrics = latest_metrics(&exporter);
let histogram_metric =
find_metric(&resource_metrics, TOOL_CALL_DURATION_METRIC).expect("tool duration metric");
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = histogram_metric.data() else {
panic!("expected duration histogram");
};
let expected = histogram
.data_points()
.filter_map(|point| {
let mut tags = attributes_to_map(point.attributes());
if tags.remove("tool").as_deref() != Some("exec_command")
|| (tags["shell_snapshot_mode"] == "v1" && tags["success"] == "false")
{
return None;
}
Some((tags, (point.count(), point.sum() as u64)))
})
.collect::<BTreeMap<_, _>>();
let mut snapshots = BTreeMap::new();
for name in [
EXEC_COMMAND_COUNT_METRIC,
EXEC_COMMAND_DURATION_TOTAL_METRIC,
] {
let metric = find_metric(&resource_metrics, name).expect("exec counter");
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("expected counter");
};
let points = sum
.data_points()
.map(|point| (attributes_to_map(point.attributes()), point.value()))
.collect::<BTreeMap<_, _>>();
snapshots.insert(name, points);
}
assert_eq!(
snapshots[EXEC_COMMAND_COUNT_METRIC],
expected
.iter()
.map(|(tags, (count, _))| (tags.clone(), *count))
.collect(),
);
assert_eq!(
snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC],
expected
.iter()
.map(|(tags, (_, total_ms))| (tags.clone(), *total_ms))
.collect(),
);
assert!(
snapshots[EXEC_COMMAND_DURATION_TOTAL_METRIC]
.values()
.all(|total_ms| *total_ms > 0)
);
assert_eq!(
snapshots[EXEC_COMMAND_COUNT_METRIC],
BTreeMap::from([
(
BTreeMap::from([
("product_sku".to_string(), "codex".to_string()),
("shell_snapshot_mode".to_string(), "v1".to_string()),
("success".to_string(), "true".to_string()),
]),
2
),
(
BTreeMap::from([
("product_sku".to_string(), "codex".to_string()),
("shell_snapshot_mode".to_string(), "v2".to_string()),
("success".to_string(), "false".to_string()),
]),
2
),
]),
);
Ok(())
}
#[test] #[test]
fn tool_metrics_keep_product_skus_separate_on_a_shared_client() -> Result<()> { fn tool_metrics_keep_product_skus_separate_on_a_shared_client() -> Result<()> {
let (metrics, exporter) = build_metrics_with_defaults(&[])?; let (metrics, exporter) = build_metrics_with_defaults(&[])?;