Record reasoning replay removal metrics (#61017)

## What changed

Parse reasoning removals from response `input_transformations` and carry the counts through completion events to sampling telemetry.

- Add `codex.responses.reasoning_replay` to distinguish unavailable reports, no reported removals, prefix mismatches, and other removals.
- Add `codex.responses.reasoning_items_removed` histograms by bounded removal reason, grouping unknown or missing reasons under `other`.
- Include zero counts for usable reports; exclude missing, null, or malformed reports from histograms without failing the response.
- Use the sampling step's session telemetry when recording response events.

## Testing

Add SSE and WebSocket integration coverage for report availability, removal counts, unknown reasons, zero observations, and `product_sku` tags in exported metrics.

GitOrigin-RevId: 83c70ca6067fa4b34bf5c463958b4632a7a0a188
This commit is contained in:
Alex Kira
2026-11-12 04:31:25 +00:00
committed by copyberry
parent b75ec48a2c
commit 64b8d019c8
12 changed files with 304 additions and 1 deletions
+2
View File
@@ -2842,6 +2842,7 @@ where
response_id,
token_usage,
usage_metadata,
reasoning_removals,
end_turn,
}) => {
feedback_tags!(last_model_response_id = &response_id);
@@ -2866,6 +2867,7 @@ where
response_id,
token_usage,
usage_metadata,
reasoning_removals,
end_turn,
}))
.await
+2
View File
@@ -1820,6 +1820,7 @@ async fn response_stream_records_last_model_feedback_ids() {
response_id: "resp-123".to_string(),
token_usage: None,
usage_metadata: None,
reasoning_removals: None,
end_turn: Some(true),
}),
]);
@@ -2864,6 +2865,7 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow:
response_id: "response".into(),
token_usage: None,
usage_metadata: None,
reasoning_removals: None,
end_turn: None,
}))
.await?;
+2
View File
@@ -1433,6 +1433,7 @@ mod tests {
response_id: "resp-compact".into(),
token_usage: Some(usage.clone()),
usage_metadata: None,
reasoning_removals: None,
end_turn: Some(true),
})]);
let result = collect_compaction_output(
@@ -1485,6 +1486,7 @@ mod tests {
amount: Some("0.125".to_string()),
metadata: Some(serde_json::json!({ "extra": { "label": "example" } })),
}),
reasoning_removals: None,
end_turn: Some(true),
}),
]);
+2 -1
View File
@@ -3178,7 +3178,7 @@ async fn try_run_sampling_request(
let Some(event) = resampling.filter(event, &step_context.tool_router) else {
continue;
};
sess.services
step_context
.session_telemetry
.record_responses(&handle_responses, &event);
@@ -3532,6 +3532,7 @@ async fn try_run_sampling_request(
response_id,
token_usage,
usage_metadata,
reasoning_removals: _,
end_turn,
} => {
if let Some(accepted) = pending_audio.take() {
+1
View File
@@ -101,6 +101,7 @@ mod guardian_authorization_refresh;
mod guardian_cached_score;
#[path = "guardian_checkpoint_migration_tests.rs"]
mod guardian_checkpoint_migration;
mod reasoning_replay_metrics;
// Uses the same command-approval harness as guardian_review below.
mod builtin_model_catalog;
mod canonical_plugin_connectors;
@@ -0,0 +1,185 @@
//! Verifies reported CoT removals through both sampling transports and the metrics exporter.
use anyhow::Result;
use codex_otel::MetricsClient;
use codex_otel::MetricsConfig;
use codex_otel::REASONING_ITEMS_REMOVED_METRIC;
use codex_otel::REASONING_REPLAY_COUNT_METRIC;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::test_codex::test_codex;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::BTreeMap;
#[test_case::test_case(false; "sse")]
#[test_case::test_case(true; "websocket")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn reasoning_replay_metrics_preserve_report_availability(use_websocket: bool) -> Result<()> {
skip_if_no_network!(Ok(()));
let reports = [
None,
Some(json!([])),
Some(json!([
{"type": "reasoning", "id": "rs_same", "action": "removed", "reason": "prefix_mismatch"},
{"type": "reasoning", "id": "rs_same", "action": "removed", "reason": "prefix_mismatch"},
{"type": "reasoning", "action": "removed", "reason": "organization_mismatch"},
{"type": "reasoning", "action": "removed", "reason": "end_user_mismatch"},
{"type": "reasoning", "action": "removed", "reason": "model_mismatch"},
{"type": "reasoning", "action": "removed", "reason": "turn_boundary"},
{"type": "reasoning", "action": "removed", "reason": "future-private-reason"},
{"type": "reasoning", "action": "removed"},
{"type": "message", "action": "removed", "reason": "prefix_mismatch"},
{"type": "reasoning", "action": "changed", "reason": "prefix_mismatch"},
null
])),
Some(json!([{"type": "reasoning", "action": "removed", "reason": "turn_boundary"}])),
Some(json!(null)),
Some(json!({"malformed": true})),
];
let events = reports
.iter()
.enumerate()
.map(|(index, report)| {
let id = format!("resp-{index}");
let mut completed = responses::ev_completed(&id);
if let Some(report) = report {
completed["response"]["input_transformations"] = report.clone();
}
let mut in_progress = completed.clone();
in_progress["type"] = json!("response.in_progress");
vec![responses::ev_response_created(&id), in_progress, completed]
})
.collect::<Vec<_>>();
let server = responses::start_mock_server().await;
let http_requests = if use_websocket {
None
} else {
Some(
responses::mount_sse_sequence(
&server,
events.iter().cloned().map(responses::sse).collect(),
)
.await,
)
};
let websocket_server = if use_websocket {
let mut frames = vec![vec![
responses::ev_response_created("prewarm"),
responses::ev_completed("prewarm"),
]];
frames.extend(events);
Some(responses::start_websocket_server(vec![frames]).await)
} else {
None
};
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"reasoning-replay",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)?;
let mut builder = test_codex().with_config(|config| {
config.apps_mcp_product_sku = Some("codex".to_string());
});
let mut test = builder.build_with_auto_env(&server).await?;
let mut options = test.start_thread_options().await;
test.codex.shutdown_and_wait().await?;
// Only the instrumented thread may prewarm the scripted WebSocket connection.
if let Some(server) = &websocket_server {
options.config.model_provider.base_url = Some(format!("{}/v1", server.uri()));
options.config.model_provider.supports_websockets = true;
}
options.thread_extension_init.insert(metrics.clone());
test.codex = test.thread_manager.start_thread(options).await?.thread;
for _ in &reports {
test.submit_text_turn("continue").await?;
}
if let Some(requests) = http_requests {
assert_eq!(requests.requests().len(), reports.len());
}
let snapshot = metrics.snapshot()?;
let mut reports = BTreeMap::new();
let mut removals = BTreeMap::new();
for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) {
match metric.name() {
REASONING_REPLAY_COUNT_METRIC => {
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("report availability must be a counter");
};
for point in sum.data_points() {
let tags = point
.attributes()
.map(|attribute| {
(attribute.key.as_str(), attribute.value.as_str().to_string())
})
.collect::<BTreeMap<_, _>>();
assert_eq!(tags["product_sku"], "codex");
reports.insert(tags["report"].clone(), point.value());
}
}
REASONING_ITEMS_REMOVED_METRIC => {
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
panic!("reasoning removal counts must be histograms");
};
for point in histogram.data_points() {
let tags = point
.attributes()
.map(|attribute| {
(attribute.key.as_str(), attribute.value.as_str().to_string())
})
.collect::<BTreeMap<_, _>>();
assert_eq!(tags["product_sku"], "codex");
removals.insert(
tags["reason"].clone(),
(point.count(), point.sum(), point.min(), point.max()),
);
}
}
_ => {}
}
}
assert_eq!(
reports,
BTreeMap::from([
("unavailable".to_string(), 3),
("no_removals_reported".to_string(), 1),
("prefix_mismatch_reported".to_string(), 1),
("removals_reported".to_string(), 1),
])
);
// Three usable reports contribute to every reason, including their zero counts.
// Missing, null, and malformed reports contribute no histogram observations.
assert_eq!(
removals,
BTreeMap::from([
(
"prefix_mismatch".to_string(),
(3, 2.0, Some(0.0), Some(2.0))
),
(
"organization_mismatch".to_string(),
(3, 1.0, Some(0.0), Some(1.0))
),
(
"end_user_mismatch".to_string(),
(3, 1.0, Some(0.0), Some(1.0))
),
("model_mismatch".to_string(), (3, 1.0, Some(0.0), Some(1.0))),
("turn_boundary".to_string(), (3, 2.0, Some(0.0), Some(1.0))),
("other".to_string(), (3, 2.0, Some(0.0), Some(2.0))),
])
);
test.codex.shutdown_and_wait().await?;
if let Some(server) = websocket_server {
server.shutdown().await;
}
Ok(())
}