Attribute turn metrics to the models used during the turn (#44656)

## Why

Turn completion metrics used session-level telemetry, which could label usage with the wrong model after a model switch. A turn can also include compaction and responses from different models, so a single token total cannot attribute usage correctly.

## What changed

- Accumulate token usage by model using each response's resolved step settings, and emit one histogram sample per model and token type per turn.
- Preserve zero-valued samples when no usage is reported, without adding a sample for an unused selected model when only compaction ran.
- Use turn telemetry for tool-call, memory, network-proxy, and running-process metrics.

## Testing

Add regression tests for model switches, step-level model changes, compaction followed by a response or an aborted turn, and turns without responses. Verify repeated requests aggregate into one sample per model and earlier session usage is excluded.

GitOrigin-RevId: 03ab1dcde7da0b16b3a500a9a05e4c86583ee585
This commit is contained in:
Andrei Eternal
2026-09-10 21:58:58 +00:00
committed by copyberry
parent 8570091e14
commit c8a8295e79
8 changed files with 352 additions and 46 deletions
+15 -42
View File
@@ -44,7 +44,6 @@ use codex_otel::SessionTelemetry;
use codex_otel::TURN_E2E_DURATION_METRIC;
use codex_otel::TURN_MEMORY_METRIC;
use codex_otel::TURN_NETWORK_PROXY_METRIC;
use codex_otel::TURN_TOKEN_USAGE_METRIC;
use codex_otel::TURN_TOOL_CALL_METRIC;
use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC;
use codex_protocol::models::ResponseItem;
@@ -631,12 +630,18 @@ impl Session {
.input_queue
.take_pending_input_for_turn_state(turn_state.as_ref())
.await;
let (turn_had_memory_citation, turn_tool_calls, token_usage_at_turn_start) = {
let ts = turn_state.lock().await;
let (
turn_had_memory_citation,
turn_tool_calls,
token_usage_at_turn_start,
token_usage_by_model,
) = {
let mut ts = turn_state.lock().await;
(
ts.has_memory_citation,
ts.tool_calls,
ts.token_usage_at_turn_start.clone(),
std::mem::take(&mut ts.token_usage_by_model),
)
};
run_hooks_and_record_inputs(
@@ -647,6 +652,7 @@ impl Session {
PersistContext::Standard,
)
.await;
let turn_telemetry = &turn_context.session_telemetry;
// Emit token usage metrics.
{
// TODO(jif): drop this
@@ -673,12 +679,8 @@ impl Session {
}
None => false,
};
emit_turn_network_proxy_metric(
&self.services.session_telemetry,
network_proxy_active,
tmp_mem,
);
self.services.session_telemetry.histogram(
emit_turn_network_proxy_metric(turn_telemetry, network_proxy_active, tmp_mem);
turn_telemetry.histogram(
TURN_TOOL_CALL_METRIC,
i64::try_from(turn_tool_calls).unwrap_or(i64::MAX),
&[tmp_mem],
@@ -739,46 +741,17 @@ impl Session {
.track_turn_token_usage(TurnTokenUsageFact {
turn_id: turn_context.sub_id.clone(),
thread_id: self.thread_id.to_string(),
token_usage: turn_token_usage.clone(),
token_usage: turn_token_usage,
});
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.total_tokens,
&[("token_type", "total"), tmp_mem],
);
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.input_tokens,
&[("token_type", "input"), tmp_mem],
);
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.cached_input(),
&[("token_type", "cached_input"), tmp_mem],
);
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.cache_write_input_tokens,
&[("token_type", "cache_write_input"), tmp_mem],
);
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.output_tokens,
&[("token_type", "output"), tmp_mem],
);
self.services.session_telemetry.histogram(
TURN_TOKEN_USAGE_METRIC,
turn_token_usage.reasoning_output_tokens,
&[("token_type", "reasoning_output"), tmp_mem],
);
token_usage_by_model.emit(turn_telemetry, tmp_mem);
}
emit_turn_memory_metric(
&self.services.session_telemetry,
turn_telemetry,
turn_context.config.features.enabled(Feature::MemoryTool),
turn_context.config.memories.use_memories,
turn_had_memory_citation,
);
self.services.session_telemetry.counter(
turn_telemetry.counter(
TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC,
i64::try_from(self.list_background_terminals().await.len()).unwrap_or(i64::MAX),
&[],