mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Attribute turn metrics to the models used during the turn (#44656)
## Why Turn completion metrics used session-level telemetry, which could label usage with the wrong model after a model switch. A turn can also include compaction and responses from different models, so a single token total cannot attribute usage correctly. ## What changed - Accumulate token usage by model using each response's resolved step settings, and emit one histogram sample per model and token type per turn. - Preserve zero-valued samples when no usage is reported, without adding a sample for an unused selected model when only compaction ran. - Use turn telemetry for tool-call, memory, network-proxy, and running-process metrics. ## Testing Add regression tests for model switches, step-level model changes, compaction followed by a response or an aborted turn, and turns without responses. Verify repeated requests aggregate into one sample per model and earlier session usage is excluded. GitOrigin-RevId: 03ab1dcde7da0b16b3a500a9a05e4c86583ee585
This commit is contained in:
committed by
copyberry
parent
8570091e14
commit
c8a8295e79
@@ -44,7 +44,6 @@ use codex_otel::SessionTelemetry;
|
||||
use codex_otel::TURN_E2E_DURATION_METRIC;
|
||||
use codex_otel::TURN_MEMORY_METRIC;
|
||||
use codex_otel::TURN_NETWORK_PROXY_METRIC;
|
||||
use codex_otel::TURN_TOKEN_USAGE_METRIC;
|
||||
use codex_otel::TURN_TOOL_CALL_METRIC;
|
||||
use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -631,12 +630,18 @@ impl Session {
|
||||
.input_queue
|
||||
.take_pending_input_for_turn_state(turn_state.as_ref())
|
||||
.await;
|
||||
let (turn_had_memory_citation, turn_tool_calls, token_usage_at_turn_start) = {
|
||||
let ts = turn_state.lock().await;
|
||||
let (
|
||||
turn_had_memory_citation,
|
||||
turn_tool_calls,
|
||||
token_usage_at_turn_start,
|
||||
token_usage_by_model,
|
||||
) = {
|
||||
let mut ts = turn_state.lock().await;
|
||||
(
|
||||
ts.has_memory_citation,
|
||||
ts.tool_calls,
|
||||
ts.token_usage_at_turn_start.clone(),
|
||||
std::mem::take(&mut ts.token_usage_by_model),
|
||||
)
|
||||
};
|
||||
run_hooks_and_record_inputs(
|
||||
@@ -647,6 +652,7 @@ impl Session {
|
||||
PersistContext::Standard,
|
||||
)
|
||||
.await;
|
||||
let turn_telemetry = &turn_context.session_telemetry;
|
||||
// Emit token usage metrics.
|
||||
{
|
||||
// TODO(jif): drop this
|
||||
@@ -673,12 +679,8 @@ impl Session {
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
emit_turn_network_proxy_metric(
|
||||
&self.services.session_telemetry,
|
||||
network_proxy_active,
|
||||
tmp_mem,
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
emit_turn_network_proxy_metric(turn_telemetry, network_proxy_active, tmp_mem);
|
||||
turn_telemetry.histogram(
|
||||
TURN_TOOL_CALL_METRIC,
|
||||
i64::try_from(turn_tool_calls).unwrap_or(i64::MAX),
|
||||
&[tmp_mem],
|
||||
@@ -739,46 +741,17 @@ impl Session {
|
||||
.track_turn_token_usage(TurnTokenUsageFact {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
thread_id: self.thread_id.to_string(),
|
||||
token_usage: turn_token_usage.clone(),
|
||||
token_usage: turn_token_usage,
|
||||
});
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.total_tokens,
|
||||
&[("token_type", "total"), tmp_mem],
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.input_tokens,
|
||||
&[("token_type", "input"), tmp_mem],
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.cached_input(),
|
||||
&[("token_type", "cached_input"), tmp_mem],
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.cache_write_input_tokens,
|
||||
&[("token_type", "cache_write_input"), tmp_mem],
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.output_tokens,
|
||||
&[("token_type", "output"), tmp_mem],
|
||||
);
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.reasoning_output_tokens,
|
||||
&[("token_type", "reasoning_output"), tmp_mem],
|
||||
);
|
||||
token_usage_by_model.emit(turn_telemetry, tmp_mem);
|
||||
}
|
||||
emit_turn_memory_metric(
|
||||
&self.services.session_telemetry,
|
||||
turn_telemetry,
|
||||
turn_context.config.features.enabled(Feature::MemoryTool),
|
||||
turn_context.config.memories.use_memories,
|
||||
turn_had_memory_citation,
|
||||
);
|
||||
self.services.session_telemetry.counter(
|
||||
turn_telemetry.counter(
|
||||
TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC,
|
||||
i64::try_from(self.list_background_terminals().await.len()).unwrap_or(i64::MAX),
|
||||
&[],
|
||||
|
||||
Reference in New Issue
Block a user