Track inference tool changes in turn analytics (#50964)

## What changed

Add `tools_change_count` to turn profiles and turn analytics events. Compare the full model-visible tool list before each sampling request, retaining the previous list across turns and connection resets. The first list establishes a baseline; subsequent changes increment the current turn's count.

## Testing

Add coverage for schema changes, tool additions, ordering changes, empty lists, and tracking across sessions and transport fallback. Extend the turn event serialization test to include `tools_change_count`.

GitOrigin-RevId: 44a15c42dc7d0b35c166666faeb9627b718b33dd
This commit is contained in:
Ahmed Ibrahim
2026-10-04 21:19:05 +00:00
committed by copyberry
parent 335c7f8eca
commit 4ad985e2ca
10 changed files with 91 additions and 0 deletions
+1
View File
@@ -1140,6 +1140,7 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) after_last_sampling_ms: u64,
pub(crate) sampling_request_count: u32,
pub(crate) sampling_retry_count: u32,
pub(crate) tools_change_count: u32,
pub(crate) duration_ms: Option<u64>,
pub(crate) started_at: Option<u64>,
pub(crate) completed_at: Option<u64>,
+1
View File
@@ -252,6 +252,7 @@ pub struct TurnProfile {
pub after_last_sampling_ms: u64,
pub sampling_request_count: u32,
pub sampling_retry_count: u32,
pub tools_change_count: u32,
}
#[derive(Clone)]
+2
View File
@@ -3667,6 +3667,7 @@ fn codex_turn_event_params(
after_last_sampling_ms,
sampling_request_count,
sampling_retry_count,
tools_change_count,
} = profile;
let token_usage = turn_state.token_usage.clone();
let codex_error = turn_state.codex_error.as_ref();
@@ -3750,6 +3751,7 @@ fn codex_turn_event_params(
after_last_sampling_ms,
sampling_request_count,
sampling_retry_count,
tools_change_count,
duration_ms: completed.duration_ms,
started_at,
completed_at: Some(completed.completed_at),
@@ -173,6 +173,7 @@ fn turn_event_serializes_expected_shape() {
after_last_sampling_ms: 94,
sampling_request_count: 2,
sampling_retry_count: 1,
tools_change_count: 2,
duration_ms: Some(1234),
started_at: Some(455),
completed_at: Some(456),
@@ -264,6 +265,7 @@ fn turn_event_serializes_expected_shape() {
"after_last_sampling_ms": 94,
"sampling_request_count": 2,
"sampling_retry_count": 1,
"tools_change_count": 2,
"duration_ms": 1234,
"started_at": 455,
"completed_at": 456
+1
View File
@@ -378,6 +378,7 @@ pub(super) fn sample_turn_profile() -> TurnProfile {
after_last_sampling_ms: 94,
sampling_request_count: 2,
sampling_retry_count: 1,
tools_change_count: 2,
}
}