Track inference tool changes in turn analytics (#50964)

## What changed

Add `tools_change_count` to turn profiles and turn analytics events. Compare the full model-visible tool list before each sampling request, retaining the previous list across turns and connection resets. The first list establishes a baseline; subsequent changes increment the current turn's count.

## Testing

Add coverage for schema changes, tool additions, ordering changes, empty lists, and tracking across sessions and transport fallback. Extend the turn event serialization test to include `tools_change_count`.

GitOrigin-RevId: 44a15c42dc7d0b35c166666faeb9627b718b33dd
This commit is contained in:
Ahmed Ibrahim
2026-10-04 21:19:05 +00:00
committed by copyberry
parent 335c7f8eca
commit 4ad985e2ca
10 changed files with 91 additions and 0 deletions
+55
View File
@@ -70,6 +70,9 @@ use codex_rollout_trace::RawTraceEventPayload;
use codex_rollout_trace::RolloutTrace;
use codex_rollout_trace::TraceWriter;
use codex_rollout_trace::replay_bundle;
use codex_tools::JsonSchema;
use codex_tools::ResponsesApiTool;
use codex_tools::ToolSpec;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -2130,3 +2133,55 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow:
assert_eq!(recorded["output_items"], serde_json::to_value(&delivered)?);
Ok(())
}
#[tokio::test]
async fn inference_tools_changes_follow_full_specs_across_turns() {
let client = test_model_client(SessionSource::Cli);
let telemetry = test_session_telemetry();
let model_info = test_model_info();
let alpha = ResponsesApiTool {
name: "alpha".into(),
description: "Original".into(),
strict: false,
defer_loading: None,
parameters: JsonSchema::default(),
output_schema: None,
};
let beta = ResponsesApiTool {
name: "beta".into(),
..alpha.clone()
};
let mut output_changed = alpha.clone();
output_changed.output_schema = Some(json!({"type": "object"}).into());
let mut params_changed = beta.clone();
params_changed.parameters = JsonSchema::string(Some("Changed parameters".into()));
let mut session = client.new_session();
for (index, (tools, expected)) in [
(vec![alpha.clone()], false),
(vec![output_changed.clone()], true),
(vec![output_changed.clone(), beta.clone()], true),
(vec![output_changed, beta.clone()], false),
(vec![beta, alpha.clone()], true),
(vec![params_changed, alpha], true),
(vec![], true),
(vec![], false),
]
.into_iter()
.enumerate()
{
if index == 4 {
session.try_switch_fallback_transport(&telemetry, &model_info);
drop(session);
session = client.new_session();
}
let specs = tools
.into_iter()
.map(ToolSpec::Function)
.collect::<Arc<[_]>>();
assert_eq!(
session.inference_tools_changed(&specs),
expected,
"step {index}"
);
}
}