mirror of
https://github.com/openai/codex.git
synced 2026-11-07 20:30:33 +00:00
Track inference tool changes in turn analytics (#50964)
## What changed Add `tools_change_count` to turn profiles and turn analytics events. Compare the full model-visible tool list before each sampling request, retaining the previous list across turns and connection resets. The first list establishes a baseline; subsequent changes increment the current turn's count. ## Testing Add coverage for schema changes, tool additions, ordering changes, empty lists, and tracking across sessions and transport fallback. Extend the turn event serialization test to include `tools_change_count`. GitOrigin-RevId: 44a15c42dc7d0b35c166666faeb9627b718b33dd
This commit is contained in:
@@ -70,6 +70,9 @@ use codex_rollout_trace::RawTraceEventPayload;
|
||||
use codex_rollout_trace::RolloutTrace;
|
||||
use codex_rollout_trace::TraceWriter;
|
||||
use codex_rollout_trace::replay_bundle;
|
||||
use codex_tools::JsonSchema;
|
||||
use codex_tools::ResponsesApiTool;
|
||||
use codex_tools::ToolSpec;
|
||||
use futures::StreamExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
@@ -2130,3 +2133,55 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow:
|
||||
assert_eq!(recorded["output_items"], serde_json::to_value(&delivered)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inference_tools_changes_follow_full_specs_across_turns() {
|
||||
let client = test_model_client(SessionSource::Cli);
|
||||
let telemetry = test_session_telemetry();
|
||||
let model_info = test_model_info();
|
||||
let alpha = ResponsesApiTool {
|
||||
name: "alpha".into(),
|
||||
description: "Original".into(),
|
||||
strict: false,
|
||||
defer_loading: None,
|
||||
parameters: JsonSchema::default(),
|
||||
output_schema: None,
|
||||
};
|
||||
let beta = ResponsesApiTool {
|
||||
name: "beta".into(),
|
||||
..alpha.clone()
|
||||
};
|
||||
let mut output_changed = alpha.clone();
|
||||
output_changed.output_schema = Some(json!({"type": "object"}).into());
|
||||
let mut params_changed = beta.clone();
|
||||
params_changed.parameters = JsonSchema::string(Some("Changed parameters".into()));
|
||||
let mut session = client.new_session();
|
||||
for (index, (tools, expected)) in [
|
||||
(vec![alpha.clone()], false),
|
||||
(vec![output_changed.clone()], true),
|
||||
(vec![output_changed.clone(), beta.clone()], true),
|
||||
(vec![output_changed, beta.clone()], false),
|
||||
(vec![beta, alpha.clone()], true),
|
||||
(vec![params_changed, alpha], true),
|
||||
(vec![], true),
|
||||
(vec![], false),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
if index == 4 {
|
||||
session.try_switch_fallback_transport(&telemetry, &model_info);
|
||||
drop(session);
|
||||
session = client.new_session();
|
||||
}
|
||||
let specs = tools
|
||||
.into_iter()
|
||||
.map(ToolSpec::Function)
|
||||
.collect::<Arc<[_]>>();
|
||||
assert_eq!(
|
||||
session.inference_tools_changed(&specs),
|
||||
expected,
|
||||
"step {index}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user