diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 033717960d..e318a9282e 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1140,6 +1140,7 @@ pub(crate) struct CodexTurnEventParams { pub(crate) after_last_sampling_ms: u64, pub(crate) sampling_request_count: u32, pub(crate) sampling_retry_count: u32, + pub(crate) tools_change_count: u32, pub(crate) duration_ms: Option, pub(crate) started_at: Option, pub(crate) completed_at: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 442fb00dd5..25f4c185fa 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -252,6 +252,7 @@ pub struct TurnProfile { pub after_last_sampling_ms: u64, pub sampling_request_count: u32, pub sampling_retry_count: u32, + pub tools_change_count: u32, } #[derive(Clone)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 7640223e42..edbb046c15 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -3667,6 +3667,7 @@ fn codex_turn_event_params( after_last_sampling_ms, sampling_request_count, sampling_retry_count, + tools_change_count, } = profile; let token_usage = turn_state.token_usage.clone(); let codex_error = turn_state.codex_error.as_ref(); @@ -3750,6 +3751,7 @@ fn codex_turn_event_params( after_last_sampling_ms, sampling_request_count, sampling_retry_count, + tools_change_count, duration_ms: completed.duration_ms, started_at, completed_at: Some(completed.completed_at), diff --git a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs index 25dfdd4a0e..59250f93ea 100644 --- a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs +++ b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs @@ -173,6 +173,7 @@ fn turn_event_serializes_expected_shape() { after_last_sampling_ms: 94, sampling_request_count: 2, sampling_retry_count: 1, + tools_change_count: 2, duration_ms: Some(1234), started_at: Some(455), completed_at: Some(456), @@ -264,6 +265,7 @@ fn turn_event_serializes_expected_shape() { "after_last_sampling_ms": 94, "sampling_request_count": 2, "sampling_retry_count": 1, + "tools_change_count": 2, "duration_ms": 1234, "started_at": 455, "completed_at": 456 diff --git a/codex-rs/analytics/src/tests/support.rs b/codex-rs/analytics/src/tests/support.rs index 5ce1c77216..7343ff3144 100644 --- a/codex-rs/analytics/src/tests/support.rs +++ b/codex-rs/analytics/src/tests/support.rs @@ -378,6 +378,7 @@ pub(super) fn sample_turn_profile() -> TurnProfile { after_last_sampling_ms: 94, sampling_request_count: 2, sampling_retry_count: 1, + tools_change_count: 2, } } diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 3b6c84d4d6..270fa96695 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -99,6 +99,7 @@ use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::W3cTraceContext; use codex_rollout_trace::InferenceTraceAttempt; use codex_rollout_trace::InferenceTraceContext; +use codex_tools::ToolSpec; use codex_tools::create_tools_json_for_responses_lite; use codex_tools::create_tools_raw_json_for_responses_api; use eventsource_stream::Event; @@ -217,6 +218,8 @@ struct ModelClientState { disable_websockets: AtomicBool, agent_identity_session_fallback: AgentIdentitySessionFallback, cached_websocket_session: StdMutex, + /// Last full tool list used for sampling, retained across turns and connection resets. + last_inference_tools: StdMutex>>, } enum ClientRouting { @@ -533,6 +536,7 @@ impl ModelClient { disable_websockets: AtomicBool::new(false), agent_identity_session_fallback: AgentIdentitySessionFallback::default(), cached_websocket_session: StdMutex::new(WebsocketSession::default()), + last_inference_tools: StdMutex::new(None), }), agent_identity_policy, prompt_cache_key_override: None, @@ -1350,6 +1354,17 @@ impl Drop for ModelClientSession { } impl ModelClientSession { + pub(crate) fn inference_tools_changed(&self, tools: &Arc<[ToolSpec]>) -> bool { + let previous = self + .client + .state + .last_inference_tools + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .replace(Arc::clone(tools)); + previous.is_some_and(|previous| previous != *tools) + } + #[allow(clippy::too_many_arguments)] /// Builds shared Responses API transport options and request-body options. /// diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index 38f58dbde9..5e50bb3acc 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -70,6 +70,9 @@ use codex_rollout_trace::RawTraceEventPayload; use codex_rollout_trace::RolloutTrace; use codex_rollout_trace::TraceWriter; use codex_rollout_trace::replay_bundle; +use codex_tools::JsonSchema; +use codex_tools::ResponsesApiTool; +use codex_tools::ToolSpec; use futures::StreamExt; use pretty_assertions::assert_eq; use serde_json::json; @@ -2130,3 +2133,55 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow: assert_eq!(recorded["output_items"], serde_json::to_value(&delivered)?); Ok(()) } + +#[tokio::test] +async fn inference_tools_changes_follow_full_specs_across_turns() { + let client = test_model_client(SessionSource::Cli); + let telemetry = test_session_telemetry(); + let model_info = test_model_info(); + let alpha = ResponsesApiTool { + name: "alpha".into(), + description: "Original".into(), + strict: false, + defer_loading: None, + parameters: JsonSchema::default(), + output_schema: None, + }; + let beta = ResponsesApiTool { + name: "beta".into(), + ..alpha.clone() + }; + let mut output_changed = alpha.clone(); + output_changed.output_schema = Some(json!({"type": "object"}).into()); + let mut params_changed = beta.clone(); + params_changed.parameters = JsonSchema::string(Some("Changed parameters".into())); + let mut session = client.new_session(); + for (index, (tools, expected)) in [ + (vec![alpha.clone()], false), + (vec![output_changed.clone()], true), + (vec![output_changed.clone(), beta.clone()], true), + (vec![output_changed, beta.clone()], false), + (vec![beta, alpha.clone()], true), + (vec![params_changed, alpha], true), + (vec![], true), + (vec![], false), + ] + .into_iter() + .enumerate() + { + if index == 4 { + session.try_switch_fallback_transport(&telemetry, &model_info); + drop(session); + session = client.new_session(); + } + let specs = tools + .into_iter() + .map(ToolSpec::Function) + .collect::>(); + assert_eq!( + session.inference_tools_changed(&specs), + expected, + "step {index}" + ); + } +} diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 96456c77eb..b5147caa73 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -2568,6 +2568,9 @@ async fn try_run_sampling_request( turn_context.provider.info().name.as_str(), ); let sampling_timing_guard = turn_context.turn_timing_state.begin_sampling(); + if client_session.inference_tools_changed(&step_context.tool_router.model_visible_specs()) { + turn_context.turn_timing_state.record_tools_change(); + } // Do not enter this span: overlapping tools must not retain it past sampling. let sampling_span = trace_span!( "codex.sampling", diff --git a/codex-rs/core/src/turn_timing.rs b/codex-rs/core/src/turn_timing.rs index bc90f81a11..75a2bf9bf0 100644 --- a/codex-rs/core/src/turn_timing.rs +++ b/codex-rs/core/src/turn_timing.rs @@ -69,6 +69,7 @@ struct TurnProfileState { pending_idle_after_sampling: Duration, sampling_request_count: u32, sampling_retry_count: u32, + tools_change_count: u32, completed_profile: Option, } @@ -155,6 +156,13 @@ impl TurnTimingState { self.profile_state().record_sampling_retry(); } + pub(crate) fn record_tools_change(&self) { + let mut profile = self.profile_state(); + if profile.completed_profile.is_none() && profile.started_at.is_some() { + profile.tools_change_count = profile.tools_change_count.saturating_add(1); + } + } + pub(crate) fn begin_compaction(self: &Arc) -> TurnProfileTimingGuard { let active = self.profile_state().begin_compaction(Instant::now()); TurnProfileTimingGuard { @@ -324,6 +332,7 @@ impl TurnProfileState { after_last_sampling_ms: duration_to_u64_ms(after_last_sampling), sampling_request_count: self.sampling_request_count, sampling_retry_count: self.sampling_retry_count, + tools_change_count: self.tools_change_count, }; let total_ms = self .started_at diff --git a/codex-rs/core/src/turn_timing_tests.rs b/codex-rs/core/src/turn_timing_tests.rs index 6b6cd8ac38..65185ecea2 100644 --- a/codex-rs/core/src/turn_timing_tests.rs +++ b/codex-rs/core/src/turn_timing_tests.rs @@ -237,6 +237,7 @@ fn turn_profile_breaks_down_sampling_blocking_and_retry_overhead() { after_last_sampling_ms: 100, sampling_request_count: 2, sampling_retry_count: 1, + tools_change_count: 0, } ); } @@ -270,6 +271,7 @@ fn turn_profile_counts_compaction_as_an_exclusive_phase() { after_last_sampling_ms: 100, sampling_request_count: 1, sampling_retry_count: 0, + tools_change_count: 0, } ); }