diff --git a/codex-rs/codex-mcp/src/binding.rs b/codex-rs/codex-mcp/src/binding.rs index 294a81d44f..8bf47bea87 100644 --- a/codex-rs/codex-mcp/src/binding.rs +++ b/codex-rs/codex-mcp/src/binding.rs @@ -417,6 +417,22 @@ impl PreparedMcpCall { requested_timeout: Option, prepare: F, ) -> Result + where + F: FnOnce() -> Fut, + Fut: Future, Option)>>, + { + self.call_with_admission(requested_timeout, || Ok(()), prepare) + .await + } + + /// Checks admission on the negotiated session immediately before handing a tool request + /// to the peer. An in-flight request can complete even if new work is stopped. + pub async fn call_with_admission( + &self, + requested_timeout: Option, + admit: impl Fn() -> Result<()> + Send + Sync, + prepare: F, + ) -> Result where F: FnOnce() -> Fut, Fut: Future, Option)>>, @@ -466,16 +482,21 @@ impl PreparedMcpCall { } None => None, }; + let policy_meta = self.connections.network_policy_meta_for_call( + &self.client, + Some(self.permission_profile()), + self.server_environment_id(), + meta, + ); self.client .call_tool( tool_name.clone(), arguments, - self.connections.network_policy_meta_for_call( - &self.client, - Some(self.permission_profile()), - self.server_environment_id(), - meta, - ), + |capabilities| { + let meta = policy_meta(capabilities)?; + admit()?; + Ok(meta) + }, remaining_timeout, ) .await diff --git a/codex-rs/core/src/agent/host/usage.rs b/codex-rs/core/src/agent/host/usage.rs index 073e15bbaf..0ecf617948 100644 --- a/codex-rs/core/src/agent/host/usage.rs +++ b/codex-rs/core/src/agent/host/usage.rs @@ -19,6 +19,7 @@ pub struct AgentUsageReport { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum UsageAccountingDecision { Continue, + /// Do not start more tool handlers in this local rollout tree. Already-running work continues. Exhausted, } diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index a90aa1c9ad..c9c66bac4c 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -1472,11 +1472,13 @@ mod tests { assert!( matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted)) ); + assert!(!sess.rollout_budget().tools_stopped()); assert!(accounting.reports.lock().unwrap().is_empty()); return; } assert!(matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_)))); + assert!(sess.rollout_budget().tools_stopped()); assert_eq!( *accounting.reports.lock().unwrap(), vec![AgentUsageReport { diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 4084f21dbd..8350baba82 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -8,6 +8,7 @@ use crate::config::Config; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::connectors; +use crate::function_tool::FunctionCallError; use crate::guardian::GuardianApprovalRequest; use crate::guardian::GuardianMcpAnnotations; use crate::guardian::GuardianReviewContext; @@ -167,7 +168,7 @@ pub(crate) async fn handle_mcp_tool_call( invocation_tool_name: ToolName, source: &crate::tools::context::ToolCallSource, arguments: String, -) -> HandledMcpToolCall { +) -> Result { let turn_context = &step_context.turn; let server = advertised_tool.server_name().to_owned(); let tool_name = advertised_tool.tool.name.to_string(); @@ -175,10 +176,10 @@ pub(crate) async fn handle_mcp_tool_call( Ok(value) => value, Err(e) => { error!("failed to parse tool call arguments: {e}"); - return HandledMcpToolCall { + return Ok(CompletedMcpToolCall { result: CallToolResult::from_error_text(format!("err: {e}")), tool_input: JsonValue::Object(serde_json::Map::new()), - }; + }); } }; @@ -230,11 +231,11 @@ pub(crate) async fn handle_mcp_tool_call( /*already_started*/ false, ) .await; - return HandledMcpToolCall { + return Ok(CompletedMcpToolCall { result: CallToolResult::from_result(result), tool_input: arguments_value .unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())), - }; + }); } }; let item_metadata = McpToolCallItemMetadata::from_tool_metadata(&server, Some(&metadata)); @@ -263,11 +264,11 @@ pub(crate) async fn handle_mcp_tool_call( connector_name.as_deref(), /*duration*/ None, ); - return HandledMcpToolCall { + return Ok(CompletedMcpToolCall { result: CallToolResult::from_result(result), tool_input: arguments_value .unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())), - }; + }); }; let _approval_metadata = sess.register_mcp_tool_approval_metadata( &call_id, @@ -396,11 +397,11 @@ pub(crate) async fn handle_mcp_tool_call( /*duration*/ None, ); - return HandledMcpToolCall { + return Ok(CompletedMcpToolCall { result: CallToolResult::from_result(result), tool_input: arguments_value .unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())), - }; + }); } handle_approved_mcp_tool_call( @@ -418,11 +419,16 @@ pub(crate) async fn handle_mcp_tool_call( .await } -pub(crate) struct HandledMcpToolCall { +pub(crate) struct CompletedMcpToolCall { pub(crate) result: CallToolResult, pub(crate) tool_input: JsonValue, } +/// Preserves the local admission failure through RMCP's transport error context. +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +struct McpAdmissionError(FunctionCallError); + #[derive(Clone, Debug, PartialEq, Eq)] struct McpToolCallItemMetadata { connector_id: Option, @@ -480,7 +486,7 @@ async fn handle_approved_mcp_tool_call( item_metadata: McpToolCallItemMetadata, source: &crate::tools::context::ToolCallSource, approval_application: McpToolApprovalApplication, -) -> HandledMcpToolCall { +) -> Result { let turn_context = step_context.turn.as_ref(); let server = invocation.server.clone(); let tool_name = invocation.tool.clone(); @@ -521,17 +527,31 @@ async fn handle_approved_mcp_tool_call( /*already_started*/ true, ) .await; - return HandledMcpToolCall { + return Ok(CompletedMcpToolCall { result: CallToolResult::from_result(result), tool_input, - }; + }); } let mut elicitation_type = None; let mut request_error_code = None; + let mut admission_error = None; + let admit = || { + let reason = sess + .rollout_budget() + .tools_stopped() + .then(|| "rollout budget exhausted".to_owned()); + if let Some(reason) = reason { + return Err(McpAdmissionError(FunctionCallError::RespondToModel(format!( + "aborted before MCP handoff: {reason}" + ))) + .into()); + } + Ok(()) + }; let result = async { let result = async { let result = prepared_call - .call_with_preparation(/*requested_timeout*/ None, || async { + .call_with_admission(/*requested_timeout*/ None, admit, || async { if let McpToolApprovalApplication::Apply { decision, policy } = &approval_application { @@ -680,9 +700,16 @@ async fn handle_approved_mcp_tool_call( Ok((rewritten_arguments, request_meta)) }) .await - .map_err(|error| { - request_error_code = codex_rmcp_client::request_error_code(&error); - format!("tool call error: {error:?}") + .map_err(|error| match error.downcast::() { + Ok(McpAdmissionError(error)) => { + let message = error.to_string(); + admission_error = Some(error); + message + } + Err(error) => { + request_error_code = codex_rmcp_client::request_error_code(&error); + format!("tool call error: {error:?}") + } }); // Keep the backend response intact, before callbacks or model/UI limits can change it. sess.services.rollout_thread_trace.record_mcp_tool_result( @@ -747,6 +774,19 @@ async fn handle_approved_mcp_tool_call( }, )) .await; + if let Some(error) = admission_error { + notify_mcp_tool_call_completed( + sess, + turn_context, + call_id, + invocation, + item_metadata, + Duration::ZERO, + truncate_mcp_tool_result_for_event(&result), + ) + .await; + return Err(error); + } if let Err(error) = &result { tracing::warn!("MCP tool call error: {error:?}"); } @@ -792,10 +832,10 @@ async fn handle_approved_mcp_tool_call( Some(duration), ); - HandledMcpToolCall { + Ok(CompletedMcpToolCall { result: CallToolResult::from_result(result), tool_input, - } + }) } fn mcp_tool_call_span( diff --git a/codex-rs/core/src/session/turn_start_mcp_tests.rs b/codex-rs/core/src/session/turn_start_mcp_tests.rs index 21c806cd16..48f265d78f 100644 --- a/codex-rs/core/src/session/turn_start_mcp_tests.rs +++ b/codex-rs/core/src/session/turn_start_mcp_tests.rs @@ -196,3 +196,171 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() { .expect("completed refresh must release the catalog lock"), ); } + +#[derive(Clone, Copy)] +enum McpPause { + Start, + Result, +} + +struct PauseMcpCall { + phase: McpPause, + entered: Notify, + release: Notify, + outcome: std::sync::OnceLock, +} + +impl codex_extension_api::ToolLifecycleContributor for PauseMcpCall { + fn on_tool_start<'a>( + &'a self, + input: codex_extension_api::ToolStartInput<'a>, + ) -> codex_extension_api::ToolLifecycleFuture<'a> { + Box::pin(async move { + assert_eq!(input.mcp_tool.unwrap().tool_info().tool.name, "probe"); + if matches!(self.phase, McpPause::Start) { + self.entered.notify_one(); + self.release.notified().await; + } + }) + } + + fn on_mcp_tool_result<'a>( + &'a self, + _input: codex_extension_api::McpToolResultInput<'a>, + ) -> codex_extension_api::ToolLifecycleFuture<'a> { + Box::pin(async move { + if matches!(self.phase, McpPause::Result) { + self.entered.notify_one(); + self.release.notified().await; + } + }) + } + + fn on_tool_finish<'a>( + &'a self, + input: codex_extension_api::ToolFinishInput<'a>, + ) -> codex_extension_api::ToolLifecycleFuture<'a> { + Box::pin(async move { self.outcome.set(input.outcome).expect("one finish") }) + } +} + +#[test_case::test_case(McpPause::Start; "stop_before_send")] +#[test_case::test_case(McpPause::Result; "stop_after_send")] +#[tokio::test] +async fn mcp_budget_stop_at_handoff(phase: McpPause) -> anyhow::Result<()> { + use crate::tools::parallel::ToolCallRuntime; + use crate::tools::router::ToolCall; + use codex_extension_api::ToolCallOutcome; + use core_test_support::hooks::mount_hook_mcp; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + core_test_support::skip_if_no_network!(Ok(())); + let endpoint = wiremock::MockServer::start().await; + let calls = Arc::new(AtomicUsize::new(0)); + let received = Arc::clone(&calls); + mount_hook_mcp(&endpoint, "handoff", &["probe"], move |_| { + received.fetch_add(1, Ordering::Relaxed); + json!({"content": [{"type":"text", "text":"peer result"}]}) + }) + .await; + let observer = Arc::new(PauseMcpCall { + phase, + entered: Notify::new(), + release: Notify::new(), + outcome: Default::default(), + }); + let (mut session, _) = make_session_and_context().await; + let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new(); + extensions.tool_lifecycle_contributor(observer.clone()); + session.services.extensions = Arc::new(extensions.build()); + session + .rollout_budget() + .configure(crate::config::RolloutBudgetConfig { + enforce_limit: true, + limit_tokens: 1, + reminder_at_remaining_tokens: Vec::new(), + sampling_token_weight: 1.0, + prefill_token_weight: 1.0, + }); + { + let mut state = session.state.lock().await; + let config = Arc::make_mut(&mut state.session_configuration.original_config_do_not_use); + config + .mcp_servers + .set(serde_json::from_value(json!({"handoff": { + "url": format!("{}/repl", endpoint.uri()), + "default_tools_approval_mode": "approve" + }}))?)?; + } + let session = Arc::new(session); + session.mark_mcp_runtime_dirty(McpRefreshReason::ExplicitInvalidation); + session + .wait_for_mcp_server(&codex_mcp::McpServerKey::unscoped("handoff")) + .await; + let turn = session.new_default_turn().await; + let cancel = CancellationToken::new(); + let step = session + .capture_step_context_with_required_mcp_servers( + turn, + &cancel, + &["handoff".to_owned()], + &HashSet::new(), + ) + .await?; + let tool = step + .mcp + .tools() + .iter() + .find(|tool| tool.server_name() == "handoff" && tool.tool.name == "probe") + .expect("MCP server advertised probe"); + let call = ToolCall { + tool_name: tool.canonical_tool_name(), + call_id: "handoff-probe".into(), + payload: ToolPayload::Function { + arguments: "{}".into(), + }, + encrypted_function_args: None, + native_call: None, + }; + let runtime = ToolCallRuntime::new( + session.clone(), + step, + Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())), + ); + let response = runtime.handle_tool_call(call, cancel); + tokio::pin!(response); + tokio::select! { + () = observer.entered.notified() => {} + reply = &mut response => anyhow::bail!("MCP call finished before the observer paused: {reply:?}"), + } + let usage = codex_protocol::protocol::TokenUsage { + output_tokens: 2, + ..Default::default() + }; + session + .record_rollout_budget_usage("peer-turn", &usage.into()) + .await?; + observer.release.notify_one(); + let reply = response.await?; + let (count, outcome) = match phase { + McpPause::Start => { + assert_eq!( + reply.item, + ResponseInputItem::FunctionCallOutput { + call_id: "handoff-probe".into(), + output: FunctionCallOutputPayload::from_text( + "aborted before MCP handoff: rollout budget exhausted".into() + ), + } + .into() + ); + (0, ToolCallOutcome::Aborted) + } + McpPause::Result => (1, ToolCallOutcome::Completed { success: true }), + }; + assert_eq!(calls.load(Ordering::Relaxed), count); + assert_eq!(observer.outcome.get(), Some(&outcome)); + session.services.mcp_runtime.shutdown().await; + Ok(()) +} diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index c5734f6c40..6dc92b0774 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -382,14 +382,10 @@ pub(crate) async fn handle_output_item_done( let cancellation_token = ctx.cancellation_token.child_token(); let runtime = ctx.tool_runtime.clone(); let dispatch = |allowed: bool| -> InFlightFuture<'static> { - if allowed - && !ctx.sess.rollout_budget().tools_stopped() - && !cancellation_token.is_cancelled() - { + if allowed && !cancellation_token.is_cancelled() { Box::pin(runtime.handle_tool_call(call, cancellation_token)) } else { - let output = - ToolCallRuntime::aborted_response(&call, /*elapsed_secs*/ None); + let output = ToolCallRuntime::aborted_before_dispatch(&call); Box::pin(std::future::ready(Ok( output.into_response(/*file_backed_tool_audio*/ false) ))) diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index 5b2845698d..d70e2998c5 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -6,6 +6,7 @@ use crate::session::turn_context::TurnEnvironment; use crate::turn_diff_tracker::TurnDiffTracker; use crate::unified_exec::output::CapturedStream; use crate::unified_exec::resolve_max_tokens; +use codex_extension_api::ToolCallOutcome; use codex_protocol::ResponseItemId; use codex_protocol::mcp::CallToolResult; use codex_protocol::models::FunctionCallOutputBody; @@ -25,8 +26,8 @@ use serde::Serialize; use serde_json::Value as JsonValue; use std::sync::Arc; use std::sync::OnceLock; -use std::sync::atomic::AtomicBool; use std::time::Duration; +use std::time::Instant; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; @@ -46,7 +47,8 @@ pub type SharedTurnDiffTracker = Arc>; /// Delivery does not finish the call: post-tool hooks can still be cancelled. #[derive(Default)] pub(crate) struct ToolCallState { - pub(crate) terminal_outcome_reached: AtomicBool, + pub(crate) terminal_outcome: OnceLock, + pub(crate) handler_started_at: OnceLock, pub(crate) delivered_assistant_message: OnceLock, pub(crate) invocation: crate::agent::host::InvocationReservation, } @@ -396,6 +398,10 @@ pub struct AbortedToolOutput { } impl ToolOutput for AbortedToolOutput { + fn was_aborted(&self) -> bool { + true + } + fn log_output(&self) -> String { self.message.clone() } diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index bd083a3ef7..b386b3e895 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -7,9 +7,11 @@ use crate::context::NodeReplReviewEvidenceMode; use crate::context::node_repl_review_evidence_mode; use crate::function_tool::FunctionCallError; use crate::image_preparation::prompt_image_decoder; +use crate::mcp_tool_call::CompletedMcpToolCall; use crate::mcp_tool_call::handle_mcp_tool_call; use crate::original_image_detail::can_request_original_image_detail; use crate::session::session::Session; +use crate::tools::context::AbortedToolOutput; use crate::tools::context::McpToolOutput; use crate::tools::context::ToolCallSource; use crate::tools::context::ToolEnvironments; @@ -314,9 +316,16 @@ impl McpHandler { )) .await; + let (result, tool_input) = match result { + Ok(CompletedMcpToolCall { result, tool_input }) => (result, tool_input), + Err(FunctionCallError::RespondToModel(message)) => { + return Ok(boxed_tool_output(AbortedToolOutput { message })); + } + Err(error) => return Err(error), + }; Ok(boxed_tool_output(McpToolOutput { - result: result.result, - tool_input: result.tool_input, + result, + tool_input, result_metadata_capture_allowed, wall_time: started.elapsed(), original_image_detail_supported: can_request_original_image_detail( diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 98eeb26405..18b6e16f52 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1,6 +1,4 @@ use std::sync::Arc; -use std::sync::OnceLock; -use std::sync::atomic::Ordering; use std::time::Instant; use tokio::sync::RwLock; @@ -30,6 +28,7 @@ use crate::tools::router::ToolCallSource; use codex_direct_tool_timing::ToolAdmission; use codex_direct_tool_timing::ToolBatchTiming; use codex_direct_tool_timing::ToolGateTiming; +use codex_extension_api::ToolCallOutcome; use codex_history::ResponseItemEnvelope; use codex_protocol::error::CodexErr; use codex_protocol::models::ResponseInputItem; @@ -37,7 +36,7 @@ use codex_protocol::models::ToolResultMetadata; struct ToolCallTimingGuard { started_at: Option, - execution_started_at: Arc>, + call_state: Arc, conversation_id: String, turn_id: String, call_id: String, @@ -61,6 +60,23 @@ struct DirectToolTiming { batch: ToolBatchTiming, } +enum ToolDispatchError { + Call(FunctionCallError), + Aborted(ToolAbort), +} + +#[derive(Clone, Copy)] +enum ToolAbort { + BeforeDispatch, + User(f32), +} + +impl From for ToolDispatchError { + fn from(error: FunctionCallError) -> Self { + Self::Call(error) + } +} + // Direct spawn and wait calls use this turn's tool time; child work stays on its own turn. // CodeMode and follow-up agent operations still lack runtime capture. fn direct_path_supported(tool_name: &codex_tools::ToolName, source: &ToolCallSource) -> bool { @@ -235,6 +251,7 @@ impl ToolCallRuntime { &turn.sub_id, &call, &source, + Arc::clone(&call_state), Arc::clone(&session.services.extensions), ); if let Some(guard) = tool_call_timing_guard.as_mut() { @@ -247,9 +264,6 @@ impl ToolCallRuntime { }); } } - let execution_started_at = tool_call_timing_guard - .as_ref() - .map(|timing| Arc::clone(&timing.execution_started_at)); let abort_session = Arc::clone(&session); let abort_source = source.clone(); let abort_turn = Arc::clone(&turn); @@ -264,6 +278,17 @@ impl ToolCallRuntime { }; let dispatch_tool_name = call.tool_name.clone(); let dispatch_call_id = call.call_id.clone(); + let record_result_ready = + move |turn_id: &str, tool_name: &codex_tools::ToolName, call_id: &str| { + call_trace::result_ready( + thread_id, + turn_id, + tool_name, + call_id, + trace_source, + started.elapsed(), + ); + }; // Code-mode callbacks can resume outside the turn's local span ancestry. let dispatch_span = trace_span!( @@ -312,6 +337,12 @@ impl ToolCallRuntime { } .or_cancel_tool(&invocation_cancellation_token) .await?; + // A peer may have exhausted the account while this call waited for its slot. + if session.rollout_budget().tools_stopped() { + tracing::Span::current().record("aborted", true); + record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id); + return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch)); + } // Publish the completed path before releasing the admission lock, // including when cancellation drops the handler future. let _admitted_timing = work_timing.map(|(gate, work)| { @@ -325,12 +356,6 @@ impl ToolCallRuntime { admission_started, ) }); - // Admission through the parallel-execution gate marks the end - // of dispatch waiting and the start of handler execution. - if let Some(execution_started_at) = execution_started_at { - let _ = execution_started_at.set(Instant::now()); - } - let result = router .dispatch_tool_call_with_state( session, @@ -349,24 +374,17 @@ impl ToolCallRuntime { // A fatal error still propagates to the caller instead of producing a tool // result; unlike a normal tool failure, it has no readiness event. if !matches!(&result, Err(FunctionCallError::Fatal(_))) { - call_trace::result_ready( - thread_id, - &turn.sub_id, - &dispatch_tool_name, - &dispatch_call_id, - trace_source, - started.elapsed(), - ); + record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id); } // Finished calls release their IDs even if their results have not been collected. // Otherwise the outer task may still need the ID for its abort callback. - if dispatch_call_state - .terminal_outcome_reached - .load(Ordering::Acquire) - { + if let Some(outcome) = dispatch_call_state.terminal_outcome.get() { + if *outcome == ToolCallOutcome::Aborted { + tracing::Span::current().record("aborted", true); + } drop(dispatch_call_state.invocation.take()); } - result + result.map_err(ToolDispatchError::Call) } .instrument(dispatch_span.clone().or_current()), )); @@ -382,7 +400,7 @@ impl ToolCallRuntime { _ = cancellation_token.cancelled() => { // Dynamic handlers observe the token and publish their terminal item themselves. if finishes_on_cancellation - || call_state.terminal_outcome_reached.load(Ordering::Acquire) + || call_state.terminal_outcome.get().is_some() || dispatch_handle.is_finished() { dispatch_handle.await.map_err(join_error)? @@ -393,28 +411,10 @@ impl ToolCallRuntime { match dispatch_handle.await { Ok(result) => result, Err(err) if err.is_cancelled() => { - let _reservation = call_state.invocation.take(); - let response = Self::aborted_response(&call, Some(secs)); - call_trace::result_ready( - thread_id, - &abort_turn.sub_id, - &call.tool_name, - &call.call_id, - trace_source, - started.elapsed(), - ); - call_state.terminal_outcome_reached.store(true, Ordering::Release); - notify_tool_aborted( - abort_session.as_ref(), - abort_turn.as_ref(), - call.call_id.as_str(), - &call.tool_name, - abort_source.clone(), - ) - .await; - Ok(response) + record_result_ready(&abort_turn.sub_id, &call.tool_name, &call.call_id); + Err(ToolDispatchError::Aborted(ToolAbort::User(secs))) } - Err(err) => Err(Self::tool_task_join_error(err)), + Err(err) => Err(Self::tool_task_join_error(err).into()), } } }, @@ -422,25 +422,32 @@ impl ToolCallRuntime { let _reservation = call_state.invocation.take(); // Cancellation outside the handler can return before the registry reports an outcome. if cancellation_token.is_cancelled() - && !call_state - .terminal_outcome_reached - .swap(true, Ordering::AcqRel) + && !matches!(result, Err(ToolDispatchError::Aborted(_))) + && call_state.terminal_outcome.get().is_none() { - notify_tool_aborted( - abort_session.as_ref(), - abort_turn.as_ref(), - call.call_id.as_str(), - &call.tool_name, - abort_source, - ) - .await; - result = Ok(Self::aborted_response( - &call, - Some(started.elapsed().as_secs_f32().max(0.1)), - )); + result = Err(ToolDispatchError::Aborted(ToolAbort::User( + started.elapsed().as_secs_f32().max(0.1), + ))); } + let mut result = match result { + Ok(output) => Ok(output), + Err(ToolDispatchError::Call(error)) => Err(error), + Err(ToolDispatchError::Aborted(stop)) => { + if call_state.terminal_outcome.set(ToolCallOutcome::Aborted).is_ok() { + notify_tool_aborted( + &abort_session, + &abort_turn, + &call.call_id, + &call.tool_name, + abort_source, + ) + .await; + } + Ok(Self::stopped_response(&call, stop)) + } + }; // Use one completion measurement for logging and response formatting. - // Measuring inside a handler would omit routing and output processing. + // Completion includes output processing after the handler returns. if let Some(timing) = tool_call_timing_guard.as_mut() && let Some(handler_duration_ms) = timing.finish() && let Ok(result) = &mut result @@ -490,27 +497,28 @@ impl ToolCallRuntime { } } - pub(crate) fn aborted_response(call: &ToolCall, elapsed_secs: Option) -> AnyToolResult { + pub(crate) fn aborted_before_dispatch(call: &ToolCall) -> AnyToolResult { + Self::stopped_response(call, ToolAbort::BeforeDispatch) + } + + fn stopped_response(call: &ToolCall, stop: ToolAbort) -> AnyToolResult { + let message = match stop { + ToolAbort::BeforeDispatch => "aborted before dispatch".to_string(), + ToolAbort::User(secs) + if call.tool_name.is_default_namespace() + && call.tool_name.name == "exec_command" => + { + format!("Wall time: {secs:.1} seconds\naborted by user") + } + ToolAbort::User(secs) => format!("aborted by user after {secs:.1}s"), + }; AnyToolResult { call_id: call.call_id.clone(), payload: call.payload.clone(), - result: Box::new(AbortedToolOutput { - message: Self::abort_message(call, elapsed_secs), - }), + result: Box::new(AbortedToolOutput { message }), post_tool_use_payload: None, } } - - fn abort_message(call: &ToolCall, elapsed_secs: Option) -> String { - let Some(secs) = elapsed_secs else { - return "aborted before dispatch".to_string(); - }; - if call.tool_name.is_default_namespace() && call.tool_name.name == "exec_command" { - format!("Wall time: {secs:.1} seconds\naborted by user") - } else { - format!("aborted by user after {secs:.1}s") - } - } } impl ToolCallTimingGuard { @@ -520,6 +528,7 @@ impl ToolCallTimingGuard { turn_id: &str, call: &ToolCall, source: &ToolCallSource, + call_state: Arc, extensions: Arc>, ) -> Option { // Code-mode calls are nested within a direct code-mode tool call whose @@ -534,7 +543,7 @@ impl ToolCallTimingGuard { Some(Self { started_at: Some(started_at), - execution_started_at: Arc::new(OnceLock::new()), + call_state, conversation_id: conversation_id.to_string(), turn_id: turn_id.to_string(), call_id: call.call_id.clone(), @@ -551,7 +560,8 @@ impl ToolCallTimingGuard { // Snapshot once so a concurrently-starting dispatch cannot make one // event internally inconsistent. let execution_started_at = self - .execution_started_at + .call_state + .handler_started_at .get() .copied() .filter(|execution_started_at| *execution_started_at <= completed_at); @@ -622,6 +632,8 @@ impl Drop for ToolCallTimingGuard { mod tests { use super::*; use std::collections::BTreeMap; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; use std::time::Duration; use crate::session::step_context::StepContext; @@ -632,13 +644,11 @@ mod tests { use crate::tools::registry::ToolRegistry; use crate::tools::router::ToolRouter; use crate::turn_diff_tracker::TurnDiffTracker; - use codex_extension_api::ToolCallOutcome; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::openai_models::ToolMode; use pretty_assertions::assert_eq; use tokio::sync::Notify; - use tokio::sync::oneshot; use tracing_test::internal::MockWriter; #[test] @@ -704,6 +714,7 @@ mod tests { "turn-id", &call, &ToolCallSource::Direct, + Arc::new(ToolCallState::default()), codex_extension_api::empty_extension_registry(), ); assert!( @@ -721,6 +732,7 @@ mod tests { cell_id: "cell-1".to_string(), runtime_tool_call_id: "runtime-call-1".to_string(), }, + Arc::new(ToolCallState::default()), codex_extension_api::empty_extension_registry(), ); assert!( @@ -758,11 +770,13 @@ mod tests { "turn-id", &call, &ToolCallSource::Direct, + Arc::new(ToolCallState::default()), codex_extension_api::empty_extension_registry(), ) .expect("model-visible timing must not depend on INFO logging"); timing - .execution_started_at + .call_state + .handler_started_at .set(started_at + Duration::from_secs(/*secs*/ 4)) .expect("execution start is unset"); let handler_ms = timing.finish().expect("completed handler duration"); @@ -831,6 +845,8 @@ mod tests { } else { Arc::new(ImmediateHandler { tool_name: tool_name.clone(), + supports_parallel: false, + calls: AtomicUsize::default(), }) }; let step_context = StepContext::for_test(Arc::clone(&turn_context)); @@ -935,9 +951,15 @@ mod tests { struct ImmediateHandler { tool_name: codex_tools::ToolName, + supports_parallel: bool, + calls: AtomicUsize, } impl ToolExecutor for ImmediateHandler { + fn supports_parallel_tool_calls(&self) -> bool { + self.supports_parallel + } + fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() } @@ -957,6 +979,7 @@ mod tests { where ToolInvocation: 'a, { + self.calls.fetch_add(1, Ordering::Relaxed); Box::pin(async { Ok( Box::new(FunctionToolOutput::from_text("ok".to_string(), Some(true))) @@ -968,63 +991,82 @@ mod tests { impl CoreToolRuntime for ImmediateHandler {} - struct BlockingFinishContributor { - records: Arc>>, - finish_started: std::sync::Mutex>>, - allow_finish: Arc, + #[derive(Default)] + struct PausedToolLifecycle { + finishes: std::sync::Mutex>, + at_start: Notify, + release_start: Notify, + at_finish: Notify, + release_finish: Notify, } - impl codex_extension_api::ToolLifecycleContributor for BlockingFinishContributor { + impl codex_extension_api::ToolLifecycleContributor for PausedToolLifecycle { + fn on_tool_start<'a>( + &'a self, + input: codex_extension_api::ToolStartInput<'a>, + ) -> codex_extension_api::ToolLifecycleFuture<'a> { + Box::pin(async move { + if input.call_id == "held-at-start" { + self.at_start.notify_one(); + self.release_start.notified().await; + } + }) + } + fn on_tool_finish<'a>( &'a self, input: codex_extension_api::ToolFinishInput<'a>, ) -> codex_extension_api::ToolLifecycleFuture<'a> { - let records = Arc::clone(&self.records); - let allow_finish = Arc::clone(&self.allow_finish); - let finish_started = self - .finish_started - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); - let outcome = input.outcome; Box::pin(async move { - if let Some(finish_started) = finish_started { - let _ = finish_started.send(()); - } - allow_finish.notified().await; - records + self.finishes .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .push(outcome); + .push((input.call_id.to_owned(), input.outcome)); + if input.call_id == "call-1" { + self.at_finish.notify_one(); + self.release_finish.notified().await; + } }) } } #[tokio::test] - async fn cancellation_after_handler_finishes_preserves_completed_lifecycle() - -> anyhow::Result<()> { + async fn completed_and_budget_stopped_tools_report_their_own_lifecycle() -> anyhow::Result<()> { let (mut session, turn_context) = crate::session::tests::make_session_and_context().await; - let records = Arc::new(std::sync::Mutex::new(Vec::new())); - let (finish_started_tx, finish_started_rx) = oneshot::channel(); - let allow_finish = Arc::new(Notify::new()); + session + .rollout_budget() + .configure(crate::config::RolloutBudgetConfig { + enforce_limit: true, + limit_tokens: 1, + reminder_at_remaining_tokens: Vec::new(), + sampling_token_weight: 1.0, + prefill_token_weight: 1.0, + }); + let lifecycle = Arc::new(PausedToolLifecycle::default()); let mut builder = codex_extension_api::ExtensionRegistryBuilder::::new(); - builder.tool_lifecycle_contributor(Arc::new(BlockingFinishContributor { - records: Arc::clone(&records), - finish_started: std::sync::Mutex::new(Some(finish_started_tx)), - allow_finish: Arc::clone(&allow_finish), - })); + builder.tool_lifecycle_contributor(lifecycle.clone()); session.services.extensions = Arc::new(builder.build()); let session = Arc::new(session); let turn_context = Arc::new(turn_context); - let tool_name = codex_tools::ToolName::plain("test_tool"); - let handler = Arc::new(ImmediateHandler { + let tool_name = codex_tools::ToolName::plain("parallel_tool"); + let queued_name = codex_tools::ToolName::plain("exclusive_tool"); + let parallel_handler = Arc::new(ImmediateHandler { tool_name: tool_name.clone(), - }) as Arc; + supports_parallel: true, + calls: AtomicUsize::default(), + }); + let queued_handler = Arc::new(ImmediateHandler { + tool_name: queued_name.clone(), + supports_parallel: false, + calls: AtomicUsize::default(), + }); + let handlers: [Arc; 2] = + [parallel_handler.clone(), queued_handler.clone()]; let step_context = StepContext::for_test(Arc::clone(&turn_context)); let router = Arc::new(ToolRouter::from_parts( - ToolRegistry::from_tools([handler]), + ToolRegistry::from_tools(handlers), Vec::new(), ToolMode::Direct, BTreeMap::new(), @@ -1034,7 +1076,8 @@ mod tests { )); let step_context = step_context.with_tool_router_for_test(router); let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); - let runtime = ToolCallRuntime::new(session, step_context, tracker); + let runtime = ToolCallRuntime::new(Arc::clone(&session), step_context, tracker); + let queued_runtime = runtime.clone(); let cancellation_token = CancellationToken::new(); let call = ToolCall { tool_name, @@ -1046,20 +1089,69 @@ mod tests { native_call: None, }; - let response_task = - tokio::spawn(runtime.handle_tool_call(call, cancellation_token.clone())); - tokio::time::timeout(Duration::from_secs(1), finish_started_rx) - .await - .expect("timed out waiting for lifecycle notification to start") - .expect("lifecycle notification should start"); - cancellation_token.cancel(); - tokio::time::sleep(Duration::from_millis(10)).await; - allow_finish.notify_waiters(); + let queued_call = ToolCall { + tool_name: queued_name, + call_id: "call-2".to_string(), + ..call.clone() + }; + let held_call = ToolCall { + call_id: "held-at-start".to_string(), + ..call.clone() + }; + let response = runtime + .clone() + .handle_tool_call(call, cancellation_token.clone()); + tokio::pin!(response); + tokio::time::timeout(Duration::from_secs(1), lifecycle.at_finish.notified()).await?; - let response = tokio::time::timeout(Duration::from_secs(1), response_task) - .await - .expect("timed out waiting for tool response") - .expect("tool response task should join")?; + // This call has a shared slot, but its start observer hasn't released the handler. + let held = runtime.handle_tool_call(held_call, CancellationToken::new()); + tokio::time::timeout(Duration::from_secs(1), lifecycle.at_start.notified()).await?; + + // The first call keeps its shared slot until its finish callback returns. New + // readers stop getting slots only once the exclusive call has actually queued. + let gate = Arc::clone(&queued_runtime.parallel_execution); + assert!(gate.try_read().is_ok()); + let expected_abort = ResponseInputItem::FunctionCallOutput { + call_id: queued_call.call_id.clone(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text("aborted before dispatch".to_string()), + success: None, + }, + }; + let queued = queued_runtime.handle_tool_call(queued_call, CancellationToken::new()); + tokio::time::timeout(Duration::from_secs(1), async { + while gate.try_read().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("the tool should be queued behind the held slot"); + let usage = codex_protocol::protocol::TokenUsage { + output_tokens: 1, + ..Default::default() + }; + session + .record_rollout_budget_usage("peer-turn", &usage.into()) + .await?; + cancellation_token.cancel(); + // Let the caller observe cancellation while the already-finished handler is held. + assert!(futures::poll!(response.as_mut()).is_pending()); + lifecycle.release_start.notify_one(); + let held_reply = tokio::time::timeout(Duration::from_secs(1), held).await??; + assert_eq!( + held_reply.item, + ResponseInputItem::FunctionCallOutput { + call_id: "held-at-start".into(), + output: FunctionCallOutputPayload::from_text( + "aborted before handler start: rollout budget exhausted".into(), + ), + } + .into() + ); + lifecycle.release_finish.notify_one(); + + let response = tokio::time::timeout(Duration::from_secs(1), response).await??; let expected_response = ResponseInputItem::FunctionCallOutput { call_id: "call-1".to_string(), output: FunctionCallOutputPayload { @@ -1071,13 +1163,30 @@ mod tests { ResponseItemEnvelope::new(expected_response.into()), response ); + assert_eq!( + ResponseItemEnvelope::new(expected_abort.into()), + tokio::time::timeout(Duration::from_secs(1), queued).await?? + ); + assert_eq!(parallel_handler.calls.load(Ordering::Relaxed), 1); + assert_eq!(queued_handler.calls.load(Ordering::Relaxed), 0); - let actual = records + let actual = lifecycle + .finishes .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .drain(..) .collect::>(); - assert_eq!(vec![ToolCallOutcome::Completed { success: true }], actual); + assert_eq!( + vec![ + ( + "call-1".to_string(), + ToolCallOutcome::Completed { success: true } + ), + ("held-at-start".to_string(), ToolCallOutcome::Aborted), + ("call-2".to_string(), ToolCallOutcome::Aborted), + ], + actual + ); Ok(()) } diff --git a/codex-rs/core/src/tools/parallel_observation_tests.rs b/codex-rs/core/src/tools/parallel_observation_tests.rs index 132294fb2e..36fc8fdf34 100644 --- a/codex-rs/core/src/tools/parallel_observation_tests.rs +++ b/codex-rs/core/src/tools/parallel_observation_tests.rs @@ -36,6 +36,7 @@ fn cancelled_dispatch_reports_zero_handler_work_even_without_tracing() { "turn", &call, &ToolCallSource::Direct, + Arc::new(ToolCallState::default()), Arc::new(extensions.build()), ) .unwrap(); diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index e9d6e86ee1..ac8288c7e4 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::sync::Arc; -use std::sync::atomic::Ordering; use std::time::Duration; +use std::time::Instant; use crate::function_tool::FunctionCallError; use crate::function_tool::OrCancelToolExt; @@ -13,6 +13,7 @@ use crate::memory_usage::emit_metric_for_tool_read; use crate::memory_usage::shell_script_for_invocation; use crate::session::session::Session; use crate::session::turn_context::TurnContext; +use crate::tools::context::AbortedToolOutput; use crate::tools::context::FunctionToolOutput; use crate::tools::context::ToolCallState; use crate::tools::context::ToolEnvironments; @@ -39,6 +40,7 @@ use codex_protocol::models::ResponseItem; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::protocol::EventMsg; use codex_rollout::state_db; +use codex_rollout_trace::ExecutionStatus; use codex_shell_command::parse_command::parse_shell_script; use codex_tools::FunctionsNamespaceFunctionPrefixes; use codex_tools::ToolName; @@ -930,12 +932,40 @@ impl ToolRegistry { return Err(fail_before_execution(dispatch_trace, &invocation, err).await); } - let invocation_guard = reservation.activate(&invocation); - if tool.mcp_server_name().is_none() { - notify_tool_start(&invocation, /*mcp_tool*/ None) - .or_cancel_tool(&cancellation_token) - .await?; + // Environment resolution, pre-tool hooks and Guardian may all have waited. + let invocation_guard = if invocation.session.rollout_budget().tools_stopped() { + None + } else { + let guard = reservation.activate(&invocation); + if tool.mcp_server_name().is_none() { + notify_tool_start(&invocation, /*mcp_tool*/ None) + .or_cancel_tool(&cancellation_token) + .await?; + } + guard + }; + // A start observer can also wait. This is the last check before polling the + // handler; a stop after admission does not cancel an already-running tool. + if invocation.session.rollout_budget().tools_stopped() { + drop(invocation_guard); + let result = AnyToolResult { + call_id: invocation.call_id.clone(), + payload: invocation.payload.clone(), + result: Box::new(AbortedToolOutput { + message: "aborted before handler start: rollout budget exhausted".into(), + }), + post_tool_use_payload: None, + }; + dispatch_trace.record_result(&invocation, &result, ExecutionStatus::Aborted); + notify_tool_finish_if_unclaimed( + &invocation, + call_state.as_deref(), + ToolCallOutcome::Aborted, + ) + .await; + return Ok(result); } + let mut control_tool_analytics = tool .is_builtin_control_tool() .then(|| ControlToolCallGuard::new(&invocation)); @@ -1025,6 +1055,7 @@ impl ToolRegistry { { ToolCallOutcome::Aborted } + Ok(result) if result.result.was_aborted() => ToolCallOutcome::Aborted, Ok(_) => ToolCallOutcome::Completed { success }, Err(_) => ToolCallOutcome::Failed { handler_executed: true, @@ -1035,6 +1066,10 @@ impl ToolRegistry { .await; match result { + Ok(result) if result.result.was_aborted() => { + dispatch_trace.record_result(&invocation, &result, ExecutionStatus::Aborted); + Ok(result) + } Ok(mut result) => { if let Some(outcome) = post_tool_use_outcome { if outcome.should_block { @@ -1056,12 +1091,12 @@ impl ToolRegistry { } } tool.on_tool_result_accepted(&invocation, result.result.as_ref()); - dispatch_trace.record_completed( - &invocation, - &result.call_id, - &result.payload, - result.result.as_ref(), - ); + let status = if success { + ExecutionStatus::Completed + } else { + ExecutionStatus::Failed + }; + dispatch_trace.record_result(&invocation, &result, status); Ok(result) } Err(err) => { @@ -1077,7 +1112,7 @@ async fn notify_tool_finish_if_unclaimed( call_state: Option<&ToolCallState>, outcome: ToolCallOutcome, ) -> bool { - if call_state.is_some_and(|state| state.terminal_outcome_reached.swap(true, Ordering::AcqRel)) { + if call_state.is_some_and(|state| state.terminal_outcome.set(outcome).is_err()) { return false; } @@ -1092,6 +1127,9 @@ async fn handle_any_tool( ) -> Result { let call_id = invocation.call_id.clone(); let payload = invocation.payload.clone(); + if let Some(state) = call_state { + let _ = state.handler_started_at.set(Instant::now()); + } let output = tool.handle(invocation.clone()).await?; let post_tool_use_payload = CoreToolRuntime::post_tool_use_payload(tool, &invocation, output.as_ref()); diff --git a/codex-rs/core/src/tools/tool_dispatch_trace.rs b/codex-rs/core/src/tools/tool_dispatch_trace.rs index a16e70055d..a0eb4da44e 100644 --- a/codex-rs/core/src/tools/tool_dispatch_trace.rs +++ b/codex-rs/core/src/tools/tool_dispatch_trace.rs @@ -9,6 +9,7 @@ use crate::tools::context::ToolCallSource; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; +use crate::tools::registry::AnyToolResult; use codex_rollout_trace::ExecutionStatus; use codex_rollout_trace::ToolDispatchInvocation; use codex_rollout_trace::ToolDispatchPayload; @@ -31,26 +32,24 @@ impl ToolDispatchTrace { Self { context } } - pub(crate) fn record_completed( + pub(crate) fn record_result( self, invocation: &ToolInvocation, - call_id: &str, - payload: &ToolPayload, - result: &dyn ToolOutput, + result: &AnyToolResult, + status: ExecutionStatus, ) { if !self.context.is_enabled() { return; } - let Some(result_payload) = tool_dispatch_result(invocation, call_id, payload, result) - else { + let Some(result_payload) = tool_dispatch_result( + invocation, + &result.call_id, + &result.payload, + result.result.as_ref(), + ) else { return; }; - let status = if result.success_for_logging() { - ExecutionStatus::Completed - } else { - ExecutionStatus::Failed - }; self.context.record_completed(status, result_payload); } diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index 5f6f80d34d..e80421c258 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -19,6 +19,11 @@ pub trait ToolOutput: Send { fn success_for_logging(&self) -> bool; + /// Whether the call was refused before its underlying operation started. + fn was_aborted(&self) -> bool { + false + } + /// Finalizes output using the same completed handler duration reported in tool-call logs. /// Called before recording model-visible history; implementations must not measure time here. fn set_handler_duration_ms(&mut self, _handler_duration_ms: u64) {} @@ -92,6 +97,10 @@ where (**self).success_for_logging() } + fn was_aborted(&self) -> bool { + (**self).was_aborted() + } + fn set_handler_duration_ms(&mut self, handler_duration_ms: u64) { (**self).set_handler_duration_ms(handler_duration_ms); }