diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 3312f0bfcb..5d619e91b1 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -3420,6 +3420,10 @@ "hide_spawn_agent_metadata": { "type": "boolean" }, + "interrupt_at_item_boundary": { + "description": "Let a subagent finish its current output item while cancelling tools immediately.", + "type": "boolean" + }, "max_concurrent_threads_per_session": { "format": "uint", "minimum": 1.0, diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index aa227fee4d..f6c37735c8 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -334,17 +334,14 @@ impl LocalAgentControl { } /// Interrupt the current task for an existing agent thread. - pub(crate) async fn interrupt_agent(&self, agent_id: ThreadId) -> CodexResult { + pub(crate) async fn interrupt_agent(&self, agent_id: ThreadId, op: Op) -> CodexResult { let state = self.runtime.upgrade()?; self.handle_thread_request_result( agent_id, &state, state .send_op( - agent_id, - Op::Interrupt, - /*parent_turn_id*/ None, - /*root_turn_id*/ None, + agent_id, op, /*parent_turn_id*/ None, /*root_turn_id*/ None, /*message_delivery*/ None, ) .await, diff --git a/codex-rs/core/src/agent/control/api.rs b/codex-rs/core/src/agent/control/api.rs index 22e43a59cb..ba4ab17781 100644 --- a/codex-rs/core/src/agent/control/api.rs +++ b/codex-rs/core/src/agent/control/api.rs @@ -26,6 +26,7 @@ use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::error::Result; use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::Op; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TokenUsage; use codex_rollout_trace::ThreadTraceContext; @@ -211,7 +212,7 @@ impl AgentControl for LocalAgentControl { match version { MultiAgentVersion::Disabled | MultiAgentVersion::V1 => { let snapshot = self.inspect_agent(target).await?; - self.interrupt_agent(target).await?; + self.interrupt_agent(target, Op::Interrupt).await?; Ok(snapshot) } MultiAgentVersion::V2 => self.interrupt_spawned_agent(caller, target).await, diff --git a/codex-rs/core/src/agent/control/interrupt.rs b/codex-rs/core/src/agent/control/interrupt.rs index 5ad4c19351..110a7518ae 100644 --- a/codex-rs/core/src/agent/control/interrupt.rs +++ b/codex-rs/core/src/agent/control/interrupt.rs @@ -10,6 +10,7 @@ use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::error::CodexErrorDetails; use codex_protocol::error::Result as CodexResult; +use codex_protocol::protocol::Op; impl LocalAgentControl { /// Interrupts a spawned agent's current task, preserving the status observed before dispatch. @@ -38,7 +39,7 @@ impl LocalAgentControl { CodexErr::UnsupportedOperation("target agent is missing an agent_path".to_string()) })?; let snapshot = self.inspect_agent(target).await?; - match self.interrupt_agent(target).await { + match self.interrupt_agent(target, Op::InterruptAgent).await { Ok(_) => {} Err(err) if matches!( diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 410bc7239b..96b86fce61 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -12870,6 +12870,7 @@ tool_namespace = "agents" hide_spawn_agent_metadata = true expose_spawn_agent_model_overrides = false wait_agent_enabled = false +interrupt_at_item_boundary = true disable_native_tools = true disable_direct_message = true message_board_in_memory = true @@ -12929,6 +12930,7 @@ max_concurrent_threads_per_session = 9 assert!(!config.multi_agent_v2.preserve_fork_prefix); assert!(!config.multi_agent_v2.expose_spawn_agent_model_overrides); assert!(!config.multi_agent_v2.wait_agent_enabled); + assert!(config.multi_agent_v2.interrupt_at_item_boundary); assert!(config.multi_agent_v2.disable_native_tools); assert!(config.multi_agent_v2.disable_direct_message); assert!(config.multi_agent_v2.message_board_in_memory); diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index a095bce3a7..1006e07ec0 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1457,6 +1457,7 @@ pub struct MultiAgentV2Config { pub hide_spawn_agent_metadata: bool, pub expose_spawn_agent_model_overrides: bool, pub wait_agent_enabled: bool, + pub interrupt_at_item_boundary: bool, pub disable_native_tools: bool, pub disable_direct_message: bool, pub message_board_in_memory: bool, @@ -1481,6 +1482,7 @@ impl MultiAgentV2Config { hide_spawn_agent_metadata: true, expose_spawn_agent_model_overrides: true, wait_agent_enabled: true, + interrupt_at_item_boundary: false, disable_native_tools: false, disable_direct_message: false, message_board_in_memory: false, @@ -2968,6 +2970,9 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config let wait_agent_enabled = base .and_then(|config| config.wait_agent_enabled) .unwrap_or(default.wait_agent_enabled); + let interrupt_at_item_boundary = base + .and_then(|config| config.interrupt_at_item_boundary) + .unwrap_or(default.interrupt_at_item_boundary); let disable_native_tools = base .and_then(|config| config.disable_native_tools) .unwrap_or(default.disable_native_tools); @@ -3010,6 +3015,7 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config hide_spawn_agent_metadata, expose_spawn_agent_model_overrides, wait_agent_enabled, + interrupt_at_item_boundary, disable_native_tools, disable_direct_message, message_board_in_memory, diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 7a91460967..7f0bbbd612 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -1,4 +1,5 @@ use super::Submission; +use super::sampling_interrupt::SamplingInterrupt; use crate::McpRefreshReason; use crate::WithTurnExtensionData; use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; @@ -60,7 +61,7 @@ use tracing::info; use tracing::warn; pub async fn interrupt(sess: &Arc) { - sess.interrupt_task().await; + sess.interrupt_task(sess.active_turn.lock().await).await; } pub async fn clean_background_terminals(sess: &Arc) { @@ -615,6 +616,10 @@ pub(super) async fn submission_loop( interrupt(&sess).await; false } + Op::InterruptAgent => { + SamplingInterrupt::request(&sess).await; + false + } Op::InterruptIfNoPendingInput { turn_id, reply } => { sess.interrupt_turn_if_no_pending_input(&turn_id, reply) .await; diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index 6576985ab1..37ea40f065 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -1039,10 +1039,15 @@ async fn review_guardian_mcp_elicitation( session: Arc, request: ElicitationReviewRequest, ) -> anyhow::Result> { - let Some((turn_context, cancellation_token)) = - session.active_turn_context_and_cancellation_token().await - else { - return Ok(None); + let (turn_context, cancellation_token) = { + let active = session.active_turn.lock().await; + let Some(task) = active.as_ref().and_then(|turn| turn.task.as_ref()) else { + return Ok(None); + }; + ( + Arc::clone(&task.turn_context), + task.mcp_elicitation_cancellation.child_token(), + ) }; let Some(mcp_config) = session.services.mcp_runtime.current_config() else { diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 6ff243beb0..84408e7d19 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -281,6 +281,8 @@ mod retained_context; mod review; mod rollout_budget; mod rollout_reconstruction; +mod sampling_interrupt; +pub(crate) use sampling_interrupt::SamplingInterrupt; #[allow(clippy::module_inception)] pub(crate) mod session; pub(crate) mod startup_prewarm; @@ -6043,15 +6045,6 @@ impl Session { turn_state.lock().await.has_memory_citation = true; } - pub async fn interrupt_task(self: &Arc) { - info!("interrupt received: abort current task, if any"); - let had_active_turn = self.active_turn.lock().await.is_some(); - self.abort_all_tasks(TurnAbortReason::Interrupted).await; - if !had_active_turn { - self.cancel_mcp_startup(); - } - } - pub(crate) fn hooks(&self) -> Arc { self.services.hooks.load_full() } diff --git a/codex-rs/core/src/session/sampling_interrupt.rs b/codex-rs/core/src/session/sampling_interrupt.rs new file mode 100644 index 0000000000..960914d6e1 --- /dev/null +++ b/codex-rs/core/src/session/sampling_interrupt.rs @@ -0,0 +1,116 @@ +//! Lets an opted-in child finish the sampled item while its tools are cancelled. + +use super::session::Session; +use super::turn_context::TurnContext; +use crate::state::ActiveTurn; +use codex_features::Feature; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; +use codex_protocol::protocol::MultiAgentVersion; +use std::ops::ControlFlow; +use std::sync::Arc; +use tokio_util::sync::CancellationToken; + +/// Owned by the active turn through publication of its terminal event. +pub(crate) struct SamplingInterrupt { + tools: CancellationToken, + pub(super) finished: CancellationToken, +} + +impl Drop for SamplingInterrupt { + fn drop(&mut self) { + self.finished.cancel(); + } +} + +impl SamplingInterrupt { + pub(crate) fn enabled(turn: &TurnContext) -> bool { + turn.config.multi_agent_v2.interrupt_at_item_boundary + && turn.multi_agent_version == MultiAgentVersion::V2 + && turn.parent_thread_id.is_some() + } + + pub(super) async fn register( + session: &Session, + turn: &TurnContext, + tools: &CancellationToken, + ) -> CodexResult<()> { + let mut active = session.active_turn.lock().await; + let active = matching_turn(&mut active, turn).ok_or(CodexErr::TurnAborted)?; + if tools.is_cancelled() { + return Err(CodexErr::TurnAborted); + } + active.sampling_interrupt = Some(Self { + tools: tools.clone(), + finished: CancellationToken::new(), + }); + Ok(()) + } + + pub(super) async fn unregister(session: &Session, turn: &TurnContext) { + let mut active = session.active_turn.lock().await; + if let Some(active) = matching_turn(&mut active, turn) + && !active + .sampling_interrupt + .as_ref() + .is_some_and(Self::is_requested) + { + active.sampling_interrupt = None; + } + } + + pub(crate) fn is_requested(&self) -> bool { + self.tools.is_cancelled() + } + + pub(super) async fn request(session: &Arc) { + let action = { + let active = session.active_turn.lock().await; + if let Some(ActiveTurn { + sampling_interrupt: Some(interrupt), + task, + .. + }) = active.as_ref() + { + if interrupt.is_requested() { + return; + } + interrupt.tools.cancel(); + let Some(task) = task else { return }; + task.mcp_elicitation_cancellation.cancel(); + ControlFlow::Continue( + task.turn_context + .config + .features + .enabled(Feature::CodeModeInterrupt), + ) + } else { + // Claim the hard abort under the same lock that admits a sampling attempt. + ControlFlow::Break(session.interrupt_task(active)) + } + }; + match action { + ControlFlow::Break(abort) => abort.await, + ControlFlow::Continue(true) => { + session + .services + .code_mode_service + .interrupt_active_cells() + .await; + } + ControlFlow::Continue(false) => {} + } + } +} + +fn matching_turn<'a>( + active: &'a mut Option, + turn: &TurnContext, +) -> Option<&'a mut ActiveTurn> { + active.as_mut().filter(|active| { + active + .task + .as_ref() + .is_some_and(|task| task.turn_context.sub_id == turn.sub_id) + }) +} diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 0deab079e8..fea7a5d249 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -46,13 +46,13 @@ use crate::responses_retry::is_terminal_sampling_outcome; use crate::session::PreviousTurnSettings; use crate::session::TurnInput; use crate::session::daemon_recovery::RecordedTurnInput; +use crate::session::sampling_interrupt::SamplingInterrupt; use crate::session::session::Session; use crate::session::step_context::StepContext; use crate::session::turn_context::TurnContext; use crate::skills::emit_explicit_skill_invocations; use crate::stream_events_utils::AssistantText; use crate::stream_events_utils::HandleOutputCtx; -use crate::stream_events_utils::InFlightFuture; use crate::stream_events_utils::TurnItemContributorPolicy; use crate::stream_events_utils::finalize_non_tool_response_item; use crate::stream_events_utils::handle_non_tool_response_item; @@ -88,6 +88,7 @@ use codex_features::Feature; use codex_file_system::EnvironmentAccess; use codex_file_system::FindUpErrorPolicy; use codex_file_system::find_nearest_ancestor_with_markers; +use codex_history::ResponseItemEnvelope; use codex_hooks::SamplingKind; use codex_login::CodexAuth; use codex_model_provider::RemoteCompactionSupport; @@ -2066,7 +2067,8 @@ async fn run_sampling_request( (submitted, pending) }; let mut draining_for_mailbox = false; - let err = match try_run_sampling_request( + let tool_cancellation = cancellation_token.child_token(); + let result = try_run_sampling_request( &mut sampling_request, tool_runtime.clone(), Arc::clone(&sess), @@ -2080,9 +2082,13 @@ async fn run_sampling_request( unfinished_plan_id, cancellation_token.child_token(), &mut draining_for_mailbox, + &tool_cancellation, ) - .await - { + .await; + if SamplingInterrupt::enabled(&turn_context) && tool_cancellation.is_cancelled() { + return Err(CodexErr::TurnAborted); + } + let err = match result { Ok(output) => { return Ok((output, notify_messages)); } @@ -2901,7 +2907,7 @@ async fn handle_assistant_item_done_in_plan_mode( #[instrument(level = "trace", skip_all)] async fn drain_in_flight( - in_flight: &mut FuturesOrdered>, + in_flight: &mut (impl Stream> + Unpin), sess: Arc, step_context: &StepContext, ) { @@ -2951,6 +2957,7 @@ async fn try_run_sampling_request( unfinished_plan_id: &mut Option, cancellation_token: CancellationToken, draining_for_mailbox: &mut bool, + tool_cancellation: &CancellationToken, ) -> CodexResult { let turn_context = Arc::clone(&step_context.turn); #[expect(deprecated, reason = "Classify feedback using the initial turn cwd.")] @@ -3015,6 +3022,7 @@ async fn try_run_sampling_request( }; let mut resampling = crate::guardian::ResponseResampling::new(&sess); let prompt = Arc::new(prompt); + let interrupt_at_boundary = SamplingInterrupt::enabled(&turn_context); let mut stream_request = async |prompt: &Arc| { let send = client_session .open_model_response( @@ -3030,6 +3038,12 @@ async fn try_run_sampling_request( &inference_trace, ) .instrument(trace_span!("stream_request")); + let send = async { + if interrupt_at_boundary { + SamplingInterrupt::register(&sess, &turn_context, tool_cancellation).await?; + } + send.await + }; sampling_request .open( &sess, @@ -3040,9 +3054,18 @@ async fn try_run_sampling_request( ) .await }; - let mut stream = stream_request(&prompt) + let mut stream = match stream_request(&prompt) .or_cancel(&cancellation_token) - .await??; + .await? + { + Ok(stream) => stream, + Err(error) => { + if interrupt_at_boundary { + SamplingInterrupt::unregister(&sess, &turn_context).await; + } + return Err(error); + } + }; let can_interrupt = turn_context .config .features @@ -3084,6 +3107,10 @@ async fn try_run_sampling_request( !sess.services.extensions.turn_item_contributors().is_empty(); let mut active_item_is_streaming_to_client = false; let receiving_span = trace_span!("receiving_stream"); + // Sending the request starts the first item, even before the first stream event arrives. + let mut between_items = false; + let mut stopping_tools = false; + let mut completed_tools = Vec::new(); let outcome: CodexResult = loop { if response_completed && resampling.accepted_output.is_empty() { if resampling.update_usage_boundary { @@ -3112,12 +3139,25 @@ async fn try_run_sampling_request( ); let receive_started = Instant::now(); - let event = resampling + let receive = resampling .next(&mut stream) .instrument(trace_span!(parent: &handle_responses, "receiving")) .or_cancel(&preempt) - .or_cancel(&cancellation_token) - .await; + .or_cancel(&cancellation_token); + let event = tokio::select! { + biased; + () = tool_cancellation.cancelled(), if interrupt_at_boundary && between_items => break Err(CodexErr::TurnAborted), + () = tool_cancellation.cancelled(), if interrupt_at_boundary && !stopping_tools => { + stopping_tools = true; + continue; + } + Some(output) = in_flight.next(), if stopping_tools && !in_flight.is_empty() => { + // Drive cancellation now and record the outputs after the retained item. + completed_tools.push(output); + continue; + } + event = receive => event, + }; // Split the existing iteration span without adding another span per streamed delta. handle_responses.record( "codex.response.receive_wait_ms", @@ -3126,7 +3166,9 @@ async fn try_run_sampling_request( let event = match event { Ok(Ok(event)) => event, Ok(Err(_)) => { - if *draining_for_mailbox { + if *draining_for_mailbox + || (interrupt_at_boundary && tool_cancellation.is_cancelled()) + { preempt = CancellationToken::new(); continue; } @@ -3225,6 +3267,7 @@ async fn try_run_sampling_request( } } ResponseEvent::OutputItemDone(mut item) => { + between_items = true; // Serving emits one final value immediately before this item's completion. // After this handoff, the invocation/cell owns it independently of the stream. let assessment = pending_assessment @@ -3311,7 +3354,7 @@ async fn try_run_sampling_request( step_context: Arc::clone(&step_context), turn_store: Arc::clone(&turn_store), tool_runtime, - cancellation_token: cancellation_token.child_token(), + cancellation_token: tool_cancellation.child_token(), }; if *draining_for_mailbox { // Retain buffered calls in history, but use the existing pre-dispatch @@ -3368,6 +3411,7 @@ async fn try_run_sampling_request( // Hosts can keep the current response intact and deliver mail before the next // model request instead of cutting off its remaining tool calls. if preempt_for_mailbox_mail + && !(interrupt_at_boundary && tool_cancellation.is_cancelled()) && !*draining_for_mailbox && !turn_context .config @@ -3409,6 +3453,7 @@ async fn try_run_sampling_request( } } ResponseEvent::OutputItemAdded(mut item) => { + between_items = false; sess.assign_missing_response_item_id(&mut item); sess.reserve_assistant_message_order(&turn_context, &item) .await; @@ -3785,7 +3830,10 @@ async fn try_run_sampling_request( } } }; - if cancellation_token.is_cancelled() && !response_completed && can_interrupt { + if interrupt_at_boundary { + SamplingInterrupt::unregister(&sess, &turn_context).await; + } + if tool_cancellation.is_cancelled() && !response_completed && can_interrupt { let deadline = tokio::time::Instant::now() + Duration::from_secs(1); if let Some(interrupt) = stream.interrupt.take() { let _ = interrupt.send(codex_api::ResponseInterrupt::Cancel { deadline }); @@ -3805,7 +3853,7 @@ async fn try_run_sampling_request( *unfinished_plan_id = state.plan_item_state.unfinished_id(); } - if !in_flight.is_empty() { + if !in_flight.is_empty() || !completed_tools.is_empty() { let _tool_blocking_timing_guard = turn_context .turn_timing_state .begin_tool_blocking(tool_runtime.batch_timing()); @@ -3815,7 +3863,8 @@ async fn try_run_sampling_request( conversation.id = %sess.thread_id, turn.id = %turn_context.sub_id, ); - drain_in_flight(&mut in_flight, Arc::clone(&sess), &step_context).await; + let mut outputs = futures::stream::iter(completed_tools).chain(in_flight); + drain_in_flight(&mut outputs, Arc::clone(&sess), &step_context).await; } if should_emit_token_count { diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index e958098928..ad890b2c59 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -1002,7 +1002,26 @@ impl Session { responsesapi_client_metadata: ResponsesApiClientMetadataUpdate, input_metadata: UserInputMetadata, ) -> Result<(String, String), NotSubmittedReason> { - let mut active = self.active_turn.lock().await; + let mut active = loop { + let active = self.active_turn.lock().await; + let Some(interrupt) = active + .as_ref() + .and_then(|turn| turn.sampling_interrupt.as_ref()) + .filter(|interrupt| interrupt.is_requested()) + else { + break active; + }; + if expected_turn_id.is_some() { + return Err(NotSubmittedReason::NoActiveTurn); + } + // Start-or-steer can open a new turn once the old terminal event is published. + let finished = interrupt.finished.clone(); + drop(active); + tokio::select! { + () = finished.cancelled() => {} + () = self.services.local_agent_runtime.shutdown.cancelled() => return Err(NotSubmittedReason::ServerDraining), + } + }; let Some(active_turn) = active.as_mut() else { return Err(NotSubmittedReason::NoActiveTurn); }; diff --git a/codex-rs/core/src/state/turn.rs b/codex-rs/core/src/state/turn.rs index 6d34c18300..d54dbff5a3 100644 --- a/codex-rs/core/src/state/turn.rs +++ b/codex-rs/core/src/state/turn.rs @@ -22,6 +22,7 @@ use tokio::sync::oneshot; use super::TurnTokenUsage; use crate::agent::types::AgentExecutionGuard; +use crate::session::SamplingInterrupt; use crate::session::TurnInput; use crate::session::TurnInputQueue; use crate::session::step_context::StepContext; @@ -36,6 +37,7 @@ use codex_protocol::protocol::TokenUsage; /// Metadata about the currently running turn. pub(crate) struct ActiveTurn { pub(crate) task: Option, + pub(crate) sampling_interrupt: Option, /// Present when input admission is closed; cancelled when the final batch is recorded. /// Reservations that have not started still accept inputs. pub(crate) finishing: Option, @@ -66,6 +68,7 @@ impl Default for ActiveTurn { fn default() -> Self { Self { task: None, + sampling_interrupt: None, finishing: None, turn_state: Arc::new(Mutex::new(TurnState::default())), } @@ -84,6 +87,8 @@ pub(crate) struct RunningTask { pub(crate) kind: TaskKind, pub(crate) task: Arc, pub(crate) cancellation_token: CancellationToken, + /// MCP reviews can span sampling attempts and must stop even while the sampler drains. + pub(crate) mcp_elicitation_cancellation: CancellationToken, pub(crate) handle: AbortOnDropHandle<()>, pub(crate) turn_context: Arc, pub(crate) _agent_execution_guard: Option, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 9debf1956e..56e9658546 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -6,6 +6,7 @@ mod review; mod user_shell; use crate::McpRefreshReason; +use std::future::Future; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -16,12 +17,14 @@ use codex_extension_api::ThreadIdleCause; use futures::future::BoxFuture; use tokio::select; use tokio::sync::Mutex; +use tokio::sync::MutexGuard; use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; use tracing::Span; use tracing::field; +use tracing::info; use tracing::info_span; use tracing::trace; use tracing::trace_span; @@ -32,6 +35,7 @@ use crate::codex_thread::BackgroundTerminalInfo; use crate::config::Config; use crate::context::ContextualUserFragment; use crate::hook_runtime::run_turn_interrupt_hooks; +use crate::session::SamplingInterrupt; use crate::session::TurnInput; use crate::session::session::Session; use crate::session::turn::McpStartupRequirements; @@ -521,6 +525,11 @@ impl Session { handle: AbortOnDropHandle::new(handle), kind: task_kind, task, + mcp_elicitation_cancellation: if SamplingInterrupt::enabled(&turn_context) { + cancellation_token.child_token() + } else { + cancellation_token.clone() + }, cancellation_token, turn_context: Arc::clone(&turn_context), _agent_execution_guard: agent_execution_guard, @@ -648,10 +657,38 @@ impl Session { .await; } + pub(crate) fn interrupt_task( + self: &Arc, + active: MutexGuard<'_, Option>, + ) -> impl Future + Send + use<> { + info!("interrupt received: abort current task, if any"); + // Release the admission lock before creating the asynchronous cleanup. + let turn = self.take_active_turn(active, &TurnAbortReason::Interrupted); + let session = Arc::clone(self); + async move { + let had_active_turn = turn.is_some(); + session + .abort_taken_tasks(turn, TurnAbortReason::Interrupted) + .await; + if !had_active_turn { + session.cancel_mcp_startup(); + } + } + } + pub async fn abort_all_tasks(self: &Arc, reason: TurnAbortReason) { + let active = self.take_active_turn(self.active_turn.lock().await, &reason); + self.abort_taken_tasks(active, reason).await; + } + + async fn abort_taken_tasks( + self: &Arc, + active: Option, + reason: TurnAbortReason, + ) { let mut aborted_turn = false; let mut active_turn_to_clear = None; - if let Some(mut active_turn) = self.take_active_turn(&reason).await { + if let Some(mut active_turn) = active { let task = active_turn.task.take(); aborted_turn = task.is_some(); if let Some(task) = task { @@ -759,7 +796,7 @@ impl Session { && !self.work_budget().is_exhausted(); let finishing = CancellationToken::new(); let finish_recording = finishing.clone().drop_guard(); - let (turn_state, unadmitted_task, mut pending_input, outcome_handoff) = { + let (turn_state, unadmitted_task, mut pending_input, outcome_handoff, sampling_interrupted) = { let mut active = self.active_turn.lock().await; let active_turn = active.as_mut()?; if !Arc::ptr_eq(&active_turn.task.as_ref()?.turn_context, &turn_context) { @@ -782,6 +819,10 @@ impl Session { .then_some(task.task), task.pending_initial_input.unwrap_or_default(), task.outcome_handoff, + active_turn + .sampling_interrupt + .as_ref() + .is_some_and(SamplingInterrupt::is_requested), ) }; turn_context @@ -835,8 +876,20 @@ impl Session { }) .filter(is_automatic_rollout_stop) }; - if let Some(cause) = &stop_cause { - self.record_interrupted_turn_history(&turn_context, Some(cause)) + if sampling_interrupted + && turn_context + .config + .features + .enabled(Feature::CodeModeInterrupt) + { + // A cell can finish registering while its dispatch is being cancelled. + self.services + .code_mode_service + .interrupt_active_cells() + .await; + } + if sampling_interrupted || stop_cause.is_some() { + self.record_interrupted_turn_history(&turn_context, stop_cause.as_ref()) .await; } drop(finish_recording); @@ -1039,7 +1092,8 @@ impl Session { self.send_event(turn_context.as_ref(), event).await; } drop(outcome_handoff); - if cleared_active_turn { + // A parent interrupt must not let an idle contributor restart the child. + if cleared_active_turn && !sampling_interrupted { self.emit_thread_idle_lifecycle_if_idle(idle_cause).await; } // Private reviewers already flushed the terminal event before delivering it. @@ -1065,8 +1119,11 @@ impl Session { } } - async fn take_active_turn(&self, reason: &TurnAbortReason) -> Option { - let mut active = self.active_turn.lock().await; + fn take_active_turn( + &self, + mut active: MutexGuard<'_, Option>, + reason: &TurnAbortReason, + ) -> Option { if matches!( reason, TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 7887c29103..a6e32a22e9 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -158,6 +158,7 @@ type AgentControlFactory = fn capture_test_op(op: &Op) -> Option { match op { Op::Interrupt => Some(Op::Interrupt), + Op::InterruptAgent => Some(Op::InterruptAgent), Op::InterAgentCommunication { communication, start_options, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index eaad60b9fe..49d82f572f 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -3950,14 +3950,14 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() { let ops = manager.captured_ops(); assert!( ops.iter() - .any(|(thread_id, op)| *thread_id == agent_id && matches!(op, Op::Interrupt)) + .any(|(thread_id, op)| *thread_id == agent_id && matches!(op, Op::InterruptAgent)) ); assert!(!ops.iter().any(|(thread_id, op)| { (*thread_id == agent_id || *thread_id == child_id) && matches!(op, Op::Shutdown) })); assert!( !ops.iter() - .any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::Interrupt)) + .any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::InterruptAgent)) ); } diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 7fa6a3d646..6484fdcaae 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -1,4 +1,4 @@ -//! Pending mail survives eviction without loading idle recipients. +//! Child history survives interruption and pending mail survives eviction. use super::*; use crate::suite::settings_commits::COMMITTED_MODEL; @@ -10,12 +10,16 @@ use codex_extension_api::MessageLifecycleInput; use codex_protocol::ThreadId; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::Op; +use codex_protocol::protocol::TurnAbortReason; use core_test_support::ThreadIdle; +use core_test_support::streaming_sse::StreamingSseChunk; +use core_test_support::streaming_sse::start_streaming_sse_server; use pretty_assertions::assert_eq; use std::collections::HashSet; use std::sync::Arc; use std::sync::Mutex; use std::sync::mpsc; +use test_case::test_case; use tokio::sync::oneshot; #[derive(Default)] @@ -34,6 +38,161 @@ impl MessageLifecycleContributor for Messages { } } +#[test_case(ev_assistant_message("sampled", "Keep this answer."); "message")] +#[test_case(ev_function_call_with_namespace("sampled", MULTI_AGENT_V2_NAMESPACE, "send_message", + r#"{"target":"/root","message":"Should never be sent."}"#); "tool")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Result<()> { + skip_if_no_network!(Ok(())); + done["item"]["id"] = json!("sampled"); + if done["item"]["type"] == "message" { + done["item"]["phase"] = json!("final_answer"); + } + let chunk = |gate, events| StreamingSseChunk { + gate, + body: sse(events), + }; + let complete = |id| vec![chunk(None, vec![ev_completed(id)])]; + let call = + |name, args| ev_function_call_with_namespace(name, MULTI_AGENT_V2_NAMESPACE, name, args); + let (spawned, spawn_gate) = oneshot::channel(); + let (release_item, item_gate) = oneshot::channel(); + let (_finish_response, response_gate) = oneshot::channel(); + let (server, _) = start_streaming_sse_server(vec![ + vec![ + chunk( + None, + vec![call( + "spawn_agent", + r#"{"task_name":"worker","message":"Work","fork_turns":"none"}"#, + )], + ), + chunk(Some(spawn_gate), vec![ev_completed("spawn")]), + ], + vec![ + chunk(None, vec![ev_response_created("child")]), + chunk( + Some(item_gate), + vec![done.clone(), ev_assistant_message("later", "Discard this.")], + ), + chunk(Some(response_gate), vec![ev_completed("child")]), + ], + vec![chunk( + None, + vec![ + call("interrupt_agent", r#"{"target":"worker"}"#), + ev_completed("interrupt"), + ], + )], + complete("interrupt-result"), + complete("resume"), + ]) + .await; + let test = test_codex() + .with_model("gpt-5.6-sol") + .with_config(|config| { + config + .features + .enable(Feature::MultiAgentV2) + .expect("enable the parent collaboration tools"); + config + .features + .disable(Feature::EnableRequestCompression) + .expect("capture uncompressed request bodies"); + config + .features + .disable(Feature::WakeOnAgentMessages) + .expect("keep the parent idle after interrupting the child"); + config.multi_agent_v2.interrupt_at_item_boundary = true; + }) + .build_with_streaming_server(&server) + .await?; + let mut created = test.thread_manager.subscribe_thread_created(); + let (spawn, ()) = tokio::join!(test.submit_turn(FIRST_PROMPT), async { + server.wait_for_request_count(/*count*/ 2).await; + spawned + .send(()) + .expect("release the parent after the child starts sampling"); + }); + spawn?; + let child = test + .thread_manager + .get_thread(created.recv().await?) + .await?; + // The settings reply confirms the child accepted the interrupt while its item is held. + child + .update_thread_settings(ThreadSettingsOverrides::default()) + .await?; + wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await; + let (reply, resumed) = oneshot::channel(); + child + .submit(Op::TurnInput { + request: Box::new(codex_core::TurnInputRequest::user_input(vec![ + codex_protocol::user_input::UserInput::Text { + text: "Resume.".into(), + text_elements: vec![], + }, + ])), + mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer, + reply, + }) + .await?; + assert_eq!(child.agent_status().await, AgentStatus::Running); + release_item.send(()).expect("release the sampled item"); + let next = |event: &EventMsg| match event { + EventMsg::RawResponseItem(raw) => raw.item.id().is_some_and(|id| id.as_str() == "sampled"), + EventMsg::TurnStarted(_) | EventMsg::TurnAborted(_) | EventMsg::TurnComplete(_) => true, + _ => false, + }; + assert!(matches!( + wait_for_event(&child, next).await, + EventMsg::RawResponseItem(_) + )); + assert!(matches!( + wait_for_event(&child, next).await, + EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted + )); + assert!(matches!( + resumed.await??, + codex_core::TurnInputSubmission::Started { .. } + )); + wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await; + wait_for_event(&child, |event| matches!(event, EventMsg::TurnComplete(_))).await; + let requests = server.requests().await; + assert_eq!(requests.len(), 5); + let first: serde_json::Value = serde_json::from_slice(&requests[1])?; + let resumed: serde_json::Value = serde_json::from_slice(&requests[4])?; + let prefix = first["input"].as_array().context("child input")?; + let items = resumed["input"].as_array().context("resumed input")?; + assert_eq!(items.get(..prefix.len()), Some(prefix.as_slice())); + assert_eq!( + items + .iter() + .filter(|item| item["role"] == "user" + && item["content"] == json!([{"type": "input_text", "text": "Resume."}])) + .count(), + 1 + ); + let retained = &items[prefix.len()]; + if done["item"]["type"] == "message" { + assert_eq!(retained["content"], done["item"]["content"]); + assert_eq!(retained["phase"], done["item"]["phase"]); + } else { + assert_eq!(retained["arguments"], done["item"]["arguments"]); + assert_eq!(retained["call_id"], "sampled"); + let output = &items[prefix.len() + 1]; + assert_eq!(output["call_id"], "sampled"); + assert_eq!(output["output"], "aborted before dispatch"); + } + assert!( + !items + .iter() + .any(|item| item.to_string().contains("Discard this.")) + ); + server.shutdown().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn idle_mail_survives_eviction_and_preserves_followup_order() -> Result<()> { const MAIL: &str = "Keep this note for later"; diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs index 0e0eb6313b..f7d7335151 100644 --- a/codex-rs/features/src/feature_configs.rs +++ b/codex-rs/features/src/feature_configs.rs @@ -371,6 +371,9 @@ pub struct MultiAgentV2ConfigToml { /// Expose the multi-agent v2 `wait_agent` tool. #[serde(skip_serializing_if = "Option::is_none")] pub wait_agent_enabled: Option, + /// Let a subagent finish its current output item while cancelling tools immediately. + #[serde(skip_serializing_if = "Option::is_none")] + pub interrupt_at_item_boundary: Option, /// Hide native multi-agent tools while retaining the V2 runtime for host-provided tools. #[serde(skip_serializing_if = "Option::is_none")] pub disable_native_tools: Option, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 3133e1751c..7b2975aff5 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -858,6 +858,7 @@ non_code_mode_only = true hide_spawn_agent_metadata: Some(true), expose_spawn_agent_model_overrides: Some(true), wait_agent_enabled: Some(false), + interrupt_at_item_boundary: None, disable_native_tools: Some(true), disable_direct_message: Some(true), message_board_in_memory: Some(true), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 97bbc7a561..7083f21285 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -728,6 +728,9 @@ pub enum Op { /// This server sends [`EventMsg::TurnAborted`] in response. Interrupt, + /// Interrupt a spawned agent, respecting its configured sampling boundary. + InterruptAgent, + /// Interrupt the named turn only if no input is queued for it. /// The decision is acknowledged before cancellation finishes. #[debug("InterruptIfNoPendingInput {{ turn_id: {turn_id:?} }}")] @@ -1134,6 +1137,7 @@ impl Op { pub fn kind(&self) -> &'static str { match self { Self::Interrupt => "interrupt", + Self::InterruptAgent => "interrupt_agent", Self::InterruptIfNoPendingInput { .. } => "interrupt_if_no_pending_input", Self::CleanBackgroundTerminals => "clean_background_terminals", Self::RealtimeConversationStart(_) => "realtime_conversation_start",