Add opt-in item-boundary interrupts for subagents (#61909)

## What changed

Add the multi-agent v2 setting `interrupt_at_item_boundary`, disabled by default. When enabled, `interrupt_agent` lets a subagent finish and retain its current output item while cancelling tools immediately. Later output items are discarded, and retained tool calls that have not run receive an aborted result.

Wait for the interrupted turn's terminal event before starting follow-up input, and prevent idle contributors from restarting the interrupted child.

## Testing

Add streaming tests for retaining an assistant message or tool call, discarding later output, and preserving history and follow-up input when the child resumes. Update configuration and interrupt-routing tests.

GitOrigin-RevId: 8b2b9b1383bfb4eefea7369a40012bebc5fff07b
This commit is contained in:
jif
2026-11-16 23:26:07 +00:00
committed by copyberry
parent 837f8b50be
commit cff061fca9
20 changed files with 475 additions and 47 deletions
+4
View File
@@ -3420,6 +3420,10 @@
"hide_spawn_agent_metadata": {
"type": "boolean"
},
"interrupt_at_item_boundary": {
"description": "Let a subagent finish its current output item while cancelling tools immediately.",
"type": "boolean"
},
"max_concurrent_threads_per_session": {
"format": "uint",
"minimum": 1.0,
+2 -5
View File
@@ -334,17 +334,14 @@ impl LocalAgentControl {
}
/// Interrupt the current task for an existing agent thread.
pub(crate) async fn interrupt_agent(&self, agent_id: ThreadId) -> CodexResult<String> {
pub(crate) async fn interrupt_agent(&self, agent_id: ThreadId, op: Op) -> CodexResult<String> {
let state = self.runtime.upgrade()?;
self.handle_thread_request_result(
agent_id,
&state,
state
.send_op(
agent_id,
Op::Interrupt,
/*parent_turn_id*/ None,
/*root_turn_id*/ None,
agent_id, op, /*parent_turn_id*/ None, /*root_turn_id*/ None,
/*message_delivery*/ None,
)
.await,
+2 -1
View File
@@ -26,6 +26,7 @@ use codex_protocol::ThreadId;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::TokenUsage;
use codex_rollout_trace::ThreadTraceContext;
@@ -211,7 +212,7 @@ impl AgentControl for LocalAgentControl {
match version {
MultiAgentVersion::Disabled | MultiAgentVersion::V1 => {
let snapshot = self.inspect_agent(target).await?;
self.interrupt_agent(target).await?;
self.interrupt_agent(target, Op::Interrupt).await?;
Ok(snapshot)
}
MultiAgentVersion::V2 => self.interrupt_spawned_agent(caller, target).await,
+2 -1
View File
@@ -10,6 +10,7 @@ use codex_protocol::ThreadId;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::Op;
impl LocalAgentControl {
/// Interrupts a spawned agent's current task, preserving the status observed before dispatch.
@@ -38,7 +39,7 @@ impl LocalAgentControl {
CodexErr::UnsupportedOperation("target agent is missing an agent_path".to_string())
})?;
let snapshot = self.inspect_agent(target).await?;
match self.interrupt_agent(target).await {
match self.interrupt_agent(target, Op::InterruptAgent).await {
Ok(_) => {}
Err(err)
if matches!(
+2
View File
@@ -12870,6 +12870,7 @@ tool_namespace = "agents"
hide_spawn_agent_metadata = true
expose_spawn_agent_model_overrides = false
wait_agent_enabled = false
interrupt_at_item_boundary = true
disable_native_tools = true
disable_direct_message = true
message_board_in_memory = true
@@ -12929,6 +12930,7 @@ max_concurrent_threads_per_session = 9
assert!(!config.multi_agent_v2.preserve_fork_prefix);
assert!(!config.multi_agent_v2.expose_spawn_agent_model_overrides);
assert!(!config.multi_agent_v2.wait_agent_enabled);
assert!(config.multi_agent_v2.interrupt_at_item_boundary);
assert!(config.multi_agent_v2.disable_native_tools);
assert!(config.multi_agent_v2.disable_direct_message);
assert!(config.multi_agent_v2.message_board_in_memory);
+6
View File
@@ -1457,6 +1457,7 @@ pub struct MultiAgentV2Config {
pub hide_spawn_agent_metadata: bool,
pub expose_spawn_agent_model_overrides: bool,
pub wait_agent_enabled: bool,
pub interrupt_at_item_boundary: bool,
pub disable_native_tools: bool,
pub disable_direct_message: bool,
pub message_board_in_memory: bool,
@@ -1481,6 +1482,7 @@ impl MultiAgentV2Config {
hide_spawn_agent_metadata: true,
expose_spawn_agent_model_overrides: true,
wait_agent_enabled: true,
interrupt_at_item_boundary: false,
disable_native_tools: false,
disable_direct_message: false,
message_board_in_memory: false,
@@ -2968,6 +2970,9 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config
let wait_agent_enabled = base
.and_then(|config| config.wait_agent_enabled)
.unwrap_or(default.wait_agent_enabled);
let interrupt_at_item_boundary = base
.and_then(|config| config.interrupt_at_item_boundary)
.unwrap_or(default.interrupt_at_item_boundary);
let disable_native_tools = base
.and_then(|config| config.disable_native_tools)
.unwrap_or(default.disable_native_tools);
@@ -3010,6 +3015,7 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config
hide_spawn_agent_metadata,
expose_spawn_agent_model_overrides,
wait_agent_enabled,
interrupt_at_item_boundary,
disable_native_tools,
disable_direct_message,
message_board_in_memory,
+6 -1
View File
@@ -1,4 +1,5 @@
use super::Submission;
use super::sampling_interrupt::SamplingInterrupt;
use crate::McpRefreshReason;
use crate::WithTurnExtensionData;
use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio;
@@ -60,7 +61,7 @@ use tracing::info;
use tracing::warn;
pub async fn interrupt(sess: &Arc<Session>) {
sess.interrupt_task().await;
sess.interrupt_task(sess.active_turn.lock().await).await;
}
pub async fn clean_background_terminals(sess: &Arc<Session>) {
@@ -615,6 +616,10 @@ pub(super) async fn submission_loop(
interrupt(&sess).await;
false
}
Op::InterruptAgent => {
SamplingInterrupt::request(&sess).await;
false
}
Op::InterruptIfNoPendingInput { turn_id, reply } => {
sess.interrupt_turn_if_no_pending_input(&turn_id, reply)
.await;
+9 -4
View File
@@ -1039,10 +1039,15 @@ async fn review_guardian_mcp_elicitation(
session: Arc<Session>,
request: ElicitationReviewRequest,
) -> anyhow::Result<Option<ElicitationResponse>> {
let Some((turn_context, cancellation_token)) =
session.active_turn_context_and_cancellation_token().await
else {
return Ok(None);
let (turn_context, cancellation_token) = {
let active = session.active_turn.lock().await;
let Some(task) = active.as_ref().and_then(|turn| turn.task.as_ref()) else {
return Ok(None);
};
(
Arc::clone(&task.turn_context),
task.mcp_elicitation_cancellation.child_token(),
)
};
let Some(mcp_config) = session.services.mcp_runtime.current_config() else {
+2 -9
View File
@@ -281,6 +281,8 @@ mod retained_context;
mod review;
mod rollout_budget;
mod rollout_reconstruction;
mod sampling_interrupt;
pub(crate) use sampling_interrupt::SamplingInterrupt;
#[allow(clippy::module_inception)]
pub(crate) mod session;
pub(crate) mod startup_prewarm;
@@ -6043,15 +6045,6 @@ impl Session {
turn_state.lock().await.has_memory_citation = true;
}
pub async fn interrupt_task(self: &Arc<Self>) {
info!("interrupt received: abort current task, if any");
let had_active_turn = self.active_turn.lock().await.is_some();
self.abort_all_tasks(TurnAbortReason::Interrupted).await;
if !had_active_turn {
self.cancel_mcp_startup();
}
}
pub(crate) fn hooks(&self) -> Arc<Hooks> {
self.services.hooks.load_full()
}
@@ -0,0 +1,116 @@
//! Lets an opted-in child finish the sampled item while its tools are cancelled.
use super::session::Session;
use super::turn_context::TurnContext;
use crate::state::ActiveTurn;
use codex_features::Feature;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::MultiAgentVersion;
use std::ops::ControlFlow;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
/// Owned by the active turn through publication of its terminal event.
pub(crate) struct SamplingInterrupt {
tools: CancellationToken,
pub(super) finished: CancellationToken,
}
impl Drop for SamplingInterrupt {
fn drop(&mut self) {
self.finished.cancel();
}
}
impl SamplingInterrupt {
pub(crate) fn enabled(turn: &TurnContext) -> bool {
turn.config.multi_agent_v2.interrupt_at_item_boundary
&& turn.multi_agent_version == MultiAgentVersion::V2
&& turn.parent_thread_id.is_some()
}
pub(super) async fn register(
session: &Session,
turn: &TurnContext,
tools: &CancellationToken,
) -> CodexResult<()> {
let mut active = session.active_turn.lock().await;
let active = matching_turn(&mut active, turn).ok_or(CodexErr::TurnAborted)?;
if tools.is_cancelled() {
return Err(CodexErr::TurnAborted);
}
active.sampling_interrupt = Some(Self {
tools: tools.clone(),
finished: CancellationToken::new(),
});
Ok(())
}
pub(super) async fn unregister(session: &Session, turn: &TurnContext) {
let mut active = session.active_turn.lock().await;
if let Some(active) = matching_turn(&mut active, turn)
&& !active
.sampling_interrupt
.as_ref()
.is_some_and(Self::is_requested)
{
active.sampling_interrupt = None;
}
}
pub(crate) fn is_requested(&self) -> bool {
self.tools.is_cancelled()
}
pub(super) async fn request(session: &Arc<Session>) {
let action = {
let active = session.active_turn.lock().await;
if let Some(ActiveTurn {
sampling_interrupt: Some(interrupt),
task,
..
}) = active.as_ref()
{
if interrupt.is_requested() {
return;
}
interrupt.tools.cancel();
let Some(task) = task else { return };
task.mcp_elicitation_cancellation.cancel();
ControlFlow::Continue(
task.turn_context
.config
.features
.enabled(Feature::CodeModeInterrupt),
)
} else {
// Claim the hard abort under the same lock that admits a sampling attempt.
ControlFlow::Break(session.interrupt_task(active))
}
};
match action {
ControlFlow::Break(abort) => abort.await,
ControlFlow::Continue(true) => {
session
.services
.code_mode_service
.interrupt_active_cells()
.await;
}
ControlFlow::Continue(false) => {}
}
}
}
fn matching_turn<'a>(
active: &'a mut Option<ActiveTurn>,
turn: &TurnContext,
) -> Option<&'a mut ActiveTurn> {
active.as_mut().filter(|active| {
active
.task
.as_ref()
.is_some_and(|task| task.turn_context.sub_id == turn.sub_id)
})
}
+64 -15
View File
@@ -46,13 +46,13 @@ use crate::responses_retry::is_terminal_sampling_outcome;
use crate::session::PreviousTurnSettings;
use crate::session::TurnInput;
use crate::session::daemon_recovery::RecordedTurnInput;
use crate::session::sampling_interrupt::SamplingInterrupt;
use crate::session::session::Session;
use crate::session::step_context::StepContext;
use crate::session::turn_context::TurnContext;
use crate::skills::emit_explicit_skill_invocations;
use crate::stream_events_utils::AssistantText;
use crate::stream_events_utils::HandleOutputCtx;
use crate::stream_events_utils::InFlightFuture;
use crate::stream_events_utils::TurnItemContributorPolicy;
use crate::stream_events_utils::finalize_non_tool_response_item;
use crate::stream_events_utils::handle_non_tool_response_item;
@@ -88,6 +88,7 @@ use codex_features::Feature;
use codex_file_system::EnvironmentAccess;
use codex_file_system::FindUpErrorPolicy;
use codex_file_system::find_nearest_ancestor_with_markers;
use codex_history::ResponseItemEnvelope;
use codex_hooks::SamplingKind;
use codex_login::CodexAuth;
use codex_model_provider::RemoteCompactionSupport;
@@ -2066,7 +2067,8 @@ async fn run_sampling_request(
(submitted, pending)
};
let mut draining_for_mailbox = false;
let err = match try_run_sampling_request(
let tool_cancellation = cancellation_token.child_token();
let result = try_run_sampling_request(
&mut sampling_request,
tool_runtime.clone(),
Arc::clone(&sess),
@@ -2080,9 +2082,13 @@ async fn run_sampling_request(
unfinished_plan_id,
cancellation_token.child_token(),
&mut draining_for_mailbox,
&tool_cancellation,
)
.await
{
.await;
if SamplingInterrupt::enabled(&turn_context) && tool_cancellation.is_cancelled() {
return Err(CodexErr::TurnAborted);
}
let err = match result {
Ok(output) => {
return Ok((output, notify_messages));
}
@@ -2901,7 +2907,7 @@ async fn handle_assistant_item_done_in_plan_mode(
#[instrument(level = "trace", skip_all)]
async fn drain_in_flight(
in_flight: &mut FuturesOrdered<InFlightFuture<'static>>,
in_flight: &mut (impl Stream<Item = CodexResult<ResponseItemEnvelope>> + Unpin),
sess: Arc<Session>,
step_context: &StepContext,
) {
@@ -2951,6 +2957,7 @@ async fn try_run_sampling_request(
unfinished_plan_id: &mut Option<String>,
cancellation_token: CancellationToken,
draining_for_mailbox: &mut bool,
tool_cancellation: &CancellationToken,
) -> CodexResult<SamplingRequestResult> {
let turn_context = Arc::clone(&step_context.turn);
#[expect(deprecated, reason = "Classify feedback using the initial turn cwd.")]
@@ -3015,6 +3022,7 @@ async fn try_run_sampling_request(
};
let mut resampling = crate::guardian::ResponseResampling::new(&sess);
let prompt = Arc::new(prompt);
let interrupt_at_boundary = SamplingInterrupt::enabled(&turn_context);
let mut stream_request = async |prompt: &Arc<Prompt>| {
let send = client_session
.open_model_response(
@@ -3030,6 +3038,12 @@ async fn try_run_sampling_request(
&inference_trace,
)
.instrument(trace_span!("stream_request"));
let send = async {
if interrupt_at_boundary {
SamplingInterrupt::register(&sess, &turn_context, tool_cancellation).await?;
}
send.await
};
sampling_request
.open(
&sess,
@@ -3040,9 +3054,18 @@ async fn try_run_sampling_request(
)
.await
};
let mut stream = stream_request(&prompt)
let mut stream = match stream_request(&prompt)
.or_cancel(&cancellation_token)
.await??;
.await?
{
Ok(stream) => stream,
Err(error) => {
if interrupt_at_boundary {
SamplingInterrupt::unregister(&sess, &turn_context).await;
}
return Err(error);
}
};
let can_interrupt = turn_context
.config
.features
@@ -3084,6 +3107,10 @@ async fn try_run_sampling_request(
!sess.services.extensions.turn_item_contributors().is_empty();
let mut active_item_is_streaming_to_client = false;
let receiving_span = trace_span!("receiving_stream");
// Sending the request starts the first item, even before the first stream event arrives.
let mut between_items = false;
let mut stopping_tools = false;
let mut completed_tools = Vec::new();
let outcome: CodexResult<SamplingRequestResult> = loop {
if response_completed && resampling.accepted_output.is_empty() {
if resampling.update_usage_boundary {
@@ -3112,12 +3139,25 @@ async fn try_run_sampling_request(
);
let receive_started = Instant::now();
let event = resampling
let receive = resampling
.next(&mut stream)
.instrument(trace_span!(parent: &handle_responses, "receiving"))
.or_cancel(&preempt)
.or_cancel(&cancellation_token)
.await;
.or_cancel(&cancellation_token);
let event = tokio::select! {
biased;
() = tool_cancellation.cancelled(), if interrupt_at_boundary && between_items => break Err(CodexErr::TurnAborted),
() = tool_cancellation.cancelled(), if interrupt_at_boundary && !stopping_tools => {
stopping_tools = true;
continue;
}
Some(output) = in_flight.next(), if stopping_tools && !in_flight.is_empty() => {
// Drive cancellation now and record the outputs after the retained item.
completed_tools.push(output);
continue;
}
event = receive => event,
};
// Split the existing iteration span without adding another span per streamed delta.
handle_responses.record(
"codex.response.receive_wait_ms",
@@ -3126,7 +3166,9 @@ async fn try_run_sampling_request(
let event = match event {
Ok(Ok(event)) => event,
Ok(Err(_)) => {
if *draining_for_mailbox {
if *draining_for_mailbox
|| (interrupt_at_boundary && tool_cancellation.is_cancelled())
{
preempt = CancellationToken::new();
continue;
}
@@ -3225,6 +3267,7 @@ async fn try_run_sampling_request(
}
}
ResponseEvent::OutputItemDone(mut item) => {
between_items = true;
// Serving emits one final value immediately before this item's completion.
// After this handoff, the invocation/cell owns it independently of the stream.
let assessment = pending_assessment
@@ -3311,7 +3354,7 @@ async fn try_run_sampling_request(
step_context: Arc::clone(&step_context),
turn_store: Arc::clone(&turn_store),
tool_runtime,
cancellation_token: cancellation_token.child_token(),
cancellation_token: tool_cancellation.child_token(),
};
if *draining_for_mailbox {
// Retain buffered calls in history, but use the existing pre-dispatch
@@ -3368,6 +3411,7 @@ async fn try_run_sampling_request(
// Hosts can keep the current response intact and deliver mail before the next
// model request instead of cutting off its remaining tool calls.
if preempt_for_mailbox_mail
&& !(interrupt_at_boundary && tool_cancellation.is_cancelled())
&& !*draining_for_mailbox
&& !turn_context
.config
@@ -3409,6 +3453,7 @@ async fn try_run_sampling_request(
}
}
ResponseEvent::OutputItemAdded(mut item) => {
between_items = false;
sess.assign_missing_response_item_id(&mut item);
sess.reserve_assistant_message_order(&turn_context, &item)
.await;
@@ -3785,7 +3830,10 @@ async fn try_run_sampling_request(
}
}
};
if cancellation_token.is_cancelled() && !response_completed && can_interrupt {
if interrupt_at_boundary {
SamplingInterrupt::unregister(&sess, &turn_context).await;
}
if tool_cancellation.is_cancelled() && !response_completed && can_interrupt {
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
if let Some(interrupt) = stream.interrupt.take() {
let _ = interrupt.send(codex_api::ResponseInterrupt::Cancel { deadline });
@@ -3805,7 +3853,7 @@ async fn try_run_sampling_request(
*unfinished_plan_id = state.plan_item_state.unfinished_id();
}
if !in_flight.is_empty() {
if !in_flight.is_empty() || !completed_tools.is_empty() {
let _tool_blocking_timing_guard = turn_context
.turn_timing_state
.begin_tool_blocking(tool_runtime.batch_timing());
@@ -3815,7 +3863,8 @@ async fn try_run_sampling_request(
conversation.id = %sess.thread_id,
turn.id = %turn_context.sub_id,
);
drain_in_flight(&mut in_flight, Arc::clone(&sess), &step_context).await;
let mut outputs = futures::stream::iter(completed_tools).chain(in_flight);
drain_in_flight(&mut outputs, Arc::clone(&sess), &step_context).await;
}
if should_emit_token_count {
+20 -1
View File
@@ -1002,7 +1002,26 @@ impl Session {
responsesapi_client_metadata: ResponsesApiClientMetadataUpdate,
input_metadata: UserInputMetadata,
) -> Result<(String, String), NotSubmittedReason> {
let mut active = self.active_turn.lock().await;
let mut active = loop {
let active = self.active_turn.lock().await;
let Some(interrupt) = active
.as_ref()
.and_then(|turn| turn.sampling_interrupt.as_ref())
.filter(|interrupt| interrupt.is_requested())
else {
break active;
};
if expected_turn_id.is_some() {
return Err(NotSubmittedReason::NoActiveTurn);
}
// Start-or-steer can open a new turn once the old terminal event is published.
let finished = interrupt.finished.clone();
drop(active);
tokio::select! {
() = finished.cancelled() => {}
() = self.services.local_agent_runtime.shutdown.cancelled() => return Err(NotSubmittedReason::ServerDraining),
}
};
let Some(active_turn) = active.as_mut() else {
return Err(NotSubmittedReason::NoActiveTurn);
};
+5
View File
@@ -22,6 +22,7 @@ use tokio::sync::oneshot;
use super::TurnTokenUsage;
use crate::agent::types::AgentExecutionGuard;
use crate::session::SamplingInterrupt;
use crate::session::TurnInput;
use crate::session::TurnInputQueue;
use crate::session::step_context::StepContext;
@@ -36,6 +37,7 @@ use codex_protocol::protocol::TokenUsage;
/// Metadata about the currently running turn.
pub(crate) struct ActiveTurn {
pub(crate) task: Option<RunningTask>,
pub(crate) sampling_interrupt: Option<SamplingInterrupt>,
/// Present when input admission is closed; cancelled when the final batch is recorded.
/// Reservations that have not started still accept inputs.
pub(crate) finishing: Option<CancellationToken>,
@@ -66,6 +68,7 @@ impl Default for ActiveTurn {
fn default() -> Self {
Self {
task: None,
sampling_interrupt: None,
finishing: None,
turn_state: Arc::new(Mutex::new(TurnState::default())),
}
@@ -84,6 +87,8 @@ pub(crate) struct RunningTask {
pub(crate) kind: TaskKind,
pub(crate) task: Arc<dyn AnySessionTask>,
pub(crate) cancellation_token: CancellationToken,
/// MCP reviews can span sampling attempts and must stop even while the sampler drains.
pub(crate) mcp_elicitation_cancellation: CancellationToken,
pub(crate) handle: AbortOnDropHandle<()>,
pub(crate) turn_context: Arc<TurnContext>,
pub(crate) _agent_execution_guard: Option<AgentExecutionGuard>,
+64 -7
View File
@@ -6,6 +6,7 @@ mod review;
mod user_shell;
use crate::McpRefreshReason;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use std::time::Instant;
@@ -16,12 +17,14 @@ use codex_extension_api::ThreadIdleCause;
use futures::future::BoxFuture;
use tokio::select;
use tokio::sync::Mutex;
use tokio::sync::MutexGuard;
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::Instrument;
use tracing::Span;
use tracing::field;
use tracing::info;
use tracing::info_span;
use tracing::trace;
use tracing::trace_span;
@@ -32,6 +35,7 @@ use crate::codex_thread::BackgroundTerminalInfo;
use crate::config::Config;
use crate::context::ContextualUserFragment;
use crate::hook_runtime::run_turn_interrupt_hooks;
use crate::session::SamplingInterrupt;
use crate::session::TurnInput;
use crate::session::session::Session;
use crate::session::turn::McpStartupRequirements;
@@ -521,6 +525,11 @@ impl Session {
handle: AbortOnDropHandle::new(handle),
kind: task_kind,
task,
mcp_elicitation_cancellation: if SamplingInterrupt::enabled(&turn_context) {
cancellation_token.child_token()
} else {
cancellation_token.clone()
},
cancellation_token,
turn_context: Arc::clone(&turn_context),
_agent_execution_guard: agent_execution_guard,
@@ -648,10 +657,38 @@ impl Session {
.await;
}
pub(crate) fn interrupt_task(
self: &Arc<Self>,
active: MutexGuard<'_, Option<ActiveTurn>>,
) -> impl Future<Output = ()> + Send + use<> {
info!("interrupt received: abort current task, if any");
// Release the admission lock before creating the asynchronous cleanup.
let turn = self.take_active_turn(active, &TurnAbortReason::Interrupted);
let session = Arc::clone(self);
async move {
let had_active_turn = turn.is_some();
session
.abort_taken_tasks(turn, TurnAbortReason::Interrupted)
.await;
if !had_active_turn {
session.cancel_mcp_startup();
}
}
}
pub async fn abort_all_tasks(self: &Arc<Self>, reason: TurnAbortReason) {
let active = self.take_active_turn(self.active_turn.lock().await, &reason);
self.abort_taken_tasks(active, reason).await;
}
async fn abort_taken_tasks(
self: &Arc<Self>,
active: Option<ActiveTurn>,
reason: TurnAbortReason,
) {
let mut aborted_turn = false;
let mut active_turn_to_clear = None;
if let Some(mut active_turn) = self.take_active_turn(&reason).await {
if let Some(mut active_turn) = active {
let task = active_turn.task.take();
aborted_turn = task.is_some();
if let Some(task) = task {
@@ -759,7 +796,7 @@ impl Session {
&& !self.work_budget().is_exhausted();
let finishing = CancellationToken::new();
let finish_recording = finishing.clone().drop_guard();
let (turn_state, unadmitted_task, mut pending_input, outcome_handoff) = {
let (turn_state, unadmitted_task, mut pending_input, outcome_handoff, sampling_interrupted) = {
let mut active = self.active_turn.lock().await;
let active_turn = active.as_mut()?;
if !Arc::ptr_eq(&active_turn.task.as_ref()?.turn_context, &turn_context) {
@@ -782,6 +819,10 @@ impl Session {
.then_some(task.task),
task.pending_initial_input.unwrap_or_default(),
task.outcome_handoff,
active_turn
.sampling_interrupt
.as_ref()
.is_some_and(SamplingInterrupt::is_requested),
)
};
turn_context
@@ -835,8 +876,20 @@ impl Session {
})
.filter(is_automatic_rollout_stop)
};
if let Some(cause) = &stop_cause {
self.record_interrupted_turn_history(&turn_context, Some(cause))
if sampling_interrupted
&& turn_context
.config
.features
.enabled(Feature::CodeModeInterrupt)
{
// A cell can finish registering while its dispatch is being cancelled.
self.services
.code_mode_service
.interrupt_active_cells()
.await;
}
if sampling_interrupted || stop_cause.is_some() {
self.record_interrupted_turn_history(&turn_context, stop_cause.as_ref())
.await;
}
drop(finish_recording);
@@ -1039,7 +1092,8 @@ impl Session {
self.send_event(turn_context.as_ref(), event).await;
}
drop(outcome_handoff);
if cleared_active_turn {
// A parent interrupt must not let an idle contributor restart the child.
if cleared_active_turn && !sampling_interrupted {
self.emit_thread_idle_lifecycle_if_idle(idle_cause).await;
}
// Private reviewers already flushed the terminal event before delivering it.
@@ -1065,8 +1119,11 @@ impl Session {
}
}
async fn take_active_turn(&self, reason: &TurnAbortReason) -> Option<ActiveTurn> {
let mut active = self.active_turn.lock().await;
fn take_active_turn(
&self,
mut active: MutexGuard<'_, Option<ActiveTurn>>,
reason: &TurnAbortReason,
) -> Option<ActiveTurn> {
if matches!(
reason,
TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited
+1
View File
@@ -158,6 +158,7 @@ type AgentControlFactory =
fn capture_test_op(op: &Op) -> Option<Op> {
match op {
Op::Interrupt => Some(Op::Interrupt),
Op::InterruptAgent => Some(Op::InterruptAgent),
Op::InterAgentCommunication {
communication,
start_options,
@@ -3950,14 +3950,14 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
let ops = manager.captured_ops();
assert!(
ops.iter()
.any(|(thread_id, op)| *thread_id == agent_id && matches!(op, Op::Interrupt))
.any(|(thread_id, op)| *thread_id == agent_id && matches!(op, Op::InterruptAgent))
);
assert!(!ops.iter().any(|(thread_id, op)| {
(*thread_id == agent_id || *thread_id == child_id) && matches!(op, Op::Shutdown)
}));
assert!(
!ops.iter()
.any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::Interrupt))
.any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::InterruptAgent))
);
}
+160 -1
View File
@@ -1,4 +1,4 @@
//! Pending mail survives eviction without loading idle recipients.
//! Child history survives interruption and pending mail survives eviction.
use super::*;
use crate::suite::settings_commits::COMMITTED_MODEL;
@@ -10,12 +10,16 @@ use codex_extension_api::MessageLifecycleInput;
use codex_protocol::ThreadId;
use codex_protocol::protocol::AgentStatus;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::TurnAbortReason;
use core_test_support::ThreadIdle;
use core_test_support::streaming_sse::StreamingSseChunk;
use core_test_support::streaming_sse::start_streaming_sse_server;
use pretty_assertions::assert_eq;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::mpsc;
use test_case::test_case;
use tokio::sync::oneshot;
#[derive(Default)]
@@ -34,6 +38,161 @@ impl MessageLifecycleContributor for Messages {
}
}
#[test_case(ev_assistant_message("sampled", "Keep this answer."); "message")]
#[test_case(ev_function_call_with_namespace("sampled", MULTI_AGENT_V2_NAMESPACE, "send_message",
r#"{"target":"/root","message":"Should never be sent."}"#); "tool")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Result<()> {
skip_if_no_network!(Ok(()));
done["item"]["id"] = json!("sampled");
if done["item"]["type"] == "message" {
done["item"]["phase"] = json!("final_answer");
}
let chunk = |gate, events| StreamingSseChunk {
gate,
body: sse(events),
};
let complete = |id| vec![chunk(None, vec![ev_completed(id)])];
let call =
|name, args| ev_function_call_with_namespace(name, MULTI_AGENT_V2_NAMESPACE, name, args);
let (spawned, spawn_gate) = oneshot::channel();
let (release_item, item_gate) = oneshot::channel();
let (_finish_response, response_gate) = oneshot::channel();
let (server, _) = start_streaming_sse_server(vec![
vec![
chunk(
None,
vec![call(
"spawn_agent",
r#"{"task_name":"worker","message":"Work","fork_turns":"none"}"#,
)],
),
chunk(Some(spawn_gate), vec![ev_completed("spawn")]),
],
vec![
chunk(None, vec![ev_response_created("child")]),
chunk(
Some(item_gate),
vec![done.clone(), ev_assistant_message("later", "Discard this.")],
),
chunk(Some(response_gate), vec![ev_completed("child")]),
],
vec![chunk(
None,
vec![
call("interrupt_agent", r#"{"target":"worker"}"#),
ev_completed("interrupt"),
],
)],
complete("interrupt-result"),
complete("resume"),
])
.await;
let test = test_codex()
.with_model("gpt-5.6-sol")
.with_config(|config| {
config
.features
.enable(Feature::MultiAgentV2)
.expect("enable the parent collaboration tools");
config
.features
.disable(Feature::EnableRequestCompression)
.expect("capture uncompressed request bodies");
config
.features
.disable(Feature::WakeOnAgentMessages)
.expect("keep the parent idle after interrupting the child");
config.multi_agent_v2.interrupt_at_item_boundary = true;
})
.build_with_streaming_server(&server)
.await?;
let mut created = test.thread_manager.subscribe_thread_created();
let (spawn, ()) = tokio::join!(test.submit_turn(FIRST_PROMPT), async {
server.wait_for_request_count(/*count*/ 2).await;
spawned
.send(())
.expect("release the parent after the child starts sampling");
});
spawn?;
let child = test
.thread_manager
.get_thread(created.recv().await?)
.await?;
// The settings reply confirms the child accepted the interrupt while its item is held.
child
.update_thread_settings(ThreadSettingsOverrides::default())
.await?;
wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await;
let (reply, resumed) = oneshot::channel();
child
.submit(Op::TurnInput {
request: Box::new(codex_core::TurnInputRequest::user_input(vec![
codex_protocol::user_input::UserInput::Text {
text: "Resume.".into(),
text_elements: vec![],
},
])),
mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer,
reply,
})
.await?;
assert_eq!(child.agent_status().await, AgentStatus::Running);
release_item.send(()).expect("release the sampled item");
let next = |event: &EventMsg| match event {
EventMsg::RawResponseItem(raw) => raw.item.id().is_some_and(|id| id.as_str() == "sampled"),
EventMsg::TurnStarted(_) | EventMsg::TurnAborted(_) | EventMsg::TurnComplete(_) => true,
_ => false,
};
assert!(matches!(
wait_for_event(&child, next).await,
EventMsg::RawResponseItem(_)
));
assert!(matches!(
wait_for_event(&child, next).await,
EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted
));
assert!(matches!(
resumed.await??,
codex_core::TurnInputSubmission::Started { .. }
));
wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await;
wait_for_event(&child, |event| matches!(event, EventMsg::TurnComplete(_))).await;
let requests = server.requests().await;
assert_eq!(requests.len(), 5);
let first: serde_json::Value = serde_json::from_slice(&requests[1])?;
let resumed: serde_json::Value = serde_json::from_slice(&requests[4])?;
let prefix = first["input"].as_array().context("child input")?;
let items = resumed["input"].as_array().context("resumed input")?;
assert_eq!(items.get(..prefix.len()), Some(prefix.as_slice()));
assert_eq!(
items
.iter()
.filter(|item| item["role"] == "user"
&& item["content"] == json!([{"type": "input_text", "text": "Resume."}]))
.count(),
1
);
let retained = &items[prefix.len()];
if done["item"]["type"] == "message" {
assert_eq!(retained["content"], done["item"]["content"]);
assert_eq!(retained["phase"], done["item"]["phase"]);
} else {
assert_eq!(retained["arguments"], done["item"]["arguments"]);
assert_eq!(retained["call_id"], "sampled");
let output = &items[prefix.len() + 1];
assert_eq!(output["call_id"], "sampled");
assert_eq!(output["output"], "aborted before dispatch");
}
assert!(
!items
.iter()
.any(|item| item.to_string().contains("Discard this."))
);
server.shutdown().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn idle_mail_survives_eviction_and_preserves_followup_order() -> Result<()> {
const MAIL: &str = "Keep this note for later";
+3
View File
@@ -371,6 +371,9 @@ pub struct MultiAgentV2ConfigToml {
/// Expose the multi-agent v2 `wait_agent` tool.
#[serde(skip_serializing_if = "Option::is_none")]
pub wait_agent_enabled: Option<bool>,
/// Let a subagent finish its current output item while cancelling tools immediately.
#[serde(skip_serializing_if = "Option::is_none")]
pub interrupt_at_item_boundary: Option<bool>,
/// Hide native multi-agent tools while retaining the V2 runtime for host-provided tools.
#[serde(skip_serializing_if = "Option::is_none")]
pub disable_native_tools: Option<bool>,
+1
View File
@@ -858,6 +858,7 @@ non_code_mode_only = true
hide_spawn_agent_metadata: Some(true),
expose_spawn_agent_model_overrides: Some(true),
wait_agent_enabled: Some(false),
interrupt_at_item_boundary: None,
disable_native_tools: Some(true),
disable_direct_message: Some(true),
message_board_in_memory: Some(true),
+4
View File
@@ -728,6 +728,9 @@ pub enum Op {
/// This server sends [`EventMsg::TurnAborted`] in response.
Interrupt,
/// Interrupt a spawned agent, respecting its configured sampling boundary.
InterruptAgent,
/// Interrupt the named turn only if no input is queued for it.
/// The decision is acknowledged before cancellation finishes.
#[debug("InterruptIfNoPendingInput {{ turn_id: {turn_id:?} }}")]
@@ -1134,6 +1137,7 @@ impl Op {
pub fn kind(&self) -> &'static str {
match self {
Self::Interrupt => "interrupt",
Self::InterruptAgent => "interrupt_agent",
Self::InterruptIfNoPendingInput { .. } => "interrupt_if_no_pending_input",
Self::CleanBackgroundTerminals => "clean_background_terminals",
Self::RealtimeConversationStart(_) => "realtime_conversation_start",