diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index d4797bf2ea..92d6d486e8 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -36,6 +36,7 @@ use crate::responses_retry::ResponsesStreamRequest; use crate::responses_retry::ResponsesStreamRetryState; use crate::responses_retry::handle_response_stream_error; use crate::session::RequestEffortUsage; +use crate::session::SamplingInterrupt; use crate::session::session::Session; use crate::session::step_context::StepContext; use crate::session::turn_context::TurnContext; @@ -244,7 +245,10 @@ async fn run_remote_compact_task_inner( fallback, )) .await; - if matches!(result, Ok(false)) { + // Keep the registration through installation, not just through receiving the summary. + let interrupted = SamplingInterrupt::enabled(turn_context) + && SamplingInterrupt::unregister(sess, turn_context).await; + if matches!(result, Ok(false)) && !interrupted { if let Some(item) = started_item { sess.emit_turn_item_completed(turn_context, item).await; } @@ -252,7 +256,7 @@ async fn run_remote_compact_task_inner( } let status = compaction_status_from_result(&result); let codex_error = result.as_ref().err(); - if result.is_ok() { + if result.is_ok() && !interrupted { let post_compact_outcome = run_post_compact_hooks(sess, turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { attempt @@ -264,6 +268,11 @@ async fn run_remote_compact_task_inner( attempt .track(sess.as_ref(), status, codex_error, analytics_details) .await; + let result = if interrupted { + Err(CodexErr::TurnAborted) + } else { + result + }; let result = sess.finish_compaction(turn_context, phase, result); match result { Ok(installed) => Ok(installed), @@ -352,27 +361,36 @@ async fn run_remote_compaction_request_v2( .min(MAX_REMOTE_COMPACTION_V2_STREAM_RETRIES); let mut retry_state = ResponsesStreamRetryState::default(); let cancellation = &step_context.turn_cancellation; + let interrupt_request = cancellation.child_token(); + let interrupt_at_boundary = SamplingInterrupt::enabled(turn_context); loop { + let trace = InferenceTraceContext::disabled(); + let send = client_session.open_model_response( + prompt, + &settings.model_info, + &turn_context.session_telemetry, + SamplingOptions { + reasoning_effort: sess + .reasoning_effort_for_request(settings, RequestEffortUsage::Compaction) + .await, + reasoning_summary: settings.reasoning_summary, + service_tier: step_context.settings.service_tier.clone(), + }, + responses_metadata, + &trace, + ); let result = match request .open( sess, step_context, &settings.model_info.slug, cancellation, - client_session.open_model_response( - prompt, - &settings.model_info, - &turn_context.session_telemetry, - SamplingOptions { - reasoning_effort: sess - .reasoning_effort_for_request(settings, RequestEffortUsage::Compaction) - .await, - reasoning_summary: settings.reasoning_summary, - service_tier: step_context.settings.service_tier.clone(), - }, - responses_metadata, - &InferenceTraceContext::disabled(), - ), + async { + if interrupt_at_boundary { + SamplingInterrupt::register(sess, turn_context, &interrupt_request).await?; + } + send.await + }, ) .or_cancel(cancellation) .await? @@ -388,6 +406,12 @@ async fn run_remote_compaction_request_v2( } Err(err) => Err(err), }; + if result.is_err() + && interrupt_at_boundary + && SamplingInterrupt::unregister(sess, turn_context).await + { + return Err(CodexErr::TurnAborted); + } match result { Ok(compaction_output) => return Ok(compaction_output), Err(err) diff --git a/codex-rs/core/src/session/sampling_interrupt.rs b/codex-rs/core/src/session/sampling_interrupt.rs index 960914d6e1..261edaabb4 100644 --- a/codex-rs/core/src/session/sampling_interrupt.rs +++ b/codex-rs/core/src/session/sampling_interrupt.rs @@ -1,4 +1,4 @@ -//! Lets an opted-in child finish the sampled item while its tools are cancelled. +//! Cancels an opted-in child's tools while it retains its sampled item or installs its summary. use super::session::Session; use super::turn_context::TurnContext; @@ -30,7 +30,7 @@ impl SamplingInterrupt { && turn.parent_thread_id.is_some() } - pub(super) async fn register( + pub(crate) async fn register( session: &Session, turn: &TurnContext, tools: &CancellationToken, @@ -47,22 +47,34 @@ impl SamplingInterrupt { Ok(()) } - pub(super) async fn unregister(session: &Session, turn: &TurnContext) { + /// Releases an uninterrupted attempt; returns whether a stop is pending instead. + pub(crate) async fn unregister(session: &Session, turn: &TurnContext) -> bool { let mut active = session.active_turn.lock().await; - if let Some(active) = matching_turn(&mut active, turn) - && !active - .sampling_interrupt - .as_ref() - .is_some_and(Self::is_requested) + let Some(active) = matching_turn(&mut active, turn) else { + return false; + }; + if active + .sampling_interrupt + .as_ref() + .is_some_and(Self::is_requested) { - active.sampling_interrupt = None; + return true; } + active.sampling_interrupt = None; + false } pub(crate) fn is_requested(&self) -> bool { self.tools.is_cancelled() } + pub(crate) async fn is_requested_for(session: &Session, turn: &TurnContext) -> bool { + let mut active = session.active_turn.lock().await; + matching_turn(&mut active, turn) + .and_then(|active| active.sampling_interrupt.as_ref()) + .is_some_and(Self::is_requested) + } + pub(super) async fn request(session: &Arc) { let action = { let active = session.active_turn.lock().await; diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index fea7a5d249..04e3aab6c0 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -911,6 +911,11 @@ pub(crate) async fn run_turn( if matches!(err.details(), CodexErrorDetails::Interrupted) || crate::tasks::ends_turn_as_interrupted(&err) { + // The answer preceded this optional compaction. Deliver it while + // the interrupt state keeps follow-ups in the next turn. + if SamplingInterrupt::is_requested_for(&sess, &turn_context).await { + return Ok(last_agent_message); + } return Err(err); } let error = err.to_codex_protocol_error(); diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 56e9658546..af1f14d657 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -802,7 +802,11 @@ impl Session { if !Arc::ptr_eq(&active_turn.task.as_ref()?.turn_context, &turn_context) { return None; } - if can_continue { + let sampling_interrupted = active_turn + .sampling_interrupt + .as_ref() + .is_some_and(SamplingInterrupt::is_requested); + if can_continue && !sampling_interrupted { let state = active_turn.turn_state.lock().await; if state.accepts_mailbox_delivery_for_current_turn() && !state.pending_input.is_empty() @@ -819,10 +823,7 @@ impl Session { .then_some(task.task), task.pending_initial_input.unwrap_or_default(), task.outcome_handoff, - active_turn - .sampling_interrupt - .as_ref() - .is_some_and(SamplingInterrupt::is_requested), + sampling_interrupted, ) }; turn_context diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 0658a06d4a..aafe10b854 100644 --- a/codex-rs/core/src/tasks/regular.rs +++ b/codex-rs/core/src/tasks/regular.rs @@ -10,6 +10,7 @@ use futures::future::BoxFuture; use tokio_util::sync::CancellationToken; use crate::client::ModelClientSession; +use crate::session::SamplingInterrupt; use crate::session::TurnInput; use crate::session::session::Session; use crate::session::startup_prewarm::SessionStartupPrewarmResolution; @@ -160,10 +161,11 @@ impl RegularTask { )) .instrument(run_turn_span.clone()) .await?; - // Terminal errors are already reported. Let task completion preserve pending - // input instead of restarting the failed turn for that same input. + // Preserve pending input for the next turn after an error, exhausted budget, + // or interrupt instead of reopening this turn. if ctx.terminal_error.lock().await.is_some() || sess.work_budget().is_exhausted() + || SamplingInterrupt::is_requested_for(&sess, &ctx).await || !sess.input_queue.has_pending_input(&sess.active_turn).await { return Ok(Some(RegularTaskOutput { diff --git a/codex-rs/core/tests/suite/scenarios_compaction_tests.rs b/codex-rs/core/tests/suite/scenarios_compaction_tests.rs index ec46adecd4..c84cdaca40 100644 --- a/codex-rs/core/tests/suite/scenarios_compaction_tests.rs +++ b/codex-rs/core/tests/suite/scenarios_compaction_tests.rs @@ -1,9 +1,233 @@ -//! Snapshots pending mail across remote compaction, restart, and normal inference. +//! Snapshots context retention across compaction, interruption, restart, and normal inference. use super::*; +use codex_core::TurnInputSubmission; +use codex_protocol::AgentPath; +use codex_protocol::protocol::AgentStatus; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::TurnAbortReason; use codex_protocol::turn_input::InjectedResponseItem; +use core_test_support::responses::ev_function_call; use core_test_support::responses::strip_metadata_from_json; +use core_test_support::streaming_sse::StreamingSseChunk; +use core_test_support::streaming_sse::start_streaming_sse_server; use pretty_assertions::assert_eq; +use tokio::sync::oneshot; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn interrupt_during_remote_compaction_preserves_summary_on_resume() -> Result<()> { + skip_if_no_network!(Ok(())); + const SUMMARY: &str = "REMOTE_COMPACTION_CHECKPOINT"; + const POST_TURN_SUMMARY: &str = "POST_TURN_COMPACTION_CHECKPOINT"; + const FINAL_ANSWER: &str = "Fixed escaping in parser.rs; all tests pass."; + let chunk = |gate, events| StreamingSseChunk { + gate, + body: sse(events), + }; + let (release_summary, summary_gate) = oneshot::channel(); + let (release_post_summary, post_summary_gate) = oneshot::channel(); + let (server, _) = start_streaming_sse_server(vec![ + vec![chunk( + None, + vec![ + ev_function_call( + "plan", + "update_plan", + r#"{"plan":[{"step":"Fix escaping in parser.rs","status":"in_progress"}]}"#, + ), + ev_completed_with_tokens("first", /*total_tokens*/ 200_000), + ], + )], + vec![ + chunk(None, vec![ev_response_created("compact")]), + chunk( + Some(summary_gate), + vec![ + compaction_output(SUMMARY), + ev_completed_with_tokens("compact", /*total_tokens*/ 50), + ], + ), + ], + vec![chunk( + None, + vec![ + ev_assistant_message("answer", FINAL_ANSWER), + ev_completed_with_tokens("answer", /*total_tokens*/ 200_000), + ], + )], + vec![ + chunk(None, vec![ev_response_created("post-compact")]), + chunk( + Some(post_summary_gate), + vec![ + compaction_output(POST_TURN_SUMMARY), + ev_completed_with_tokens("post-compact", /*total_tokens*/ 50), + ], + ), + ], + vec![chunk( + None, + vec![ + ev_assistant_message("resumed", "Continuing with the escaping fix."), + ev_completed_with_tokens("resume", /*total_tokens*/ 60), + ], + )], + ]) + .await; + let config_server = start_mock_server().await; + let base_url = format!("{}/v1", server.uri()); + let test = test_codex() + .with_model("gpt-5.4") + .with_config(move |config| { + config.model_provider.base_url = Some(base_url); + config.model_provider.supports_websockets = false; + config.model_auto_compact_token_limit = Some(100_000); + config.model_post_turn_compact_threshold_percent = 50; + config.update_plan_enabled = true; + config.features.enable(Feature::MultiAgentV2).unwrap(); + config + .features + .disable(Feature::WakeOnAgentMessages) + .unwrap(); + config + .features + .disable(Feature::EnableRequestCompression) + .unwrap(); + config.multi_agent_v2.interrupt_at_item_boundary = true; + }) + .build_with_auto_env(&config_server) + .await?; + let child = test + .thread_manager + .start_thread(StartThreadOptions { + session_source: Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: test.session_configured.thread_id, + depth: 1, + agent_path: Some(AgentPath::try_from("/root/worker").expect("child path")), + agent_nickname: None, + agent_role: None, + })), + ..test.start_thread_options().await + }) + .await? + .thread; + child + .start_or_steer_turn(TurnInputRequest::user_input(vec![text( + "Find and fix the bug.", + )])) + .await?; + let EventMsg::TurnStarted(started) = + wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await + else { + unreachable!("waited for turn start"); + }; + let mut previous_turn_id = started.turn_id; + let next = |event: &EventMsg| { + assert!(!matches!(event, EventMsg::Error(_)), "{event:?}"); + matches!(event, EventMsg::ItemCompleted(item) if matches!(item.item, TurnItem::ContextCompaction(_))) + || matches!( + event, + EventMsg::TurnAborted(_) | EventMsg::TurnStarted(_) | EventMsg::TurnComplete(_) + ) + }; + // The same child stops during mid-turn compaction, then after its completed answer. + for (request_count, release_summary, expected_answer) in [ + (2, release_summary, None), + (4, release_post_summary, Some(FINAL_ANSWER)), + ] { + server.wait_for_request_count(request_count).await; + child.submit(Op::InterruptAgent).await?; + // This reply confirms the interrupt was accepted while the summary is still sampling. + child + .update_thread_settings(ThreadSettingsOverrides::default()) + .await?; + assert_eq!(child.agent_status().await, AgentStatus::Running); + let (reply, resumed) = oneshot::channel(); + child + .submit(Op::TurnInput { + request: Box::new(TurnInputRequest::user_input(vec![text("Resume.")])), + mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer, + reply, + }) + .await?; + release_summary + .send(()) + .expect("finish sampling the summary"); + assert!(matches!( + wait_for_event(&child, next).await, + EventMsg::ItemCompleted(_) + )); + let terminal = wait_for_event(&child, next).await; + if let Some(answer) = expected_answer { + let EventMsg::TurnComplete(completed) = terminal else { + panic!("completed answer was lost: {terminal:?}"); + }; + assert_eq!(completed.last_agent_message.as_deref(), Some(answer)); + assert_eq!(completed.turn_id, previous_turn_id); + } else { + assert!(matches!(terminal, + EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted)); + } + let TurnInputSubmission::Started { turn_id, .. } = resumed.await?? else { + panic!("Resume must start a new turn"); + }; + assert_ne!(turn_id, previous_turn_id); + let EventMsg::TurnStarted(started) = wait_for_event(&child, next).await else { + panic!("expected resumed turn start"); + }; + assert_eq!(started.turn_id, turn_id); + previous_turn_id = turn_id; + } + assert!(matches!( + wait_for_event(&child, next).await, + EventMsg::TurnComplete(_) + )); + + let requests = server + .requests() + .await + .iter() + .map(|body| serde_json::from_slice::(body)) + .collect::>>()?; + assert_eq!(requests.len(), 5); + for (request_index, summary) in [(2, SUMMARY), (4, POST_TURN_SUMMARY)] { + assert_eq!( + requests[request_index - 1]["input"] + .as_array() + .expect("compaction input") + .last(), + Some(&json!({"type": "compaction_trigger"})), + ); + let resumed_input = requests[request_index]["input"] + .as_array() + .expect("resumed input"); + let summaries = resumed_input + .iter() + .filter(|item| item["type"] == "compaction") + .map(|item| item["encrypted_content"].clone()) + .collect::>(); + assert_eq!(summaries, vec![json!(summary)]); + let resume_messages = resumed_input + .iter() + .filter(|item| item["role"] == "user") + .filter_map(|item| item["content"].as_array()) + .flatten() + .filter_map(|content| content["text"].as_str()) + .filter(|text| *text == "Resume.") + .collect::>(); + assert_eq!(resume_messages, vec!["Resume."; request_index / 2]); + } + let entries = requests.iter().map(SnapshotEntry::body).collect::>(); + insta::assert_snapshot!(context_snapshot::format_context_snapshot( + "A child interrupted during mid-turn and post-turn remote compaction keeps both checkpoints, delivers its completed answer, and handles each Resume in a separate turn.", + &entries, + &ContextSnapshotOptions::default().rewrite_known_segments(), + )); + child.shutdown_and_wait().await?; + server.shutdown().await; + Ok(()) +} #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_slash_compact_preserves_context_across_windows() -> Result<()> { diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__compaction__interrupt_during_remote_compaction_preserves_summary_on_resume.snap b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__compaction__interrupt_during_remote_compaction_preserves_summary_on_resume.snap new file mode 100644 index 0000000000..b813e8d925 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__compaction__interrupt_during_remote_compaction_preserves_summary_on_resume.snap @@ -0,0 +1,122 @@ +--- +source: core/tests/suite/scenarios_compaction_tests.rs +expression: "context_snapshot::format_context_snapshot(\"A child interrupted during mid-turn and post-turn remote compaction keeps both checkpoints, delivers its completed answer, and handles each Resume in a separate turn.\",\n&entries, &ContextSnapshotOptions::default().rewrite_known_segments(),)" +--- +Scenario: A child interrupted during mid-turn and post-turn remote compaction keeps both checkpoints, delivers its completed answer, and handles each Resume in a separate turn. + +## Window 1 +-- request 1 (request) -- +00:message/developer: + +01:message/developer: + +02:message/developer: + You are an agent in a team of agents collaborating to complete a task. + + You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents. All ag...nd capable, and have access to the same set of tools. [hash=876a93c54d3f61d6] + + You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task an... `send_message` to pass a message to a running agent. [hash=824c12425c7626dc] + Child agents can also spawn their own sub-agents. + + When you provide a response in the final channel, that content is immediately delivered back to your parent agent. + + - All agents use the same current working directory. + - As a result, edits made by one agent are immediately visible to all other agents. + + When calling `wait_agent`, prefer longer waits (minutes) to avoid busy polling. + + There are 4 available concurrency slots, meaning that up to 4 agents can be active at once, including you. + + Full-history forks (`fork_turns` omitted or `"all"`) inherit the parent model and reasoning effort and d...uctions; when doing so, set `fork_turns` to `"none"`. [hash=18ccf7e93ecabab9] +03:message/developer: + +04:message/user: + +05:message/user: + Find and fix the bug. +-- request 2 (request) -- +06:function_call/update_plan:{"plan":[{"status":"in_progress","step":"Fix escaping in parser.rs"}]} +07:function_call_output:Plan updated +08:compaction_trigger + +## Window 2 (after request 2: input diverged at item 06) +-- request 3 (request) -- +00:message/developer: + +01:message/developer: + +02:message/developer: + You are an agent in a team of agents collaborating to complete a task. + + You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents. All ag...nd capable, and have access to the same set of tools. [hash=876a93c54d3f61d6] + + You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task an... `send_message` to pass a message to a running agent. [hash=824c12425c7626dc] + Child agents can also spawn their own sub-agents. + + When you provide a response in the final channel, that content is immediately delivered back to your parent agent. + + - All agents use the same current working directory. + - As a result, edits made by one agent are immediately visible to all other agents. + + When calling `wait_agent`, prefer longer waits (minutes) to avoid busy polling. + + There are 4 available concurrency slots, meaning that up to 4 agents can be active at once, including you. + + Full-history forks (`fork_turns` omitted or `"all"`) inherit the parent model and reasoning effort and d...uctions; when doing so, set `fork_turns` to `"none"`. [hash=18ccf7e93ecabab9] +03:message/developer: + +04:message/user: + +05:message/user: + Find and fix the bug. +06:compaction:encrypted=true; chars=28; hash=f898585a8fffdd12 +07:message/developer: + + The previous turn was interrupted on purpose. Any running unified exec processes may still be running in...mands were aborted, they may have partially executed. [hash=cee256d92b55ad65] + +08:message/user: + Resume. +-- request 4 (request) -- +09:message/assistant: + Fixed escaping in parser.rs; all tests pass. +10:compaction_trigger + +## Window 3 (after request 4: input diverged at item 01) +-- request 5 (request) -- +00:message/developer: + +01:message/user: + Find and fix the bug. +02:message/developer: + +03:message/developer: + You are an agent in a team of agents collaborating to complete a task. + + You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents. All ag...nd capable, and have access to the same set of tools. [hash=876a93c54d3f61d6] + + You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task an... `send_message` to pass a message to a running agent. [hash=824c12425c7626dc] + Child agents can also spawn their own sub-agents. + + When you provide a response in the final channel, that content is immediately delivered back to your parent agent. + + - All agents use the same current working directory. + - As a result, edits made by one agent are immediately visible to all other agents. + + When calling `wait_agent`, prefer longer waits (minutes) to avoid busy polling. + + There are 4 available concurrency slots, meaning that up to 4 agents can be active at once, including you. + + Full-history forks (`fork_turns` omitted or `"all"`) inherit the parent model and reasoning effort and d...uctions; when doing so, set `fork_turns` to `"none"`. [hash=18ccf7e93ecabab9] +04:message/developer: + +05:message/user: + +06:message/user: + Resume. +07:compaction:encrypted=true; chars=31; hash=19888c8bed116fb6 +08:message/developer: + + The previous turn was interrupted on purpose. Any running unified exec processes may still be running in...mands were aborted, they may have partially executed. [hash=cee256d92b55ad65] + +09:message/user: + Resume.