Add unfinished root turn suspension (#40038)

## Why

An active root turn needs to be stopped without marking it complete or aborted
before another runtime can recover the same turn ID.

## What changed

- Add `CodexThread::suspend_turn_and_shutdown` and `SuspendTurnOutcome`.
- Flush history, stop the active regular task, close the history writer, and
  shut down the session without recording a terminal turn event.
- Reject suspension when no supported turn is active or the loaded agent
  subtree still contains a live descendant.

## Testing

- Verify that suspension preserves unfinished history and allows the turn to be
  recovered under its original ID after the descendant guard is cleared.

GitOrigin-RevId: 1f9b019d07c51474ec2d991d263bc15cdd4f89ad
This commit is contained in:
richardopenai
2026-08-22 00:42:19 +00:00
committed by copyberry
parent 970b7f2ff4
commit 4f39251a01
10 changed files with 345 additions and 3 deletions
+46
View File
@@ -3,6 +3,7 @@ use crate::config::ConstraintResult;
use crate::elicitation::ElicitationRegistration;
use crate::session::SessionIo;
use crate::session::SessionSettingsUpdate;
use crate::session::new_submission_id;
use crate::session::session::Session;
use codex_diagnostics::Gauge;
use codex_diagnostics::GaugeGuard;
@@ -12,6 +13,7 @@ use codex_extension_api::ThreadIdleCause;
use codex_features::Feature;
use codex_history::RolloutItem;
use codex_otel::SessionTelemetry;
use codex_otel::current_span_w3c_trace_context;
use codex_protocol::ThreadId;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::CollaborationMode;
@@ -36,6 +38,7 @@ use codex_protocol::protocol::Op;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionConfiguredEvent;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::Submission;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_protocol::protocol::ThreadSettingsSnapshot;
@@ -47,6 +50,7 @@ use codex_protocol::protocol::W3cTraceContext;
use codex_protocol::turn_input::RecoverTurnRequest;
use codex_protocol::turn_input::StartIfIdleSubmission;
use codex_protocol::turn_input::SteerSubmission;
use codex_protocol::turn_input::SuspendTurnOutcome;
use codex_protocol::turn_input::TurnInputMode;
use codex_protocol::turn_input::TurnInputRequest;
use codex_protocol::turn_input::TurnInputSubmission;
@@ -63,6 +67,7 @@ use rmcp::model::ReadResourceRequestParams;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::Mutex;
use tokio::sync::oneshot;
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
@@ -397,6 +402,47 @@ impl CodexThread {
}
}
/// Stops the active unfinished root turn without recording TurnAborted or
/// TurnComplete, so another worker can recover its original turn ID.
///
/// Suspension is refused while a currently loaded descendant exists. Past
/// descendants do not prevent recovery, and concurrent descendant admission
/// is not sealed. Queued user input and outstanding approval, elicitation,
/// or server-request waiters remain best effort and may be discarded.
///
/// The session processes an accepted request even if its caller disconnects.
/// Callers must not transfer ownership until suspension succeeds, which
/// requires stopping execution, flushing history, and closing its writer.
pub async fn suspend_turn_and_shutdown(&self) -> CodexResult<SuspendTurnOutcome> {
if self.session_source.is_non_root_agent() {
return Err(CodexErr::UnsupportedOperation(
"turn suspension requires the owning root thread".to_string(),
));
}
// The session owns accepted suspension, so dropping this caller cannot interrupt
// cancellation, persistence, or writer shutdown halfway through a handoff.
let (reply, result) = oneshot::channel();
self.io
.tx_sub
.send(Submission {
id: new_submission_id(),
op: Op::SuspendTurnAndShutdown { reply },
trace: current_span_w3c_trace_context(),
parent_turn_id: None,
root_turn_id: None,
})
.await
.map_err(|_| CodexErr::Fatal("thread session has stopped".to_string()))?;
let outcome = result
.await
.map_err(|_| CodexErr::Fatal("thread suspension reply was lost".to_string()))??;
if matches!(&outcome, SuspendTurnOutcome::Suspended { .. }) {
self.io.session_loop_termination.clone().await;
}
Ok(outcome)
}
/// Steers only if `expected_turn_id` is still the active regular turn.
pub async fn steer_turn(
&self,
+1
View File
@@ -19,6 +19,7 @@ pub use codex_protocol::turn_input::NotSubmittedReason;
pub use codex_protocol::turn_input::RecoverTurnRequest;
pub use codex_protocol::turn_input::StartIfIdleSubmission;
pub use codex_protocol::turn_input::SteerSubmission;
pub use codex_protocol::turn_input::SuspendTurnOutcome;
pub use codex_protocol::turn_input::TurnInput;
pub use codex_protocol::turn_input::TurnInputRequest;
pub use codex_protocol::turn_input::TurnInputSubmission;
+15 -2
View File
@@ -394,7 +394,7 @@ pub async fn set_thread_memory_mode(sess: &Arc<Session>, sub_id: String, mode: T
}
}
async fn shutdown_session_runtime(sess: &Arc<Session>) {
pub(super) async fn shutdown_session_runtime(sess: &Arc<Session>) {
if let Some(startup_prewarm) = sess.take_session_startup_prewarm().await {
startup_prewarm.abort().await;
}
@@ -421,7 +421,7 @@ async fn shutdown_session_runtime(sess: &Arc<Session>) {
crate::hook_runtime::run_session_end_hooks(sess).await;
}
async fn emit_thread_stop_lifecycle(sess: &Session) {
pub(super) async fn emit_thread_stop_lifecycle(sess: &Session) {
for contributor in sess.services.extensions.thread_lifecycle_contributors() {
contributor
.on_thread_stop(codex_extension_api::ThreadStopInput {
@@ -585,6 +585,19 @@ pub(super) async fn submission_loop(
let _ = reply.send(result);
false
}
Op::SuspendTurnAndShutdown { reply } => {
let result =
super::turn_suspension::suspend_turn_and_shutdown(&sess, sub.id.clone())
.await;
// Exit only after history is durable and its writer has closed; an error
// must leave responsibility for the thread with the current worker.
let should_exit = matches!(
&result,
Ok(codex_protocol::turn_input::SuspendTurnOutcome::Suspended { .. })
);
let _ = reply.send(result);
should_exit
}
Op::ThreadSettings { thread_settings } => {
thread_settings::update(&sess, sub.id.clone(), thread_settings).await;
false
+1
View File
@@ -229,6 +229,7 @@ mod token_budget;
pub(crate) mod turn;
pub(crate) mod turn_context;
mod turn_input;
mod turn_suspension;
mod world_state;
use self::code_mode_warning::unsupported_code_mode_warning;
#[cfg(test)]
@@ -0,0 +1,120 @@
use super::handlers;
use super::session::Session;
use crate::state::TaskKind;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::turn_input::SuspendTurnOutcome;
use std::sync::Arc;
use std::time::Duration;
use tracing::warn;
pub(super) async fn suspend_turn_and_shutdown(
session: &Arc<Session>,
submission_id: String,
) -> CodexResult<SuspendTurnOutcome> {
{
let active = session.active_turn.lock().await;
let Some(task) = active.as_ref().and_then(|turn| turn.task.as_ref()) else {
return Ok(SuspendTurnOutcome::NotActive);
};
if task.kind != TaskKind::Regular {
return Ok(SuspendTurnOutcome::UnsupportedTask);
}
}
// This is a snapshot of currently loaded descendants, not a spawn-admission seal.
// Previously closed descendants and concurrent future spawns remain best effort.
if session
.services
.agent_control
.list_live_agent_subtree_thread_ids(session.thread_id)
.await?
.len()
> 1
{
return Ok(SuspendTurnOutcome::HasLiveDescendants);
}
let live_thread = session
.live_thread_for_persistence("suspend an unfinished root turn")
.map_err(|error| CodexErr::Fatal(error.to_string()))?;
// Flush before canceling execution so a persistence failure leaves the original turn running.
live_thread.flush().await.map_err(|error| {
CodexErr::Fatal(format!("flush before root turn suspension failed: {error}"))
})?;
// The flush can yield while the active turn completes or changes. Recheck its
// kind under the same lock used to remove it.
let mut turn = {
let mut active = session.active_turn.lock().await;
let Some(active_turn) = active.as_ref() else {
return Ok(SuspendTurnOutcome::NotActive);
};
let Some(task) = active_turn.task.as_ref() else {
return Ok(SuspendTurnOutcome::NotActive);
};
if task.kind != TaskKind::Regular {
return Ok(SuspendTurnOutcome::UnsupportedTask);
}
active.take().ok_or_else(|| {
CodexErr::Fatal("accepted root turn suspension had no running turn".to_string())
})?
};
let task = turn.task.take().ok_or_else(|| {
CodexErr::Fatal("accepted root turn suspension had no running task".to_string())
})?;
let turn_id = task.turn_context.sub_id.clone();
// Normal shutdown records a terminal turn event, preventing another worker from
// recovering this turn under its original ID. Cancel the task without that event.
task.cancellation_token.cancel();
task.turn_context
.turn_metadata_state
.cancel_git_enrichment_task();
let mut task_handle = task.handle.detach();
match tokio::time::timeout(
Duration::from_millis(crate::tasks::GRACEFULL_INTERRUPTION_TIMEOUT_MS),
&mut task_handle,
)
.await
{
Ok(Ok(())) => {}
Ok(Err(error)) => {
warn!(thread_id = %session.thread_id, %error, "suspended turn task exited abnormally");
}
Err(_) => {
warn!(
thread_id = %session.thread_id,
"suspended turn task did not stop gracefully; aborting it"
);
task_handle.abort();
let _ = task_handle.await;
}
}
// Pending accepted input and interactive waiters live only in this process. Handoff
// intentionally drops that state; persisting or replaying it needs a separate protocol.
session.input_queue.clear_pending(&turn).await;
// Stop all producers before flushing their final history and closing its writer.
// If either persistence step fails, do not report success: the current worker
// retains ownership until worker-failure recovery can take responsibility.
handlers::shutdown_session_runtime(session).await;
live_thread.flush().await.map_err(|error| {
CodexErr::Fatal(format!("flush after root turn suspension failed: {error}"))
})?;
live_thread.shutdown().await.map_err(|error| {
CodexErr::Fatal(format!("close suspended root turn writer failed: {error}"))
})?;
// Announce thread shutdown only after its writer closes so a replacement worker
// cannot write the same thread concurrently.
handlers::emit_thread_stop_lifecycle(session.as_ref()).await;
session
.deliver_event_raw(Event {
id: submission_id,
msg: EventMsg::ShutdownComplete,
})
.await;
Ok(SuspendTurnOutcome::Suspended { turn_id })
}
+1 -1
View File
@@ -63,7 +63,7 @@ pub(crate) use user_shell::UserShellCommandMode;
pub(crate) use user_shell::UserShellCommandTask;
pub(crate) use user_shell::execute_user_shell_command;
const GRACEFULL_INTERRUPTION_TIMEOUT_MS: u64 = 100;
pub(crate) const GRACEFULL_INTERRUPTION_TIMEOUT_MS: u64 = 100;
const TASK_COMPACT_METRIC: &str = "codex.task.compact";
static ACTIVE_TURNS: Gauge = Gauge::new("core.turns.active");