Add graceful agent-tree suspension and recovery (#61198)

## Why

Suspending a locally controlled agent tree must preserve unfinished turns and child results that arrive during teardown so they remain available after recovery.

## What changed

- Add `ThreadManager::suspend_agent_tree` to stop durable trees, close their writers, and return unfinished root and named V2 child turn IDs plus any pending root wake.
- Persist terminal child results before suspension succeeds, including results queued for evicted parents. Report failed or unconfirmed writes.
- Add `load_suspended_agent` to restore named V2 descendants through their recorded ancestors without starting model turns.
- Allow empty `continue_turn_if_idle` input to wake on saved results, guarded by the preceding turn ID. Keep mail queued until task registration so suspension can preserve it.

## Testing

Add coverage for completion and wake races, child errors, parent writer failures, evicted parents, descendant restoration, repeated suspension after caller cancellation, and ephemeral roots remaining active.

GitOrigin-RevId: 3041791f51bff2555210f6b2d2443f9f05f68330
This commit is contained in:
Bryan Ashley
2026-11-12 22:43:14 +00:00
committed by copyberry
parent cdfcc18dba
commit fae87361f4
30 changed files with 1480 additions and 320 deletions
+68 -1
View File
@@ -316,6 +316,9 @@ pub(super) async fn shutdown_session_runtime(sess: &Arc<Session>) {
if let Some(reporting) = &sess.outcome_reporting {
reporting.handoffs.close();
}
let runtime = &sess.services.local_agent_runtime;
let wakes_root = runtime.suspension_root() == Some(sess.thread_id)
&& (sess.has_outstanding_durable_sleep() || sess.should_wake_on_agent_messages().await);
let startup_prewarm = {
let mut state = sess.state.lock().await;
// Stop admission and take the current warmup together so resume cannot replace it.
@@ -330,6 +333,49 @@ pub(super) async fn shutdown_session_runtime(sess: &Arc<Session>) {
if let Some(reporting) = &sess.outcome_reporting {
reporting.handoffs.wait().await;
}
if runtime.suspension_root().is_some() {
let pending: Vec<_> = sess
.input_queue
.drain_mailbox()
.await
.into_iter()
.map(|mail| mail.communication)
.filter(crate::PendingAgentMessage::is_terminal_result)
.collect();
if !pending.is_empty() && sess.live_thread().is_some() {
let turn_id = {
let state = sess.state.lock().await;
state.last_started_turn_id.clone().or_else(|| {
state
.turn_attribution
.as_ref()
.map(|turn| turn.turn_id.clone())
})
};
if wakes_root {
if let Some(turn_id) = turn_id.as_ref() {
runtime.record_root_wake_after(turn_id.clone());
} else {
runtime.record_shutdown_failure(AgentTreeShutdownFailure::operation_failed(
"tree_suspension",
"record_root_wake",
Some(sess.thread_id),
"missing_previous_turn",
));
}
}
let context = sess
.new_turn_with_default_settings(
turn_id.unwrap_or_else(super::new_submission_id),
Default::default(),
)
.await;
for result in pending {
sess.record_inter_agent_communication(&context, context.model_info(), result)
.await;
}
}
}
super::async_compaction::shutdown(sess).await;
let shell_snapshot_prewarm = sess.state.lock().await.shell_snapshot_prewarm.take();
if let Some(shell_snapshot_prewarm) = shell_snapshot_prewarm {
@@ -491,7 +537,28 @@ pub(super) async fn submission_loop(
let sub = tokio::select! {
biased;
_ = sess.services.local_agent_runtime.shutdown.cancelled() => {
shutdown_received = shutdown(&sess, super::new_submission_id()).await;
if sess.services.local_agent_runtime.suspension_root().is_some() {
// Keep accepting results until all direct children stop, then drain
// buffered submissions before deciding which parent work to hand off.
let runtime = &sess.services.local_agent_runtime;
tokio::join!(
async {
runtime.wait_for_children(sess.thread_id).await;
rx_sub.close();
},
async {
while let Ok(sub) = rx_sub.recv().await {
if let Op::InterAgentCommunication { communication, start_options } = sub.op
&& sub.message_delivery.as_ref().is_some_and(crate::agent_communication::MessageDelivery::awaits_result_write)
{
let mail = crate::PendingAgentMessage::new(communication, sub.message_delivery);
inter_agent_communication(&sess, sub.id, mail, start_options).await;
}
}
},
);
}
shutdown_received = super::turn_suspension::stop_tree_member(&sess).await;
break;
}
update = async {
+49 -10
View File
@@ -269,15 +269,25 @@ impl InputQueue {
pending.drain(..).collect()
}
pub(crate) async fn drain_mailbox_input_items(&self) -> (Vec<TurnInput>, TurnStartOptions) {
let pending_mails = self.drain_mailbox().await;
pub(crate) async fn pending_wake_options(&self) -> Option<TurnStartOptions> {
let mut mails = self.mailbox_pending_mails.lock().await;
self.read_mailbox(&mut mails);
let mut options = Self::trigger_turn_options(&mails)?;
if let Some(mail) = mails.iter().find(|mail| mail.communication.trigger_turn) {
options.initiating_agent_path = Some(mail.communication.author.clone());
}
Some(options)
}
fn trigger_turn_options(
pending_mails: &VecDeque<PendingMailboxCommunication>,
) -> Option<TurnStartOptions> {
// A later follow-up supersedes the earlier choice, including an omitted choice.
let mut start_options = pending_mails
.iter()
.rev()
.find(|mail| mail.communication.trigger_turn)
.map(|mail| mail.start_options.clone())
.unwrap_or_default();
.map(|mail| mail.start_options.clone())?;
start_options.parent_turn_id = pending_mails
.iter()
.filter(|mail| mail.communication.trigger_turn)
@@ -296,8 +306,15 @@ impl InputQueue {
.filter(|id| !id.trim().is_empty())
})
.map(str::to_string);
Some(start_options)
}
pub(crate) async fn drain_mailbox_input_items(&self) -> (Vec<TurnInput>, TurnStartOptions) {
let mut pending_mails = self.mailbox_pending_mails.lock().await;
self.read_mailbox(&mut pending_mails);
let start_options = Self::trigger_turn_options(&pending_mails).unwrap_or_default();
let items = pending_mails
.into_iter()
.drain(..)
.map(|mail| TurnInput::InterAgentCommunication(mail.communication))
.collect();
(items, start_options)
@@ -343,12 +360,34 @@ impl InputQueue {
/// Clear any pending waiters and input buffered for the current turn.
pub(crate) async fn clear_pending(&self, active_turn: &ActiveTurn) {
let mut turn_state = active_turn.turn_state.lock().await;
turn_state.clear_pending_waiters();
// Pending mail can own extension attachments whose destructors need the lock released.
let pending_input = std::mem::take(&mut turn_state.pending_input);
drop(turn_state);
drop(pending_input);
let pending_input = {
let mut turn_state = active_turn.turn_state.lock().await;
turn_state.clear_pending_waiters();
std::mem::take(&mut turn_state.pending_input)
};
// A terminal result still in the queue has no writer yet. Keep it available
// to the next turn or to the source writer during a graceful drain.
let results: Vec<_> = pending_input
.items
.into_iter()
.filter_map(|input| match input {
TurnInput::InterAgentCommunication(message) if message.is_terminal_result() => {
Some(message)
}
_ => None,
})
.collect();
if !results.is_empty() {
let mut pending = self.mailbox_pending_mails.lock().await;
for communication in results.into_iter().rev() {
pending.push_front(PendingMailboxCommunication {
communication,
start_options: TurnStartOptions::default(),
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
});
}
}
}
pub(crate) async fn defer_mailbox_delivery_to_next_turn(
+48 -20
View File
@@ -195,7 +195,6 @@ use codex_thread_store::LocalThreadStore;
use codex_thread_store::PersistContext;
use codex_thread_store::ReadThreadParams;
use codex_thread_store::ResumeThreadParams;
use codex_thread_store::ThreadPersistenceMetadata;
use codex_thread_store::ThreadStore;
use codex_thread_store::ThreadStoreResumePreparation;
use codex_utils_audio::prepare_response_items as prepare_audio_response_items;
@@ -396,6 +395,7 @@ use codex_protocol::protocol::DeprecationNoticeEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ExecApprovalRequestEvent;
use codex_protocol::protocol::InterAgentCommunication;
use codex_protocol::protocol::ModelRerouteEvent;
use codex_protocol::protocol::ModelRerouteReason;
use codex_protocol::protocol::ModelVerification;
@@ -2832,7 +2832,7 @@ impl Session {
let reporter = self
.outcome_reporting
.as_ref()
.map(|reporting| &reporting.reporter);
.and_then(|reporting| reporting.reporter.as_ref());
if !notify_native_parent && reporter.is_none() {
return;
}
@@ -4656,13 +4656,11 @@ impl Session {
}))
}
pub(crate) async fn record_inter_agent_communication(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
communication: impl Into<crate::PendingAgentMessage>,
) {
let communication = communication.into();
/// Builds the same history item whether the recipient is live or has been evicted.
pub(crate) fn inter_agent_history_item(
communication: &InterAgentCommunication,
config: &Config,
) -> ResponseItem {
let mut response_item = communication.to_model_input_item();
if let ResponseItem::AgentMessage {
content,
@@ -4676,8 +4674,7 @@ impl Session {
.get_or_insert_with(|| {
vec![
ContentItemMetadata::tool(Some(
turn_context
.config
config
.multi_agent_v2
.tool_namespace
.clone()
@@ -4688,6 +4685,17 @@ impl Session {
]
});
}
response_item
}
pub(crate) async fn record_inter_agent_communication(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
communication: impl Into<crate::PendingAgentMessage>,
) {
let communication = communication.into();
let response_item = Self::inter_agent_history_item(&communication, &turn_context.config);
let (items, _) = self
.prepare_conversation_items_for_history(
turn_context,
@@ -4728,16 +4736,21 @@ impl Session {
model_info.truncation_policy.into(),
);
}
if let Some(delivery) = &communication.delivery {
delivery.consumed(self.thread_id);
let result_write = communication.consumed(self.thread_id);
let saved = self
.persist_rollout_items(&[
RolloutItem::InterAgentCommunicationMetadata {
trigger_turn: communication.trigger_turn,
},
RolloutItem::ResponseItem(response_item),
])
.await;
if saved
&& self.live_thread().is_some()
&& let Some(write) = result_write
{
write.confirm();
}
self.persist_rollout_items(&[
RolloutItem::InterAgentCommunicationMetadata {
trigger_turn: communication.trigger_turn,
},
RolloutItem::ResponseItem(response_item),
])
.await;
drop(boundary);
self.send_raw_response_items(turn_context, &items).await;
}
@@ -5394,6 +5407,21 @@ impl Session {
&& let Err(e) = live_thread.append_items(items).await
{
error!("failed to record rollout items: {e:#}");
if self
.services
.local_agent_runtime
.suspension_root()
.is_some()
{
self.services.local_agent_runtime.record_shutdown_failure(
crate::thread_manager::AgentTreeShutdownFailure::operation_failed(
"tree_suspension",
"append_history",
Some(self.thread_id),
crate::thread_manager::thread_store_error_kind(&e),
),
);
}
return false;
}
true
+21 -27
View File
@@ -60,9 +60,10 @@ use tokio_util::task::TaskTracker;
type McpToolApprovalMetadataMap =
HashMap<(codex_mcp::McpServerKey, String), std::sync::Weak<McpToolApprovalContext>>;
/// A thread's installed reporter and the tasks that may still hand it an outcome.
/// Joins terminal publishers before their session writer closes. Native child results
/// and optional host reporters both use this barrier.
pub(crate) struct AgentOutcomeReporting {
pub(super) reporter: Arc<dyn AgentTurnOutcomeReporter>,
pub(crate) reporter: Option<Arc<dyn AgentTurnOutcomeReporter>>,
pub(crate) handoffs: TaskTracker,
}
@@ -634,10 +635,15 @@ async fn warm_plugins_and_skills_for_session_init(
}
impl Session {
pub(crate) fn outcome_handoffs_closed(&self) -> bool {
self.outcome_reporting
.as_ref()
.is_some_and(|reporting| reporting.handoffs.is_closed())
pub(crate) fn turn_starts_closed(&self) -> bool {
self.services
.local_agent_runtime
.suspension_root()
.is_some()
|| self
.outcome_reporting
.as_ref()
.is_some_and(|reporting| reporting.handoffs.is_closed())
}
pub(crate) fn new_item_id_generator() -> ItemIdGenerator {
@@ -1143,15 +1149,7 @@ impl Session {
.window_id
.clone(),
runtime_workspace_roots: Some(config.workspace_roots.clone()),
metadata: ThreadPersistenceMetadata {
cwd: Some(config.cwd.to_path_buf()),
model_provider: config.model_provider_id.clone(),
memory_mode: if config.memories.generate_memories {
ThreadMemoryMode::Enabled
} else {
ThreadMemoryMode::Disabled
},
},
metadata: config.thread_persistence_metadata(),
};
if is_paginated_subagent
&& matches!(
@@ -1180,15 +1178,7 @@ impl Session {
rollout_path: resumed_history.rollout_path.clone(),
history: Some(resumed_history.history.clone()),
include_archived: true,
metadata: ThreadPersistenceMetadata {
cwd: Some(config.cwd.to_path_buf()),
model_provider: config.model_provider_id.clone(),
memory_mode: if config.memories.generate_memories {
ThreadMemoryMode::Enabled
} else {
ThreadMemoryMode::Disabled
},
},
metadata: config.thread_persistence_metadata(),
};
let store = Arc::clone(&thread_store);
let history_mode = session_configuration.history_mode;
@@ -1797,10 +1787,14 @@ impl Session {
.await;
}
let outcome_reporting = thread_extension_data
let reporter = thread_extension_data
.get::<Arc<dyn AgentTurnOutcomeReporter>>()
.map(|reporter| AgentOutcomeReporting {
reporter: Arc::clone(&reporter),
.map(|reporter| Arc::clone(reporter.as_ref()));
// The turn's resolved version determines whether it publishes a native result.
let local_durable = live_thread.is_some() && !has_provided_agent_control;
let outcome_reporting =
(reporter.is_some() || local_durable).then(|| AgentOutcomeReporting {
reporter,
handoffs: TaskTracker::new(),
});
let (agent_runtime, mut startup_error) =
+7 -8
View File
@@ -2,6 +2,7 @@ use crate::McpRefreshReason;
use crate::agent::LocalAgentControl;
use codex_mcp::McpServerKey;
use codex_protocol::ContextWindowId;
use codex_thread_store::ThreadPersistenceMetadata;
#[path = "compaction_replay_tests.rs"]
mod compaction_replay_tests;
@@ -6254,12 +6255,10 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc
.tx_event
.send(Event {
id: "occupied".into(),
msg: EventMsg::ThreadSettingsApplied(
codex_protocol::protocol::ThreadSettingsAppliedEvent {
thread_id: Some(session.thread_id()),
thread_settings: session.thread_settings_snapshot().await,
},
),
msg: EventMsg::ThreadSettingsApplied(ThreadSettingsAppliedEvent {
thread_id: Some(session.thread_id()),
thread_settings: session.thread_settings_snapshot().await,
}),
})
.await
.expect("fill event channel");
@@ -6271,7 +6270,7 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc
id: "settings".into(),
turn_extension_init: None,
op: Op::ThreadSettings {
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides::default(),
thread_settings: ThreadSettingsOverrides::default(),
reply: Some(reply),
},
trace: None,
@@ -6356,7 +6355,7 @@ async fn compaction_persists_resume_metadata_and_companion_records() {
}),
previous_turn_settings: Some(previous_turn_settings),
};
let expected_settings = codex_protocol::protocol::ThreadSettingsAppliedEvent {
let expected_settings = ThreadSettingsAppliedEvent {
thread_id: Some(session.thread_id()),
thread_settings: session.thread_settings_snapshot().await,
};
+52 -42
View File
@@ -306,13 +306,20 @@ pub(super) async fn handle(
TurnInputMode::ContinueIfIdle {
expected_previous_turn_id,
} => {
if !matches!(
&request.request.input,
SubmittedTurnInput::ResponseItem(_) | SubmittedTurnInput::AnnotatedResponseItem(_)
) {
return Err(CodexErr::InvalidRequest(
"continuation requires internal response input".to_string(),
));
let mut request = request;
match &request.request.input {
SubmittedTurnInput::ResponseItem(_)
| SubmittedTurnInput::AnnotatedResponseItem(_) => {}
SubmittedTurnInput::UserInput { content, .. } if content.is_empty() => {
request.request.start = session.agent_message_wake_options().await;
}
SubmittedTurnInput::UserInput { .. }
| SubmittedTurnInput::InterAgentCommunication(_) => {
return Err(CodexErr::InvalidRequest(
"continuation requires internal response input or an empty wake"
.to_string(),
));
}
}
start_if_idle(
session,
@@ -501,13 +508,9 @@ async fn start_or_steer(
.await,
);
}
session
.spawn_task(Arc::clone(&turn_context), task_input, RegularTask::new())
.await;
Ok(TurnInputSubmission::Started {
turn_id: submission_id,
root_turn_id: turn_context.root_turn_id(),
})
Ok(session
.spawn_task(turn_context, task_input, RegularTask::new())
.await)
}
Err(reason) => Ok(TurnInputSubmission::NotSubmitted { reason }),
}
@@ -537,7 +540,9 @@ async fn start_if_idle(
responsesapi_client_metadata,
..
} = request;
if session.input_queue.has_trigger_turn_mailbox_items().await {
if expected_previous_turn_id.is_none()
&& session.input_queue.has_trigger_turn_mailbox_items().await
{
return Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::PendingTriggerTurn,
});
@@ -575,18 +580,23 @@ async fn start_if_idle(
let turn_state = {
let mut active_turn = session.active_turn.lock().await;
if let Some(expected) = expected_previous_turn_id {
let state = session.state.lock().await;
// A registered successor already owns this work, even if it is still running.
// An unregistered reservation can still fail to start and must be retried.
if state.last_started_turn_id.as_ref() != Some(&expected)
&& (state.last_started_turn_id.is_some() || active_turn.is_none())
{
return Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::Superseded,
});
}
}
if active_turn.is_some() {
return Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::NotIdle,
});
}
if let Some(expected) = expected_previous_turn_id
&& session.state.lock().await.last_started_turn_id.as_ref() != Some(&expected)
{
return Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::Superseded,
});
}
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
Arc::clone(&active_turn.turn_state)
};
@@ -680,13 +690,9 @@ async fn start_if_idle(
}
}
}
session
.start_task(Arc::clone(&turn_context), task_input, RegularTask::new())
.await;
Ok(TurnInputSubmission::Started {
turn_id: submission_id,
root_turn_id: turn_context.root_turn_id(),
})
Ok(session
.start_task(turn_context, task_input, RegularTask::new())
.await)
}
async fn steer(
@@ -814,7 +820,7 @@ impl Session {
&self,
) -> Option<(Arc<tokio::sync::Mutex<TurnState>>, TurnStartOptions)> {
let mut active_turn = self.active_turn.lock().await;
if active_turn.is_some() || self.outcome_handoffs_closed() {
if active_turn.is_some() || self.turn_starts_closed() {
return None;
}
let has_trigger_turn = self.input_queue.has_trigger_turn_mailbox_items().await;
@@ -828,17 +834,24 @@ impl Session {
let previous_options = if has_trigger_turn {
Default::default()
} else {
self.state
.lock()
.await
.turn_attribution
.as_ref()
.map(codex_history::TurnAttribution::start_options)
.unwrap_or_default()
self.agent_message_wake_options().await
};
Some((Arc::clone(&active_turn.turn_state), previous_options))
}
async fn agent_message_wake_options(&self) -> TurnStartOptions {
let state = self.state.lock().await;
let mut options = state
.turn_attribution
.as_ref()
.map(codex_history::TurnAttribution::start_options)
.unwrap_or_default();
options.cyber_access_program = state
.reference_context_item()
.and_then(|context| context.cyber_access_program);
options
}
/// Revalidates a reserved wake and selects its settings before any mail is consumed.
#[expect(
clippy::await_holding_invalid_type,
@@ -871,11 +884,8 @@ impl Session {
Some(configuration.clone())
}
async fn should_wake_on_agent_messages(&self) -> bool {
if !self.features.enabled(Feature::WakeOnAgentMessages)
|| self.is_interrupted()
|| !self.input_queue.has_pending_mailbox_items().await
{
pub(crate) async fn should_wake_on_agent_messages(&self) -> bool {
if !self.features.enabled(Feature::WakeOnAgentMessages) || self.is_interrupted() {
return false;
}
@@ -250,6 +250,29 @@ async fn start_only_rejects_active_turn_without_injecting() {
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
}
#[tokio::test]
async fn saved_message_wake_checks_the_last_started_turn_during_admission() {
let (session, _, _rx) = make_session_and_context_with_rx().await;
*session.active_turn.lock().await = Some(ActiveTurn::default());
for (last_started, reason) in [
("original", NotSubmittedReason::NotIdle),
("successor", NotSubmittedReason::Superseded),
] {
session.state.lock().await.last_started_turn_id = Some(last_started.into());
let result = handle(
&session,
TurnInputRequest::user_input(vec![]),
TurnInputMode::ContinueIfIdle {
expected_previous_turn_id: "original".into(),
},
"wake".into(),
)
.await
.unwrap();
assert_eq!(result, TurnInputSubmission::NotSubmitted { reason });
}
}
#[tokio::test]
async fn recovery_rejects_active_turn_without_injecting_or_applying_settings() {
let (session, turn_context, _rx) = make_session_and_context_with_rx().await;
@@ -1,10 +1,12 @@
use super::handlers;
use super::session::Session;
use crate::state::TaskKind;
use crate::thread_manager::AgentTreeShutdownFailure;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::turn_input::SuspendTurnOutcome;
use std::sync::Arc;
use std::time::Duration;
@@ -120,3 +122,39 @@ pub(super) async fn suspend_turn_and_shutdown(
.await;
Ok(SuspendTurnOutcome::Suspended { turn_id })
}
pub(super) async fn stop_tree_member(session: &Arc<Session>) -> bool {
let runtime = &session.services.local_agent_runtime;
let submission_id = super::new_submission_id();
if let Some(root) = runtime.suspension_root()
&& session.live_thread().is_some()
&& (root == session.thread_id
|| session.multi_agent_version() == Some(MultiAgentVersion::V2)
&& session
.state
.lock()
.await
.session_configuration
.session_source
.get_agent_path()
.is_some())
{
match suspend_turn_and_shutdown(session, submission_id.clone()).await {
Ok(SuspendTurnOutcome::Suspended { turn_id }) => {
runtime.record_suspended_turn(session.thread_id, turn_id);
return true;
}
Ok(SuspendTurnOutcome::NotActive | SuspendTurnOutcome::UnsupportedTask) => {}
Err(error) => {
runtime.record_shutdown_failure(AgentTreeShutdownFailure::operation_failed(
"tree_suspension",
"suspend_regular_turn",
Some(session.thread_id),
"suspension_failed",
));
warn!(thread_id = %session.thread_id, %error, "failed to suspend agent turn");
}
}
}
handlers::shutdown(session, submission_id).await
}