Add graceful agent-tree suspension and recovery (#61198)

## Why

Suspending a locally controlled agent tree must preserve unfinished turns and child results that arrive during teardown so they remain available after recovery.

## What changed

- Add `ThreadManager::suspend_agent_tree` to stop durable trees, close their writers, and return unfinished root and named V2 child turn IDs plus any pending root wake.
- Persist terminal child results before suspension succeeds, including results queued for evicted parents. Report failed or unconfirmed writes.
- Add `load_suspended_agent` to restore named V2 descendants through their recorded ancestors without starting model turns.
- Allow empty `continue_turn_if_idle` input to wake on saved results, guarded by the preceding turn ID. Keep mail queued until task registration so suspension can preserve it.

## Testing

Add coverage for completion and wake races, child errors, parent writer failures, evicted parents, descendant restoration, repeated suspension after caller cancellation, and ephemeral roots remaining active.

GitOrigin-RevId: 3041791f51bff2555210f6b2d2443f9f05f68330
This commit is contained in:
Bryan Ashley
2026-11-12 22:43:14 +00:00
committed by copyberry
parent cdfcc18dba
commit fae87361f4
30 changed files with 1480 additions and 320 deletions
+49 -10
View File
@@ -7,6 +7,8 @@ pub use shutdown::AgentTreeShutdown;
pub use shutdown::AgentTreeShutdownFailure;
pub use shutdown::AgentTreeShutdownFailureReason;
pub use shutdown::AgentTreeShutdownReport;
pub use shutdown::LoadedSuspendedAgent;
pub use shutdown::SuspendedAgentTree;
pub(crate) use shutdown::thread_store_error_kind;
use crate::CodexAppsToolsCache;
@@ -2083,6 +2085,22 @@ impl ThreadManagerState {
})
}
pub(crate) async fn load_legacy_model_context(
&self,
params: LoadThreadHistoryParams,
) -> CodexResult<StoredModelContext> {
let history = self
.thread_store
.load_history(params)
.await
.map_err(thread_store_rollout_read_error)?;
Ok(StoredModelContext {
thread_id: history.thread_id,
revision: history.revision,
items: history.items,
})
}
pub(crate) async fn load_latest_model_context(
&self,
params: LoadThreadHistoryParams,
@@ -2121,11 +2139,31 @@ impl ThreadManagerState {
} else {
None
};
self.send_pinned_op(
&thread,
op,
parent_turn_id,
root_turn_id,
residency_guard,
message_delivery,
)
.await
}
pub(crate) async fn send_pinned_op(
&self,
thread: &CodexThread,
op: Op,
parent_turn_id: Option<String>,
root_turn_id: Option<String>,
residency_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
message_delivery: Option<crate::agent_communication::MessageDelivery>,
) -> CodexResult<String> {
if let Some(ops_log) = &self.ops_log
&& let Ok(mut log) = ops_log.lock()
&& let Some(captured_op) = capture_test_op(&op)
{
log.push((thread_id, captured_op));
log.push((thread.session.thread_id, captured_op));
}
thread
.io
@@ -2461,7 +2499,16 @@ impl ThreadManagerState {
.config_layer_stack
.with_mcp_ema_cancellation(cancellation);
let runtime = request.agent_control.runtime().clone();
let membership = runtime.admit_start()?;
let options = &request.options;
let source = options
.session_source
.as_ref()
.unwrap_or(&self.session_source);
request.parent_thread_id = request
.parent_thread_id
.or_else(|| options.initial_history.get_resumed_parent_thread_id())
.or_else(|| source.parent_thread_id());
let membership = runtime.admit_session(request.parent_thread_id)?;
let owns_startup = request.startup.is_none();
let startup_state = Arc::clone(
request
@@ -2658,14 +2705,6 @@ impl ThreadManagerState {
}
(_, control) => control,
};
let parent_thread_id = parent_thread_id
.or_else(|| initial_history.get_resumed_parent_thread_id())
.or(match &session_source {
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
parent_thread_id, ..
}) => Some(*parent_thread_id),
_ => None,
});
// Both resume entry points must restore identities before children can be loaded lazily.
if let InitialHistory::Resumed(resumed) = &initial_history
&& initial_history.get_multi_agent_version() == Some(MultiAgentVersion::V2)