mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add graceful agent-tree suspension and recovery (#61198)
## Why Suspending a locally controlled agent tree must preserve unfinished turns and child results that arrive during teardown so they remain available after recovery. ## What changed - Add `ThreadManager::suspend_agent_tree` to stop durable trees, close their writers, and return unfinished root and named V2 child turn IDs plus any pending root wake. - Persist terminal child results before suspension succeeds, including results queued for evicted parents. Report failed or unconfirmed writes. - Add `load_suspended_agent` to restore named V2 descendants through their recorded ancestors without starting model turns. - Allow empty `continue_turn_if_idle` input to wake on saved results, guarded by the preceding turn ID. Keep mail queued until task registration so suspension can preserve it. ## Testing Add coverage for completion and wake races, child errors, parent writer failures, evicted parents, descendant restoration, repeated suspension after caller cancellation, and ephemeral roots remaining active. GitOrigin-RevId: 3041791f51bff2555210f6b2d2443f9f05f68330
This commit is contained in:
@@ -264,21 +264,37 @@ impl LocalAgentControl {
|
||||
context: AgentCommunicationContext,
|
||||
start_options: TurnStartOptions,
|
||||
) -> CodexResult<String> {
|
||||
let observers = state.extensions.message_lifecycle_contributors();
|
||||
let delivery = (!observers.is_empty())
|
||||
.then(|| MessageDelivery::publish(context.sender_thread_id, observers));
|
||||
let terminal_result = context.kind == AgentCommunicationKind::Result;
|
||||
let publish = || {
|
||||
MessageDelivery::publish(
|
||||
context.sender_thread_id,
|
||||
state.extensions.message_lifecycle_contributors(),
|
||||
terminal_result.then_some(&self.runtime.unconfirmed_results),
|
||||
)
|
||||
};
|
||||
let log_content = crate::agent_communication::send_log_content(&communication);
|
||||
{
|
||||
// Keep unloaded delivery atomic with publication of a reloaded session.
|
||||
let (thread, guard) = {
|
||||
// Choose the loaded submission queue or the eviction mailbox while reload is fenced.
|
||||
let threads = state.threads.read().await;
|
||||
if !communication.trigger_turn && !threads.contains_key(&agent_id) {
|
||||
let _membership = self.runtime.admit_start()?;
|
||||
self.runtime.ensure_agent_known(agent_id)?;
|
||||
let submission_id = uuid::Uuid::now_v7().to_string();
|
||||
let thread = threads.get(&agent_id);
|
||||
let guard = thread
|
||||
.filter(|_| terminal_result)
|
||||
.and_then(|thread| Arc::clone(&thread.residency_gate).try_read_owned().ok());
|
||||
if !communication.trigger_turn
|
||||
&& (thread.is_none() || (terminal_result && guard.is_none()))
|
||||
{
|
||||
// A reporting child already holds membership until it publishes the result.
|
||||
let _membership = (!terminal_result)
|
||||
.then(|| self.runtime.admit_start())
|
||||
.transpose()?;
|
||||
if thread.is_none() {
|
||||
self.runtime.ensure_agent_known(agent_id)?;
|
||||
}
|
||||
let submission_id = Uuid::now_v7().to_string();
|
||||
self.runtime.mailboxes.enqueue(
|
||||
agent_id,
|
||||
Some(submission_id.clone()),
|
||||
vec![crate::PendingAgentMessage::new(communication, delivery)],
|
||||
vec![crate::PendingAgentMessage::new(communication, publish())],
|
||||
)?;
|
||||
drop(threads);
|
||||
crate::agent_communication::emit_agent_communication_send(
|
||||
@@ -289,8 +305,19 @@ impl LocalAgentControl {
|
||||
);
|
||||
return Ok(submission_id);
|
||||
}
|
||||
}
|
||||
// Loaded recipients retain submission ordering with follow-ups and interrupts.
|
||||
(
|
||||
thread.cloned().ok_or(CodexErr::ThreadNotFound(agent_id))?,
|
||||
guard,
|
||||
)
|
||||
};
|
||||
let guard = match guard {
|
||||
Some(guard) => {
|
||||
self.runtime.touch_v2_recipient(&thread);
|
||||
Some(guard)
|
||||
}
|
||||
None => self.runtime.pin_v2_residency(&thread).await?,
|
||||
};
|
||||
let delivery = publish();
|
||||
let result = submit_loaded_agent_communication(
|
||||
agent_id,
|
||||
communication,
|
||||
@@ -298,7 +325,7 @@ impl LocalAgentControl {
|
||||
log_content.as_deref(),
|
||||
start_options,
|
||||
|op, parent_turn_id, root_turn_id| {
|
||||
state.send_op(agent_id, op, parent_turn_id, root_turn_id, delivery)
|
||||
state.send_pinned_op(&thread, op, parent_turn_id, root_turn_id, guard, delivery)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -519,8 +546,10 @@ impl LocalAgentControl {
|
||||
message,
|
||||
/*trigger_turn*/ false,
|
||||
);
|
||||
let context =
|
||||
AgentCommunicationContext::new(AgentCommunicationKind::Result, child_thread_id);
|
||||
let context = AgentCommunicationContext::new(
|
||||
AgentCommunicationKind::CompletionWatcher,
|
||||
child_thread_id,
|
||||
);
|
||||
let _ = control
|
||||
.send_inter_agent_communication(
|
||||
parent_thread_id,
|
||||
|
||||
@@ -179,7 +179,7 @@ impl AgentControl for LocalAgentControl {
|
||||
}
|
||||
|
||||
fn take_mailbox(&self, agent: ThreadId) -> Vec<crate::PendingAgentMessage> {
|
||||
self.runtime.mailboxes.take(agent)
|
||||
self.runtime.mailboxes.take(agent, |_| true)
|
||||
}
|
||||
|
||||
fn watch_mailbox(&self, agent: ThreadId) -> tokio::sync::watch::Receiver<bool> {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Delivers terminal child results and completion activity to the agent tree.
|
||||
//!
|
||||
//! Sessions capture terminal state; the controller owns routing and queue-only delivery.
|
||||
//! Delivery remains best effort; metrics distinguish queue acceptance from delivery failure.
|
||||
//! Unconsumed results remain available to the source writer through graceful suspension.
|
||||
|
||||
use super::LocalAgentControl;
|
||||
use crate::TurnStartOptions;
|
||||
@@ -11,6 +11,7 @@ use crate::agent_communication::AgentCommunicationKind;
|
||||
use crate::session_prefix::format_guardian_interruption_message;
|
||||
use crate::session_prefix::format_inter_agent_completion_message;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ResponseItemId;
|
||||
use codex_protocol::items::SubAgentActivityItem;
|
||||
use codex_protocol::protocol::AgentStatus;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
@@ -106,23 +107,20 @@ impl LocalAgentControl {
|
||||
let Some(message) = message else {
|
||||
return;
|
||||
};
|
||||
// `communication` owns the message. Keep a second copy only when the
|
||||
// recorder will actually need it after parent delivery succeeds.
|
||||
let trace_message = trace.is_enabled().then(|| message.clone());
|
||||
let communication = InterAgentCommunication::new(
|
||||
let mut communication = InterAgentCommunication::new(
|
||||
child_agent_path.clone(),
|
||||
parent_agent_path,
|
||||
Vec::new(),
|
||||
message,
|
||||
/*trigger_turn*/ false,
|
||||
);
|
||||
let context =
|
||||
AgentCommunicationContext::new(AgentCommunicationKind::Result, outcome.thread_id);
|
||||
communication.id = Some(ResponseItemId::with_suffix("amsg", uuid::Uuid::now_v7()));
|
||||
let delivery = self
|
||||
.send_inter_agent_communication(
|
||||
parent_thread_id,
|
||||
communication,
|
||||
context,
|
||||
AgentCommunicationContext::new(AgentCommunicationKind::Result, outcome.thread_id),
|
||||
TurnStartOptions::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -45,18 +45,26 @@ impl Mailboxes {
|
||||
.ok_or_else(|| CodexErr::InvalidRequest("agent runtime is shutting down".into()))?
|
||||
.entry(agent)
|
||||
.or_default();
|
||||
mailbox
|
||||
.pending
|
||||
.extend(messages.into_iter().map(|communication| StoredMail {
|
||||
id: id.clone(),
|
||||
communication,
|
||||
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
|
||||
}));
|
||||
let mail = messages.into_iter().map(|communication| StoredMail {
|
||||
id: id.clone(),
|
||||
communication,
|
||||
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
|
||||
});
|
||||
if id.is_some() {
|
||||
mailbox.pending.extend(mail);
|
||||
} else {
|
||||
// Mail returned by eviction predates results published while its writer closed.
|
||||
mailbox.pending.splice(..0, mail);
|
||||
}
|
||||
mailbox.state.send_replace(!mailbox.pending.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn take(&self, agent: ThreadId) -> Vec<PendingAgentMessage> {
|
||||
pub(super) fn take(
|
||||
&self,
|
||||
agent: ThreadId,
|
||||
accept: impl Fn(&PendingAgentMessage) -> bool,
|
||||
) -> Vec<PendingAgentMessage> {
|
||||
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let Some(mailbox) = mailboxes
|
||||
.as_mut()
|
||||
@@ -66,8 +74,10 @@ impl Mailboxes {
|
||||
};
|
||||
let pending = std::mem::take(&mut mailbox.pending);
|
||||
mailbox.state.send_if_modified(std::mem::take);
|
||||
drop(mailboxes);
|
||||
pending
|
||||
.into_iter()
|
||||
.filter(|mail| accept(&mail.communication))
|
||||
.map(|mail| {
|
||||
// Mail transferred during eviction was already received by the old session.
|
||||
if let Some(id) = mail.id {
|
||||
|
||||
@@ -66,8 +66,8 @@ impl ThreadManagerState {
|
||||
teardown.complete();
|
||||
return Err(err);
|
||||
}
|
||||
// Dispatch has stopped and the residency guard excludes senders. A tree
|
||||
// shutdown deliberately discards its mailbox instead of retaining it.
|
||||
// Dispatch has stopped. Put previously received mail ahead of results
|
||||
// published directly to the mailbox while eviction owned the residency guard.
|
||||
let mail = thread
|
||||
.session
|
||||
.input_queue
|
||||
@@ -80,10 +80,11 @@ impl ThreadManagerState {
|
||||
let _ = runtime.mailboxes.enqueue(thread_id, /*id*/ None, mail);
|
||||
// Reuse the same effective configuration snapshot as a fork. This includes live
|
||||
// permissions and network restrictions that a profile name cannot reconstruct.
|
||||
let (config, ..) = thread.session.fork_config().await;
|
||||
let (config, snapshot, ..) = thread.session.fork_config().await;
|
||||
// Preserve configured bindings across eviction without retaining turn-only attachments.
|
||||
// Raw bindings keep the original pending owner configuration alive.
|
||||
let environments = EvictedEnvironments {
|
||||
history_mode: snapshot.history_mode,
|
||||
config: Arc::new(config),
|
||||
has_independent_service_tier: matches!(
|
||||
&thread.session.agent_runtime,
|
||||
@@ -174,6 +175,12 @@ impl LocalAgentControl {
|
||||
|
||||
impl LocalAgentRuntime {
|
||||
/// Pins and touches the registered runtime without waiting for unrelated eviction.
|
||||
pub(super) fn touch_v2_recipient(&self, thread: &CodexThread) {
|
||||
if is_resident_candidate(thread) {
|
||||
self.residency.touch(thread.session.thread_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn pin_v2_residency(
|
||||
&self,
|
||||
thread: &CodexThread,
|
||||
@@ -187,7 +194,7 @@ impl LocalAgentRuntime {
|
||||
if !std::ptr::eq(thread, registered.as_ref()) {
|
||||
return Err(CodexErr::ThreadNotFound(thread_id));
|
||||
}
|
||||
self.residency.touch(thread_id);
|
||||
self.touch_v2_recipient(thread);
|
||||
Ok(Some(guard))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ async fn residency_slot_reservation_unloads_oldest_idle_v2_agent() {
|
||||
.services
|
||||
.local_agent_runtime
|
||||
.mailboxes
|
||||
.take(first.thread_id);
|
||||
.take(first.thread_id, |_| true);
|
||||
assert_eq!(retained, vec![communication.into()]);
|
||||
let second = spawn_v2_subagent(&control, &state, config, root.thread_id, "worker-2").await;
|
||||
second_slot.commit(second.thread_id);
|
||||
|
||||
@@ -1,16 +1,102 @@
|
||||
//! Reopens an agent only when its runtime is absent, then returns its effective state.
|
||||
//! Rollout and descendant restoration remain in the existing local startup path.
|
||||
//! Restores recorded ancestor ownership before loading a suspended descendant.
|
||||
|
||||
use super::LocalAgentControl;
|
||||
use crate::CodexThread;
|
||||
use crate::agent::types::LiveAgent;
|
||||
use crate::codex_thread::ThreadConfigSnapshot;
|
||||
use crate::config::Config;
|
||||
use crate::thread_manager::LoadedSuspendedAgent;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
impl LocalAgentControl {
|
||||
/// Load ancestors first so each child inherits its actual parent's runtime and permissions.
|
||||
pub(crate) async fn load_suspended_child(
|
||||
&self,
|
||||
root: &Arc<CodexThread>,
|
||||
child_id: ThreadId,
|
||||
) -> CodexResult<LoadedSuspendedAgent> {
|
||||
let root_id = root.startup_metadata().thread_id;
|
||||
if child_id == root_id || root.multi_agent_version() != Some(MultiAgentVersion::V2) {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"expected a named V2 descendant".to_owned(),
|
||||
));
|
||||
}
|
||||
let state = self.runtime.upgrade()?;
|
||||
let mut chain = Vec::new();
|
||||
let mut visited = HashSet::new();
|
||||
let mut id = child_id;
|
||||
while id != root_id {
|
||||
if !visited.insert(id) {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"cycle in stored agent ancestry".into(),
|
||||
));
|
||||
}
|
||||
let stored = state
|
||||
.read_stored_thread(ReadThreadParams {
|
||||
thread_id: id,
|
||||
include_archived: false,
|
||||
include_history: false,
|
||||
})
|
||||
.await?;
|
||||
let SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id,
|
||||
agent_path: Some(path),
|
||||
..
|
||||
}) = &stored.source
|
||||
else {
|
||||
return Err(CodexErr::InvalidRequest(format!(
|
||||
"thread {id} is not a named child of {root_id}"
|
||||
)));
|
||||
};
|
||||
if stored
|
||||
.parent_thread_id
|
||||
.is_some_and(|parent| parent != *parent_thread_id)
|
||||
|| stored
|
||||
.agent_path
|
||||
.as_deref()
|
||||
.is_some_and(|name| name != path.as_str())
|
||||
{
|
||||
return Err(CodexErr::InvalidRequest(format!(
|
||||
"inconsistent stored ancestry for {id}"
|
||||
)));
|
||||
}
|
||||
chain.push(id);
|
||||
id = *parent_thread_id;
|
||||
}
|
||||
chain.reverse();
|
||||
let mut thread = Arc::clone(root);
|
||||
let mut residency = None;
|
||||
for id in chain {
|
||||
let config = thread.session.get_config().await;
|
||||
// Usually root resume registered the name from the graph. This also handles
|
||||
// stores without a graph, using the same identity restoration path.
|
||||
self.restore_agent_identities(&config, &state, vec![id])
|
||||
.await;
|
||||
let pin = self
|
||||
.ensure_v2_agent_loaded(config.as_ref().clone(), id, Some(thread))
|
||||
.await?;
|
||||
thread = state.get_thread(id).await?;
|
||||
// Allow an idle ancestor to give up its slot while its child reloads at capacity.
|
||||
if id == child_id {
|
||||
residency = pin;
|
||||
}
|
||||
}
|
||||
Ok(LoadedSuspendedAgent {
|
||||
thread,
|
||||
_residency: residency,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn resume_agent(
|
||||
&self,
|
||||
config: Config,
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::rollout_budget::RolloutBudget;
|
||||
use crate::thread_manager::AgentTreeShutdownFailure;
|
||||
use crate::thread_manager::AgentTreeShutdownFailureReason;
|
||||
use crate::thread_manager::AgentTreeShutdownReport;
|
||||
use crate::thread_manager::SuspendedAgentTree;
|
||||
use crate::thread_manager::ThreadIdGenerator;
|
||||
use crate::thread_manager::ThreadManagerState;
|
||||
use arc_swap::ArcSwapOption;
|
||||
@@ -24,11 +25,14 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_rollout_trace::ThreadTraceContext;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::PoisonError;
|
||||
use std::sync::Weak;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
use tokio_util::task::task_tracker::TaskTrackerToken;
|
||||
@@ -41,12 +45,27 @@ const MAX_RETAINED_SHUTDOWN_FAILURES: usize = 64;
|
||||
pub(crate) struct AgentTreeShutdownState {
|
||||
members: TaskTracker,
|
||||
report: Mutex<AgentTreeShutdownReport>,
|
||||
disposition: OnceLock<TreeDisposition>,
|
||||
suspended: Mutex<SuspendedAgentTree>,
|
||||
children: Mutex<HashMap<ThreadId, TaskTracker>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum TreeDisposition {
|
||||
Shutdown,
|
||||
Suspend {
|
||||
root: ThreadId,
|
||||
evicted_writers: TaskTracker,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for AgentTreeShutdownState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
members: TaskTracker::new(),
|
||||
disposition: OnceLock::new(),
|
||||
suspended: Mutex::default(),
|
||||
children: Mutex::default(),
|
||||
report: Mutex::new(AgentTreeShutdownReport {
|
||||
tree_id: Uuid::now_v7(),
|
||||
failures: Vec::new(),
|
||||
@@ -65,8 +84,14 @@ impl AgentTreeShutdownState {
|
||||
|
||||
pub(crate) async fn wait_detailed(&self) -> Result<(), AgentTreeShutdownReport> {
|
||||
self.members.wait().await;
|
||||
// Reporters hold a membership until after recording failures. Once the tracker drains,
|
||||
// no admitted operation can add a failure after the snapshot.
|
||||
// The source may need to reopen an evicted parent's writer after all live
|
||||
// sessions stop. Both shutdown waiters and suspension callers join that write.
|
||||
if let Some(TreeDisposition::Suspend {
|
||||
evicted_writers, ..
|
||||
}) = self.disposition.get()
|
||||
{
|
||||
evicted_writers.wait().await;
|
||||
}
|
||||
let report = self
|
||||
.report
|
||||
.lock()
|
||||
@@ -79,6 +104,15 @@ impl AgentTreeShutdownState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn suspended_tree(&self) -> CodexResult<SuspendedAgentTree> {
|
||||
self.wait().await?;
|
||||
Ok(self
|
||||
.suspended
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.clone())
|
||||
}
|
||||
|
||||
fn record_failure(&self, failure: AgentTreeShutdownFailure) {
|
||||
let tree_id = {
|
||||
let mut report = self.report.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
@@ -112,6 +146,7 @@ impl AgentTreeShutdownState {
|
||||
pub(crate) struct AgentTreeMembership {
|
||||
state: Arc<AgentTreeShutdownState>,
|
||||
_member: TaskTrackerToken,
|
||||
_child: Option<TaskTrackerToken>,
|
||||
}
|
||||
|
||||
impl AgentTreeMembership {
|
||||
@@ -202,6 +237,7 @@ pub(crate) struct LocalAgentRuntime {
|
||||
Arc<OnceLock<Arc<dyn ThreadInstructionsProvider>>>,
|
||||
pub(super) registry: Arc<AgentRegistry>,
|
||||
pub(super) residency: Arc<V2Residency>,
|
||||
pub(super) unconfirmed_results: Arc<AtomicUsize>,
|
||||
pub(super) mailboxes: Arc<super::mailbox::Mailboxes>,
|
||||
/// Shared by every session in this tree, including private delegates.
|
||||
pub(crate) shutdown: CancellationToken,
|
||||
@@ -219,6 +255,7 @@ impl LocalAgentRuntime {
|
||||
thread_id_generator,
|
||||
registry: Arc::default(),
|
||||
residency: Arc::default(),
|
||||
unconfirmed_results: Arc::default(),
|
||||
mailboxes: Arc::default(),
|
||||
shutdown: CancellationToken::new(),
|
||||
shutdown_state: Arc::default(),
|
||||
@@ -233,6 +270,35 @@ impl LocalAgentRuntime {
|
||||
runtime
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_children(&self, parent: ThreadId) {
|
||||
let children = self
|
||||
.shutdown_state
|
||||
.children
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.get(&parent)
|
||||
.cloned();
|
||||
if let Some(children) = children {
|
||||
children.close();
|
||||
children.wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admit_session(
|
||||
&self,
|
||||
parent: Option<ThreadId>,
|
||||
) -> CodexResult<AgentTreeMembership> {
|
||||
let mut children = self
|
||||
.shutdown_state
|
||||
.children
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner);
|
||||
let mut membership = self.admit_start()?;
|
||||
children.retain(|_, tracker| !tracker.is_empty());
|
||||
membership._child = parent.map(|id| children.entry(id).or_default().token());
|
||||
Ok(membership)
|
||||
}
|
||||
|
||||
pub(crate) fn tree_id(&self) -> Uuid {
|
||||
self.shutdown_state
|
||||
.report
|
||||
@@ -294,6 +360,7 @@ impl LocalAgentRuntime {
|
||||
let membership = AgentTreeMembership {
|
||||
state: Arc::clone(&self.shutdown_state),
|
||||
_member: self.shutdown_state.members.token(),
|
||||
_child: None,
|
||||
};
|
||||
// Closing a TaskTracker does not reject new tokens. Recheck so a start racing with
|
||||
// shutdown is either admitted before the fence or rejected after it.
|
||||
@@ -307,12 +374,145 @@ impl LocalAgentRuntime {
|
||||
}
|
||||
|
||||
pub(crate) fn request_shutdown(&self) -> Arc<AgentTreeShutdownState> {
|
||||
let disposition = self
|
||||
.shutdown_state
|
||||
.disposition
|
||||
.get_or_init(|| TreeDisposition::Shutdown);
|
||||
self.shutdown_state.members.close();
|
||||
self.shutdown.cancel();
|
||||
self.mailboxes.close();
|
||||
if matches!(disposition, TreeDisposition::Shutdown) {
|
||||
self.mailboxes.close();
|
||||
}
|
||||
Arc::clone(&self.shutdown_state)
|
||||
}
|
||||
|
||||
pub(crate) fn request_suspension(
|
||||
&self,
|
||||
root: ThreadId,
|
||||
) -> CodexResult<Arc<AgentTreeShutdownState>> {
|
||||
let disposition = self.shutdown_state.disposition.get_or_init(|| {
|
||||
let evicted_writers = TaskTracker::new();
|
||||
let guard = AgentTreeMembership {
|
||||
state: Arc::clone(&self.shutdown_state),
|
||||
_member: evicted_writers.token(),
|
||||
_child: None,
|
||||
}
|
||||
.into_teardown_guard("save_evicted_parent_results", Some(root));
|
||||
evicted_writers.close();
|
||||
let runtime = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(failure) = runtime.save_results_after_eviction().await {
|
||||
runtime.record_shutdown_failure(failure);
|
||||
}
|
||||
runtime.mailboxes.close();
|
||||
guard.complete();
|
||||
});
|
||||
TreeDisposition::Suspend {
|
||||
root,
|
||||
evicted_writers,
|
||||
}
|
||||
});
|
||||
if !matches!(disposition, TreeDisposition::Suspend { root: owner, .. } if *owner == root) {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"agent tree shutdown already has a different disposition".into(),
|
||||
));
|
||||
}
|
||||
Ok(self.request_shutdown())
|
||||
}
|
||||
|
||||
async fn save_results_after_eviction(&self) -> Result<(), AgentTreeShutdownFailure> {
|
||||
self.shutdown_state.members.wait().await;
|
||||
if self.unconfirmed_results.load(Ordering::Acquire) == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
// A failed teardown may still own a writer. Never reopen it.
|
||||
if !self
|
||||
.shutdown_state
|
||||
.report
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.failures
|
||||
.is_empty()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
for parent in self
|
||||
.registry
|
||||
.live_agents()
|
||||
.into_iter()
|
||||
.filter_map(|agent| agent.agent_id)
|
||||
{
|
||||
let results = self
|
||||
.mailboxes
|
||||
.take(parent, crate::PendingAgentMessage::is_terminal_result);
|
||||
if results.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let failed = |phase, kind| {
|
||||
AgentTreeShutdownFailure::operation_failed(
|
||||
"save_evicted_parent_results",
|
||||
phase,
|
||||
Some(parent),
|
||||
kind,
|
||||
)
|
||||
};
|
||||
let owner = self
|
||||
.registry
|
||||
.evicted_environments(parent)
|
||||
.ok_or_else(|| failed("locate_parent", "parent_not_evicted"))?;
|
||||
self.upgrade()
|
||||
.map_err(|_| failed("open_writer", "manager_unavailable"))?
|
||||
.save_evicted_parent_results(parent, &owner.config, owner.history_mode, &results)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
failed(
|
||||
"persist",
|
||||
crate::thread_manager::thread_store_error_kind(&err),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
// A task that claimed a result and stopped before its append returned leaves
|
||||
// an unconfirmed write. Only its writer knows whether the append took effect.
|
||||
if self.unconfirmed_results.load(Ordering::Acquire) != 0 {
|
||||
return Err(AgentTreeShutdownFailure::operation_failed(
|
||||
"tree_suspension",
|
||||
"persist_child_result",
|
||||
/*thread_id*/ None,
|
||||
"unconfirmed_write",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn suspension_root(&self) -> Option<ThreadId> {
|
||||
match self.shutdown_state.disposition.get() {
|
||||
Some(TreeDisposition::Suspend { root, .. }) => Some(*root),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_root_wake_after(&self, previous_turn_id: String) {
|
||||
self.shutdown_state
|
||||
.suspended
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.wake_root_after = Some(previous_turn_id);
|
||||
}
|
||||
|
||||
pub(crate) fn record_suspended_turn(&self, thread_id: ThreadId, turn_id: String) {
|
||||
let mut tree = self
|
||||
.shutdown_state
|
||||
.suspended
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner);
|
||||
if self.suspension_root() == Some(thread_id) {
|
||||
tree.root_turn_id = Some(turn_id);
|
||||
tree.wake_root_after = None;
|
||||
} else {
|
||||
tree.child_turns.insert(thread_id, turn_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// The caller must hold tree membership until this record has been written. This keeps
|
||||
/// the completed shutdown report stable for every waiter.
|
||||
pub(crate) fn record_shutdown_failure(&self, failure: AgentTreeShutdownFailure) {
|
||||
|
||||
@@ -139,23 +139,31 @@ impl LocalAgentControl {
|
||||
}
|
||||
};
|
||||
|
||||
self.restore_agent_identities(config, &state, descendant_ids)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn restore_agent_identities(
|
||||
&self,
|
||||
config: &Config,
|
||||
state: &ThreadManagerState,
|
||||
descendant_ids: Vec<ThreadId>,
|
||||
) {
|
||||
let registry = &self.runtime.registry;
|
||||
// Overlap storage reads, but reserve paths and nicknames in graph order.
|
||||
let mut stored_threads = stream::iter(
|
||||
descendant_ids
|
||||
.into_iter()
|
||||
.filter(|thread_id| registry.agent_metadata_for_thread(*thread_id).is_none())
|
||||
.map(|thread_id| {
|
||||
let state = &state;
|
||||
async move {
|
||||
let stored_thread = state
|
||||
.read_stored_thread(ReadThreadParams {
|
||||
thread_id,
|
||||
include_archived: true,
|
||||
include_history: false,
|
||||
})
|
||||
.await;
|
||||
(thread_id, stored_thread)
|
||||
}
|
||||
.map(|thread_id| async move {
|
||||
let stored_thread = state
|
||||
.read_stored_thread(ReadThreadParams {
|
||||
thread_id,
|
||||
include_archived: true,
|
||||
include_history: false,
|
||||
})
|
||||
.await;
|
||||
(thread_id, stored_thread)
|
||||
}),
|
||||
)
|
||||
.buffered(/*n*/ 8);
|
||||
@@ -306,9 +314,8 @@ impl LocalAgentControl {
|
||||
let stored_reasoning_effort = stored_thread.reasoning_effort.clone();
|
||||
let stored_source = stored_thread.source.clone();
|
||||
let stored_parent_thread_id = stored_thread.parent_thread_id;
|
||||
let history = load_agent_model_context(&state, thread_id, stored_thread.history_mode)
|
||||
.await?
|
||||
.ok_or(CodexErr::ThreadNotFound(thread_id))?;
|
||||
let history =
|
||||
load_agent_model_context(&state, thread_id, stored_thread.history_mode).await?;
|
||||
let initial_history = InitialHistory::Resumed(ResumedHistory {
|
||||
history_revision: history.revision,
|
||||
conversation_id: thread_id,
|
||||
@@ -568,9 +575,30 @@ impl LocalAgentControl {
|
||||
};
|
||||
// Reserving a slot can evict an idle nested parent. Capture its instructions and
|
||||
// analytics metadata alongside its authority before the live parent can disappear.
|
||||
let residency_slot = self
|
||||
.reserve_v2_residency_slot(&state, &config, &membership, Some(thread_id))
|
||||
.await?;
|
||||
let reserve =
|
||||
|| self.reserve_v2_residency_slot(&state, &config, &membership, Some(thread_id));
|
||||
let residency_slot = match reserve().await {
|
||||
Err(limit)
|
||||
if matches!(limit.details(), CodexErrorDetails::AgentLimitReached { .. }) =>
|
||||
{
|
||||
// A cold parent can report PendingInit despite having spawned this stored child.
|
||||
// Try that exact parent only after the ordinary eviction policy found no slot.
|
||||
if let Some((parent, _)) = &parent
|
||||
&& matches!(parent.agent_status().await, AgentStatus::PendingInit)
|
||||
&& matches!(
|
||||
state
|
||||
.try_evict_v2_thread(Arc::clone(parent), &membership)
|
||||
.await,
|
||||
Ok(ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)
|
||||
)
|
||||
{
|
||||
reserve().await?
|
||||
} else {
|
||||
return Err(limit);
|
||||
}
|
||||
}
|
||||
result => result?,
|
||||
};
|
||||
|
||||
match state
|
||||
.resume_thread_with_history_with_source(ResumeThreadWithHistoryOptions {
|
||||
@@ -1137,9 +1165,8 @@ impl LocalAgentControl {
|
||||
.map_err(|err| CodexErr::InvalidRequest(format!("invalid stored agent path: {err}")))?;
|
||||
let resumed_agent_nickname = stored_thread.agent_nickname.clone();
|
||||
let resumed_agent_role = stored_thread.agent_role.clone();
|
||||
let history = load_agent_model_context(&state, thread_id, stored_thread.history_mode)
|
||||
.await?
|
||||
.ok_or(CodexErr::ThreadNotFound(thread_id))?;
|
||||
let history =
|
||||
load_agent_model_context(&state, thread_id, stored_thread.history_mode).await?;
|
||||
let initial_history = InitialHistory::Resumed(ResumedHistory {
|
||||
history_revision: history.revision,
|
||||
conversation_id: thread_id,
|
||||
|
||||
@@ -4588,6 +4588,8 @@ async fn multi_agent_v2_completion_ignores_dead_direct_parent() {
|
||||
.await
|
||||
.expect("tester thread should exist");
|
||||
let tester_turn = tester_thread.session.new_default_turn().await;
|
||||
let writes = &harness.control.runtime.unconfirmed_results;
|
||||
let before = writes.load(std::sync::atomic::Ordering::Acquire);
|
||||
tester_thread
|
||||
.session
|
||||
.send_event(
|
||||
@@ -4605,6 +4607,7 @@ async fn multi_agent_v2_completion_ignores_dead_direct_parent() {
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(writes.load(std::sync::atomic::Ordering::Acquire), before);
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
|
||||
assert!(
|
||||
@@ -4638,6 +4641,18 @@ async fn multi_agent_v2_completion_ignores_dead_direct_parent() {
|
||||
assert!(!has_subagent_notification(root_history.raw_items()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_result_writes_require_confirmation_after_the_parent_claims_mail() {
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::default());
|
||||
let publish = || MessageDelivery::publish(ThreadId::new(), &[], Some(&count)).unwrap();
|
||||
let parent = ThreadId::new();
|
||||
drop(publish().consumed(parent));
|
||||
let saved = publish();
|
||||
saved.consumed(parent).unwrap().confirm();
|
||||
assert!(saved.consumed(parent).is_none());
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_completion_queues_message_for_direct_parent() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
@@ -5122,6 +5137,11 @@ async fn resume_thread_subagent_restores_stored_metadata() {
|
||||
Vec::new()
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
original_snapshot.history_mode,
|
||||
codex_protocol::protocol::ThreadHistoryMode::Legacy
|
||||
);
|
||||
let before_resume = thread_store.calls().await;
|
||||
let resumed_thread_id = harness
|
||||
.control
|
||||
.resume_agent_from_rollout(
|
||||
@@ -5137,6 +5157,13 @@ async fn resume_thread_subagent_restores_stored_metadata() {
|
||||
)
|
||||
.await
|
||||
.expect("resume should succeed");
|
||||
// Remote stores reject unbounded reads on their metadata API.
|
||||
let after_resume = thread_store.calls().await;
|
||||
assert_eq!(
|
||||
after_resume.read_thread_with_history,
|
||||
before_resume.read_thread_with_history
|
||||
);
|
||||
assert!(after_resume.load_history > before_resume.load_history);
|
||||
assert_eq!(resumed_thread_id, child_thread_id);
|
||||
|
||||
let resumed_thread = harness
|
||||
|
||||
@@ -38,7 +38,6 @@ use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_thread_store::LoadThreadHistoryParams;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
|
||||
fn keep_forked_rollout_item(item: &RolloutItem, preserve_context_baselines: bool) -> bool {
|
||||
match item {
|
||||
@@ -132,29 +131,14 @@ pub(super) async fn load_agent_model_context(
|
||||
state: &ThreadManagerState,
|
||||
thread_id: ThreadId,
|
||||
history_mode: ThreadHistoryMode,
|
||||
) -> CodexResult<Option<codex_thread_store::StoredModelContext>> {
|
||||
) -> CodexResult<codex_thread_store::StoredModelContext> {
|
||||
let params = LoadThreadHistoryParams {
|
||||
thread_id,
|
||||
include_archived: true,
|
||||
};
|
||||
match history_mode {
|
||||
ThreadHistoryMode::Legacy => Ok(state
|
||||
.read_stored_thread(ReadThreadParams {
|
||||
thread_id,
|
||||
include_archived: true,
|
||||
include_history: true,
|
||||
})
|
||||
.await?
|
||||
.history
|
||||
.map(|history| codex_thread_store::StoredModelContext {
|
||||
thread_id,
|
||||
revision: history.revision,
|
||||
items: history.items,
|
||||
})),
|
||||
ThreadHistoryMode::Paginated => Ok(Some(
|
||||
state
|
||||
.load_latest_model_context(LoadThreadHistoryParams {
|
||||
thread_id,
|
||||
include_archived: true,
|
||||
})
|
||||
.await?,
|
||||
)),
|
||||
ThreadHistoryMode::Legacy => state.load_legacy_model_context(params).await,
|
||||
ThreadHistoryMode::Paginated => state.load_latest_model_context(params).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,11 +211,6 @@ pub(super) async fn prepare_forked_history(
|
||||
let mut forked_rollout_items =
|
||||
load_agent_model_context(state, parent_thread_id, parent_history_mode)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
CodexErr::Fatal(format!(
|
||||
"parent thread history unavailable for fork: {parent_thread_id}"
|
||||
))
|
||||
})?
|
||||
.items;
|
||||
|
||||
let multi_agent_v2_usage_hint_texts_to_filter: Vec<String> =
|
||||
|
||||
@@ -42,6 +42,7 @@ struct ActiveAgents {
|
||||
/// Effective configuration and executor bindings retained while a child's runtime is unloaded.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct EvictedEnvironments {
|
||||
pub(crate) history_mode: codex_protocol::protocol::ThreadHistoryMode,
|
||||
pub(crate) config: Arc<Config>,
|
||||
pub(crate) has_independent_service_tier: bool,
|
||||
pub(crate) selections: Vec<TurnEnvironmentSelection>,
|
||||
|
||||
Reference in New Issue
Block a user