mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add graceful agent-tree suspension and recovery (#61198)
## Why Suspending a locally controlled agent tree must preserve unfinished turns and child results that arrive during teardown so they remain available after recovery. ## What changed - Add `ThreadManager::suspend_agent_tree` to stop durable trees, close their writers, and return unfinished root and named V2 child turn IDs plus any pending root wake. - Persist terminal child results before suspension succeeds, including results queued for evicted parents. Report failed or unconfirmed writes. - Add `load_suspended_agent` to restore named V2 descendants through their recorded ancestors without starting model turns. - Allow empty `continue_turn_if_idle` input to wake on saved results, guarded by the preceding turn ID. Keep mail queued until task registration so suspension can preserve it. ## Testing Add coverage for completion and wake races, child errors, parent writer failures, evicted parents, descendant restoration, repeated suspension after caller cancellation, and ephemeral roots remaining active. GitOrigin-RevId: 3041791f51bff2555210f6b2d2443f9f05f68330
This commit is contained in:
@@ -61,6 +61,8 @@ use codex_protocol::protocol::TurnAbortReason;
|
||||
use codex_protocol::protocol::TurnAbortedEvent;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
use codex_protocol::turn_input::NotSubmittedReason;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use codex_thread_store::PersistContext;
|
||||
|
||||
pub(crate) use admission::HostTurnStarted;
|
||||
@@ -280,22 +282,22 @@ impl Session {
|
||||
turn_context: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
task: T,
|
||||
) {
|
||||
) -> TurnInputSubmission {
|
||||
self.abort_all_tasks(TurnAbortReason::Replaced).await;
|
||||
self.clear_connector_selection().await;
|
||||
self.start_task(turn_context, input, task).await;
|
||||
self.start_task(turn_context, input, task).await
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "record the started turn atomically with task registration"
|
||||
reason = "register the turn and its pending input atomically with the suspension fence"
|
||||
)]
|
||||
pub(crate) async fn start_task<T: SessionTask>(
|
||||
self: &Arc<Self>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
task: T,
|
||||
) {
|
||||
) -> TurnInputSubmission {
|
||||
// Replace the cache; in-flight acquisitions retain their previous Arc.
|
||||
self.services
|
||||
.thread_extension_data
|
||||
@@ -335,11 +337,12 @@ impl Session {
|
||||
let cancellation_token = CancellationToken::new();
|
||||
let done = Arc::new(Notify::new());
|
||||
|
||||
let (pending_items, _) = self.input_queue.drain_mailbox_input_items().await;
|
||||
let turn_state = {
|
||||
let mut active = self.active_turn.lock().await;
|
||||
if self.outcome_handoffs_closed() {
|
||||
return;
|
||||
if self.turn_starts_closed() {
|
||||
return TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::ServerDraining,
|
||||
};
|
||||
}
|
||||
let turn = active.get_or_insert_with(ActiveTurn::default);
|
||||
debug_assert!(turn.task.is_none());
|
||||
@@ -347,9 +350,6 @@ impl Session {
|
||||
Arc::clone(&turn.turn_state)
|
||||
};
|
||||
turn_state.lock().await.token_usage_at_turn_start = token_usage_at_turn_start.clone();
|
||||
self.input_queue
|
||||
.extend_pending_input_for_turn_state(turn_state.as_ref(), pending_items)
|
||||
.await;
|
||||
self.emit_turn_start_lifecycle(
|
||||
turn_context.as_ref(),
|
||||
Some(&token_usage_at_turn_start),
|
||||
@@ -359,16 +359,35 @@ impl Session {
|
||||
|
||||
let mut active = self.active_turn.lock().await;
|
||||
// Shutdown closes registration before taking this same lock to abort active work.
|
||||
let outcome_handoff = if let Some(reporting) = &self.outcome_reporting {
|
||||
if reporting.handoffs.is_closed() {
|
||||
return;
|
||||
}
|
||||
Some(reporting.handoffs.token())
|
||||
} else {
|
||||
None
|
||||
if self.turn_starts_closed() {
|
||||
return TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::ServerDraining,
|
||||
};
|
||||
}
|
||||
let outcome_handoff = self
|
||||
.outcome_reporting
|
||||
.as_ref()
|
||||
.filter(|reporting| {
|
||||
reporting.reporter.is_some()
|
||||
|| turn_context.multi_agent_version == MultiAgentVersion::V2
|
||||
&& (!turn_context.session_source.is_non_root_agent()
|
||||
|| turn_context.session_source.get_agent_path().is_some())
|
||||
})
|
||||
.map(|reporting| reporting.handoffs.token());
|
||||
let Some(turn) = active
|
||||
.as_mut()
|
||||
.filter(|turn| Arc::ptr_eq(&turn.turn_state, &turn_state))
|
||||
else {
|
||||
return TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::Superseded,
|
||||
};
|
||||
};
|
||||
let turn = active.get_or_insert_with(ActiveTurn::default);
|
||||
debug_assert!(turn.task.is_none());
|
||||
// Keep mail in its queue until a task owns it. Suspension can save it if it wins this lock.
|
||||
let (pending_items, _) = self.input_queue.drain_mailbox_input_items().await;
|
||||
self.input_queue
|
||||
.extend_pending_input_for_turn_state(&turn_state, pending_items)
|
||||
.await;
|
||||
// Record lineage only when the task starts: settings changes can invalidate a reservation.
|
||||
self.record_started_turn(
|
||||
&turn_context.sub_id,
|
||||
@@ -494,6 +513,10 @@ impl Session {
|
||||
_timer: timer,
|
||||
};
|
||||
turn.task = Some(running_task);
|
||||
TurnInputSubmission::Started {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
root_turn_id: turn_context.root_turn_id(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether an extension has marked this thread as durably asleep.
|
||||
@@ -559,19 +582,10 @@ impl Session {
|
||||
}
|
||||
None
|
||||
};
|
||||
let (input, mut start_options) =
|
||||
self.input_queue.get_pending_input(&self.active_turn).await;
|
||||
let triggered = input.iter().any(
|
||||
|item| matches!(item, TurnInput::InterAgentCommunication(mail) if mail.trigger_turn),
|
||||
);
|
||||
if !triggered {
|
||||
// Queue-only wakes retain the previous turn context without selecting a new task.
|
||||
start_options = previous_options;
|
||||
start_options.cyber_access_program = self
|
||||
.reference_context_item()
|
||||
.await
|
||||
.and_then(|context| context.cyber_access_program);
|
||||
}
|
||||
// Read the routing options now and leave the mail queued until task registration.
|
||||
let wake_options = self.input_queue.pending_wake_options().await;
|
||||
let triggered = wake_options.is_some();
|
||||
let start_options = wake_options.unwrap_or(previous_options);
|
||||
let options = NewTurnContextOptions {
|
||||
direct_only_tool_namespaces: start_options.direct_only_tool_namespaces,
|
||||
final_output_json_schema: start_options.final_output_json_schema,
|
||||
@@ -590,18 +604,7 @@ impl Session {
|
||||
turn_context.turn_metadata_state.set_turn_trigger(trigger);
|
||||
}
|
||||
if let Some(id) = start_options.parent_turn_id {
|
||||
if let Some(initiating_agent_path) = input
|
||||
.iter()
|
||||
.find_map(|item| {
|
||||
let TurnInput::InterAgentCommunication(communication) = item else {
|
||||
return None;
|
||||
};
|
||||
communication
|
||||
.trigger_turn
|
||||
.then(|| communication.author.clone())
|
||||
})
|
||||
.or(start_options.initiating_agent_path)
|
||||
{
|
||||
if let Some(initiating_agent_path) = start_options.initiating_agent_path {
|
||||
turn_context
|
||||
.turn_metadata_state
|
||||
.set_initiating_agent_path(initiating_agent_path);
|
||||
@@ -613,10 +616,6 @@ impl Session {
|
||||
}
|
||||
self.maybe_emit_model_warnings_for_turn(turn_context.as_ref())
|
||||
.await;
|
||||
// Task completion must still save this mail if pre-turn compaction fails.
|
||||
self.input_queue
|
||||
.extend_pending_input_for_turn_state(turn_state.as_ref(), input)
|
||||
.await;
|
||||
// Context construction can outlive an interrupt or replacement. Do not let
|
||||
// start_task recreate a turn whose reservation is no longer ours.
|
||||
if self
|
||||
|
||||
Reference in New Issue
Block a user