mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add graceful agent-tree suspension and recovery (#61198)
## Why Suspending a locally controlled agent tree must preserve unfinished turns and child results that arrive during teardown so they remain available after recovery. ## What changed - Add `ThreadManager::suspend_agent_tree` to stop durable trees, close their writers, and return unfinished root and named V2 child turn IDs plus any pending root wake. - Persist terminal child results before suspension succeeds, including results queued for evicted parents. Report failed or unconfirmed writes. - Add `load_suspended_agent` to restore named V2 descendants through their recorded ancestors without starting model turns. - Allow empty `continue_turn_if_idle` input to wake on saved results, guarded by the preceding turn ID. Keep mail queued until task registration so suspension can preserve it. ## Testing Add coverage for completion and wake races, child errors, parent writer failures, evicted parents, descendant restoration, repeated suspension after caller cancellation, and ephemeral roots remaining active. GitOrigin-RevId: 3041791f51bff2555210f6b2d2443f9f05f68330
This commit is contained in:
@@ -316,6 +316,9 @@ pub(super) async fn shutdown_session_runtime(sess: &Arc<Session>) {
|
||||
if let Some(reporting) = &sess.outcome_reporting {
|
||||
reporting.handoffs.close();
|
||||
}
|
||||
let runtime = &sess.services.local_agent_runtime;
|
||||
let wakes_root = runtime.suspension_root() == Some(sess.thread_id)
|
||||
&& (sess.has_outstanding_durable_sleep() || sess.should_wake_on_agent_messages().await);
|
||||
let startup_prewarm = {
|
||||
let mut state = sess.state.lock().await;
|
||||
// Stop admission and take the current warmup together so resume cannot replace it.
|
||||
@@ -330,6 +333,49 @@ pub(super) async fn shutdown_session_runtime(sess: &Arc<Session>) {
|
||||
if let Some(reporting) = &sess.outcome_reporting {
|
||||
reporting.handoffs.wait().await;
|
||||
}
|
||||
if runtime.suspension_root().is_some() {
|
||||
let pending: Vec<_> = sess
|
||||
.input_queue
|
||||
.drain_mailbox()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|mail| mail.communication)
|
||||
.filter(crate::PendingAgentMessage::is_terminal_result)
|
||||
.collect();
|
||||
if !pending.is_empty() && sess.live_thread().is_some() {
|
||||
let turn_id = {
|
||||
let state = sess.state.lock().await;
|
||||
state.last_started_turn_id.clone().or_else(|| {
|
||||
state
|
||||
.turn_attribution
|
||||
.as_ref()
|
||||
.map(|turn| turn.turn_id.clone())
|
||||
})
|
||||
};
|
||||
if wakes_root {
|
||||
if let Some(turn_id) = turn_id.as_ref() {
|
||||
runtime.record_root_wake_after(turn_id.clone());
|
||||
} else {
|
||||
runtime.record_shutdown_failure(AgentTreeShutdownFailure::operation_failed(
|
||||
"tree_suspension",
|
||||
"record_root_wake",
|
||||
Some(sess.thread_id),
|
||||
"missing_previous_turn",
|
||||
));
|
||||
}
|
||||
}
|
||||
let context = sess
|
||||
.new_turn_with_default_settings(
|
||||
turn_id.unwrap_or_else(super::new_submission_id),
|
||||
Default::default(),
|
||||
)
|
||||
.await;
|
||||
for result in pending {
|
||||
sess.record_inter_agent_communication(&context, context.model_info(), result)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
super::async_compaction::shutdown(sess).await;
|
||||
let shell_snapshot_prewarm = sess.state.lock().await.shell_snapshot_prewarm.take();
|
||||
if let Some(shell_snapshot_prewarm) = shell_snapshot_prewarm {
|
||||
@@ -491,7 +537,28 @@ pub(super) async fn submission_loop(
|
||||
let sub = tokio::select! {
|
||||
biased;
|
||||
_ = sess.services.local_agent_runtime.shutdown.cancelled() => {
|
||||
shutdown_received = shutdown(&sess, super::new_submission_id()).await;
|
||||
if sess.services.local_agent_runtime.suspension_root().is_some() {
|
||||
// Keep accepting results until all direct children stop, then drain
|
||||
// buffered submissions before deciding which parent work to hand off.
|
||||
let runtime = &sess.services.local_agent_runtime;
|
||||
tokio::join!(
|
||||
async {
|
||||
runtime.wait_for_children(sess.thread_id).await;
|
||||
rx_sub.close();
|
||||
},
|
||||
async {
|
||||
while let Ok(sub) = rx_sub.recv().await {
|
||||
if let Op::InterAgentCommunication { communication, start_options } = sub.op
|
||||
&& sub.message_delivery.as_ref().is_some_and(crate::agent_communication::MessageDelivery::awaits_result_write)
|
||||
{
|
||||
let mail = crate::PendingAgentMessage::new(communication, sub.message_delivery);
|
||||
inter_agent_communication(&sess, sub.id, mail, start_options).await;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
shutdown_received = super::turn_suspension::stop_tree_member(&sess).await;
|
||||
break;
|
||||
}
|
||||
update = async {
|
||||
|
||||
@@ -269,15 +269,25 @@ impl InputQueue {
|
||||
pending.drain(..).collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn drain_mailbox_input_items(&self) -> (Vec<TurnInput>, TurnStartOptions) {
|
||||
let pending_mails = self.drain_mailbox().await;
|
||||
pub(crate) async fn pending_wake_options(&self) -> Option<TurnStartOptions> {
|
||||
let mut mails = self.mailbox_pending_mails.lock().await;
|
||||
self.read_mailbox(&mut mails);
|
||||
let mut options = Self::trigger_turn_options(&mails)?;
|
||||
if let Some(mail) = mails.iter().find(|mail| mail.communication.trigger_turn) {
|
||||
options.initiating_agent_path = Some(mail.communication.author.clone());
|
||||
}
|
||||
Some(options)
|
||||
}
|
||||
|
||||
fn trigger_turn_options(
|
||||
pending_mails: &VecDeque<PendingMailboxCommunication>,
|
||||
) -> Option<TurnStartOptions> {
|
||||
// A later follow-up supersedes the earlier choice, including an omitted choice.
|
||||
let mut start_options = pending_mails
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|mail| mail.communication.trigger_turn)
|
||||
.map(|mail| mail.start_options.clone())
|
||||
.unwrap_or_default();
|
||||
.map(|mail| mail.start_options.clone())?;
|
||||
start_options.parent_turn_id = pending_mails
|
||||
.iter()
|
||||
.filter(|mail| mail.communication.trigger_turn)
|
||||
@@ -296,8 +306,15 @@ impl InputQueue {
|
||||
.filter(|id| !id.trim().is_empty())
|
||||
})
|
||||
.map(str::to_string);
|
||||
Some(start_options)
|
||||
}
|
||||
|
||||
pub(crate) async fn drain_mailbox_input_items(&self) -> (Vec<TurnInput>, TurnStartOptions) {
|
||||
let mut pending_mails = self.mailbox_pending_mails.lock().await;
|
||||
self.read_mailbox(&mut pending_mails);
|
||||
let start_options = Self::trigger_turn_options(&pending_mails).unwrap_or_default();
|
||||
let items = pending_mails
|
||||
.into_iter()
|
||||
.drain(..)
|
||||
.map(|mail| TurnInput::InterAgentCommunication(mail.communication))
|
||||
.collect();
|
||||
(items, start_options)
|
||||
@@ -343,12 +360,34 @@ impl InputQueue {
|
||||
|
||||
/// Clear any pending waiters and input buffered for the current turn.
|
||||
pub(crate) async fn clear_pending(&self, active_turn: &ActiveTurn) {
|
||||
let mut turn_state = active_turn.turn_state.lock().await;
|
||||
turn_state.clear_pending_waiters();
|
||||
// Pending mail can own extension attachments whose destructors need the lock released.
|
||||
let pending_input = std::mem::take(&mut turn_state.pending_input);
|
||||
drop(turn_state);
|
||||
drop(pending_input);
|
||||
let pending_input = {
|
||||
let mut turn_state = active_turn.turn_state.lock().await;
|
||||
turn_state.clear_pending_waiters();
|
||||
std::mem::take(&mut turn_state.pending_input)
|
||||
};
|
||||
// A terminal result still in the queue has no writer yet. Keep it available
|
||||
// to the next turn or to the source writer during a graceful drain.
|
||||
let results: Vec<_> = pending_input
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|input| match input {
|
||||
TurnInput::InterAgentCommunication(message) if message.is_terminal_result() => {
|
||||
Some(message)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if !results.is_empty() {
|
||||
let mut pending = self.mailbox_pending_mails.lock().await;
|
||||
for communication in results.into_iter().rev() {
|
||||
pending.push_front(PendingMailboxCommunication {
|
||||
communication,
|
||||
start_options: TurnStartOptions::default(),
|
||||
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn defer_mailbox_delivery_to_next_turn(
|
||||
|
||||
@@ -195,7 +195,6 @@ use codex_thread_store::LocalThreadStore;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_thread_store::ResumeThreadParams;
|
||||
use codex_thread_store::ThreadPersistenceMetadata;
|
||||
use codex_thread_store::ThreadStore;
|
||||
use codex_thread_store::ThreadStoreResumePreparation;
|
||||
use codex_utils_audio::prepare_response_items as prepare_audio_response_items;
|
||||
@@ -396,6 +395,7 @@ use codex_protocol::protocol::DeprecationNoticeEvent;
|
||||
use codex_protocol::protocol::Event;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ExecApprovalRequestEvent;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::ModelRerouteEvent;
|
||||
use codex_protocol::protocol::ModelRerouteReason;
|
||||
use codex_protocol::protocol::ModelVerification;
|
||||
@@ -2832,7 +2832,7 @@ impl Session {
|
||||
let reporter = self
|
||||
.outcome_reporting
|
||||
.as_ref()
|
||||
.map(|reporting| &reporting.reporter);
|
||||
.and_then(|reporting| reporting.reporter.as_ref());
|
||||
if !notify_native_parent && reporter.is_none() {
|
||||
return;
|
||||
}
|
||||
@@ -4656,13 +4656,11 @@ impl Session {
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) async fn record_inter_agent_communication(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
communication: impl Into<crate::PendingAgentMessage>,
|
||||
) {
|
||||
let communication = communication.into();
|
||||
/// Builds the same history item whether the recipient is live or has been evicted.
|
||||
pub(crate) fn inter_agent_history_item(
|
||||
communication: &InterAgentCommunication,
|
||||
config: &Config,
|
||||
) -> ResponseItem {
|
||||
let mut response_item = communication.to_model_input_item();
|
||||
if let ResponseItem::AgentMessage {
|
||||
content,
|
||||
@@ -4676,8 +4674,7 @@ impl Session {
|
||||
.get_or_insert_with(|| {
|
||||
vec![
|
||||
ContentItemMetadata::tool(Some(
|
||||
turn_context
|
||||
.config
|
||||
config
|
||||
.multi_agent_v2
|
||||
.tool_namespace
|
||||
.clone()
|
||||
@@ -4688,6 +4685,17 @@ impl Session {
|
||||
]
|
||||
});
|
||||
}
|
||||
response_item
|
||||
}
|
||||
|
||||
pub(crate) async fn record_inter_agent_communication(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
communication: impl Into<crate::PendingAgentMessage>,
|
||||
) {
|
||||
let communication = communication.into();
|
||||
let response_item = Self::inter_agent_history_item(&communication, &turn_context.config);
|
||||
let (items, _) = self
|
||||
.prepare_conversation_items_for_history(
|
||||
turn_context,
|
||||
@@ -4728,16 +4736,21 @@ impl Session {
|
||||
model_info.truncation_policy.into(),
|
||||
);
|
||||
}
|
||||
if let Some(delivery) = &communication.delivery {
|
||||
delivery.consumed(self.thread_id);
|
||||
let result_write = communication.consumed(self.thread_id);
|
||||
let saved = self
|
||||
.persist_rollout_items(&[
|
||||
RolloutItem::InterAgentCommunicationMetadata {
|
||||
trigger_turn: communication.trigger_turn,
|
||||
},
|
||||
RolloutItem::ResponseItem(response_item),
|
||||
])
|
||||
.await;
|
||||
if saved
|
||||
&& self.live_thread().is_some()
|
||||
&& let Some(write) = result_write
|
||||
{
|
||||
write.confirm();
|
||||
}
|
||||
self.persist_rollout_items(&[
|
||||
RolloutItem::InterAgentCommunicationMetadata {
|
||||
trigger_turn: communication.trigger_turn,
|
||||
},
|
||||
RolloutItem::ResponseItem(response_item),
|
||||
])
|
||||
.await;
|
||||
drop(boundary);
|
||||
self.send_raw_response_items(turn_context, &items).await;
|
||||
}
|
||||
@@ -5394,6 +5407,21 @@ impl Session {
|
||||
&& let Err(e) = live_thread.append_items(items).await
|
||||
{
|
||||
error!("failed to record rollout items: {e:#}");
|
||||
if self
|
||||
.services
|
||||
.local_agent_runtime
|
||||
.suspension_root()
|
||||
.is_some()
|
||||
{
|
||||
self.services.local_agent_runtime.record_shutdown_failure(
|
||||
crate::thread_manager::AgentTreeShutdownFailure::operation_failed(
|
||||
"tree_suspension",
|
||||
"append_history",
|
||||
Some(self.thread_id),
|
||||
crate::thread_manager::thread_store_error_kind(&e),
|
||||
),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
true
|
||||
|
||||
@@ -60,9 +60,10 @@ use tokio_util::task::TaskTracker;
|
||||
type McpToolApprovalMetadataMap =
|
||||
HashMap<(codex_mcp::McpServerKey, String), std::sync::Weak<McpToolApprovalContext>>;
|
||||
|
||||
/// A thread's installed reporter and the tasks that may still hand it an outcome.
|
||||
/// Joins terminal publishers before their session writer closes. Native child results
|
||||
/// and optional host reporters both use this barrier.
|
||||
pub(crate) struct AgentOutcomeReporting {
|
||||
pub(super) reporter: Arc<dyn AgentTurnOutcomeReporter>,
|
||||
pub(crate) reporter: Option<Arc<dyn AgentTurnOutcomeReporter>>,
|
||||
pub(crate) handoffs: TaskTracker,
|
||||
}
|
||||
|
||||
@@ -634,10 +635,15 @@ async fn warm_plugins_and_skills_for_session_init(
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) fn outcome_handoffs_closed(&self) -> bool {
|
||||
self.outcome_reporting
|
||||
.as_ref()
|
||||
.is_some_and(|reporting| reporting.handoffs.is_closed())
|
||||
pub(crate) fn turn_starts_closed(&self) -> bool {
|
||||
self.services
|
||||
.local_agent_runtime
|
||||
.suspension_root()
|
||||
.is_some()
|
||||
|| self
|
||||
.outcome_reporting
|
||||
.as_ref()
|
||||
.is_some_and(|reporting| reporting.handoffs.is_closed())
|
||||
}
|
||||
|
||||
pub(crate) fn new_item_id_generator() -> ItemIdGenerator {
|
||||
@@ -1143,15 +1149,7 @@ impl Session {
|
||||
.window_id
|
||||
.clone(),
|
||||
runtime_workspace_roots: Some(config.workspace_roots.clone()),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
model_provider: config.model_provider_id.clone(),
|
||||
memory_mode: if config.memories.generate_memories {
|
||||
ThreadMemoryMode::Enabled
|
||||
} else {
|
||||
ThreadMemoryMode::Disabled
|
||||
},
|
||||
},
|
||||
metadata: config.thread_persistence_metadata(),
|
||||
};
|
||||
if is_paginated_subagent
|
||||
&& matches!(
|
||||
@@ -1180,15 +1178,7 @@ impl Session {
|
||||
rollout_path: resumed_history.rollout_path.clone(),
|
||||
history: Some(resumed_history.history.clone()),
|
||||
include_archived: true,
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
model_provider: config.model_provider_id.clone(),
|
||||
memory_mode: if config.memories.generate_memories {
|
||||
ThreadMemoryMode::Enabled
|
||||
} else {
|
||||
ThreadMemoryMode::Disabled
|
||||
},
|
||||
},
|
||||
metadata: config.thread_persistence_metadata(),
|
||||
};
|
||||
let store = Arc::clone(&thread_store);
|
||||
let history_mode = session_configuration.history_mode;
|
||||
@@ -1797,10 +1787,14 @@ impl Session {
|
||||
.await;
|
||||
}
|
||||
|
||||
let outcome_reporting = thread_extension_data
|
||||
let reporter = thread_extension_data
|
||||
.get::<Arc<dyn AgentTurnOutcomeReporter>>()
|
||||
.map(|reporter| AgentOutcomeReporting {
|
||||
reporter: Arc::clone(&reporter),
|
||||
.map(|reporter| Arc::clone(reporter.as_ref()));
|
||||
// The turn's resolved version determines whether it publishes a native result.
|
||||
let local_durable = live_thread.is_some() && !has_provided_agent_control;
|
||||
let outcome_reporting =
|
||||
(reporter.is_some() || local_durable).then(|| AgentOutcomeReporting {
|
||||
reporter,
|
||||
handoffs: TaskTracker::new(),
|
||||
});
|
||||
let (agent_runtime, mut startup_error) =
|
||||
|
||||
@@ -2,6 +2,7 @@ use crate::McpRefreshReason;
|
||||
use crate::agent::LocalAgentControl;
|
||||
use codex_mcp::McpServerKey;
|
||||
use codex_protocol::ContextWindowId;
|
||||
use codex_thread_store::ThreadPersistenceMetadata;
|
||||
#[path = "compaction_replay_tests.rs"]
|
||||
mod compaction_replay_tests;
|
||||
|
||||
@@ -6254,12 +6255,10 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc
|
||||
.tx_event
|
||||
.send(Event {
|
||||
id: "occupied".into(),
|
||||
msg: EventMsg::ThreadSettingsApplied(
|
||||
codex_protocol::protocol::ThreadSettingsAppliedEvent {
|
||||
thread_id: Some(session.thread_id()),
|
||||
thread_settings: session.thread_settings_snapshot().await,
|
||||
},
|
||||
),
|
||||
msg: EventMsg::ThreadSettingsApplied(ThreadSettingsAppliedEvent {
|
||||
thread_id: Some(session.thread_id()),
|
||||
thread_settings: session.thread_settings_snapshot().await,
|
||||
}),
|
||||
})
|
||||
.await
|
||||
.expect("fill event channel");
|
||||
@@ -6271,7 +6270,7 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc
|
||||
id: "settings".into(),
|
||||
turn_extension_init: None,
|
||||
op: Op::ThreadSettings {
|
||||
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides::default(),
|
||||
thread_settings: ThreadSettingsOverrides::default(),
|
||||
reply: Some(reply),
|
||||
},
|
||||
trace: None,
|
||||
@@ -6356,7 +6355,7 @@ async fn compaction_persists_resume_metadata_and_companion_records() {
|
||||
}),
|
||||
previous_turn_settings: Some(previous_turn_settings),
|
||||
};
|
||||
let expected_settings = codex_protocol::protocol::ThreadSettingsAppliedEvent {
|
||||
let expected_settings = ThreadSettingsAppliedEvent {
|
||||
thread_id: Some(session.thread_id()),
|
||||
thread_settings: session.thread_settings_snapshot().await,
|
||||
};
|
||||
|
||||
@@ -306,13 +306,20 @@ pub(super) async fn handle(
|
||||
TurnInputMode::ContinueIfIdle {
|
||||
expected_previous_turn_id,
|
||||
} => {
|
||||
if !matches!(
|
||||
&request.request.input,
|
||||
SubmittedTurnInput::ResponseItem(_) | SubmittedTurnInput::AnnotatedResponseItem(_)
|
||||
) {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"continuation requires internal response input".to_string(),
|
||||
));
|
||||
let mut request = request;
|
||||
match &request.request.input {
|
||||
SubmittedTurnInput::ResponseItem(_)
|
||||
| SubmittedTurnInput::AnnotatedResponseItem(_) => {}
|
||||
SubmittedTurnInput::UserInput { content, .. } if content.is_empty() => {
|
||||
request.request.start = session.agent_message_wake_options().await;
|
||||
}
|
||||
SubmittedTurnInput::UserInput { .. }
|
||||
| SubmittedTurnInput::InterAgentCommunication(_) => {
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"continuation requires internal response input or an empty wake"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
start_if_idle(
|
||||
session,
|
||||
@@ -501,13 +508,9 @@ async fn start_or_steer(
|
||||
.await,
|
||||
);
|
||||
}
|
||||
session
|
||||
.spawn_task(Arc::clone(&turn_context), task_input, RegularTask::new())
|
||||
.await;
|
||||
Ok(TurnInputSubmission::Started {
|
||||
turn_id: submission_id,
|
||||
root_turn_id: turn_context.root_turn_id(),
|
||||
})
|
||||
Ok(session
|
||||
.spawn_task(turn_context, task_input, RegularTask::new())
|
||||
.await)
|
||||
}
|
||||
Err(reason) => Ok(TurnInputSubmission::NotSubmitted { reason }),
|
||||
}
|
||||
@@ -537,7 +540,9 @@ async fn start_if_idle(
|
||||
responsesapi_client_metadata,
|
||||
..
|
||||
} = request;
|
||||
if session.input_queue.has_trigger_turn_mailbox_items().await {
|
||||
if expected_previous_turn_id.is_none()
|
||||
&& session.input_queue.has_trigger_turn_mailbox_items().await
|
||||
{
|
||||
return Ok(TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::PendingTriggerTurn,
|
||||
});
|
||||
@@ -575,18 +580,23 @@ async fn start_if_idle(
|
||||
|
||||
let turn_state = {
|
||||
let mut active_turn = session.active_turn.lock().await;
|
||||
if let Some(expected) = expected_previous_turn_id {
|
||||
let state = session.state.lock().await;
|
||||
// A registered successor already owns this work, even if it is still running.
|
||||
// An unregistered reservation can still fail to start and must be retried.
|
||||
if state.last_started_turn_id.as_ref() != Some(&expected)
|
||||
&& (state.last_started_turn_id.is_some() || active_turn.is_none())
|
||||
{
|
||||
return Ok(TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::Superseded,
|
||||
});
|
||||
}
|
||||
}
|
||||
if active_turn.is_some() {
|
||||
return Ok(TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::NotIdle,
|
||||
});
|
||||
}
|
||||
if let Some(expected) = expected_previous_turn_id
|
||||
&& session.state.lock().await.last_started_turn_id.as_ref() != Some(&expected)
|
||||
{
|
||||
return Ok(TurnInputSubmission::NotSubmitted {
|
||||
reason: NotSubmittedReason::Superseded,
|
||||
});
|
||||
}
|
||||
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
|
||||
Arc::clone(&active_turn.turn_state)
|
||||
};
|
||||
@@ -680,13 +690,9 @@ async fn start_if_idle(
|
||||
}
|
||||
}
|
||||
}
|
||||
session
|
||||
.start_task(Arc::clone(&turn_context), task_input, RegularTask::new())
|
||||
.await;
|
||||
Ok(TurnInputSubmission::Started {
|
||||
turn_id: submission_id,
|
||||
root_turn_id: turn_context.root_turn_id(),
|
||||
})
|
||||
Ok(session
|
||||
.start_task(turn_context, task_input, RegularTask::new())
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn steer(
|
||||
@@ -814,7 +820,7 @@ impl Session {
|
||||
&self,
|
||||
) -> Option<(Arc<tokio::sync::Mutex<TurnState>>, TurnStartOptions)> {
|
||||
let mut active_turn = self.active_turn.lock().await;
|
||||
if active_turn.is_some() || self.outcome_handoffs_closed() {
|
||||
if active_turn.is_some() || self.turn_starts_closed() {
|
||||
return None;
|
||||
}
|
||||
let has_trigger_turn = self.input_queue.has_trigger_turn_mailbox_items().await;
|
||||
@@ -828,17 +834,24 @@ impl Session {
|
||||
let previous_options = if has_trigger_turn {
|
||||
Default::default()
|
||||
} else {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.turn_attribution
|
||||
.as_ref()
|
||||
.map(codex_history::TurnAttribution::start_options)
|
||||
.unwrap_or_default()
|
||||
self.agent_message_wake_options().await
|
||||
};
|
||||
Some((Arc::clone(&active_turn.turn_state), previous_options))
|
||||
}
|
||||
|
||||
async fn agent_message_wake_options(&self) -> TurnStartOptions {
|
||||
let state = self.state.lock().await;
|
||||
let mut options = state
|
||||
.turn_attribution
|
||||
.as_ref()
|
||||
.map(codex_history::TurnAttribution::start_options)
|
||||
.unwrap_or_default();
|
||||
options.cyber_access_program = state
|
||||
.reference_context_item()
|
||||
.and_then(|context| context.cyber_access_program);
|
||||
options
|
||||
}
|
||||
|
||||
/// Revalidates a reserved wake and selects its settings before any mail is consumed.
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
@@ -871,11 +884,8 @@ impl Session {
|
||||
Some(configuration.clone())
|
||||
}
|
||||
|
||||
async fn should_wake_on_agent_messages(&self) -> bool {
|
||||
if !self.features.enabled(Feature::WakeOnAgentMessages)
|
||||
|| self.is_interrupted()
|
||||
|| !self.input_queue.has_pending_mailbox_items().await
|
||||
{
|
||||
pub(crate) async fn should_wake_on_agent_messages(&self) -> bool {
|
||||
if !self.features.enabled(Feature::WakeOnAgentMessages) || self.is_interrupted() {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -250,6 +250,29 @@ async fn start_only_rejects_active_turn_without_injecting() {
|
||||
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saved_message_wake_checks_the_last_started_turn_during_admission() {
|
||||
let (session, _, _rx) = make_session_and_context_with_rx().await;
|
||||
*session.active_turn.lock().await = Some(ActiveTurn::default());
|
||||
for (last_started, reason) in [
|
||||
("original", NotSubmittedReason::NotIdle),
|
||||
("successor", NotSubmittedReason::Superseded),
|
||||
] {
|
||||
session.state.lock().await.last_started_turn_id = Some(last_started.into());
|
||||
let result = handle(
|
||||
&session,
|
||||
TurnInputRequest::user_input(vec![]),
|
||||
TurnInputMode::ContinueIfIdle {
|
||||
expected_previous_turn_id: "original".into(),
|
||||
},
|
||||
"wake".into(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result, TurnInputSubmission::NotSubmitted { reason });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_rejects_active_turn_without_injecting_or_applying_settings() {
|
||||
let (session, turn_context, _rx) = make_session_and_context_with_rx().await;
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
use super::handlers;
|
||||
use super::session::Session;
|
||||
use crate::state::TaskKind;
|
||||
use crate::thread_manager::AgentTreeShutdownFailure;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::Event;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::turn_input::SuspendTurnOutcome;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -120,3 +122,39 @@ pub(super) async fn suspend_turn_and_shutdown(
|
||||
.await;
|
||||
Ok(SuspendTurnOutcome::Suspended { turn_id })
|
||||
}
|
||||
|
||||
pub(super) async fn stop_tree_member(session: &Arc<Session>) -> bool {
|
||||
let runtime = &session.services.local_agent_runtime;
|
||||
let submission_id = super::new_submission_id();
|
||||
if let Some(root) = runtime.suspension_root()
|
||||
&& session.live_thread().is_some()
|
||||
&& (root == session.thread_id
|
||||
|| session.multi_agent_version() == Some(MultiAgentVersion::V2)
|
||||
&& session
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.session_configuration
|
||||
.session_source
|
||||
.get_agent_path()
|
||||
.is_some())
|
||||
{
|
||||
match suspend_turn_and_shutdown(session, submission_id.clone()).await {
|
||||
Ok(SuspendTurnOutcome::Suspended { turn_id }) => {
|
||||
runtime.record_suspended_turn(session.thread_id, turn_id);
|
||||
return true;
|
||||
}
|
||||
Ok(SuspendTurnOutcome::NotActive | SuspendTurnOutcome::UnsupportedTask) => {}
|
||||
Err(error) => {
|
||||
runtime.record_shutdown_failure(AgentTreeShutdownFailure::operation_failed(
|
||||
"tree_suspension",
|
||||
"suspend_regular_turn",
|
||||
Some(session.thread_id),
|
||||
"suspension_failed",
|
||||
));
|
||||
warn!(thread_id = %session.thread_id, %error, "failed to suspend agent turn");
|
||||
}
|
||||
}
|
||||
}
|
||||
handlers::shutdown(session, submission_id).await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user