mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Preserve queued agent mail across session eviction (#50087)
## Why Unread queue-only messages prevented idle agents from unloading, and sending a message to an evicted agent reloaded its session. Pending mail should not require an idle recipient to occupy a loaded thread slot. ## What changed - Retain queue-only mail in an in-memory runtime mailbox when a session unloads. Queue new messages for locally evicted agents without reloading them or starting a turn. - Add `AgentControl::take_mailbox` and `AgentControl::watch_mailbox` so sessions can consume retained mail and react to host mailbox notifications. - Preserve message order across eviction, reload, and subsequent follow-ups. Keep agents with outstanding durable sleep or turn-triggering mail loaded, and discard retained mail when an agent is removed or its tree shuts down. ## Testing Add integration coverage for retaining mail across eviction, sending while unloaded, and ordering messages around a follow-up after reload. Add a host controller test verifying that mailbox notifications wake `wait_agent` and that the message is delivered exactly once. GitOrigin-RevId: 4b55647f9ee9d1aaf3753f0688863bda81617327
This commit is contained in:
@@ -136,8 +136,17 @@ impl AgentControl for LocalAgentControl {
|
||||
"target agent is missing an agent_path".to_string(),
|
||||
)
|
||||
})?;
|
||||
self.ensure_v2_agent_loaded(resume_config, target, /*parent*/ None)
|
||||
.await?;
|
||||
// Cold-restored children still reload lazily on any message. Only
|
||||
// locally evicted recipients can retain mail without reloading.
|
||||
// Loaded recipients go straight to delivery, which rejects sends
|
||||
// racing an in-progress eviction when it acquires the residency pin.
|
||||
if mode == MessageDeliveryMode::TriggerTurn
|
||||
|| (self.runtime.upgrade()?.get_thread(target).await.is_err()
|
||||
&& self.runtime.registry.evicted_environments(target).is_none())
|
||||
{
|
||||
self.ensure_v2_agent_loaded(resume_config, target, /*parent*/ None)
|
||||
.await?;
|
||||
}
|
||||
let communication = message.into_communication(author, receiver_path, mode);
|
||||
let kind = match mode {
|
||||
MessageDeliveryMode::QueueOnly => {
|
||||
@@ -165,6 +174,17 @@ impl AgentControl for LocalAgentControl {
|
||||
})
|
||||
}
|
||||
|
||||
fn take_mailbox(
|
||||
&self,
|
||||
agent: ThreadId,
|
||||
) -> Vec<codex_protocol::protocol::InterAgentCommunication> {
|
||||
self.runtime.mailboxes.take(agent)
|
||||
}
|
||||
|
||||
fn watch_mailbox(&self, agent: ThreadId) -> tokio::sync::watch::Receiver<bool> {
|
||||
self.runtime.mailboxes.watch(agent)
|
||||
}
|
||||
|
||||
fn ensure_child_loaded(&self, parent: ThreadId, child: ThreadId) -> BoxFuture<'_, Result<()>> {
|
||||
Box::pin(async move {
|
||||
let parent = self.runtime.upgrade()?.get_thread(parent).await?;
|
||||
|
||||
@@ -41,6 +41,7 @@ impl LocalAgentControl {
|
||||
let _ = state.remove_thread(&agent_id).await;
|
||||
self.forget_v2_residency(agent_id);
|
||||
self.runtime.registry.release_spawned_thread(agent_id);
|
||||
self.runtime.mailboxes.remove(agent_id);
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Retains queue-only mail while local sessions are unloaded.
|
||||
//! Eviction transfers mail before releasing the recipient's residency guard.
|
||||
|
||||
use crate::agent_communication::PENDING_MAILBOX_MESSAGES;
|
||||
use codex_diagnostics::GaugeGuard;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::PoisonError;
|
||||
use tokio::sync::watch;
|
||||
|
||||
pub(super) struct Mailboxes(Mutex<Option<HashMap<ThreadId, Mailbox>>>);
|
||||
|
||||
impl Default for Mailboxes {
|
||||
fn default() -> Self {
|
||||
Self(Mutex::new(Some(HashMap::new())))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Mailbox {
|
||||
pending: Vec<StoredMail>,
|
||||
state: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
struct StoredMail {
|
||||
id: Option<String>,
|
||||
communication: InterAgentCommunication,
|
||||
_diagnostics_guard: GaugeGuard,
|
||||
}
|
||||
|
||||
impl Mailboxes {
|
||||
pub(super) fn enqueue(
|
||||
&self,
|
||||
agent: ThreadId,
|
||||
id: Option<String>,
|
||||
messages: Vec<InterAgentCommunication>,
|
||||
) -> Result<()> {
|
||||
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let mailbox = mailboxes
|
||||
.as_mut()
|
||||
.ok_or_else(|| CodexErr::InvalidRequest("agent runtime is shutting down".into()))?
|
||||
.entry(agent)
|
||||
.or_default();
|
||||
mailbox
|
||||
.pending
|
||||
.extend(messages.into_iter().map(|communication| StoredMail {
|
||||
id: id.clone(),
|
||||
communication,
|
||||
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
|
||||
}));
|
||||
mailbox.state.send_replace(!mailbox.pending.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn take(&self, agent: ThreadId) -> Vec<InterAgentCommunication> {
|
||||
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let Some(mailbox) = mailboxes
|
||||
.as_mut()
|
||||
.and_then(|mailboxes| mailboxes.get_mut(&agent))
|
||||
else {
|
||||
return Vec::new();
|
||||
};
|
||||
let pending = std::mem::take(&mut mailbox.pending);
|
||||
mailbox.state.send_if_modified(std::mem::take);
|
||||
pending
|
||||
.into_iter()
|
||||
.map(|mail| {
|
||||
// Mail transferred during eviction was already received by the old session.
|
||||
if let Some(id) = mail.id {
|
||||
crate::agent_communication::emit_agent_communication_receive(&id);
|
||||
}
|
||||
mail.communication
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn watch(&self, agent: ThreadId) -> watch::Receiver<bool> {
|
||||
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
match mailboxes.as_mut() {
|
||||
Some(mailboxes) => {
|
||||
// Private delegates and cancelled startups can stop without explicit removal.
|
||||
// Retain unread mail and live subscriptions, but reclaim abandoned empty entries.
|
||||
mailboxes.retain(|_, mailbox| {
|
||||
!mailbox.pending.is_empty() || mailbox.state.receiver_count() != 0
|
||||
});
|
||||
mailboxes.entry(agent).or_default().state.subscribe()
|
||||
}
|
||||
None => watch::channel(/*init*/ false).1,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remove(&self, agent: ThreadId) {
|
||||
if let Some(mailboxes) = self
|
||||
.0
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.as_mut()
|
||||
{
|
||||
mailboxes.remove(&agent);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn close(&self) {
|
||||
if let Some(mailboxes) = self.0.lock().unwrap_or_else(PoisonError::into_inner).take() {
|
||||
for mailbox in mailboxes.values() {
|
||||
mailbox.state.send_replace(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,23 @@ impl V2Residency {
|
||||
teardown.complete();
|
||||
return false;
|
||||
}
|
||||
// The submission loop has stopped and the residency guard excludes senders.
|
||||
// Preserve unread queue-only mail before dropping the session that held it.
|
||||
let mail = candidate_thread
|
||||
.session
|
||||
.input_queue
|
||||
.drain_mailbox()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|mail| mail.communication)
|
||||
.collect();
|
||||
// A concurrent tree shutdown deliberately discards its unread mail.
|
||||
let _ = candidate_thread
|
||||
.session
|
||||
.services
|
||||
.local_agent_runtime
|
||||
.mailboxes
|
||||
.enqueue(candidate_thread_id, /*id*/ None, mail);
|
||||
let environments = candidate_thread.environment_selections().await;
|
||||
let mut threads = manager.threads.write().await;
|
||||
if threads
|
||||
@@ -276,7 +293,12 @@ async fn is_unloadable(thread: &CodexThread) -> bool {
|
||||
thread.agent_status().await,
|
||||
AgentStatus::Completed(_) | AgentStatus::Errored(_) | AgentStatus::Interrupted
|
||||
) && thread.session.active_turn.lock().await.is_none()
|
||||
&& !thread.session.input_queue.has_pending_mailbox_items().await
|
||||
&& !thread.session.has_outstanding_durable_sleep()
|
||||
&& !thread
|
||||
.session
|
||||
.input_queue
|
||||
.has_trigger_turn_mailbox_items()
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -109,6 +109,7 @@ pub(crate) struct LocalAgentRuntime {
|
||||
Arc<OnceLock<Arc<dyn ThreadInstructionsProvider>>>,
|
||||
pub(super) registry: Arc<AgentRegistry>,
|
||||
pub(super) residency: Arc<V2Residency>,
|
||||
pub(super) mailboxes: Arc<super::mailbox::Mailboxes>,
|
||||
/// Shared by every session in this tree, including private delegates.
|
||||
pub(crate) shutdown: CancellationToken,
|
||||
shutdown_state: Arc<AgentTreeShutdownState>,
|
||||
@@ -125,6 +126,7 @@ impl LocalAgentRuntime {
|
||||
thread_id_generator,
|
||||
registry: Arc::default(),
|
||||
residency: Arc::default(),
|
||||
mailboxes: Arc::default(),
|
||||
shutdown: CancellationToken::new(),
|
||||
shutdown_state: Arc::default(),
|
||||
agent_execution_limiter: Arc::default(),
|
||||
@@ -204,6 +206,7 @@ impl LocalAgentRuntime {
|
||||
pub(crate) fn request_shutdown(&self) -> Arc<AgentTreeShutdownState> {
|
||||
self.shutdown_state.members.close();
|
||||
self.shutdown.cancel();
|
||||
self.mailboxes.close();
|
||||
Arc::clone(&self.shutdown_state)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user