Preserve queued agent mail across session eviction (#50087)

## Why

Unread queue-only messages prevented idle agents from unloading, and sending a
message to an evicted agent reloaded its session. Pending mail should not require
an idle recipient to occupy a loaded thread slot.

## What changed

- Retain queue-only mail in an in-memory runtime mailbox when a session unloads.
  Queue new messages for locally evicted agents without reloading them or
  starting a turn.
- Add `AgentControl::take_mailbox` and `AgentControl::watch_mailbox` so sessions
  can consume retained mail and react to host mailbox notifications.
- Preserve message order across eviction, reload, and subsequent follow-ups.
  Keep agents with outstanding durable sleep or turn-triggering mail loaded,
  and discard retained mail when an agent is removed or its tree shuts down.

## Testing

Add integration coverage for retaining mail across eviction, sending while
unloaded, and ordering messages around a follow-up after reload. Add a host
controller test verifying that mailbox notifications wake `wait_agent` and
that the message is delivered exactly once.

GitOrigin-RevId: 4b55647f9ee9d1aaf3753f0688863bda81617327
This commit is contained in:
jif
2026-10-01 20:17:34 +00:00
committed by copyberry
parent 26a49b3882
commit 960e878df4
14 changed files with 573 additions and 23 deletions
+22 -2
View File
@@ -136,8 +136,17 @@ impl AgentControl for LocalAgentControl {
"target agent is missing an agent_path".to_string(),
)
})?;
self.ensure_v2_agent_loaded(resume_config, target, /*parent*/ None)
.await?;
// Cold-restored children still reload lazily on any message. Only
// locally evicted recipients can retain mail without reloading.
// Loaded recipients go straight to delivery, which rejects sends
// racing an in-progress eviction when it acquires the residency pin.
if mode == MessageDeliveryMode::TriggerTurn
|| (self.runtime.upgrade()?.get_thread(target).await.is_err()
&& self.runtime.registry.evicted_environments(target).is_none())
{
self.ensure_v2_agent_loaded(resume_config, target, /*parent*/ None)
.await?;
}
let communication = message.into_communication(author, receiver_path, mode);
let kind = match mode {
MessageDeliveryMode::QueueOnly => {
@@ -165,6 +174,17 @@ impl AgentControl for LocalAgentControl {
})
}
fn take_mailbox(
&self,
agent: ThreadId,
) -> Vec<codex_protocol::protocol::InterAgentCommunication> {
self.runtime.mailboxes.take(agent)
}
fn watch_mailbox(&self, agent: ThreadId) -> tokio::sync::watch::Receiver<bool> {
self.runtime.mailboxes.watch(agent)
}
fn ensure_child_loaded(&self, parent: ThreadId, child: ThreadId) -> BoxFuture<'_, Result<()>> {
Box::pin(async move {
let parent = self.runtime.upgrade()?.get_thread(parent).await?;
@@ -41,6 +41,7 @@ impl LocalAgentControl {
let _ = state.remove_thread(&agent_id).await;
self.forget_v2_residency(agent_id);
self.runtime.registry.release_spawned_thread(agent_id);
self.runtime.mailboxes.remove(agent_id);
result
}
+114
View File
@@ -0,0 +1,114 @@
//! Retains queue-only mail while local sessions are unloaded.
//! Eviction transfers mail before releasing the recipient's residency guard.
use crate::agent_communication::PENDING_MAILBOX_MESSAGES;
use codex_diagnostics::GaugeGuard;
use codex_protocol::ThreadId;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result;
use codex_protocol::protocol::InterAgentCommunication;
use std::collections::HashMap;
use std::sync::Mutex;
use std::sync::PoisonError;
use tokio::sync::watch;
pub(super) struct Mailboxes(Mutex<Option<HashMap<ThreadId, Mailbox>>>);
impl Default for Mailboxes {
fn default() -> Self {
Self(Mutex::new(Some(HashMap::new())))
}
}
#[derive(Default)]
struct Mailbox {
pending: Vec<StoredMail>,
state: watch::Sender<bool>,
}
struct StoredMail {
id: Option<String>,
communication: InterAgentCommunication,
_diagnostics_guard: GaugeGuard,
}
impl Mailboxes {
pub(super) fn enqueue(
&self,
agent: ThreadId,
id: Option<String>,
messages: Vec<InterAgentCommunication>,
) -> Result<()> {
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
let mailbox = mailboxes
.as_mut()
.ok_or_else(|| CodexErr::InvalidRequest("agent runtime is shutting down".into()))?
.entry(agent)
.or_default();
mailbox
.pending
.extend(messages.into_iter().map(|communication| StoredMail {
id: id.clone(),
communication,
_diagnostics_guard: PENDING_MAILBOX_MESSAGES.track(),
}));
mailbox.state.send_replace(!mailbox.pending.is_empty());
Ok(())
}
pub(super) fn take(&self, agent: ThreadId) -> Vec<InterAgentCommunication> {
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
let Some(mailbox) = mailboxes
.as_mut()
.and_then(|mailboxes| mailboxes.get_mut(&agent))
else {
return Vec::new();
};
let pending = std::mem::take(&mut mailbox.pending);
mailbox.state.send_if_modified(std::mem::take);
pending
.into_iter()
.map(|mail| {
// Mail transferred during eviction was already received by the old session.
if let Some(id) = mail.id {
crate::agent_communication::emit_agent_communication_receive(&id);
}
mail.communication
})
.collect()
}
pub(super) fn watch(&self, agent: ThreadId) -> watch::Receiver<bool> {
let mut mailboxes = self.0.lock().unwrap_or_else(PoisonError::into_inner);
match mailboxes.as_mut() {
Some(mailboxes) => {
// Private delegates and cancelled startups can stop without explicit removal.
// Retain unread mail and live subscriptions, but reclaim abandoned empty entries.
mailboxes.retain(|_, mailbox| {
!mailbox.pending.is_empty() || mailbox.state.receiver_count() != 0
});
mailboxes.entry(agent).or_default().state.subscribe()
}
None => watch::channel(/*init*/ false).1,
}
}
pub(super) fn remove(&self, agent: ThreadId) {
if let Some(mailboxes) = self
.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_mut()
{
mailboxes.remove(&agent);
}
}
pub(super) fn close(&self) {
if let Some(mailboxes) = self.0.lock().unwrap_or_else(PoisonError::into_inner).take() {
for mailbox in mailboxes.values() {
mailbox.state.send_replace(false);
}
}
}
}
+23 -1
View File
@@ -193,6 +193,23 @@ impl V2Residency {
teardown.complete();
return false;
}
// The submission loop has stopped and the residency guard excludes senders.
// Preserve unread queue-only mail before dropping the session that held it.
let mail = candidate_thread
.session
.input_queue
.drain_mailbox()
.await
.into_iter()
.map(|mail| mail.communication)
.collect();
// A concurrent tree shutdown deliberately discards its unread mail.
let _ = candidate_thread
.session
.services
.local_agent_runtime
.mailboxes
.enqueue(candidate_thread_id, /*id*/ None, mail);
let environments = candidate_thread.environment_selections().await;
let mut threads = manager.threads.write().await;
if threads
@@ -276,7 +293,12 @@ async fn is_unloadable(thread: &CodexThread) -> bool {
thread.agent_status().await,
AgentStatus::Completed(_) | AgentStatus::Errored(_) | AgentStatus::Interrupted
) && thread.session.active_turn.lock().await.is_none()
&& !thread.session.input_queue.has_pending_mailbox_items().await
&& !thread.session.has_outstanding_durable_sleep()
&& !thread
.session
.input_queue
.has_trigger_turn_mailbox_items()
.await
}
#[cfg(test)]
@@ -109,6 +109,7 @@ pub(crate) struct LocalAgentRuntime {
Arc<OnceLock<Arc<dyn ThreadInstructionsProvider>>>,
pub(super) registry: Arc<AgentRegistry>,
pub(super) residency: Arc<V2Residency>,
pub(super) mailboxes: Arc<super::mailbox::Mailboxes>,
/// Shared by every session in this tree, including private delegates.
pub(crate) shutdown: CancellationToken,
shutdown_state: Arc<AgentTreeShutdownState>,
@@ -125,6 +126,7 @@ impl LocalAgentRuntime {
thread_id_generator,
registry: Arc::default(),
residency: Arc::default(),
mailboxes: Arc::default(),
shutdown: CancellationToken::new(),
shutdown_state: Arc::default(),
agent_execution_limiter: Arc::default(),
@@ -204,6 +206,7 @@ impl LocalAgentRuntime {
pub(crate) fn request_shutdown(&self) -> Arc<AgentTreeShutdownState> {
self.shutdown_state.members.close();
self.shutdown.cancel();
self.mailboxes.close();
Arc::clone(&self.shutdown_state)
}