From bd3eaf7829187f18d05205222bf2f67c0af80fb4 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 18 Nov 2026 14:40:03 +0000 Subject: [PATCH] Route injected plaintext agent mail through the native mailbox (#62292) ## Why Raw response injection bypasses the native mailbox wake policy, so injected agent messages cannot wake a sleeping thread through that policy. ## What changed In native Multi-Agent v2, route batches consisting entirely of single-text agent messages with valid agent paths through the mailbox as queue-only mail. Hold a residency pin across the batch so capacity eviction cannot interrupt delivery between submissions. Mixed batches, multipart or encrypted messages, and configurations with native tools disabled continue to use raw response injection. Only preempt model sampling for pending mail when the matching turn still accepts mailbox delivery. ## Testing Extend coverage for waking live and reloaded sleeping threads, delivering older queued mail before a new task, and retaining mail in order when a batch exceeds the submission channel capacity and eviction is attempted. GitOrigin-RevId: 3df60d2a4a9e7b193ef298e01f31c565ad8a2785 --- .../core/src/agent/control/residency_tests.rs | 44 +++++++++++- codex-rs/core/src/codex_thread.rs | 66 +++++++++++++++++- codex-rs/core/src/session/input_queue.rs | 26 ++++--- .../core/tests/suite/agent_message_wake.rs | 67 ++++++++++++++++--- codex-rs/core/tests/suite/pending_input.rs | 48 +++++++++++-- .../suite/scenarios_injected_agent_mail.rs | 7 ++ .../scenarios_turn_input_agent_message.rs | 10 +-- 7 files changed, 231 insertions(+), 37 deletions(-) diff --git a/codex-rs/core/src/agent/control/residency_tests.rs b/codex-rs/core/src/agent/control/residency_tests.rs index bb052e9d24..8057f32264 100644 --- a/codex-rs/core/src/agent/control/residency_tests.rs +++ b/codex-rs/core/src/agent/control/residency_tests.rs @@ -1,9 +1,11 @@ +use super::ThreadEvictionOutcome; use crate::StartThreadOptions; use crate::ThreadManager; use crate::agent::LocalAgentControl; use crate::codex_thread::CodexThread; use crate::config::Config; use crate::config::test_config; +use crate::session::SUBMISSION_CHANNEL_CAPACITY; use crate::thread_manager::ThreadManagerState; use codex_features::Feature; use codex_login::CodexAuth; @@ -75,10 +77,46 @@ async fn residency_slot_reservation_unloads_oldest_idle_v2_agent() { communication: communication.clone(), start_options: Default::default(), }); + let injected = (0..=SUBMISSION_CHANNEL_CAPACITY) + .map(|index| { + let mut message = communication.clone(); + message.content = format!("injected notice {index}"); + message + }) + .collect::>(); + let injection = first.thread.inject_response_items( + injected + .iter() + .map(InterAgentCommunication::to_model_input_item) + .collect(), + ); tokio::pin!(delivery); + tokio::pin!(injection); assert!(futures::poll!(delivery.as_mut()).is_pending()); + assert!(futures::poll!(injection.as_mut()).is_pending()); drop(eviction_gate); delivery.await.expect("submit after eviction gate releases"); + // Drain each accepted prefix while the batch yields, then attempt capacity eviction. + loop { + tokio::task::yield_now().await; + tokio::task::consume_budget().await; + if let std::task::Poll::Ready(result) = futures::poll!(injection.as_mut()) { + result.expect("inject after eviction gate releases"); + break; + } + first + .thread + .update_thread_settings(ThreadSettingsOverrides::default()) + .await + .expect("dispatch the accepted prefix"); + assert_eq!( + state + .try_evict_v2_thread(Arc::clone(&first.thread), &membership) + .await + .unwrap(), + ThreadEvictionOutcome::Busy, + ); + } first .thread .update_thread_settings(ThreadSettingsOverrides::default()) @@ -108,7 +146,11 @@ async fn residency_slot_reservation_unloads_oldest_idle_v2_agent() { .local_agent_runtime .mailboxes .take(first.thread_id, |_| true); - assert_eq!(retained, vec![communication.into()]); + let expected = std::iter::once(communication) + .chain(injected) + .map(Into::into) + .collect::>(); + assert_eq!(retained, expected); let second = spawn_v2_subagent(&control, &state, config, root.thread_id, "worker-2").await; second_slot.commit(second.thread_id); diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 3e815d612e..8b078b5061 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -28,6 +28,7 @@ use codex_history::RolloutItem; use codex_models_manager::ModelsManagerConfig; use codex_otel::SessionTelemetry; use codex_otel::current_span_w3c_trace_context; +use codex_protocol::AgentPath; use codex_protocol::RequestCountry; use codex_protocol::ThreadId; use codex_protocol::config_types::ApprovalsReviewer; @@ -40,6 +41,7 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::mcp::CallToolResult; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::models::ActivePermissionProfile; +use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ProfileWorkspaceRoot; use codex_protocol::models::ResponseItem; @@ -50,6 +52,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EnvironmentConfig; use codex_protocol::protocol::EnvironmentConfigState; use codex_protocol::protocol::Event; +use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; @@ -1129,8 +1132,69 @@ impl CodexThread { crate::guardian::review_tool_call(&self.session, history, &tool_call, cancellation).await } - /// Record raw Responses API items without starting a new turn. + /// Inject raw Responses items. Plaintext agent-mail batches in a native MAv2 + /// runtime use queue-only mailbox delivery and its existing wake policy. + /// Other batches retain history-injection semantics. pub async fn inject_response_items(&self, items: Vec) -> CodexResult<()> { + let config = self.session.get_config().await; + // Native mail can represent this shape without changing its payload. Keep + // mixed history and multipart/encrypted items on the raw injection path. + if !config.multi_agent_v2.disable_native_tools + && !items.is_empty() + && let Some(messages) = items + .iter() + .map(|item| { + let ResponseItem::AgentMessage { + id, + author, + recipient, + content, + internal_chat_message_metadata_passthrough, + } = item + else { + return None; + }; + let [AgentMessageInputContent::InputText { text }] = content.as_slice() else { + return None; + }; + Some(InterAgentCommunication { + id: id.clone(), + author: AgentPath::try_from(author.as_str()).ok()?, + recipient: AgentPath::try_from(recipient.as_str()).ok()?, + other_recipients: Vec::new(), + content: text.clone(), + encrypted_content: None, + internal_chat_message_metadata_passthrough: + internal_chat_message_metadata_passthrough.clone(), + trigger_turn: false, + }) + }) + .collect::>>() + && match self.multi_agent_version() { + Some(version) => version, + None => self.session.resolve_multi_agent_version_for_model( + self.selected_model_info().await.as_ref(), + &config, + ), + } == MultiAgentVersion::V2 + { + // Keep capacity eviction out between individual submissions. Each + // submission retains its own pin through dispatch after this batch returns. + let _batch_residency = self + .session + .services + .local_agent_runtime + .pin_v2_residency(self) + .await?; + for communication in messages { + self.submit(Op::InterAgentCommunication { + communication, + start_options: TurnStartOptions::default(), + }) + .await?; + } + return Ok(()); + } self.inject_response_items_with_retention(items).await } diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index 3f12dbbbe2..3e76288c90 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -234,30 +234,28 @@ impl InputQueue { .is_some_and(|(_, _, updates)| *updates.borrow()) } - /// Native mailbox backlog qualifies regardless of turn ID or delivery phase. - /// Injected mail qualifies only while its matching turn still accepts mail. + /// Native and injected mail qualify only while the matching turn still accepts mail. pub(crate) async fn should_preempt_sampling_for_mail( &self, active_turn: &Mutex>, sub_id: &str, ) -> bool { - if self.has_pending_mailbox_items().await { - return true; - } + let native_mail_pending = self.has_pending_mailbox_items().await; let Some(turn_state) = self.turn_state_for_sub_id(active_turn, sub_id).await else { return false; }; let turn_state = turn_state.lock().await; turn_state.accepts_mailbox_delivery_for_current_turn() - && turn_state.pending_input.items.iter().any(|input| { - matches!( - input, - TurnInput::ResponseItem(ResponseItemEnvelope { - item: ResponseItem::AgentMessage { .. }, - .. - }) - ) - }) + && (native_mail_pending + || turn_state.pending_input.items.iter().any(|input| { + matches!( + input, + TurnInput::ResponseItem(ResponseItemEnvelope { + item: ResponseItem::AgentMessage { .. }, + .. + }) + ) + })) } pub(crate) async fn has_trigger_turn_mailbox_items(&self) -> bool { diff --git a/codex-rs/core/tests/suite/agent_message_wake.rs b/codex-rs/core/tests/suite/agent_message_wake.rs index e2fd1b5ad9..f792afb2ce 100644 --- a/codex-rs/core/tests/suite/agent_message_wake.rs +++ b/codex-rs/core/tests/suite/agent_message_wake.rs @@ -21,6 +21,7 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::Op; use codex_protocol::protocol::ThreadSettingsOverrides; use codex_protocol::user_input::UserInput; @@ -179,7 +180,17 @@ async fn pending_agent_message_wake_respects_control_changes( }; let first_mail = "First deferred worker update."; let second_mail = "Worker update after stop."; - test.codex.submit(mail(first_mail)).await?; + let first = mail(first_mail); + if wakes_on_agent_messages { + let Op::InterAgentCommunication { communication, .. } = first else { + unreachable!("mail constructs an agent communication"); + }; + test.codex + .inject_response_items(vec![communication.to_model_input_item()]) + .await?; + } else { + test.codex.submit(first).await?; + } // The settings reply fences the preceding mailbox operation while the old // turn still owns admission, so only its completion can start the wake. test.codex @@ -358,8 +369,16 @@ async fn interrupt_during_completion_preserves_running_status( let mut extensions = ExtensionRegistryBuilder::new(); extensions.turn_lifecycle_contributor(barriers.clone()); let test = test_codex() + .with_model("gpt-5.4") + .with_model_info_override("gpt-5.4", |model| { + model.multi_agent_version = Some(MultiAgentVersion::V2); + }) .with_extensions(Arc::new(extensions.build())) .with_config(move |config| { + config + .features + .disable(Feature::MultiAgentV2) + .expect("select runtime from model metadata"); match setting { AgentMessageWake::Enabled => config.features.enable(Feature::WakeOnAgentMessages), AgentMessageWake::Disabled => config.features.disable(Feature::WakeOnAgentMessages), @@ -368,15 +387,43 @@ async fn interrupt_during_completion_preserves_running_status( }) .build_with_auto_env(&server) .await?; - let TurnInputSubmission::Started { turn_id, .. } = test - .codex - .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { - text: "Finish the work.".to_owned(), - text_elements: Vec::new(), - }])) - .await? - else { - anyhow::bail!("the initial input must start a turn"); + let turn_id = match setting { + AgentMessageWake::Enabled => { + assert_eq!(test.codex.multi_agent_version(), None); + test.codex + .inject_response_items(vec![ + InterAgentCommunication::new( + AgentPath::root().join("worker").expect("valid worker path"), + AgentPath::root(), + Vec::new(), + "Finish the work.".to_owned(), + /*trigger_turn*/ false, + ) + .to_model_input_item(), + ]) + .await?; + let EventMsg::TurnStarted(started) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnStarted(_)) + }) + .await + else { + unreachable!("waited for turn start") + }; + started.turn_id + } + AgentMessageWake::Disabled => { + let TurnInputSubmission::Started { turn_id, .. } = test + .codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "Finish the work.".to_owned(), + text_elements: Vec::new(), + }])) + .await? + else { + anyhow::bail!("the initial input must start a turn"); + }; + turn_id + } }; // The stop callback runs after the task is detached, before TurnComplete // is persisted. This is a finishing turn, not an unstarted wake reservation. diff --git a/codex-rs/core/tests/suite/pending_input.rs b/codex-rs/core/tests/suite/pending_input.rs index f0a82da3ff..34ad59be79 100644 --- a/codex-rs/core/tests/suite/pending_input.rs +++ b/codex-rs/core/tests/suite/pending_input.rs @@ -516,17 +516,26 @@ impl codex_extension_api::ThreadLifecycleContributor } } -#[test_case(false, Some("automation"), None, false; "live_root")] -#[test_case(true, Some("automation"), None, false; "reloaded_root")] -#[test_case(true, Some("automation"), Some("original-parent"), false; "reloaded_delegated")] -#[test_case(true, None, None, false; "reloaded_unknown")] -#[test_case(false, Some("automation"), Some("original-parent"), true; "settings_while_asleep")] +#[derive(Clone, Copy)] +enum MailDelivery { + Native, + Injected, +} + +#[test_case(false, Some("automation"), None, false, MailDelivery::Native; "live_root")] +#[test_case(true, Some("automation"), None, false, MailDelivery::Native; "reloaded_root")] +#[test_case(true, Some("automation"), Some("original-parent"), false, MailDelivery::Native; "reloaded_delegated")] +#[test_case(true, None, None, false, MailDelivery::Native; "reloaded_unknown")] +#[test_case(false, Some("automation"), Some("original-parent"), true, MailDelivery::Native; "settings_while_asleep")] +#[test_case(false, Some("automation"), None, false, MailDelivery::Injected; "injected_live_root")] +#[test_case(true, Some("automation"), Some("original-parent"), false, MailDelivery::Injected; "injected_reloaded_root")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn queue_only_agent_mail_wakes_sleeping_root_with_previous_turn_context( reload: bool, trigger: Option<&str>, parent: Option<&str>, update_settings: bool, + delivery: MailDelivery, ) -> anyhow::Result<()> { const CHILD_MESSAGE: &str = "worker completed"; @@ -546,6 +555,18 @@ async fn queue_only_agent_mail_wakes_sleeping_root_with_previous_turn_context( let mut builder = test_codex() .with_model("gpt-5.4") .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + if matches!(delivery, MailDelivery::Injected) { + config + .features + .enable(Feature::MultiAgentV2) + .expect("enable native mail"); + config + .features + .disable(Feature::WakeOnAgentMessages) + .expect("wake only for sleep"); + } + }) .with_extensions(Arc::new(extensions.build())); let mut test = builder.build_with_auto_env(&server).await?; @@ -595,7 +616,22 @@ async fn queue_only_agent_mail_wakes_sleeping_root_with_previous_turn_context( ) .await?; } - enqueue_queue_only_agent_mail(&test.codex, CHILD_MESSAGE).await; + if matches!(delivery, MailDelivery::Injected) { + test.codex + .inject_response_items(vec![ + InterAgentCommunication::new( + AgentPath::try_from("/root/worker").expect("worker path"), + AgentPath::root(), + Vec::new(), + CHILD_MESSAGE.to_owned(), + /*trigger_turn*/ false, + ) + .to_model_input_item(), + ]) + .await?; + } else { + enqueue_queue_only_agent_mail(&test.codex, CHILD_MESSAGE).await; + } wait_for_turn_complete(&test.codex).await; let captured = requests.requests(); diff --git a/codex-rs/core/tests/suite/scenarios_injected_agent_mail.rs b/codex-rs/core/tests/suite/scenarios_injected_agent_mail.rs index c2ded73902..7f22668287 100644 --- a/codex-rs/core/tests/suite/scenarios_injected_agent_mail.rs +++ b/codex-rs/core/tests/suite/scenarios_injected_agent_mail.rs @@ -75,6 +75,13 @@ async fn injected_mail_respects_delivery_boundaries() -> anyhow::Result<()> { .with_config(move |config| { config.model_provider.base_url = Some(base_url); config.update_plan_enabled = true; + // Hosted MAv2.5 owns delivery even when the model selects MAv2. + config + .features + .enable(Feature::MultiAgentV2) + .expect("use MAv2"); + config.multi_agent_v2.disable_native_tools = true; + config.multi_agent_v2.multi_agent_mode_hint_text = Some(String::new()); config .features .disable(Feature::DeferMailboxPreemption) diff --git a/codex-rs/core/tests/suite/scenarios_turn_input_agent_message.rs b/codex-rs/core/tests/suite/scenarios_turn_input_agent_message.rs index 236c583de3..f52ed44a43 100644 --- a/codex-rs/core/tests/suite/scenarios_turn_input_agent_message.rs +++ b/codex-rs/core/tests/suite/scenarios_turn_input_agent_message.rs @@ -134,16 +134,16 @@ async fn agent_followup_preserves_native_boundaries( .thread; // An idle recipient must see its older queue-only mail before the new task. child - .submit(Op::InterAgentCommunication { - communication: InterAgentCommunication::new( + .inject_response_items(vec![ + InterAgentCommunication::new( AgentPath::root(), child_path.clone(), Vec::new(), "Older queued mail".to_owned(), /*trigger_turn*/ false, - ), - start_options: Default::default(), - }) + ) + .to_model_input_item(), + ]) .await?; child.submit(Op::RealtimeConversationListVoices).await?; wait_for_event(&child, |event| {