Preserve agent state when unloading idle multi-agent v2 children (#52034)

## What changed

- Route app-server idle unloading of multi-agent v2 children through local eviction that preserves unread mail and environment selections for reload.
- Hold residency pins through input submission so newly reloaded children cannot be evicted before receiving their work.
- Guard listener cleanup against replacement runtimes and subscriptions, and reject local eviction of host-controlled threads.

## Testing

Add coverage for unsubscribe-driven eviction followed by `followup_task`, verifying restored conversation context and exactly-once delivery of queued mail. Extend tests to check that residency pins block eviction and that rejected host eviction preserves the thread and mailbox.

GitOrigin-RevId: e36723730af683d2081e495e637f0e6d23548b8d
This commit is contained in:
jif
2026-10-08 09:39:33 +00:00
committed by copyberry
parent ea27864f99
commit d2a6dac20b
12 changed files with 493 additions and 124 deletions
@@ -1,8 +1,11 @@
use super::*;
use crate::extensions::send_thread_warning;
use codex_app_server_protocol::ThreadQueueChangedNotification;
use codex_core::ThreadEvictionOutcome;
use codex_extension_api::ThreadIdleCause;
use codex_protocol::config_types::MultiAgentMode;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::SessionSource;
#[derive(Clone)]
pub(super) struct ListenerTaskContext {
@@ -93,6 +96,7 @@ impl UnloadingState {
if let Some(target) = unloading_target
&& target <= Instant::now()
{
tokio::task::yield_now().await;
return true;
}
let unloading_sleep = async {
@@ -222,6 +226,10 @@ pub(super) fn log_listener_attach_result(
}
}
#[expect(
clippy::await_holding_invalid_type,
reason = "listener removal must be serialized against new subscriptions"
)]
pub(super) async fn ensure_listener_task_running(
listener_task_context: ListenerTaskContext,
conversation_id: ThreadId,
@@ -251,6 +259,8 @@ pub(super) async fn ensure_listener_task_running(
)
.await;
let config_snapshot = conversation.config_snapshot().await;
let is_v2_child = conversation.multi_agent_version() == Some(MultiAgentVersion::V2)
&& matches!(config_snapshot.session_source, SessionSource::SubAgent(_));
let thread_settings_baseline = thread_settings_from_config_snapshot(&config_snapshot);
let (mut listener_command_rx, listener_generation) = {
let mut thread_state = thread_state.lock().await;
@@ -392,6 +402,60 @@ pub(super) async fn ensure_listener_task_running(
}
pending_thread_unloads.insert(conversation_id);
}
if is_v2_child {
let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation));
tokio::pin!(eviction);
let result = match tokio::time::timeout(
Duration::from_secs(/*secs*/ 10), &mut eviction,
).await {
Ok(result) => result,
Err(_) => {
warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait");
eviction.await
}
};
let outcome = match result {
Ok(ThreadEvictionOutcome::Busy) => {
pending_thread_unloads.lock().await.remove(&conversation_id);
unloading_state.note_thread_activity_observed();
continue;
}
Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome,
Err(err) => {
warn!("failed to evict thread {conversation_id}: {err}");
pending_thread_unloads.lock().await.remove(&conversation_id);
unloading_state.note_thread_activity_observed();
continue;
}
};
{
let mut pending = pending_thread_unloads.lock().await;
if !thread_state_manager.remove_unsubscribed_listener(
conversation_id, &thread_state, listener_generation,
).await {
pending.remove(&conversation_id);
break;
}
}
// Release the status receiver before removing its watch entry.
drop(unloading_state);
outgoing_for_task
.cancel_requests_for_thread(conversation_id, /*error*/ None)
.await;
// A runtime removed by someone else may already have a replacement.
if outcome == ThreadEvictionOutcome::Evicted {
thread_watch_manager
.remove_thread(&conversation_id.to_string())
.await;
outgoing_for_task
.send_server_notification(ServerNotification::ThreadClosed(
ThreadClosedNotification { thread_id: conversation_id.to_string() },
))
.await;
}
pending_thread_unloads.lock().await.remove(&conversation_id);
return;
}
unload_thread_without_subscribers(
thread_manager.clone(),
outgoing_for_task.clone(),
+26
View File
@@ -501,6 +501,32 @@ impl ThreadStateManager {
}
}
/// Removes an unsubscribed listener's entry without clearing a replacement's state.
#[expect(
clippy::await_holding_invalid_type,
reason = "the listener generation must stay fixed while removing its map entry"
)]
pub(crate) async fn remove_unsubscribed_listener(
&self,
thread_id: ThreadId,
expected: &Arc<Mutex<ThreadState>>,
generation: u64,
) -> bool {
let mut thread_state = expected.lock().await;
let mut state = self.state.lock().await;
if thread_state.listener_generation != generation
|| !state.threads.get(&thread_id).is_some_and(|entry| {
Arc::ptr_eq(&entry.state, expected) && entry.connection_ids.is_empty()
})
{
return false;
}
state.threads.remove(&thread_id);
self.unregister_listener_command_tx(thread_id);
thread_state.clear_listener();
true
}
pub(crate) async fn clear_all_listeners(&self) {
let thread_states = {
let state = self.state.lock().await;
@@ -2,6 +2,7 @@ use anyhow::Result;
use app_test_support::MockResponsesConfig;
use app_test_support::TestAppServer;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::write_models_cache;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallParams;
@@ -28,6 +29,7 @@ use codex_app_server_protocol::ThreadUnsubscribeStatus;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::UserInput as V2UserInput;
use codex_features::Feature;
use core_test_support::responses;
use core_test_support::streaming_sse::StreamingSseChunk;
use core_test_support::streaming_sse::start_streaming_sse_server;
@@ -522,3 +524,188 @@ async fn wait_for_dynamic_tool_started(
}
}
}
#[tokio::test]
async fn v2_child_idle_eviction_preserves_mail_and_followup_context() -> Result<()> {
let server = responses::start_mock_server().await;
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri())
.with_model("gpt-5.5")
.enable_feature(Feature::MultiAgentV2)
.with_root_config("thread_unload_delay_secs = 0")
.write(codex_home.path())?;
write_models_cache(codex_home.path()).await?;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
.build_initialized()
.await?;
let parent = mcp.start_thread(ThreadStartParams::default()).await?.thread;
let mut child_id = None;
for (tool, args) in [
(
"spawn_agent",
json!({
"task_name": "worker", "fork_turns": "none", "message": "Initial child task"
}),
),
(
"send_message",
json!({"target": "worker", "message": "Unread note for later"}),
),
(
"followup_task",
json!({"target": "worker", "message": "Continue the child task"}),
),
] {
let call_id = format!("idle-eviction-{tool}");
let _call = responses::mount_sse_once_match(
&server,
wiremock::matchers::header("thread-id", parent.id.clone()),
responses::sse(vec![
responses::ev_function_call_with_namespace(
&call_id,
"collaboration",
tool,
&args.to_string(),
),
responses::ev_completed("parent-call"),
]),
)
.await;
let _parent_done = responses::mount_sse_once_match(
&server,
move |request: &wiremock::Request| {
String::from_utf8_lossy(&request.body).contains(&call_id)
},
responses::sse(vec![responses::ev_completed("parent-done")]),
)
.await;
if tool != "send_message" {
let parent_id = parent.id.clone();
let _child = responses::mount_sse_once_match(
&server,
move |request: &wiremock::Request| {
request
.headers
.get("thread-id")
.is_some_and(|id| id != parent_id.as_str())
},
responses::sse(vec![
responses::ev_assistant_message(
"child-result",
"Remember the initial child result",
),
responses::ev_completed("child-done"),
]),
)
.await;
}
timeout(
DEFAULT_READ_TIMEOUT,
mcp.start_turn_and_wait_for_completion(TurnStartParams {
thread_id: parent.id.clone(),
input: vec![V2UserInput::Text {
text: format!("Run {tool}"),
text_elements: Vec::new(),
}],
..Default::default()
}),
)
.await??;
if tool != "send_message" {
let completed = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_matching_notification(
"child turn completed",
|notification| {
notification.method == "turn/completed"
&& notification
.params
.as_ref()
.is_some_and(|params| params["threadId"] != parent.id)
},
),
)
.await??;
let params = completed.params.expect("child completion params");
let completed_child = params["threadId"].as_str().expect("child ID").to_string();
if let Some(child_id) = &child_id {
assert_eq!(&completed_child, child_id);
} else {
child_id = Some(completed_child);
}
} else {
// Mail is queued while subscribed. Unsubscribe lets the actual child listener
// evict it; the parent stays loaded so the follow-up uses the same agent tree.
let child_id = child_id.clone().expect("spawned child");
let unsubscribed: ThreadUnsubscribeResponse = mcp
.request(|request_id| ClientRequest::ThreadUnsubscribe {
request_id,
params: ThreadUnsubscribeParams {
thread_id: child_id.clone(),
},
})
.await?;
assert_eq!(unsubscribed.status, ThreadUnsubscribeStatus::Unsubscribed);
let closed: ThreadClosedNotification =
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("thread/closed")).await??;
assert_eq!(
closed,
ThreadClosedNotification {
thread_id: child_id.clone()
}
);
let loaded: ThreadLoadedListResponse = mcp
.request(|request_id| ClientRequest::ThreadLoadedList {
request_id,
params: ThreadLoadedListParams::default(),
})
.await?;
assert_eq!(loaded.data, vec![parent.id.clone()]);
assert_eq!(
responses::received_responses_requests(&server)
.await
.iter()
.filter(
|request| request.header("thread-id").as_deref() == Some(child_id.as_str())
)
.count(),
1,
"queue-only mail must not start another child turn",
);
}
}
let child_id = child_id.expect("spawned child");
let requests = responses::received_responses_requests(&server).await;
let child_requests: Vec<_> = requests
.iter()
.filter(|request| request.header("thread-id").as_deref() == Some(child_id.as_str()))
.collect();
assert_eq!(child_requests.len(), 2);
assert_eq!(
child_requests[1]
.inputs_of_type("message")
.into_iter()
.filter(|item| item["role"] == "assistant")
.collect::<Vec<_>>(),
vec![json!({
"type": "message", "role": "assistant",
"content": [{"type": "output_text", "text": "Remember the initial child result"}],
})],
);
for message in ["Unread note for later", "Continue the child task"] {
assert_eq!(
child_requests[1]
.inputs_of_type("agent_message")
.iter()
.filter(|item| item.to_string().contains(message))
.count(),
1,
"resumed child must receive {message} exactly once",
);
}
mcp.shutdown_gracefully().await?;
Ok(())
}