mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Preserve agent state when unloading idle multi-agent v2 children (#52034)
## What changed - Route app-server idle unloading of multi-agent v2 children through local eviction that preserves unread mail and environment selections for reload. - Hold residency pins through input submission so newly reloaded children cannot be evicted before receiving their work. - Guard listener cleanup against replacement runtimes and subscriptions, and reject local eviction of host-controlled threads. ## Testing Add coverage for unsubscribe-driven eviction followed by `followup_task`, verifying restored conversation context and exactly-once delivery of queued mail. Extend tests to check that residency pins block eviction and that rejected host eviction preserves the thread and mailbox. GitOrigin-RevId: e36723730af683d2081e495e637f0e6d23548b8d
This commit is contained in:
@@ -1,8 +1,11 @@
|
||||
use super::*;
|
||||
use crate::extensions::send_thread_warning;
|
||||
use codex_app_server_protocol::ThreadQueueChangedNotification;
|
||||
use codex_core::ThreadEvictionOutcome;
|
||||
use codex_extension_api::ThreadIdleCause;
|
||||
use codex_protocol::config_types::MultiAgentMode;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct ListenerTaskContext {
|
||||
@@ -93,6 +96,7 @@ impl UnloadingState {
|
||||
if let Some(target) = unloading_target
|
||||
&& target <= Instant::now()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
return true;
|
||||
}
|
||||
let unloading_sleep = async {
|
||||
@@ -222,6 +226,10 @@ pub(super) fn log_listener_attach_result(
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "listener removal must be serialized against new subscriptions"
|
||||
)]
|
||||
pub(super) async fn ensure_listener_task_running(
|
||||
listener_task_context: ListenerTaskContext,
|
||||
conversation_id: ThreadId,
|
||||
@@ -251,6 +259,8 @@ pub(super) async fn ensure_listener_task_running(
|
||||
)
|
||||
.await;
|
||||
let config_snapshot = conversation.config_snapshot().await;
|
||||
let is_v2_child = conversation.multi_agent_version() == Some(MultiAgentVersion::V2)
|
||||
&& matches!(config_snapshot.session_source, SessionSource::SubAgent(_));
|
||||
let thread_settings_baseline = thread_settings_from_config_snapshot(&config_snapshot);
|
||||
let (mut listener_command_rx, listener_generation) = {
|
||||
let mut thread_state = thread_state.lock().await;
|
||||
@@ -392,6 +402,60 @@ pub(super) async fn ensure_listener_task_running(
|
||||
}
|
||||
pending_thread_unloads.insert(conversation_id);
|
||||
}
|
||||
if is_v2_child {
|
||||
let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation));
|
||||
tokio::pin!(eviction);
|
||||
let result = match tokio::time::timeout(
|
||||
Duration::from_secs(/*secs*/ 10), &mut eviction,
|
||||
).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait");
|
||||
eviction.await
|
||||
}
|
||||
};
|
||||
let outcome = match result {
|
||||
Ok(ThreadEvictionOutcome::Busy) => {
|
||||
pending_thread_unloads.lock().await.remove(&conversation_id);
|
||||
unloading_state.note_thread_activity_observed();
|
||||
continue;
|
||||
}
|
||||
Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome,
|
||||
Err(err) => {
|
||||
warn!("failed to evict thread {conversation_id}: {err}");
|
||||
pending_thread_unloads.lock().await.remove(&conversation_id);
|
||||
unloading_state.note_thread_activity_observed();
|
||||
continue;
|
||||
}
|
||||
};
|
||||
{
|
||||
let mut pending = pending_thread_unloads.lock().await;
|
||||
if !thread_state_manager.remove_unsubscribed_listener(
|
||||
conversation_id, &thread_state, listener_generation,
|
||||
).await {
|
||||
pending.remove(&conversation_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Release the status receiver before removing its watch entry.
|
||||
drop(unloading_state);
|
||||
outgoing_for_task
|
||||
.cancel_requests_for_thread(conversation_id, /*error*/ None)
|
||||
.await;
|
||||
// A runtime removed by someone else may already have a replacement.
|
||||
if outcome == ThreadEvictionOutcome::Evicted {
|
||||
thread_watch_manager
|
||||
.remove_thread(&conversation_id.to_string())
|
||||
.await;
|
||||
outgoing_for_task
|
||||
.send_server_notification(ServerNotification::ThreadClosed(
|
||||
ThreadClosedNotification { thread_id: conversation_id.to_string() },
|
||||
))
|
||||
.await;
|
||||
}
|
||||
pending_thread_unloads.lock().await.remove(&conversation_id);
|
||||
return;
|
||||
}
|
||||
unload_thread_without_subscribers(
|
||||
thread_manager.clone(),
|
||||
outgoing_for_task.clone(),
|
||||
|
||||
@@ -501,6 +501,32 @@ impl ThreadStateManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes an unsubscribed listener's entry without clearing a replacement's state.
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "the listener generation must stay fixed while removing its map entry"
|
||||
)]
|
||||
pub(crate) async fn remove_unsubscribed_listener(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
expected: &Arc<Mutex<ThreadState>>,
|
||||
generation: u64,
|
||||
) -> bool {
|
||||
let mut thread_state = expected.lock().await;
|
||||
let mut state = self.state.lock().await;
|
||||
if thread_state.listener_generation != generation
|
||||
|| !state.threads.get(&thread_id).is_some_and(|entry| {
|
||||
Arc::ptr_eq(&entry.state, expected) && entry.connection_ids.is_empty()
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
state.threads.remove(&thread_id);
|
||||
self.unregister_listener_command_tx(thread_id);
|
||||
thread_state.clear_listener();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) async fn clear_all_listeners(&self) {
|
||||
let thread_states = {
|
||||
let state = self.state.lock().await;
|
||||
|
||||
@@ -2,6 +2,7 @@ use anyhow::Result;
|
||||
use app_test_support::MockResponsesConfig;
|
||||
use app_test_support::TestAppServer;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::write_models_cache;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallParams;
|
||||
@@ -28,6 +29,7 @@ use codex_app_server_protocol::ThreadUnsubscribeStatus;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStartResponse;
|
||||
use codex_app_server_protocol::UserInput as V2UserInput;
|
||||
use codex_features::Feature;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::streaming_sse::StreamingSseChunk;
|
||||
use core_test_support::streaming_sse::start_streaming_sse_server;
|
||||
@@ -522,3 +524,188 @@ async fn wait_for_dynamic_tool_started(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn v2_child_idle_eviction_preserves_mail_and_followup_context() -> Result<()> {
|
||||
let server = responses::start_mock_server().await;
|
||||
let codex_home = TempDir::new()?;
|
||||
MockResponsesConfig::new(&server.uri())
|
||||
.with_model("gpt-5.5")
|
||||
.enable_feature(Feature::MultiAgentV2)
|
||||
.with_root_config("thread_unload_delay_secs = 0")
|
||||
.write(codex_home.path())?;
|
||||
write_models_cache(codex_home.path()).await?;
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.build_initialized()
|
||||
.await?;
|
||||
let parent = mcp.start_thread(ThreadStartParams::default()).await?.thread;
|
||||
let mut child_id = None;
|
||||
|
||||
for (tool, args) in [
|
||||
(
|
||||
"spawn_agent",
|
||||
json!({
|
||||
"task_name": "worker", "fork_turns": "none", "message": "Initial child task"
|
||||
}),
|
||||
),
|
||||
(
|
||||
"send_message",
|
||||
json!({"target": "worker", "message": "Unread note for later"}),
|
||||
),
|
||||
(
|
||||
"followup_task",
|
||||
json!({"target": "worker", "message": "Continue the child task"}),
|
||||
),
|
||||
] {
|
||||
let call_id = format!("idle-eviction-{tool}");
|
||||
let _call = responses::mount_sse_once_match(
|
||||
&server,
|
||||
wiremock::matchers::header("thread-id", parent.id.clone()),
|
||||
responses::sse(vec![
|
||||
responses::ev_function_call_with_namespace(
|
||||
&call_id,
|
||||
"collaboration",
|
||||
tool,
|
||||
&args.to_string(),
|
||||
),
|
||||
responses::ev_completed("parent-call"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let _parent_done = responses::mount_sse_once_match(
|
||||
&server,
|
||||
move |request: &wiremock::Request| {
|
||||
String::from_utf8_lossy(&request.body).contains(&call_id)
|
||||
},
|
||||
responses::sse(vec![responses::ev_completed("parent-done")]),
|
||||
)
|
||||
.await;
|
||||
if tool != "send_message" {
|
||||
let parent_id = parent.id.clone();
|
||||
let _child = responses::mount_sse_once_match(
|
||||
&server,
|
||||
move |request: &wiremock::Request| {
|
||||
request
|
||||
.headers
|
||||
.get("thread-id")
|
||||
.is_some_and(|id| id != parent_id.as_str())
|
||||
},
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message(
|
||||
"child-result",
|
||||
"Remember the initial child result",
|
||||
),
|
||||
responses::ev_completed("child-done"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.start_turn_and_wait_for_completion(TurnStartParams {
|
||||
thread_id: parent.id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: format!("Run {tool}"),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await??;
|
||||
|
||||
if tool != "send_message" {
|
||||
let completed = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_matching_notification(
|
||||
"child turn completed",
|
||||
|notification| {
|
||||
notification.method == "turn/completed"
|
||||
&& notification
|
||||
.params
|
||||
.as_ref()
|
||||
.is_some_and(|params| params["threadId"] != parent.id)
|
||||
},
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
let params = completed.params.expect("child completion params");
|
||||
let completed_child = params["threadId"].as_str().expect("child ID").to_string();
|
||||
if let Some(child_id) = &child_id {
|
||||
assert_eq!(&completed_child, child_id);
|
||||
} else {
|
||||
child_id = Some(completed_child);
|
||||
}
|
||||
} else {
|
||||
// Mail is queued while subscribed. Unsubscribe lets the actual child listener
|
||||
// evict it; the parent stays loaded so the follow-up uses the same agent tree.
|
||||
let child_id = child_id.clone().expect("spawned child");
|
||||
let unsubscribed: ThreadUnsubscribeResponse = mcp
|
||||
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
||||
request_id,
|
||||
params: ThreadUnsubscribeParams {
|
||||
thread_id: child_id.clone(),
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(unsubscribed.status, ThreadUnsubscribeStatus::Unsubscribed);
|
||||
let closed: ThreadClosedNotification =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("thread/closed")).await??;
|
||||
assert_eq!(
|
||||
closed,
|
||||
ThreadClosedNotification {
|
||||
thread_id: child_id.clone()
|
||||
}
|
||||
);
|
||||
let loaded: ThreadLoadedListResponse = mcp
|
||||
.request(|request_id| ClientRequest::ThreadLoadedList {
|
||||
request_id,
|
||||
params: ThreadLoadedListParams::default(),
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(loaded.data, vec![parent.id.clone()]);
|
||||
assert_eq!(
|
||||
responses::received_responses_requests(&server)
|
||||
.await
|
||||
.iter()
|
||||
.filter(
|
||||
|request| request.header("thread-id").as_deref() == Some(child_id.as_str())
|
||||
)
|
||||
.count(),
|
||||
1,
|
||||
"queue-only mail must not start another child turn",
|
||||
);
|
||||
}
|
||||
}
|
||||
let child_id = child_id.expect("spawned child");
|
||||
let requests = responses::received_responses_requests(&server).await;
|
||||
let child_requests: Vec<_> = requests
|
||||
.iter()
|
||||
.filter(|request| request.header("thread-id").as_deref() == Some(child_id.as_str()))
|
||||
.collect();
|
||||
assert_eq!(child_requests.len(), 2);
|
||||
assert_eq!(
|
||||
child_requests[1]
|
||||
.inputs_of_type("message")
|
||||
.into_iter()
|
||||
.filter(|item| item["role"] == "assistant")
|
||||
.collect::<Vec<_>>(),
|
||||
vec![json!({
|
||||
"type": "message", "role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Remember the initial child result"}],
|
||||
})],
|
||||
);
|
||||
for message in ["Unread note for later", "Continue the child task"] {
|
||||
assert_eq!(
|
||||
child_requests[1]
|
||||
.inputs_of_type("agent_message")
|
||||
.iter()
|
||||
.filter(|item| item.to_string().contains(message))
|
||||
.count(),
|
||||
1,
|
||||
"resumed child must receive {message} exactly once",
|
||||
);
|
||||
}
|
||||
mcp.shutdown_gracefully().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user