Fix idle cleanup for disconnected multi-agent v2 children (#52081)

## Why

Persisted multi-agent v2 children created after the client disconnects need lifecycle listeners so their runtimes can be released once idle. Listener attachment must also wait for an earlier runtime's cleanup to finish.

## What changed

- Start lifecycle listeners for persisted v2 children even when no initialized clients are connected, and defer attachment until pending unload cleanup completes.
- Keep ephemeral children loaded so follow-up tasks retain their model context.
- Raise the default `thread_unload_delay_secs` from 60 seconds to 1800 seconds (30 minutes), updating the configuration documentation and sample.

## Testing

Add regression coverage for children spawned after disconnect with zero- and one-second unload delays, including waiting for pending cleanup, keeping running threads loaded, releasing idle runtimes, and preserving an ephemeral child's earlier reply across a follow-up. Update configuration tests for the new default and explicit overrides.

GitOrigin-RevId: 2fecc77b5c03237aa8fd7eacc4bd119b99d36e0a
This commit is contained in:
jif
2026-10-08 11:52:25 +00:00
committed by copyberry
parent be2e129ecd
commit 2fdf047c96
13 changed files with 426 additions and 26 deletions
@@ -1,4 +1,14 @@
//! Exercises thread cleanup when required MCP startup outlives its requesting connection.
//! Exercises thread cleanup when startup or child creation outlives the client connection.
use codex_app_server_protocol::TurnStartResponse;
use codex_features::Feature;
use codex_protocol::protocol::AgentStatus;
use core_test_support::responses;
use core_test_support::streaming_sse::StreamingSseChunk;
use core_test_support::streaming_sse::start_streaming_sse_server;
use pretty_assertions::assert_eq;
use test_case::test_case;
use tokio::sync::oneshot;
use super::ConnectionSessionState;
use super::MessageProcessor;
@@ -213,6 +223,293 @@ MCP_TEST_PID_FILE = {}
Ok(())
}
#[test_case(0, false; "v2_zero_delay")]
#[test_case(1, false; "v2_one_second_delay")]
#[test_case(1, true; "ephemeral_v2_followup_after_idle_deadline")]
#[tokio::test]
async fn disconnected_v2_child_lifecycle(unload_delay_secs: u64, ephemeral: bool) -> Result<()> {
let (release_spawn, spawn_gate) = oneshot::channel();
let (release_spawn_completion, spawn_completion_gate) = oneshot::channel();
let (release_first_completion, first_completion_gate) = oneshot::channel();
let (release_second_completion, second_completion_gate) = oneshot::channel();
let done = responses::sse(vec![
responses::ev_response_created("done"),
responses::ev_assistant_message("done", "Done"),
responses::ev_completed("done"),
]);
let (server, _completions) = start_streaming_sse_server(vec![
vec![
StreamingSseChunk {
gate: Some(spawn_gate),
body: responses::sse(vec![
responses::ev_response_created("spawn"),
responses::ev_function_call_with_namespace(
"spawn",
"collaboration",
"spawn_agent",
&json!({"task_name": "child", "message": "Reply done", "fork_turns": "none"})
.to_string(),
),
]),
},
StreamingSseChunk {
gate: Some(spawn_completion_gate),
body: responses::sse(vec![responses::ev_completed("spawn")]),
},
],
vec![StreamingSseChunk {
gate: Some(first_completion_gate),
body: done.clone(),
}],
vec![StreamingSseChunk {
gate: Some(second_completion_gate),
body: if ephemeral {
responses::sse(vec![
responses::ev_response_created("followup"),
responses::ev_function_call_with_namespace(
"followup",
"collaboration",
"followup_task",
&json!({"target": "child", "message": "Recall your previous reply"})
.to_string(),
),
responses::ev_completed("followup"),
])
} else {
done.clone()
},
}],
// Follow-up responses and the child's completion notification.
vec![StreamingSseChunk {
gate: None,
body: done.clone(),
}],
vec![StreamingSseChunk {
gate: None,
body: done.clone(),
}],
vec![StreamingSseChunk {
gate: None,
body: done,
}],
])
.await;
let home = TempDir::new()?;
let config = MockResponsesConfig::new(server.uri())
.with_root_config(&format!(
"thread_unload_delay_secs = {unload_delay_secs}\napprovals_reviewer = \"user\""
))
.enable_feature(Feature::Collab)
.enable_feature(Feature::MultiAgentV2);
config.write(home.path())?;
let config = Arc::new(
ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?,
);
let auth =
AuthManager::shared_from_config(config.as_ref(), /*enable_codex_api_key_env*/ false)
.await?;
let (processor, mut outgoing) = build_test_processor(config, auth).await;
let mut created = processor.thread_created_receiver();
let owner_id = ConnectionId(1);
let owner = Arc::new(ConnectionSessionState::new(ConnectionOrigin::WebSocket));
let result: Result<()> = async {
send_request(
&processor,
owner_id,
&owner,
json!({"id": 1, "method": "initialize", "params": {
"clientInfo": {"name": "parent-owner", "version": "1"},
"capabilities": {"experimentalApi": true}
}}),
)
.await;
let _: InitializeResponse =
read_response_from(&mut outgoing, owner_id, /*request_id*/ 1).await;
processor
.connection_initialized(owner_id, owner.request_attestation())
.await;
send_request(
&processor,
owner_id,
&owner,
json!({"id": 2, "method": "thread/start", "params": {"ephemeral": ephemeral}}),
)
.await;
let parent: ThreadStartResponse =
read_response_from(&mut outgoing, owner_id, /*request_id*/ 2).await;
send_request(
&processor,
owner_id,
&owner,
json!({"id": 3, "method": "turn/start", "params": {
"threadId": parent.thread.id,
"input": [{"type": "text", "text": "Spawn a child and finish"}]
}}),
)
.await;
let _: TurnStartResponse =
read_response_from(&mut outgoing, owner_id, /*request_id*/ 3).await;
timeout(
Duration::from_secs(/*secs*/ 10),
server.wait_for_request_count(/*count*/ 1),
)
.await?;
// Finish connection cleanup before the model can create a child. Dispatch the
// core creation event exactly as the app-server does with no initialized clients.
processor.connection_closed(owner_id, &owner).await;
release_spawn.send(()).expect("parent response is waiting");
let child_id = timeout(Duration::from_secs(/*secs*/ 10), created.recv()).await??;
let child_runtime = Arc::downgrade(
&processor
.thread_processor
.thread_manager
.get_thread(child_id)
.await?,
);
let attach = processor.try_attach_thread_listener(child_id, Vec::new());
tokio::pin!(attach);
// The old runtime has left core, but its app-server cleanup still owns the ID.
let pending = &processor.thread_processor.pending_thread_unloads;
pending
.lock()
.await
.insert(child_id, tokio::sync::watch::channel(()).0);
assert!(
futures::poll!(&mut attach).is_pending(),
"attachment must wait for old cleanup"
);
pending.lock().await.remove(&child_id);
attach.await;
// Keep the spawn response open until the child's model request arrives, so
// the next gated response belongs to the parent regardless of scheduling.
timeout(
Duration::from_secs(/*secs*/ 10),
server.wait_for_request_count(/*count*/ 2),
)
.await?;
release_spawn_completion
.send(())
.expect("spawn completion is waiting");
timeout(
Duration::from_secs(/*secs*/ 10),
server.wait_for_request_count(/*count*/ 3),
)
.await?;
// Installing an unsubscribed listener must not unload a still-running child.
let premature_close = timeout(Duration::from_millis(/*millis*/ 1200), async {
loop {
if let OutgoingEnvelope::Broadcast {
message: OutgoingMessage::AppServerNotification(notification),
} = outgoing.recv().await.expect("outgoing channel stays open")
&& matches!(
notification.notification,
ServerNotification::ThreadClosed(_)
)
{
break;
}
}
})
.await;
assert!(premature_close.is_err(), "running threads must stay loaded");
if ephemeral {
// Complete the child while the parent's follow-up response remains gated.
let child = processor
.thread_processor
.thread_manager
.get_thread(child_id)
.await?;
release_first_completion
.send(())
.expect("child completion is waiting");
timeout(Duration::from_secs(/*secs*/ 10), async {
while !matches!(child.agent_status().await, AgentStatus::Completed(_)) {
tokio::task::yield_now().await;
}
})
.await
.context("child did not complete")?;
tokio::time::sleep(Duration::from_secs(unload_delay_secs + 1)).await;
release_second_completion
.send(())
.expect("parent completion is waiting");
// Exercise the parent's real followup_task call and inspect the child's next
// model request: accepting a follow-up must preserve its earlier model context.
timeout(
Duration::from_secs(/*secs*/ 10),
server.wait_for_request_count(/*count*/ 5),
)
.await
.context("ephemeral child did not receive its follow-up")?;
let requests = server.requests().await;
let followup = requests[3..]
.iter()
.map(|request| serde_json::from_slice::<serde_json::Value>(request))
.collect::<std::result::Result<Vec<_>, _>>()?
.into_iter()
.find(|request| {
request["input"]
.to_string()
.contains("Recall your previous reply")
&& !request["input"].as_array().is_some_and(|input| {
input.iter().any(|item| {
item["type"] == "function_call_output" && item["call_id"] == "spawn"
})
})
})
.context("missing child follow-up request")?;
let previous_reply = followup["input"]
.as_array()
.context("follow-up input")?
.iter()
.find(|item| item["role"] == "assistant")
.context("child's previous reply must survive the idle deadline")?;
assert_eq!(
previous_reply["content"],
json!([{"type": "output_text", "text": "Done"}])
);
return Ok(());
}
release_first_completion
.send(())
.expect("first completion is waiting");
release_second_completion
.send(())
.expect("second completion is waiting");
// Check both registry cleanup and runtime release, regardless of the order
// in which the parent and child finish unloading.
timeout(Duration::from_secs(/*secs*/ 10), async {
loop {
let loaded = processor
.thread_processor
.thread_manager
.list_thread_ids()
.await;
if loaded.is_empty() && child_runtime.upgrade().is_none() {
break;
}
tokio::task::yield_now().await;
}
})
.await
.context("parent and disconnected child still retain their runtimes")?;
Ok(())
}
.await;
drop(outgoing);
processor.clear_runtime_references();
processor.shutdown_threads().await;
server.shutdown().await;
result
}
async fn send_request(
processor: &Arc<MessageProcessor>,
connection_id: ConnectionId,