mirror of
https://github.com/openai/codex.git
synced 2026-11-08 20:40:33 +00:00
cassirer-openai
68b67f0ff4
## Why Reloading an evicted sub-agent disabled its rollout tracing, leaving later turns out of the trace. Replacement Code Mode sessions could also reuse cell IDs from earlier sessions. ## What changed - Reattach reloaded children to the root rollout trace and emit `ThreadResumed` without duplicating `ThreadStarted`. - Reopen existing threads during replay, preserving their identity and history while clearing stale runtime cell and terminal process mappings. - Allocate Code Mode cell generations across sessions for both gRPC and stdio hosts to prevent ID reuse after reload. - Record an aborted trace when session startup fails before the session loop begins. ## Testing Add regression coverage for child eviction and reload, including a failed reload, continued inference tracing, distinct cell IDs, and preserved mail and follow-up context. Extend replay tests for history preservation and reused runtime handles, and host tests for cell IDs across session replacement and reconnection. GitOrigin-RevId: 6c146c04b5cda9295afb7d973c0d5474b363a822
910 lines
31 KiB
Rust
910 lines
31 KiB
Rust
use anyhow::Result;
|
|
use app_test_support::MockResponsesConfig;
|
|
use app_test_support::TestAppServer;
|
|
use app_test_support::create_mock_responses_server_repeating_assistant;
|
|
use app_test_support::write_models_cache;
|
|
use codex_app_server_protocol::ClientRequest;
|
|
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
|
use codex_app_server_protocol::DynamicToolCallParams;
|
|
use codex_app_server_protocol::DynamicToolCallResponse;
|
|
use codex_app_server_protocol::DynamicToolFunctionSpec;
|
|
use codex_app_server_protocol::DynamicToolSpec;
|
|
use codex_app_server_protocol::ItemStartedNotification;
|
|
use codex_app_server_protocol::RequestId;
|
|
use codex_app_server_protocol::ServerRequest;
|
|
use codex_app_server_protocol::ThreadClosedNotification;
|
|
use codex_app_server_protocol::ThreadItemPayload;
|
|
use codex_app_server_protocol::ThreadLoadedListParams;
|
|
use codex_app_server_protocol::ThreadLoadedListResponse;
|
|
use codex_app_server_protocol::ThreadReadParams;
|
|
use codex_app_server_protocol::ThreadReadResponse;
|
|
use codex_app_server_protocol::ThreadResumeParams;
|
|
use codex_app_server_protocol::ThreadResumeResponse;
|
|
use codex_app_server_protocol::ThreadSource;
|
|
use codex_app_server_protocol::ThreadStartParams;
|
|
use codex_app_server_protocol::ThreadStartResponse;
|
|
use codex_app_server_protocol::ThreadStatus;
|
|
use codex_app_server_protocol::ThreadStatusChangedNotification;
|
|
use codex_app_server_protocol::ThreadUnsubscribeParams;
|
|
use codex_app_server_protocol::ThreadUnsubscribeResponse;
|
|
use codex_app_server_protocol::ThreadUnsubscribeStatus;
|
|
use codex_app_server_protocol::TurnStartParams;
|
|
use codex_app_server_protocol::TurnStartResponse;
|
|
use codex_app_server_protocol::UserInput as V2UserInput;
|
|
use codex_features::Feature;
|
|
use core_test_support::apps_test_server::AppsTestServer;
|
|
use core_test_support::responses;
|
|
use core_test_support::streaming_sse::StreamingSseChunk;
|
|
use core_test_support::streaming_sse::start_streaming_sse_server;
|
|
use pretty_assertions::assert_eq;
|
|
use serde_json::json;
|
|
use tempfile::TempDir;
|
|
use test_case::test_case;
|
|
use tokio::time::timeout;
|
|
|
|
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
|
#[tokio::test]
|
|
async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<()> {
|
|
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
|
let codex_home = TempDir::new()?;
|
|
MockResponsesConfig::new(&server.uri())
|
|
.with_sandbox_mode("danger-full-access")
|
|
.with_root_config("thread_unload_delay_secs = 2")
|
|
.write(codex_home.path())?;
|
|
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.build_initialized()
|
|
.await?;
|
|
|
|
let ThreadStartResponse { thread, .. } = mcp
|
|
.start_thread(ThreadStartParams {
|
|
model: Some("mock-model".to_string()),
|
|
..Default::default()
|
|
})
|
|
.await?;
|
|
let thread_id = thread.id;
|
|
|
|
// Persist a rollout so both warm and cold resumes can find the thread.
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.start_turn_and_wait_for_completion(TurnStartParams {
|
|
thread_id: thread_id.clone(),
|
|
input: vec![V2UserInput::Text {
|
|
text: "hello".to_string(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await??;
|
|
|
|
let unsubscribe: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: thread_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(unsubscribe.status, ThreadUnsubscribeStatus::Unsubscribed);
|
|
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
let ThreadLoadedListResponse { data, next_cursor } = mcp
|
|
.request(|request_id| ClientRequest::ThreadLoadedList {
|
|
request_id,
|
|
params: ThreadLoadedListParams::default(),
|
|
})
|
|
.await?;
|
|
assert_eq!(data, vec![thread_id.clone()]);
|
|
assert_eq!(next_cursor, None);
|
|
|
|
let resume: ThreadResumeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadResume {
|
|
request_id,
|
|
params: ThreadResumeParams {
|
|
thread_id: thread_id.clone(),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(resume.thread.id, thread_id);
|
|
|
|
// Resubscribing cancels the pending unload, even after the original deadline.
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(2200),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
let _: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: thread_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
// Losing the last subscriber starts a fresh countdown.
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
let closed: ThreadClosedNotification =
|
|
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("thread/closed")).await??;
|
|
assert_eq!(
|
|
closed,
|
|
ThreadClosedNotification {
|
|
thread_id: thread_id.clone()
|
|
}
|
|
);
|
|
let status = timeout(DEFAULT_READ_TIMEOUT, async {
|
|
loop {
|
|
let status: ThreadStatusChangedNotification =
|
|
mcp.read_notification("thread/status/changed").await?;
|
|
if status.status == ThreadStatus::NotLoaded {
|
|
return anyhow::Ok(status);
|
|
}
|
|
}
|
|
})
|
|
.await??;
|
|
assert_eq!(
|
|
status,
|
|
ThreadStatusChangedNotification {
|
|
thread_id: thread_id.clone(),
|
|
status: ThreadStatus::NotLoaded,
|
|
}
|
|
);
|
|
let loaded: ThreadLoadedListResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadLoadedList {
|
|
request_id,
|
|
params: ThreadLoadedListParams::default(),
|
|
})
|
|
.await?;
|
|
assert_eq!(
|
|
loaded,
|
|
ThreadLoadedListResponse {
|
|
data: Vec::new(),
|
|
next_cursor: None
|
|
}
|
|
);
|
|
|
|
let resume: ThreadResumeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadResume {
|
|
request_id,
|
|
params: ThreadResumeParams {
|
|
thread_id: thread_id.clone(),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(resume.thread.id, thread_id);
|
|
assert_eq!(resume.thread.status, ThreadStatus::Idle);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test_case(0, 0, ThreadStartParams::default(); "zero_delay")]
|
|
#[test_case(1, 1, ThreadStartParams::default(); "one_second_delay")]
|
|
#[test_case(
|
|
0,
|
|
1,
|
|
ThreadStartParams {
|
|
ephemeral: Some(true),
|
|
config: Some([("thread_unload_delay_secs".into(), json!(1))].into()),
|
|
..Default::default()
|
|
};
|
|
"per_thread_one_second_delay"
|
|
)]
|
|
#[test_case(
|
|
1800,
|
|
0,
|
|
ThreadStartParams {
|
|
ephemeral: Some(true),
|
|
config: Some([("thread_unload_delay_secs".into(), json!(0))].into()),
|
|
..Default::default()
|
|
};
|
|
"immediate_ephemeral"
|
|
)]
|
|
#[test_case(
|
|
1800,
|
|
0,
|
|
ThreadStartParams {
|
|
config: Some([("thread_unload_delay_secs".into(), json!(0))].into()),
|
|
..Default::default()
|
|
};
|
|
"immediate_persistent"
|
|
)]
|
|
#[tokio::test]
|
|
async fn thread_unsubscribe_during_turn_keeps_turn_running(
|
|
delay_secs: u64,
|
|
expected_delay_secs: u64,
|
|
thread_params: ThreadStartParams,
|
|
) -> Result<()> {
|
|
let check_cold_resume =
|
|
thread_params.config.is_some() && !thread_params.ephemeral.unwrap_or_default();
|
|
let call_id = "deterministic-wait-call";
|
|
let tool_name = "deterministic_wait";
|
|
let tool_args = json!({});
|
|
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
|
|
|
let tmp = TempDir::new()?;
|
|
let codex_home = tmp.path().join("codex_home");
|
|
std::fs::create_dir(&codex_home)?;
|
|
|
|
let (server, mut completions) = start_streaming_sse_server(vec![
|
|
vec![StreamingSseChunk {
|
|
gate: None,
|
|
body: responses::sse(vec![
|
|
responses::ev_response_created("resp-1"),
|
|
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
|
responses::ev_completed("resp-1"),
|
|
]),
|
|
}],
|
|
vec![StreamingSseChunk {
|
|
gate: None,
|
|
body: responses::sse(vec![
|
|
responses::ev_response_created("resp-2"),
|
|
responses::ev_assistant_message("msg-1", "Done"),
|
|
responses::ev_completed("resp-2"),
|
|
]),
|
|
}],
|
|
])
|
|
.await;
|
|
let first_response_completed = completions.remove(0);
|
|
let final_response_completed = completions.remove(0);
|
|
MockResponsesConfig::new(server.uri())
|
|
.with_sandbox_mode("danger-full-access")
|
|
.with_root_config(&format!("thread_unload_delay_secs = {delay_secs}"))
|
|
.write(&codex_home)?;
|
|
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(&codex_home)
|
|
.build_initialized()
|
|
.await?;
|
|
|
|
let ThreadStartResponse { thread, .. } = mcp
|
|
.start_thread(ThreadStartParams {
|
|
model: Some("mock-model".to_string()),
|
|
dynamic_tools: Some(vec![DynamicToolSpec::Function(DynamicToolFunctionSpec {
|
|
name: tool_name.to_string(),
|
|
description: "Deterministic wait tool".to_string(),
|
|
input_schema: json!({
|
|
"type": "object",
|
|
"properties": {},
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: false,
|
|
})]),
|
|
..thread_params
|
|
})
|
|
.await?;
|
|
let thread_id = thread.id;
|
|
|
|
// A subscribed, idle thread stays loaded even with no unload delay.
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
let _: TurnStartResponse = mcp
|
|
.request(|request_id| ClientRequest::TurnStart {
|
|
request_id,
|
|
params: TurnStartParams {
|
|
thread_id: thread_id.clone(),
|
|
client_user_message_id: None,
|
|
input: vec![V2UserInput::Text {
|
|
text: "run deterministic tool".to_string(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
server.wait_for_request_count(/*count*/ 1),
|
|
)
|
|
.await?;
|
|
timeout(DEFAULT_READ_TIMEOUT, first_response_completed).await??;
|
|
|
|
let started = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
wait_for_dynamic_tool_started(&mut mcp, call_id),
|
|
)
|
|
.await??;
|
|
assert_eq!(started.thread_id, thread_id);
|
|
|
|
let request = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_request_message(),
|
|
)
|
|
.await??;
|
|
let (request_id, params) = match request {
|
|
ServerRequest::DynamicToolCall { request_id, params } => (request_id, params),
|
|
other => panic!("expected DynamicToolCall request, got {other:?}"),
|
|
};
|
|
assert_eq!(
|
|
params,
|
|
DynamicToolCallParams {
|
|
thread_id: thread_id.clone(),
|
|
turn_id: started.turn_id,
|
|
call_id: call_id.to_string(),
|
|
namespace: None,
|
|
tool: tool_name.to_string(),
|
|
arguments: tool_args,
|
|
}
|
|
);
|
|
|
|
let unsubscribe: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: thread_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(unsubscribe.status, ThreadUnsubscribeStatus::Unsubscribed);
|
|
|
|
let closed_while_tool_call_blocked = timeout(
|
|
std::time::Duration::from_millis(1200),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
);
|
|
let closed_while_tool_call_blocked = closed_while_tool_call_blocked.await;
|
|
assert!(closed_while_tool_call_blocked.is_err());
|
|
|
|
let response = DynamicToolCallResponse {
|
|
content_items: vec![DynamicToolCallOutputContentItem::InputText {
|
|
text: "dynamic-ok".to_string(),
|
|
}],
|
|
success: true,
|
|
};
|
|
mcp.send_response(request_id, serde_json::to_value(response)?)
|
|
.await?;
|
|
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
server.wait_for_request_count(/*count*/ 2),
|
|
)
|
|
.await?;
|
|
timeout(DEFAULT_READ_TIMEOUT, final_response_completed).await??;
|
|
if expected_delay_secs > 0 {
|
|
// Once the turn finishes, inactivity starts a fresh countdown.
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
let closed: ThreadClosedNotification =
|
|
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("thread/closed")).await??;
|
|
assert_eq!(
|
|
closed,
|
|
ThreadClosedNotification {
|
|
thread_id: thread_id.clone()
|
|
}
|
|
);
|
|
if check_cold_resume {
|
|
// Immediate unloading preserves durable history. A cold resume uses the
|
|
// server's default delay rather than restoring the old runtime's override.
|
|
let resume: ThreadResumeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadResume {
|
|
request_id,
|
|
params: ThreadResumeParams {
|
|
thread_id: thread_id.clone(),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(resume.thread.id, thread_id);
|
|
assert_eq!(resume.thread.turns.len(), 1);
|
|
let _: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams { thread_id },
|
|
})
|
|
.await?;
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
server.shutdown().await;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn thread_unsubscribe_preserves_cached_status_before_idle_unload() -> Result<()> {
|
|
let server = responses::start_mock_server().await;
|
|
let _response_mock = responses::mount_sse_once(
|
|
&server,
|
|
responses::sse_failed("resp-1", "server_error", "simulated failure"),
|
|
)
|
|
.await;
|
|
let codex_home = TempDir::new()?;
|
|
MockResponsesConfig::new(&server.uri())
|
|
.with_sandbox_mode("danger-full-access")
|
|
.write(codex_home.path())?;
|
|
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.build_initialized()
|
|
.await?;
|
|
|
|
let ThreadStartResponse { thread, .. } = mcp
|
|
.start_thread(ThreadStartParams {
|
|
model: Some("mock-model".to_string()),
|
|
..Default::default()
|
|
})
|
|
.await?;
|
|
let thread_id = thread.id;
|
|
|
|
let _: TurnStartResponse = mcp
|
|
.request(|request_id| ClientRequest::TurnStart {
|
|
request_id,
|
|
params: TurnStartParams {
|
|
thread_id: thread_id.clone(),
|
|
client_user_message_id: None,
|
|
input: vec![V2UserInput::Text {
|
|
text: "fail this turn".to_string(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_notification_message("error"),
|
|
)
|
|
.await??;
|
|
|
|
let ThreadReadResponse { thread, .. } = mcp
|
|
.request(|request_id| ClientRequest::ThreadRead {
|
|
request_id,
|
|
params: ThreadReadParams {
|
|
thread_id: thread_id.clone(),
|
|
include_turns: false,
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(thread.status, ThreadStatus::SystemError);
|
|
|
|
let unsubscribe: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: thread_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(unsubscribe.status, ThreadUnsubscribeStatus::Unsubscribed);
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
let resume: ThreadResumeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadResume {
|
|
request_id,
|
|
params: ThreadResumeParams {
|
|
thread_id,
|
|
cwd: Some(codex_home.path().to_string_lossy().to_string()),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(resume.thread.status, ThreadStatus::SystemError);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test_case(ThreadStartParams::default(); "persistent")]
|
|
#[test_case(ThreadStartParams {
|
|
ephemeral: Some(true),
|
|
..Default::default()
|
|
}; "ephemeral")]
|
|
#[test_case(
|
|
ThreadStartParams {
|
|
thread_source: Some(ThreadSource::Feature("composer_predictions".into())),
|
|
..Default::default()
|
|
};
|
|
"persistent_composer_prediction"
|
|
)]
|
|
#[tokio::test]
|
|
async fn thread_unsubscribe_reports_not_subscribed_before_idle_unload(
|
|
thread_params: ThreadStartParams,
|
|
) -> Result<()> {
|
|
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
|
let codex_home = TempDir::new()?;
|
|
MockResponsesConfig::new(&server.uri())
|
|
.with_sandbox_mode("danger-full-access")
|
|
.write(codex_home.path())?;
|
|
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.build_initialized()
|
|
.await?;
|
|
|
|
let ThreadStartResponse { thread, .. } = mcp
|
|
.start_thread(ThreadStartParams {
|
|
model: Some("mock-model".to_string()),
|
|
..thread_params
|
|
})
|
|
.await?;
|
|
let thread_id = thread.id;
|
|
|
|
let first_unsubscribe: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: thread_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(
|
|
first_unsubscribe.status,
|
|
ThreadUnsubscribeStatus::Unsubscribed
|
|
);
|
|
|
|
let second_unsubscribe: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams { thread_id },
|
|
})
|
|
.await?;
|
|
assert_eq!(
|
|
second_unsubscribe.status,
|
|
ThreadUnsubscribeStatus::NotSubscribed
|
|
);
|
|
assert!(
|
|
timeout(
|
|
std::time::Duration::from_millis(250),
|
|
mcp.read_stream_until_notification_message("thread/closed"),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn wait_for_dynamic_tool_started(
|
|
mcp: &mut TestAppServer,
|
|
call_id: &str,
|
|
) -> Result<ItemStartedNotification> {
|
|
loop {
|
|
let notification = mcp
|
|
.read_stream_until_notification_message("item/started")
|
|
.await?;
|
|
let Some(params) = notification.params else {
|
|
continue;
|
|
};
|
|
let started: ItemStartedNotification = serde_json::from_value(params)?;
|
|
if matches!(&started.item.payload, ThreadItemPayload::DynamicToolCall { id, .. } if id == call_id)
|
|
{
|
|
return Ok(started);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn v2_child_idle_eviction_preserves_mail_and_followup_context() -> Result<()> {
|
|
let server = responses::start_mock_server().await;
|
|
let (mcp_server, startup_control) = AppsTestServer::mount_with_startup_control(&server).await?;
|
|
let codex_home = TempDir::new()?;
|
|
let traces = TempDir::new()?;
|
|
MockResponsesConfig::new(&server.uri())
|
|
.with_model("gpt-5.5")
|
|
.enable_feature(Feature::MultiAgentV2)
|
|
.enable_feature(Feature::CodeMode)
|
|
.enable_feature(Feature::CodeModeHost)
|
|
.with_root_config("thread_unload_delay_secs = 0")
|
|
.with_extra_config(&format!(
|
|
r#"[mcp_servers.fails_once]
|
|
url = "{}/api/codex/ps/mcp"
|
|
http_headers = {{ Authorization = "Bearer synthetic-test-token" }}
|
|
required = true
|
|
# No cached tools: the child must initialize instead of taking the lazy startup path.
|
|
enabled_tools = []
|
|
"#,
|
|
mcp_server.chatgpt_base_url
|
|
))
|
|
.write(codex_home.path())?;
|
|
write_models_cache(codex_home.path()).await?;
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.with_env_overrides(&[("CODEX_ROLLOUT_TRACE_ROOT", traces.path().to_str())])
|
|
.build_initialized()
|
|
.await?;
|
|
let parent = mcp.start_thread(ThreadStartParams::default()).await?.thread;
|
|
let mut child_id = None;
|
|
|
|
for (tool, args) in [
|
|
(
|
|
"spawn_agent",
|
|
json!({
|
|
"task_name": "worker", "fork_turns": "none", "message": "Initial child task"
|
|
}),
|
|
),
|
|
(
|
|
"send_message",
|
|
json!({"target": "worker", "message": "Unread note for later"}),
|
|
),
|
|
(
|
|
"followup_task",
|
|
json!({"target": "worker", "message": "Continue the child task"}),
|
|
),
|
|
] {
|
|
let call_id = format!("idle-eviction-{tool}");
|
|
let _call = responses::mount_sse_once_match(
|
|
&server,
|
|
wiremock::matchers::header("thread-id", parent.id.clone()),
|
|
responses::sse(vec![
|
|
responses::ev_function_call_with_namespace(
|
|
&call_id,
|
|
"collaboration",
|
|
tool,
|
|
&args.to_string(),
|
|
),
|
|
responses::ev_completed("parent-call"),
|
|
]),
|
|
)
|
|
.await;
|
|
let _parent_done = responses::mount_sse_once_match(
|
|
&server,
|
|
move |request: &wiremock::Request| {
|
|
String::from_utf8_lossy(&request.body).contains(&call_id)
|
|
},
|
|
responses::sse(vec![responses::ev_completed("parent-done")]),
|
|
)
|
|
.await;
|
|
if tool != "send_message" {
|
|
let exec_call_id = format!("child-exec-{tool}");
|
|
let parent_id = parent.id.clone();
|
|
let expected_call_id = exec_call_id.clone();
|
|
let _child_exec = responses::mount_sse_once_match(
|
|
&server,
|
|
move |request: &wiremock::Request| {
|
|
request
|
|
.headers
|
|
.get("thread-id")
|
|
.is_some_and(|id| id != parent_id.as_str())
|
|
&& !String::from_utf8_lossy(&request.body).contains(&expected_call_id)
|
|
},
|
|
responses::sse(vec![
|
|
responses::ev_custom_tool_call(&exec_call_id, "exec", "text('child cell')"),
|
|
responses::ev_completed("child-exec"),
|
|
]),
|
|
)
|
|
.await;
|
|
let _child = responses::mount_sse_once_match(
|
|
&server,
|
|
move |request: &wiremock::Request| {
|
|
String::from_utf8_lossy(&request.body).contains(&exec_call_id)
|
|
},
|
|
responses::sse(vec![
|
|
responses::ev_assistant_message(
|
|
&format!("child-result-{tool}"),
|
|
"Remember the initial child result",
|
|
),
|
|
responses::ev_completed("child-done"),
|
|
]),
|
|
)
|
|
.await;
|
|
}
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.start_turn_and_wait_for_completion(TurnStartParams {
|
|
thread_id: parent.id.clone(),
|
|
input: vec![V2UserInput::Text {
|
|
text: format!("Run {tool}"),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await??;
|
|
|
|
if tool != "send_message" {
|
|
let completed = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_matching_notification(
|
|
"child turn completed",
|
|
|notification| {
|
|
notification.method == "turn/completed"
|
|
&& notification
|
|
.params
|
|
.as_ref()
|
|
.is_some_and(|params| params["threadId"] != parent.id)
|
|
},
|
|
),
|
|
)
|
|
.await??;
|
|
let params = completed.params.expect("child completion params");
|
|
let completed_child = params["threadId"].as_str().expect("child ID").to_string();
|
|
if let Some(child_id) = &child_id {
|
|
assert_eq!(&completed_child, child_id);
|
|
} else {
|
|
child_id = Some(completed_child);
|
|
}
|
|
} else {
|
|
// Mail is queued while subscribed. Unsubscribe lets the actual child listener
|
|
// evict it; the parent stays loaded so the follow-up uses the same agent tree.
|
|
let child_id = child_id.clone().expect("spawned child");
|
|
let unsubscribed: ThreadUnsubscribeResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadUnsubscribe {
|
|
request_id,
|
|
params: ThreadUnsubscribeParams {
|
|
thread_id: child_id.clone(),
|
|
},
|
|
})
|
|
.await?;
|
|
assert_eq!(unsubscribed.status, ThreadUnsubscribeStatus::Unsubscribed);
|
|
let closed: ThreadClosedNotification =
|
|
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("thread/closed")).await??;
|
|
assert_eq!(
|
|
closed,
|
|
ThreadClosedNotification {
|
|
thread_id: child_id.clone()
|
|
}
|
|
);
|
|
// Close the failed reload's trace before the follow-up reloads the child.
|
|
startup_control.fail_next_initialize_attempts(/*attempts*/ 1);
|
|
let resume_id = mcp
|
|
.send_thread_resume_request(ThreadResumeParams {
|
|
thread_id: child_id.clone(),
|
|
..Default::default()
|
|
})
|
|
.await?;
|
|
let error = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_error_message(RequestId::Integer(resume_id)),
|
|
)
|
|
.await??;
|
|
assert!(
|
|
error
|
|
.error
|
|
.message
|
|
.contains("cannot resume an unloaded multi-agent v2 sub-agent"),
|
|
"unexpected error: {}",
|
|
error.error.message
|
|
);
|
|
let loaded: ThreadLoadedListResponse = mcp
|
|
.request(|request_id| ClientRequest::ThreadLoadedList {
|
|
request_id,
|
|
params: ThreadLoadedListParams::default(),
|
|
})
|
|
.await?;
|
|
assert_eq!(loaded.data, vec![parent.id.clone()]);
|
|
assert_eq!(
|
|
responses::received_responses_requests(&server)
|
|
.await
|
|
.iter()
|
|
.filter(
|
|
|request| request.header("thread-id").as_deref() == Some(child_id.as_str())
|
|
)
|
|
.count(),
|
|
2,
|
|
"queue-only mail must not start another child turn",
|
|
);
|
|
}
|
|
}
|
|
let child_id = child_id.expect("spawned child");
|
|
let requests = responses::received_responses_requests(&server).await;
|
|
let child_requests: Vec<_> = requests
|
|
.iter()
|
|
.filter(|request| request.header("thread-id").as_deref() == Some(child_id.as_str()))
|
|
.collect();
|
|
assert_eq!(child_requests.len(), 4);
|
|
assert_eq!(
|
|
child_requests[2]
|
|
.inputs_of_type("message")
|
|
.into_iter()
|
|
.filter(|item| item["role"] == "assistant")
|
|
.collect::<Vec<_>>(),
|
|
vec![json!({
|
|
"id": "child-result-spawn_agent", "type": "message", "role": "assistant",
|
|
"content": [{"type": "output_text", "text": "Remember the initial child result"}],
|
|
})],
|
|
);
|
|
for message in ["Unread note for later", "Continue the child task"] {
|
|
assert_eq!(
|
|
child_requests[2]
|
|
.inputs_of_type("agent_message")
|
|
.iter()
|
|
.filter(|item| item.to_string().contains(message))
|
|
.count(),
|
|
1,
|
|
"resumed child must receive {message} exactly once",
|
|
);
|
|
}
|
|
// Reload must keep tracing both turns under the same logical child.
|
|
let expected_calls = child_requests
|
|
.iter()
|
|
.map(|request| {
|
|
request
|
|
.header("x-codex-inference-call-id")
|
|
.expect("child inference must remain traced after eviction")
|
|
})
|
|
.collect::<Vec<_>>();
|
|
assert!(mcp.shutdown_gracefully().await?.success());
|
|
let bundles = std::fs::read_dir(traces.path())?.collect::<std::io::Result<Vec<_>>>()?;
|
|
assert_eq!(bundles.len(), 1);
|
|
let events = std::fs::read_to_string(bundles[0].path().join("trace.jsonl"))?
|
|
.lines()
|
|
.map(serde_json::from_str::<serde_json::Value>)
|
|
.collect::<serde_json::Result<Vec<_>>>()?;
|
|
let recorded_calls = events
|
|
.iter()
|
|
.map(|event| &event["payload"])
|
|
.filter(|event| event["type"] == "inference_started" && event["thread_id"] == child_id)
|
|
.map(|event| event["inference_call_id"].as_str().unwrap().to_string())
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(recorded_calls, expected_calls);
|
|
// Reloading the child replaces its Code Mode session, but must not reuse cell IDs.
|
|
let cells = events
|
|
.iter()
|
|
.filter(|event| {
|
|
event["thread_id"] == child_id && event["payload"]["type"] == "code_cell_started"
|
|
})
|
|
.map(|event| &event["payload"])
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(cells.len(), 2);
|
|
assert_ne!(cells[0]["runtime_cell_id"], cells[1]["runtime_cell_id"]);
|
|
let lifecycle = events
|
|
.iter()
|
|
.map(|event| &event["payload"])
|
|
.filter(|event| event["thread_id"] == child_id)
|
|
.filter_map(|event| {
|
|
let kind = event["type"].as_str()?;
|
|
matches!(kind, "thread_started" | "thread_ended" | "thread_resumed")
|
|
.then(|| (kind, event["status"].as_str()))
|
|
})
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(
|
|
lifecycle,
|
|
vec![
|
|
("thread_started", None),
|
|
("thread_ended", Some("completed")),
|
|
("thread_resumed", None),
|
|
("thread_ended", Some("aborted")),
|
|
("thread_resumed", None),
|
|
("thread_ended", Some("completed")),
|
|
]
|
|
);
|
|
Ok(())
|
|
}
|