mirror of
https://github.com/openai/codex.git
synced 2026-09-10 20:26:47 +00:00
1e66885a16
## Why A code-mode `exec` wrapper and its nested tool calls each advance Guardian's score lag. Counting the unscored wrapper against its own nested approval can trigger a synchronous review sooner than needed. ## What changed Track originating response items and approval tool-call IDs so cached approvals discount their own wrapper when it is newer than the latest score. Preserve wrapper identity across yields and waits, and retain the full lag when provenance is missing or evicted. Other wrappers and scoring failures still count. Use the current `write_stdin` call's identity for approval freshness, and omit tool-call identity for `execve` intercepts that only retain the launch ID. ## Testing Add coverage for wrapper-specific discounts, already-scored wrappers, missing and evicted provenance, identity across code-mode resumes, and approval identities for stdin writes and `execve` intercepts. GitOrigin-RevId: 4002a06d64c4e1e0a13c072303503d6ddfacad09
6598 lines
224 KiB
Rust
6598 lines
224 KiB
Rust
#![allow(clippy::unwrap_used)]
|
|
|
|
use anyhow::Result;
|
|
use base64::Engine;
|
|
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
|
use codex_config::types::McpServerConfig;
|
|
use codex_config::types::McpServerTransportConfig;
|
|
use codex_core::StartThreadOptions;
|
|
use codex_core::TurnInputRequest;
|
|
use codex_core::config::Config;
|
|
use codex_core::config::Constrained;
|
|
use codex_core::config::CurrentTimeReminderConfig;
|
|
use codex_core::context::NodeReplReviewEvidence;
|
|
use codex_extension_api::ExtensionData;
|
|
use codex_extension_api::ExtensionRegistryBuilder;
|
|
use codex_extension_api::ToolCallOutcome;
|
|
use codex_extension_api::ToolContributor;
|
|
use codex_extension_api::ToolFinishInput;
|
|
use codex_extension_api::ToolLifecycleContributor;
|
|
use codex_extension_api::ToolLifecycleFuture;
|
|
use codex_extension_api::ToolStartInput;
|
|
use codex_features::CurrentTimeSource;
|
|
use codex_features::Feature;
|
|
use codex_login::CodexAuth;
|
|
use codex_models_manager::bundled_models_response;
|
|
use codex_protocol::config_types::ApprovalsReviewer;
|
|
use codex_protocol::config_types::CollaborationMode;
|
|
use codex_protocol::config_types::ModeKind;
|
|
use codex_protocol::config_types::Settings;
|
|
use codex_protocol::config_types::WebSearchMode;
|
|
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
|
|
use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;
|
|
use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
|
|
use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
|
|
use codex_protocol::dynamic_tools::DynamicToolResponse;
|
|
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
|
use codex_protocol::models::ContentItem;
|
|
use codex_protocol::models::ImageDetail;
|
|
use codex_protocol::models::PermissionProfile;
|
|
use codex_protocol::models::ResponseItem;
|
|
use codex_protocol::openai_models::InputModality;
|
|
use codex_protocol::openai_models::ToolMode;
|
|
use codex_protocol::protocol::AskForApproval;
|
|
use codex_protocol::protocol::EventMsg;
|
|
use codex_protocol::protocol::Op;
|
|
use codex_protocol::protocol::ThreadSettingsOverrides;
|
|
use codex_protocol::user_input::UserInput;
|
|
use codex_tools::FreeformTool;
|
|
use codex_tools::FreeformToolFormat;
|
|
use codex_tools::FunctionCallError;
|
|
use codex_tools::JsonToolOutput;
|
|
use codex_tools::ResponsesApiNamespace;
|
|
use codex_tools::ResponsesApiNamespaceTool;
|
|
use codex_tools::ToolCall;
|
|
use codex_tools::ToolExecutor;
|
|
use codex_tools::ToolExecutorFuture;
|
|
use codex_tools::ToolName;
|
|
use codex_tools::ToolOutput;
|
|
use codex_tools::ToolPayload;
|
|
use codex_tools::ToolSpec;
|
|
use codex_web_search_extension::install as install_web_search_extension;
|
|
use core_test_support::apps_test_server::AppsTestServer;
|
|
use core_test_support::apps_test_server::AppsTestToolLoading;
|
|
use core_test_support::apps_test_server::DIRECT_CALENDAR_APP_ONLY_TOOL;
|
|
use core_test_support::apps_test_server::recorded_apps_tool_calls;
|
|
use core_test_support::apps_test_server::search_capable_apps_builder;
|
|
use core_test_support::assert_regex_match;
|
|
use core_test_support::responses;
|
|
use core_test_support::responses::ResponseMock;
|
|
use core_test_support::responses::ResponsesRequest;
|
|
use core_test_support::responses::ev_assistant_message;
|
|
use core_test_support::responses::ev_completed;
|
|
use core_test_support::responses::ev_custom_tool_call;
|
|
use core_test_support::responses::ev_response_created;
|
|
use core_test_support::responses::namespace_child_tool;
|
|
use core_test_support::responses::sse;
|
|
use core_test_support::skip_if_no_network;
|
|
use core_test_support::skip_if_sandbox;
|
|
use core_test_support::skip_if_wine_exec;
|
|
use core_test_support::stdio_server_bin;
|
|
use core_test_support::test_codex::TestCodex;
|
|
use core_test_support::test_codex::TestCodexBuilder;
|
|
use core_test_support::test_codex::test_codex;
|
|
use core_test_support::test_codex::turn_permission_fields;
|
|
use core_test_support::wait_for_event;
|
|
use core_test_support::wait_for_event_match;
|
|
use core_test_support::wait_for_mcp_server;
|
|
use image::DynamicImage;
|
|
use image::GenericImageView;
|
|
use image::ImageBuffer;
|
|
use image::ImageDecoder;
|
|
use image::ImageEncoder;
|
|
use image::ImageFormat;
|
|
use image::ImageReader;
|
|
use image::Rgba;
|
|
use image::codecs::png::PngEncoder;
|
|
use image::metadata::Orientation;
|
|
use pretty_assertions::assert_eq;
|
|
use serde_json::Value;
|
|
use std::collections::HashMap;
|
|
use std::fs;
|
|
use std::io::Cursor;
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
use std::sync::Mutex;
|
|
use std::sync::atomic::AtomicUsize;
|
|
use std::sync::atomic::Ordering;
|
|
use std::time::Duration;
|
|
use std::time::Instant;
|
|
use test_case::test_case;
|
|
use tokio::sync::oneshot;
|
|
use wiremock::Mock;
|
|
use wiremock::MockServer;
|
|
use wiremock::ResponseTemplate;
|
|
use wiremock::matchers::method;
|
|
use wiremock::matchers::path;
|
|
|
|
use super::rmcp_client::remote_aware_environment_id;
|
|
use super::rmcp_client::remote_aware_stdio_server_bin;
|
|
|
|
fn custom_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> {
|
|
match req.custom_tool_call_output(call_id).get("output") {
|
|
Some(Value::Array(items)) => items.clone(),
|
|
Some(Value::String(text)) => {
|
|
vec![serde_json::json!({ "type": "input_text", "text": text })]
|
|
}
|
|
_ => panic!("custom tool output should be serialized as text or content items"),
|
|
}
|
|
}
|
|
|
|
fn tool_names(body: &Value) -> Vec<String> {
|
|
body.get("tools")
|
|
.and_then(Value::as_array)
|
|
.map(|tools| {
|
|
tools
|
|
.iter()
|
|
.filter_map(|tool| {
|
|
tool.get("name")
|
|
.or_else(|| tool.get("type"))
|
|
.and_then(Value::as_str)
|
|
.map(str::to_string)
|
|
})
|
|
.collect()
|
|
})
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn function_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> {
|
|
match req.function_call_output(call_id).get("output") {
|
|
Some(Value::Array(items)) => items.clone(),
|
|
Some(Value::String(text)) => {
|
|
vec![serde_json::json!({ "type": "input_text", "text": text })]
|
|
}
|
|
_ => panic!("function tool output should be serialized as text or content items"),
|
|
}
|
|
}
|
|
|
|
fn text_item(items: &[Value], index: usize) -> &str {
|
|
items[index]
|
|
.get("text")
|
|
.and_then(Value::as_str)
|
|
.expect("content item should be input_text")
|
|
}
|
|
|
|
fn extract_running_cell_id(text: &str) -> String {
|
|
text.strip_prefix("Script running with cell ID ")
|
|
.and_then(|rest| rest.split('\n').next())
|
|
.expect("running header should contain a cell ID")
|
|
.to_string()
|
|
}
|
|
|
|
fn wait_for_file_source(path: &Path) -> Result<String> {
|
|
let quoted_path = shlex::try_join([path.to_string_lossy().as_ref()])?;
|
|
let command = format!("if [ -f {quoted_path} ]; then printf ready; fi");
|
|
Ok(format!(
|
|
r#"while ((await tools.exec_command({{ cmd: {command:?} }})).output !== "ready") {{
|
|
}}"#
|
|
))
|
|
}
|
|
|
|
fn custom_tool_output_body_and_success(
|
|
req: &ResponsesRequest,
|
|
call_id: &str,
|
|
) -> (String, Option<bool>) {
|
|
let (content, success) = req
|
|
.custom_tool_call_output_content_and_success(call_id)
|
|
.expect("custom tool output should be present");
|
|
let items = custom_tool_output_items(req, call_id);
|
|
let text_items = items
|
|
.iter()
|
|
.filter_map(|item| item.get("text").and_then(Value::as_str))
|
|
.collect::<Vec<_>>();
|
|
let output = match text_items.as_slice() {
|
|
[] => content.unwrap_or_default(),
|
|
[only] => (*only).to_string(),
|
|
[_, rest @ ..] => rest.concat(),
|
|
};
|
|
(output, success)
|
|
}
|
|
|
|
fn custom_tool_output_last_non_empty_text(req: &ResponsesRequest, call_id: &str) -> Option<String> {
|
|
match req.custom_tool_call_output(call_id).get("output") {
|
|
Some(Value::String(text)) if !text.trim().is_empty() => Some(text.clone()),
|
|
Some(Value::Array(items)) => items
|
|
.iter()
|
|
.filter_map(|item| item.get("text").and_then(Value::as_str))
|
|
.rfind(|text| !text.trim().is_empty())
|
|
.map(str::to_string),
|
|
Some(Value::String(_))
|
|
| Some(Value::Object(_))
|
|
| Some(Value::Number(_))
|
|
| Some(Value::Bool(_))
|
|
| Some(Value::Null)
|
|
| None => None,
|
|
}
|
|
}
|
|
|
|
async fn run_code_mode_turn(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_config(server, prompt, code, |_| {}).await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
configure: impl FnOnce(&mut Config) + Send + 'static,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_model_and_config(server, prompt, code, "test-gpt-5.1-codex", configure)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_model_and_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
configure: impl FnOnce(&mut Config) + Send + 'static,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
let builder = test_codex().with_model(model).with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
|
|
configure(config);
|
|
});
|
|
run_code_mode_turn_with_builder(server, prompt, code, builder).await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_builder(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
mut builder: TestCodexBuilder,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
let test = builder.build(server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn(prompt).await?;
|
|
Ok((test, second_mock))
|
|
}
|
|
|
|
async fn run_unavailable_code_mode_turn(
|
|
server: &MockServer,
|
|
test: &TestCodex,
|
|
) -> Result<(Value, Vec<String>)> {
|
|
let response_mock = responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.codex
|
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
|
text: "list available tools".to_string(),
|
|
text_elements: Vec::new(),
|
|
}]))
|
|
.await?;
|
|
|
|
let mut warnings = Vec::new();
|
|
loop {
|
|
match wait_for_event(&test.codex, |_| true).await {
|
|
EventMsg::Warning(warning) => warnings.push(warning.message),
|
|
EventMsg::TurnComplete(_) => break,
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
Ok((response_mock.single_request().body_json(), warnings))
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn missing_process_host_falls_back_to_direct_tools_and_warns_once() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_code_mode_host_program("codex-code-mode-host-does-not-exist".into())
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
let (first_body, first_warnings) = run_unavailable_code_mode_turn(&server, &test).await?;
|
|
let first_tools = tool_names(&first_body);
|
|
assert!(
|
|
first_tools
|
|
.iter()
|
|
.all(|name| name != "exec" && name != "wait"),
|
|
"unavailable code mode must not expose code-mode tools: {first_tools:?}"
|
|
);
|
|
assert!(
|
|
first_warnings.iter().any(|warning| {
|
|
warning.contains("Code Mode is unavailable")
|
|
&& warning.contains("codex-code-mode-host-does-not-exist")
|
|
}),
|
|
"missing host should produce an actionable warning: {first_warnings:?}"
|
|
);
|
|
|
|
let (_, second_warnings) = run_unavailable_code_mode_turn(&server, &test).await?;
|
|
assert!(
|
|
second_warnings
|
|
.iter()
|
|
.all(|warning| !warning.contains("Code Mode is unavailable")),
|
|
"host availability warning should be emitted once per thread: {second_warnings:?}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn missing_process_host_keeps_code_mode_only_and_fails_closed() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_code_mode_host_program("codex-code-mode-host-does-not-exist".into())
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("code mode should be enabled");
|
|
});
|
|
let (_test, follow_up_mock) = run_code_mode_turn_with_builder(
|
|
&server,
|
|
"Run required code mode",
|
|
"text('unreachable')",
|
|
builder,
|
|
)
|
|
.await?;
|
|
let request = follow_up_mock.single_request();
|
|
let tools = tool_names(&request.body_json());
|
|
assert!(
|
|
tools.iter().any(|name| name == "exec") && tools.iter().any(|name| name == "wait"),
|
|
"code-mode-only must retain code-mode tools: {tools:?}"
|
|
);
|
|
assert!(
|
|
tools
|
|
.iter()
|
|
.all(|name| !matches!(name.as_str(), "shell" | "exec_command")),
|
|
"code-mode-only must never expose direct shell tools: {tools:?}"
|
|
);
|
|
let (output, _) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert!(
|
|
output.contains("codex-code-mode-host-does-not-exist"),
|
|
"code-mode-only must report the host failure: {output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn missing_process_host_fails_closed_when_direct_fallback_is_disabled() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_code_mode_host_program("codex-code-mode-host-does-not-exist".into())
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
config.code_mode.disable_in_process_fallback = true;
|
|
});
|
|
let (_test, follow_up_mock) = run_code_mode_turn_with_builder(
|
|
&server,
|
|
"Run required code mode",
|
|
"text('unreachable')",
|
|
builder,
|
|
)
|
|
.await?;
|
|
let request = follow_up_mock.single_request();
|
|
let tools = tool_names(&request.body_json());
|
|
assert!(
|
|
tools.iter().any(|name| name == "exec") && tools.iter().any(|name| name == "wait"),
|
|
"disabled fallback must retain code-mode tools: {tools:?}"
|
|
);
|
|
let (output, _) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert!(
|
|
output.contains("codex-code-mode-host-does-not-exist"),
|
|
"disabled fallback must report the host failure: {output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn disabled_process_host_with_fallback_disabled_attempts_the_host() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
config
|
|
.features
|
|
.disable(Feature::CodeModeHost)
|
|
.expect("code-mode host should be disabled");
|
|
config.code_mode.disable_in_process_fallback = true;
|
|
});
|
|
let (_test, follow_up_mock) = run_code_mode_turn_with_builder(
|
|
&server,
|
|
"Run required code mode",
|
|
"text('unreachable')",
|
|
builder,
|
|
)
|
|
.await?;
|
|
let request = follow_up_mock.single_request();
|
|
let (output, _) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert!(
|
|
output.contains("failed to spawn code-mode host"),
|
|
"disabled fallback must still attempt the standalone host: {output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_standalone_web_search() -> Result<()> {
|
|
assert_code_mode_standalone_web_search(WebSearchMode::Live, serde_json::json!(true)).await
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_indexed_standalone_web_search() -> Result<()> {
|
|
assert_code_mode_standalone_web_search(WebSearchMode::Indexed, serde_json::json!("indexed"))
|
|
.await
|
|
}
|
|
|
|
async fn assert_code_mode_standalone_web_search(
|
|
web_search_mode: WebSearchMode,
|
|
expected_external_web_access: Value,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
Mock::given(method("POST"))
|
|
.and(path("/v1/alpha/search"))
|
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
|
"output": "Search result",
|
|
})))
|
|
.expect(1)
|
|
.mount(&server)
|
|
.await;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const result = await tools.web__run({
|
|
search_query: [{ q: "standalone web search" }],
|
|
});
|
|
text(result);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let auth = CodexAuth::from_api_key("dummy");
|
|
let auth_manager = codex_core::test_support::auth_manager_from_auth(auth.clone());
|
|
let mut extension_builder = ExtensionRegistryBuilder::<Config>::new();
|
|
install_web_search_extension(&mut extension_builder, auth_manager);
|
|
let mut builder = test_codex()
|
|
.with_auth(auth)
|
|
.with_extensions(Arc::new(extension_builder.build()))
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
config
|
|
.features
|
|
.enable(Feature::StandaloneWebSearch)
|
|
.expect("standalone web search should be enabled");
|
|
config
|
|
.web_search_mode
|
|
.set(web_search_mode)
|
|
.expect("web search mode should be accepted");
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
test.submit_turn("Search the web from code mode").await?;
|
|
|
|
let search_request = server
|
|
.received_requests()
|
|
.await
|
|
.expect("received requests should be available")
|
|
.into_iter()
|
|
.find(|request| request.url.path() == "/v1/alpha/search")
|
|
.expect("standalone search request should be sent");
|
|
let search_body = search_request
|
|
.body_json::<Value>()
|
|
.expect("search request body should be JSON");
|
|
assert_eq!(
|
|
search_body["model"],
|
|
serde_json::json!("test-gpt-5.1-codex")
|
|
);
|
|
assert_eq!(
|
|
search_body["commands"],
|
|
serde_json::json!({
|
|
"search_query": [{"q": "standalone web search"}],
|
|
})
|
|
);
|
|
assert_eq!(
|
|
search_body["settings"],
|
|
serde_json::json!({
|
|
"allowed_callers": ["direct"],
|
|
"external_web_access": expected_external_web_access,
|
|
})
|
|
);
|
|
assert_eq!(
|
|
custom_tool_output_last_non_empty_text(&follow_up_mock.single_request(), "call-1"),
|
|
Some("Search result".to_string())
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_model(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_config(
|
|
server, prompt, code, model, /*code_mode_only*/ false,
|
|
/*non_prefixed_mcp_tool_names*/ false,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_mode(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
code_mode_only: bool,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_config(
|
|
server,
|
|
prompt,
|
|
code,
|
|
"test-gpt-5.1-codex",
|
|
code_mode_only,
|
|
/*non_prefixed_mcp_tool_names*/ false,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
code_mode_only: bool,
|
|
non_prefixed_mcp_tool_names: bool,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
let rmcp_test_server_bin = stdio_server_bin()?;
|
|
let mut builder = test_codex().with_model(model).with_config(move |config| {
|
|
let _ = if code_mode_only {
|
|
config.features.enable(Feature::CodeModeOnly)
|
|
} else {
|
|
config.features.enable(Feature::CodeMode)
|
|
};
|
|
if non_prefixed_mcp_tool_names {
|
|
let _ = config.features.enable(Feature::NonPrefixedMcpToolNames);
|
|
}
|
|
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
McpServerConfig {
|
|
auth: Default::default(),
|
|
transport: McpServerTransportConfig::Stdio {
|
|
command: rmcp_test_server_bin,
|
|
args: Vec::new(),
|
|
env: Some(HashMap::from([(
|
|
"MCP_TEST_VALUE".to_string(),
|
|
"propagated-env".to_string(),
|
|
)])),
|
|
env_vars: Vec::new(),
|
|
cwd: None,
|
|
},
|
|
environment_id: "local".to_string(),
|
|
enabled: true,
|
|
required: false,
|
|
supports_parallel_tool_calls: false,
|
|
omit_tools_from: None,
|
|
disabled_reason: None,
|
|
startup_timeout_sec: Some(Duration::from_secs(10)),
|
|
tool_timeout_sec: None,
|
|
default_tools_approval_mode: None,
|
|
enabled_tools: None,
|
|
disabled_tools: None,
|
|
scopes: None,
|
|
oauth: None,
|
|
oauth_resource: None,
|
|
tools: HashMap::new(),
|
|
},
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test mcp servers should accept any configuration");
|
|
});
|
|
let test = builder.build(server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
|
|
responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn(prompt).await?;
|
|
Ok((test, second_mock))
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_return_exec_command_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to run exec_command",
|
|
r#"
|
|
text(JSON.stringify(await tools.exec_command({ cmd: "printf code_mode_exec_marker" })));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
let parsed: Value = serde_json::from_str(text_item(&items, /*index*/ 1))?;
|
|
assert!(
|
|
parsed
|
|
.get("chunk_id")
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|chunk_id| !chunk_id.is_empty())
|
|
);
|
|
assert_eq!(
|
|
parsed.get("output").and_then(Value::as_str),
|
|
Some("code_mode_exec_marker"),
|
|
);
|
|
assert_eq!(parsed.get("exit_code").and_then(Value::as_i64), Some(0));
|
|
assert!(parsed.get("wall_time_seconds").is_some());
|
|
assert!(parsed.get("session_id").is_none());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn code_mode_exec_holds_captured_result_during_elicitation() -> Result<()> {
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", "text('captured');"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(
|
|
test.codex.increment_out_of_band_elicitation_count().await?,
|
|
1
|
|
);
|
|
assert_eq!(
|
|
test.codex.increment_out_of_band_elicitation_count().await?,
|
|
2
|
|
);
|
|
let release_elicitation = async {
|
|
tokio::time::timeout(Duration::from_secs(5), async {
|
|
while first_mock.requests().is_empty() {
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("initial response request should arrive");
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
assert!(
|
|
second_mock.requests().is_empty(),
|
|
"captured exec result should not return during an elicitation"
|
|
);
|
|
assert_eq!(
|
|
test.codex.decrement_out_of_band_elicitation_count().await?,
|
|
1
|
|
);
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
assert!(
|
|
second_mock.requests().is_empty(),
|
|
"captured exec result should wait for every elicitation"
|
|
);
|
|
assert_eq!(
|
|
test.codex.decrement_out_of_band_elicitation_count().await?,
|
|
0
|
|
);
|
|
Ok::<(), anyhow::Error>(())
|
|
};
|
|
|
|
tokio::try_join!(test.submit_turn("run code mode"), release_elicitation)?;
|
|
second_mock.single_request();
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_restricts_prompt_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let resp_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("list tools in code mode only").await?;
|
|
|
|
let first_body = resp_mock.single_request().body_json();
|
|
assert_eq!(
|
|
tool_names(&first_body),
|
|
vec![
|
|
"exec".to_string(),
|
|
"wait".to_string(),
|
|
"request_user_input".to_string(),
|
|
"web_search".to_string()
|
|
]
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_excludes_mcp_servers_using_their_configured_identity() -> Result<()> {
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
for omit_tools_from in [
|
|
&["code_mode"][..],
|
|
&["deferred"][..],
|
|
&["direct"][..],
|
|
&["code_mode", "deferred"][..],
|
|
&["code_mode", "direct"][..],
|
|
&["deferred", "direct"][..],
|
|
&["code_mode", "deferred", "direct"][..],
|
|
&[][..],
|
|
] {
|
|
for (code_mode_only, supports_search_tool, non_prefixed_mcp_tool_names) in [
|
|
(false, false, false),
|
|
(false, false, true),
|
|
(false, true, false),
|
|
(false, true, true),
|
|
(true, false, false),
|
|
(true, false, true),
|
|
(true, true, false),
|
|
(true, true, true),
|
|
] {
|
|
let server = responses::start_mock_server().await;
|
|
let response = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
|
let environment_id = remote_aware_environment_id();
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", move |model| {
|
|
model.supports_search_tool = supports_search_tool;
|
|
})
|
|
.with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(if code_mode_only {
|
|
Feature::CodeModeOnly
|
|
} else {
|
|
Feature::CodeMode
|
|
})
|
|
.expect("test config should allow Code Mode");
|
|
if non_prefixed_mcp_tool_names {
|
|
config
|
|
.features
|
|
.enable(Feature::NonPrefixedMcpToolNames)
|
|
.expect("test config should allow unprefixed MCP tools");
|
|
}
|
|
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
serde_json::from_value(serde_json::json!({
|
|
"command": rmcp_test_server_bin,
|
|
"environment_id": environment_id,
|
|
"cwd": config.cwd,
|
|
"omit_tools_from": omit_tools_from,
|
|
}))
|
|
.expect("test MCP server config should be valid"),
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test config should allow MCP servers");
|
|
});
|
|
let base_test = builder.build_with_auto_env(&server).await?;
|
|
let namespace = if non_prefixed_mcp_tool_names {
|
|
"rmcp"
|
|
} else {
|
|
"mcp__rmcp"
|
|
};
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread(StartThreadOptions {
|
|
dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
|
name: namespace.to_string(),
|
|
description: "An unrelated dynamic tool sharing the MCP namespace."
|
|
.to_string(),
|
|
tools: vec![DynamicToolNamespaceTool::Function(
|
|
DynamicToolFunctionSpec {
|
|
name: "unrelated".to_string(),
|
|
description: "A dynamic tool that must remain in Code Mode."
|
|
.to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {},
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: false,
|
|
},
|
|
)],
|
|
})],
|
|
..StartThreadOptions::new(base_test.config.clone())
|
|
})
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
test.submit_turn("inspect the directly callable MCP tool")
|
|
.await?;
|
|
|
|
let body = response.single_request().body_json();
|
|
let omit_code_mode = omit_tools_from.contains(&"code_mode");
|
|
let omit_deferred = omit_tools_from.contains(&"deferred");
|
|
let omit_direct = omit_tools_from.contains(&"direct");
|
|
let available_directly = !omit_direct
|
|
&& (!supports_search_tool || omit_deferred || code_mode_only && omit_code_mode);
|
|
let visible_directly = available_directly && (!code_mode_only || omit_code_mode);
|
|
let echo_tool = namespace_child_tool(&body, namespace, "echo");
|
|
assert_eq!(
|
|
echo_tool.is_some(),
|
|
visible_directly,
|
|
"MCP omissions must determine initial prompt exposure; \
|
|
omit_tools_from={omit_tools_from:?}, code_mode_only={code_mode_only}, \
|
|
supports_search_tool={supports_search_tool}, namespace={namespace}, tools={:?}",
|
|
tool_names(&body)
|
|
);
|
|
if let Some(echo_tool) = echo_tool {
|
|
assert_eq!(
|
|
echo_tool["description"]
|
|
.as_str()
|
|
.is_some_and(|description| description.contains("exec tool declaration:")),
|
|
!omit_code_mode,
|
|
"the MCP tool description must agree with Code Mode availability; \
|
|
omit_tools_from={omit_tools_from:?}, code_mode_only={code_mode_only}, \
|
|
namespace={namespace}"
|
|
);
|
|
}
|
|
|
|
let exec_description = body["tools"]
|
|
.as_array()
|
|
.expect("request should contain tools")
|
|
.iter()
|
|
.find_map(|tool| {
|
|
(tool["name"].as_str() == Some("exec"))
|
|
.then(|| tool["description"].as_str())
|
|
.flatten()
|
|
})
|
|
.expect("Code Mode exec should remain available");
|
|
let nested_tool_name = format!("{namespace}__echo");
|
|
if code_mode_only {
|
|
assert_eq!(
|
|
exec_description.contains(&nested_tool_name),
|
|
!omit_code_mode && (!supports_search_tool || omit_deferred),
|
|
"the Code Mode declaration must agree with MCP tool exposure; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
nested_tool_name={nested_tool_name}"
|
|
);
|
|
}
|
|
let unrelated_nested_tool_name = format!("{namespace}__unrelated");
|
|
if code_mode_only {
|
|
assert!(
|
|
exec_description.contains(&unrelated_nested_tool_name),
|
|
"MCP omissions must not exclude unrelated tools sharing its namespace; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
nested_tool_name={unrelated_nested_tool_name}"
|
|
);
|
|
} else {
|
|
let unrelated_tool = namespace_child_tool(&body, namespace, "unrelated")
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"MCP omissions must not hide an unrelated same-namespace tool; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, namespace={namespace}, \
|
|
tools={:?}",
|
|
tool_names(&body)
|
|
)
|
|
});
|
|
assert!(
|
|
unrelated_tool["description"]
|
|
.as_str()
|
|
.is_some_and(|description| {
|
|
description.contains(&unrelated_nested_tool_name)
|
|
&& description.contains("exec tool declaration:")
|
|
}),
|
|
"an unrelated same-namespace tool must remain available inside Code Mode; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
nested_tool_name={unrelated_nested_tool_name}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn mcp_code_mode_exclusion_does_not_change_direct_mode_tool_exposure() -> Result<()> {
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
for omit_tools_from in [
|
|
&["code_mode"][..],
|
|
&["deferred"][..],
|
|
&["direct"][..],
|
|
&["code_mode", "deferred"][..],
|
|
&["code_mode", "direct"][..],
|
|
&["deferred", "direct"][..],
|
|
&["code_mode", "deferred", "direct"][..],
|
|
&[][..],
|
|
] {
|
|
for (supports_search_tool, non_prefixed_mcp_tool_names, direct_only_namespace) in [
|
|
(false, false, false),
|
|
(false, false, true),
|
|
(false, true, false),
|
|
(false, true, true),
|
|
(true, false, false),
|
|
(true, false, true),
|
|
(true, true, false),
|
|
(true, true, true),
|
|
] {
|
|
let namespace = if non_prefixed_mcp_tool_names {
|
|
"rmcp"
|
|
} else {
|
|
"mcp__rmcp"
|
|
};
|
|
let visible_directly = !omit_tools_from.contains(&"direct")
|
|
&& (!supports_search_tool
|
|
|| omit_tools_from.contains(&"deferred")
|
|
|| direct_only_namespace);
|
|
let server = responses::start_mock_server().await;
|
|
let response = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
if visible_directly {
|
|
responses::ev_function_call_with_namespace(
|
|
"call-1",
|
|
namespace,
|
|
"echo",
|
|
r#"{"message":"ping"}"#,
|
|
)
|
|
} else {
|
|
ev_assistant_message("msg-1", "done")
|
|
},
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_response = if visible_directly {
|
|
Some(
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await,
|
|
)
|
|
} else {
|
|
None
|
|
};
|
|
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
|
let environment_id = remote_aware_environment_id();
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", move |model| {
|
|
model.supports_search_tool = supports_search_tool;
|
|
})
|
|
.with_config(move |config| {
|
|
if non_prefixed_mcp_tool_names {
|
|
config
|
|
.features
|
|
.enable(Feature::NonPrefixedMcpToolNames)
|
|
.expect("test config should allow unprefixed MCP tools");
|
|
}
|
|
if direct_only_namespace {
|
|
config.code_mode.direct_only_tool_namespaces = vec![
|
|
if non_prefixed_mcp_tool_names {
|
|
"rmcp"
|
|
} else {
|
|
"mcp__rmcp"
|
|
}
|
|
.to_string(),
|
|
];
|
|
}
|
|
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
serde_json::from_value(serde_json::json!({
|
|
"command": rmcp_test_server_bin,
|
|
"environment_id": environment_id,
|
|
"cwd": config.cwd,
|
|
"omit_tools_from": omit_tools_from,
|
|
}))
|
|
.expect("test MCP server config should be valid"),
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test config should allow MCP servers");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
test.submit_turn("inspect ordinary direct-mode MCP tool exposure")
|
|
.await?;
|
|
|
|
let body = response.single_request().body_json();
|
|
assert_eq!(
|
|
namespace_child_tool(&body, namespace, "echo").is_some(),
|
|
visible_directly,
|
|
"MCP omissions must determine direct-mode tool exposure; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
direct_only_namespace={direct_only_namespace}, namespace={namespace}, tools={:?}",
|
|
tool_names(&body)
|
|
);
|
|
assert_eq!(
|
|
tool_names(&body).iter().any(|name| name == "tool_search"),
|
|
supports_search_tool,
|
|
"MCP omissions must not disable tool search for other tools; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
direct_only_namespace={direct_only_namespace}, namespace={namespace}, tools={:?}",
|
|
tool_names(&body)
|
|
);
|
|
if let Some(follow_up_response) = follow_up_response {
|
|
let output = follow_up_response
|
|
.single_request()
|
|
.function_call_output("call-1");
|
|
assert!(
|
|
output["output"]
|
|
.as_str()
|
|
.is_some_and(|output| output.contains("ECHOING: ping")),
|
|
"the advertised MCP tool must remain callable; \
|
|
omit_tools_from={omit_tools_from:?}, \
|
|
supports_search_tool={supports_search_tool}, \
|
|
direct_only_namespace={direct_only_namespace}, \
|
|
namespace={namespace}, output={output:?}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
#[test_case(false, false, false; "disabled")]
|
|
#[test_case(true, false, false; "completed")]
|
|
#[test_case(true, true, false; "yielded")]
|
|
#[test_case(true, false, true; "oversized")]
|
|
async fn code_mode_tool_call_completeness_is_private_and_opt_in(
|
|
metadata_enabled: bool,
|
|
yielded: bool,
|
|
oversized: bool,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = if oversized {
|
|
r#"
|
|
const args = { barrier: { id: "", participants: 1 } };
|
|
args.barrier.id = "x".repeat(8192 - JSON.stringify(args).length);
|
|
for (let index = 0; index < 4; index++) await tools.test_sync_tool(args);
|
|
text("done");
|
|
yield_control();
|
|
await new Promise(() => {});
|
|
"#
|
|
} else if yielded {
|
|
r#"await tools.test_sync_tool({}); text("done"); yield_control(); await new Promise(() => {});"#
|
|
} else {
|
|
r#"await tools.test_sync_tool({}); text("done");"#
|
|
};
|
|
// run_code_mode_turn_with_config enables this feature; production keeps it off by default.
|
|
let (test, follow_up) =
|
|
run_code_mode_turn_with_config(&server, "Record a nested tool call", code, move |config| {
|
|
if !metadata_enabled {
|
|
config
|
|
.features
|
|
.disable(Feature::ExecutedToolCallMetadata)
|
|
.expect("tool call metadata should be disabled");
|
|
}
|
|
if oversized {
|
|
let _ = config.features.disable(Feature::RemoteCompactionV2);
|
|
}
|
|
})
|
|
.await?;
|
|
|
|
let request = follow_up.single_request();
|
|
let first_output = request.custom_tool_call_output("call-1");
|
|
let metadata = &first_output["internal_chat_message_metadata_passthrough"];
|
|
let calls = serde_json::json!([{ "name": "test_sync_tool", "arguments": {} }]);
|
|
let complete = Value::Bool(true);
|
|
if oversized {
|
|
assert!(
|
|
metadata["executed_tool_calls"]
|
|
.as_array()
|
|
.is_some_and(|calls| calls.iter().any(|call| call["arguments"]
|
|
.get("_codex_executed_tool_call_truncated")
|
|
.is_some()))
|
|
);
|
|
} else {
|
|
assert_eq!(
|
|
metadata.get("executed_tool_calls"),
|
|
metadata_enabled.then_some(&calls),
|
|
);
|
|
}
|
|
assert_eq!(
|
|
metadata.get("tool_calls_complete"),
|
|
(metadata_enabled && !yielded && !oversized).then_some(&complete),
|
|
);
|
|
assert_eq!(
|
|
metadata.get("cell_id").and_then(Value::as_str),
|
|
metadata_enabled.then_some("call-1"),
|
|
);
|
|
if yielded || oversized {
|
|
let first_items = custom_tool_output_items(&request, "call-1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
if oversized {
|
|
responses::mount_compact_user_history_with_summary_once(&server, "compacted history")
|
|
.await;
|
|
test.codex.submit(Op::Compact).await?;
|
|
wait_for_event(&test.codex, |event| {
|
|
matches!(event, EventMsg::TurnComplete(_))
|
|
})
|
|
.await;
|
|
|
|
let wait = responses::mount_function_call_agent_response(
|
|
&server,
|
|
"call-2",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id,
|
|
"terminate": true,
|
|
}))?,
|
|
"wait",
|
|
)
|
|
.await;
|
|
test.submit_turn("Finish the compacted cell").await?;
|
|
let final_request = wait.completion.single_request();
|
|
assert!(
|
|
final_request
|
|
.input()
|
|
.iter()
|
|
.all(|item| item["call_id"] != "call-1")
|
|
);
|
|
assert!(
|
|
final_request.function_call_output("call-2")
|
|
["internal_chat_message_metadata_passthrough"]
|
|
.get("tool_calls_complete")
|
|
.is_none()
|
|
);
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let apps_server = AppsTestServer::mount_searchable(&server).await?;
|
|
let originating_item_id = "ctc_code_mode_origin";
|
|
let mut exec_call = ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const tool = ALL_TOOLS.find(
|
|
({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99"
|
|
);
|
|
if (!tool) {
|
|
text(JSON.stringify({ found: false }));
|
|
} else {
|
|
const result = await tools[tool.name]({ timezone: "UTC" });
|
|
text(JSON.stringify({
|
|
found: true,
|
|
isError: Boolean(result.isError),
|
|
text: result.content?.[0]?.text ?? "",
|
|
}));
|
|
}
|
|
"#,
|
|
);
|
|
exec_call["item"]["id"] = serde_json::json!(originating_item_id);
|
|
let resp_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
exec_call,
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let apps_base_url = apps_server.chatgpt_base_url.clone();
|
|
let mut builder = test_codex()
|
|
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
|
.with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(Feature::Apps)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("test config should allow feature update");
|
|
let mut model_catalog =
|
|
bundled_models_response().expect("bundled models.json should parse");
|
|
let model = model_catalog
|
|
.models
|
|
.iter_mut()
|
|
.find(|model| model.slug == "gpt-5.4")
|
|
.expect("gpt-5.4 exists in bundled models.json");
|
|
config.chatgpt_base_url = apps_base_url;
|
|
config.model = Some("gpt-5.4".to_string());
|
|
model.supports_search_tool = true;
|
|
config.model_catalog = Some(model_catalog);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("inspect tools in code mode only").await?;
|
|
|
|
let first_body = resp_mock.single_request().body_json();
|
|
assert_eq!(
|
|
tool_names(&first_body),
|
|
vec![
|
|
"exec".to_string(),
|
|
"wait".to_string(),
|
|
"request_user_input".to_string(),
|
|
"web_search".to_string()
|
|
]
|
|
);
|
|
|
|
let exec_description = first_body
|
|
.get("tools")
|
|
.and_then(Value::as_array)
|
|
.and_then(|tools| {
|
|
tools.iter().find_map(|tool| {
|
|
if tool
|
|
.get("name")
|
|
.or_else(|| tool.get("type"))
|
|
.and_then(Value::as_str)
|
|
== Some("exec")
|
|
{
|
|
tool.get("description").and_then(Value::as_str)
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
})
|
|
.expect("exec description should be present");
|
|
assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`"));
|
|
assert!(exec_description.contains("Shared MCP Types:"));
|
|
assert!(!exec_description.contains("calendar_timezone_option_99"));
|
|
|
|
let request = follow_up_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only deferred app tool call failed unexpectedly: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"found": true,
|
|
"isError": false,
|
|
"text": "called calendar_timezone_option_99 for at with ",
|
|
})
|
|
);
|
|
let apps_tool_calls = recorded_apps_tool_calls(&server).await;
|
|
assert!(
|
|
apps_tool_calls.iter().any(|call| {
|
|
call.pointer("/params/_meta/itemId") == Some(&serde_json::json!(originating_item_id))
|
|
}),
|
|
"the nested MCP call should inherit its code cell's originating Responses item"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn app_only_tools_are_not_visible_or_runnable_by_code_mode_model() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let apps_server =
|
|
AppsTestServer::mount_with_app_only_tool(&server, AppsTestToolLoading::Searchable).await?;
|
|
let code = format!(
|
|
r#"
|
|
const visibleTool = ALL_TOOLS.find(({{ name }}) => name === {visible_tool_name:?});
|
|
const tool = ALL_TOOLS.find(({{ name }}) => name === {tool_name:?});
|
|
let error = null;
|
|
try {{
|
|
await tools[{tool_name:?}]({{}});
|
|
}} catch (caught) {{
|
|
error = String(caught);
|
|
}}
|
|
text(JSON.stringify({{
|
|
visibleListed: visibleTool !== undefined,
|
|
listed: tool !== undefined,
|
|
callable: typeof tools[{tool_name:?}] === "function",
|
|
error,
|
|
}}));
|
|
"#,
|
|
visible_tool_name = "mcp__codex_apps__calendar_timezone_option_99",
|
|
tool_name = DIRECT_CALENDAR_APP_ONLY_TOOL,
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = search_capable_apps_builder(apps_server.chatgpt_base_url.clone())
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("test config should allow feature update");
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("try to call the app-only calendar tool through exec")
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code mode visibility check should complete successfully: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(parsed["visibleListed"], true);
|
|
assert_eq!(parsed["listed"], false);
|
|
assert_eq!(parsed["callable"], false);
|
|
assert!(
|
|
parsed["error"]
|
|
.as_str()
|
|
.is_some_and(|error| error.contains("is not a function")),
|
|
"app-only code mode call should fail before MCP dispatch: {parsed:?}"
|
|
);
|
|
assert!(
|
|
recorded_apps_tool_calls(&server).await.is_empty(),
|
|
"app-only code mode call should not reach the MCP server"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_can_call_nested_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const output = await tools.exec_command({ cmd: "printf code_mode_only_nested_tool_marker" });
|
|
text(output.output);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("use exec to run nested tool in code mode only")
|
|
.await?;
|
|
|
|
let request = follow_up_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only nested tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "code_mode_only_nested_tool_marker");
|
|
assert!(
|
|
request
|
|
.custom_tool_call_output("call-1")
|
|
.pointer("/internal_chat_message_metadata_passthrough/executed_tool_calls")
|
|
.is_none(),
|
|
"nested tool calls must not be recorded while executed-tool-call metadata is disabled",
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_does_not_expose_update_plan_by_default() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"inspect the available tools",
|
|
r#"
|
|
text(JSON.stringify({
|
|
callable: typeof tools.update_plan === "function",
|
|
listed: ALL_TOOLS.some(({ name }) => name === "update_plan"),
|
|
}));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let (output, _) = custom_tool_output_body_and_success(&second_mock.single_request(), "call-1");
|
|
assert_eq!(
|
|
serde_json::from_str::<Value>(&output)?,
|
|
serde_json::json!({ "callable": false, "listed": false })
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_update_plan_nested_tool_result_is_empty_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_config(
|
|
&server,
|
|
"use exec to run update_plan",
|
|
r#"
|
|
const result = await tools.update_plan({
|
|
plan: [{ step: "Run update_plan from code mode", status: "in_progress" }],
|
|
});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
|config| config.update_plan_enabled = true,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec update_plan call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(parsed, serde_json::json!({}));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_get_context_remaining_returns_structured_result() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_config(
|
|
&server,
|
|
"use exec to get remaining context",
|
|
r#"
|
|
const result = await tools.get_context_remaining({});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
|config| {
|
|
config.model_context_window = Some(10_000);
|
|
config
|
|
.features
|
|
.enable(Feature::TokenBudget)
|
|
.expect("test config should allow token budget");
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec get_context_remaining call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"tokens_left": 9000,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_current_time_returns_structured_result() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_config(
|
|
&server,
|
|
"use exec to get the current time",
|
|
r#"
|
|
const result = await tools.clock__curr_time({});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CurrentTimeReminder)
|
|
.expect("test config should allow current-time reminders");
|
|
config.current_time_reminder = Some(CurrentTimeReminderConfig {
|
|
reminder_interval_seconds: 3_000,
|
|
clock_source: CurrentTimeSource::System,
|
|
..CurrentTimeReminderConfig::default()
|
|
});
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec clock.curr_time call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
let current_time = parsed
|
|
.get("current_time")
|
|
.and_then(Value::as_str)
|
|
.expect("clock.curr_time should return current_time");
|
|
assert_regex_match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} UTC$", current_time);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "flaky on windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_nested_tool_calls_can_run_in_parallel() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
let warmup_code = r#"
|
|
const args = {
|
|
sleep_after_ms: 10,
|
|
barrier: {
|
|
id: "code-mode-parallel-tools-warmup",
|
|
participants: 2,
|
|
timeout_ms: 1_000,
|
|
},
|
|
};
|
|
|
|
await Promise.all([
|
|
tools.test_sync_tool(args),
|
|
tools.test_sync_tool(args),
|
|
]);
|
|
"#;
|
|
let code = r#"
|
|
const args = {
|
|
sleep_after_ms: 300,
|
|
barrier: {
|
|
id: "code-mode-parallel-tools",
|
|
participants: 2,
|
|
timeout_ms: 1_000,
|
|
},
|
|
};
|
|
|
|
const results = await Promise.all([
|
|
tools.test_sync_tool(args),
|
|
tools.test_sync_tool(args),
|
|
]);
|
|
|
|
text(JSON.stringify(results));
|
|
"#;
|
|
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-warm-1"),
|
|
ev_custom_tool_call("call-warm-1", "exec", warmup_code),
|
|
ev_completed("resp-warm-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-warm-1", "warmup done"),
|
|
ev_completed("resp-warm-2"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("warm up nested tools in parallel").await?;
|
|
|
|
let start = Instant::now();
|
|
test.submit_turn("run nested tools in parallel").await?;
|
|
let duration = start.elapsed();
|
|
|
|
assert!(
|
|
duration < Duration::from_millis(1_600),
|
|
"expected nested tools to finish in parallel, got {duration:?}",
|
|
);
|
|
|
|
let req = response_mock
|
|
.last_request()
|
|
.expect("parallel code mode run should send a completion request");
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "[\"ok\",\"ok\"]");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_write_stdin_calls_run_in_parallel_across_sessions() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"write to independent terminal sessions in parallel",
|
|
r#"
|
|
const sessions = await Promise.all([
|
|
tools.exec_command({ cmd: "bash --noprofile --norc", tty: true, yield_time_ms: 250 }),
|
|
tools.exec_command({ cmd: "bash --noprofile --norc", tty: true, yield_time_ms: 250 }),
|
|
]);
|
|
|
|
const results = await Promise.all([
|
|
tools.write_stdin({
|
|
session_id: sessions[0].session_id,
|
|
chars: ": > .code-mode-a; while [ ! -e .code-mode-b ]; do sleep 0.01; done; printf 'code-alpha-%s\\n' ready; exit\n",
|
|
yield_time_ms: 5000,
|
|
}),
|
|
tools.write_stdin({
|
|
session_id: sessions[1].session_id,
|
|
chars: ": > .code-mode-b; while [ ! -e .code-mode-a ]; do sleep 0.01; done; printf 'code-beta-%s\\n' ready; exit\n",
|
|
yield_time_ms: 5000,
|
|
}),
|
|
]);
|
|
|
|
text(JSON.stringify([results[0].output.includes("code-alpha-ready"), results[1].output.includes("code-beta-ready")]));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let result: Value = serde_json::from_str(text_item(&items, /*index*/ 1))?;
|
|
assert_eq!(result, serde_json::json!([true, true]));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// This model uses token-based tool-output truncation, giving the downstream
|
|
// history assertions a stable `…N tokens truncated…` marker.
|
|
const TOKEN_POLICY_TEST_MODEL: &str = "gpt-5.4";
|
|
|
|
// A nested `exec_command` limit applies to `result.output` inside JavaScript.
|
|
// The outer code-mode and history budgets apply after the script calls `text`.
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_formats_truncated_result_with_warning() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"
|
|
const result = await tools.exec_command({
|
|
cmd: "printf '0123456789012345678901234567890123456789'",
|
|
max_output_tokens: 5
|
|
});
|
|
text(result.output);
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
assert_eq!(
|
|
text_item(
|
|
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
|
|
/*index*/ 1
|
|
),
|
|
"Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_preserves_result_variable_before_default_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_truncates_result_variable_when_exceeded() -> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 25000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('A' * 90000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.includes("…2500 tokens truncated…");
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The nested 20,000-token budget leaves about 80,000 characters. This
|
|
// ceiling independently proves that history applied its smaller cap.
|
|
assert!(
|
|
output.len() < 60_000,
|
|
"expected history to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
// The boolean describes the nested result; the marker below comes from
|
|
// history truncating the value emitted with `text` afterward.
|
|
assert_regex_match(
|
|
r"(?s)^Variable truncated: True\. Variable: .*…\d+ tokens truncated…A+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_preserves_result_variable_before_configured_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|config| {
|
|
config.tool_output_token_limit = Some(50);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The 50-token override must shrink this 50,000-character value far below
|
|
// what the default 10,000-token history cap would retain.
|
|
assert!(
|
|
output.len() < 1_000,
|
|
"expected configured history cap to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_default_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_configured_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|config| {
|
|
config.tool_output_token_limit = Some(50);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The 50-token override must shrink this 50,000-character value far below
|
|
// what the default 10,000-token history cap would retain.
|
|
assert!(
|
|
output.len() < 1_000,
|
|
"expected configured history cap to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// The outer directive limits output after JavaScript emits it; it does not
|
|
// limit `result.output` returned by the nested command.
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_outer_limit_truncates_emitted_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 5}
|
|
const result = await tools.exec_command({
|
|
cmd: "printf '0123456789012345678901234567890123456789'"
|
|
});
|
|
text(result.output);
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
assert_eq!(
|
|
text_item(
|
|
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
|
|
/*index*/ 1
|
|
),
|
|
"Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_returns_accumulated_output_when_script_fails() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use code_mode to surface script failures",
|
|
r#"
|
|
text("before crash");
|
|
text("still before crash");
|
|
throw new Error("boom");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
assert_eq!(items.len(), 4);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "before crash");
|
|
assert_eq!(text_item(&items, /*index*/ 2), "still before crash");
|
|
assert_regex_match(
|
|
r#"(?sx)
|
|
\A
|
|
Script\ error:\n
|
|
Error:\ boom\n
|
|
(?:\s+at\ .+\n?)+
|
|
\z
|
|
"#,
|
|
text_item(&items, /*index*/ 3),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_surfaces_handler_errors_as_exceptions() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"surface nested tool handler failures as script exceptions",
|
|
r#"
|
|
try {
|
|
await tools.exec_command({});
|
|
text("no-exception");
|
|
} catch (error) {
|
|
text(`caught:${error?.message ?? String(error)}`);
|
|
}
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let request = second_mock.single_request();
|
|
let metadata =
|
|
&request.custom_tool_call_output("call-1")["internal_chat_message_metadata_passthrough"];
|
|
assert_eq!(
|
|
metadata["executed_tool_calls"],
|
|
serde_json::json!([
|
|
{"name": "exec_command", "arguments": {}},
|
|
]),
|
|
"failed nested tool attempts remain private request metadata",
|
|
);
|
|
assert_eq!(metadata["tool_calls_complete"], Value::Bool(true));
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"script should catch the nested tool error: {output}"
|
|
);
|
|
assert!(
|
|
output.contains("caught:"),
|
|
"expected caught exception text in output: {output}"
|
|
);
|
|
assert!(
|
|
!output.contains("no-exception"),
|
|
"nested tool error should not allow success path: {output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// A stalled host wait must return its timeout to the model and reconnect for the next exec.
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn code_mode_wait_timeout_reconnects_on_next_exec() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
|
|
let first_turn = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
"yield_control(); await new Promise(() => {});",
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start a stalled code-mode cell").await?;
|
|
let first_request = first_turn
|
|
.last_request()
|
|
.expect("initial exec should be returned to the model");
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
let timeout_completion = responses::mount_function_call_agent_response(
|
|
&server,
|
|
"call-2",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id,
|
|
"yield_time_ms": 60_000,
|
|
}))?,
|
|
"wait",
|
|
)
|
|
.await
|
|
.completion;
|
|
|
|
test.codex
|
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
|
text: "wait for the stalled cell".to_string(),
|
|
text_elements: Vec::new(),
|
|
}]))
|
|
.await?;
|
|
wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::RawResponseItem(raw) => match &raw.item {
|
|
ResponseItem::FunctionCall { call_id, .. } if call_id == "call-2" => Some(()),
|
|
_ => None,
|
|
},
|
|
_ => None,
|
|
})
|
|
.await;
|
|
|
|
tokio::time::pause();
|
|
for _ in 0..130 {
|
|
tokio::time::advance(Duration::from_secs(1)).await;
|
|
if timeout_completion
|
|
.function_call_output_text("call-2")
|
|
.is_some()
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
tokio::time::resume();
|
|
wait_for_event(&test.codex, |event| {
|
|
matches!(event, EventMsg::TurnComplete(_))
|
|
})
|
|
.await;
|
|
|
|
let timeout_output = timeout_completion
|
|
.function_call_output_text("call-2")
|
|
.expect("timed-out wait should be returned to the model");
|
|
assert!(
|
|
timeout_output.contains("code-mode host timed out waiting for wait response"),
|
|
"unexpected wait output: {timeout_output}"
|
|
);
|
|
|
|
let reconnect_turn = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
ev_custom_tool_call(
|
|
"call-3",
|
|
"exec",
|
|
r#"text("reconnected"); yield_control(); await new Promise(() => {});"#,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "reconnected"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("run a cell after the timeout").await?;
|
|
let reconnect_request = reconnect_turn
|
|
.last_request()
|
|
.expect("replacement exec should be returned to the model");
|
|
let reconnect_items = custom_tool_output_items(&reconnect_request, "call-3");
|
|
assert_eq!(
|
|
extract_running_cell_id(text_item(&reconnect_items, /*index*/ 0)),
|
|
"g2:1"
|
|
);
|
|
assert_eq!(text_item(&reconnect_items, /*index*/ 1), "reconnected");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct ResponseIdObserver {
|
|
response_ids: Mutex<Vec<(String, Option<String>)>>,
|
|
originating_items: Mutex<Vec<(String, Option<codex_protocol::ResponseItemId>)>>,
|
|
wait_started: tokio::sync::Notify,
|
|
}
|
|
|
|
impl ToolLifecycleContributor for ResponseIdObserver {
|
|
fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> {
|
|
Box::pin(async move {
|
|
if matches!(input.tool_name.name.as_str(), "exec" | "exec_command") {
|
|
self.originating_items.lock().unwrap().push((
|
|
input.tool_name.name.clone(),
|
|
input.originating_item_id.cloned(),
|
|
));
|
|
}
|
|
if matches!(input.tool_name.name.as_str(), "exec_command" | "wait") {
|
|
self.response_ids.lock().unwrap().push((
|
|
input.tool_name.name.clone(),
|
|
input
|
|
.turn_store
|
|
.get::<codex_api::ResponseId>()
|
|
.map(|id| id.0.clone()),
|
|
));
|
|
if input.tool_name.name == "wait" {
|
|
self.wait_started.notify_one();
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let observer = Arc::new(ResponseIdObserver::default());
|
|
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
|
|
extensions.tool_lifecycle_contributor(observer.clone());
|
|
let mut builder = test_codex()
|
|
.with_extensions(Arc::new(extensions.build()))
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let phase_2_gate = test.workspace_path("code-mode-phase-2.ready");
|
|
let phase_3_gate = test.workspace_path("code-mode-phase-3.ready");
|
|
let phase_2_wait = wait_for_file_source(&phase_2_gate)?;
|
|
let phase_3_wait = wait_for_file_source(&phase_3_gate)?;
|
|
|
|
let code = format!(
|
|
r#"
|
|
text("phase 1");
|
|
yield_control();
|
|
{phase_2_wait}
|
|
text((await tools.exec_command({{cmd: "printf 'phase 2'"}})).output);
|
|
{phase_3_wait}
|
|
text((await tools.exec_command({{cmd: "printf 'phase 3'"}})).output);
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
responses::ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
responses::ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "still waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
tokio::try_join!(test.submit_turn("wait again"), async {
|
|
observer.wait_started.notified().await;
|
|
fs::write(&phase_2_gate, "ready")?;
|
|
anyhow::Ok(())
|
|
})?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
extract_running_cell_id(text_item(&second_items, /*index*/ 0)),
|
|
cell_id
|
|
);
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "phase 2");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
responses::ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
tokio::try_join!(test.submit_turn("wait for completion"), async {
|
|
observer.wait_started.notified().await;
|
|
fs::write(&phase_3_gate, "ready")?;
|
|
anyhow::Ok(())
|
|
})?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "phase 3");
|
|
|
|
// Resuming the cell changes the response id, but its nested calls retain
|
|
// the original exec item's identity for request-local freshness accounting.
|
|
let originating_items = observer.originating_items.lock().unwrap();
|
|
let wrapper_item = originating_items[0]
|
|
.1
|
|
.clone()
|
|
.expect("wrapper item identity");
|
|
assert!(originating_items.len() >= 3);
|
|
assert!(
|
|
originating_items
|
|
.iter()
|
|
.all(|(_, item)| item.as_ref() == Some(&wrapper_item))
|
|
);
|
|
|
|
let observed = observer.response_ids.lock().unwrap();
|
|
let mut nested_response_ids = observed
|
|
.iter()
|
|
.filter(|(tool, _)| tool == "exec_command")
|
|
.filter_map(|(_, response_id)| response_id.clone())
|
|
.skip_while(|response_id| response_id != &"resp-3".to_owned())
|
|
.collect::<Vec<_>>();
|
|
nested_response_ids.dedup();
|
|
assert_eq!(
|
|
nested_response_ids,
|
|
vec!["resp-3".to_owned(), "resp-5".to_owned()]
|
|
);
|
|
assert_eq!(
|
|
observed
|
|
.iter()
|
|
.filter(|(tool, _)| tool == "wait")
|
|
.cloned()
|
|
.collect::<Vec<_>>(),
|
|
vec![
|
|
("wait".to_owned(), Some("resp-3".to_owned())),
|
|
("wait".to_owned(), Some("resp-5".to_owned())),
|
|
]
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_yield_and_termination_are_not_starved_by_runtime_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let termination_gate = test.workspace_path("code-mode-output-termination.ready");
|
|
let termination_wait = wait_for_file_source(&termination_gate)?;
|
|
|
|
// Exact controller arbitration is covered by deterministic code-mode contract tests. Keep
|
|
// this end-to-end load bounded while exercising a substantial runtime output backlog.
|
|
let code = format!(
|
|
r#"// @exec: {{"yield_time_ms": 0, "max_output_tokens": 16}}
|
|
for (let index = 0; index < 256; index++) {{
|
|
text(`event ${{index}}`);
|
|
}}
|
|
{termination_wait}
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the bounded output backlog").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "terminated"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate it").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert!(!second_items.is_empty());
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_run_multiple_yielded_sessions() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let session_a_gate = test.workspace_path("code-mode-session-a.ready");
|
|
let session_b_gate = test.workspace_path("code-mode-session-b.ready");
|
|
let session_a_wait = wait_for_file_source(&session_a_gate)?;
|
|
let session_b_wait = wait_for_file_source(&session_b_gate)?;
|
|
|
|
let session_a_code = format!(
|
|
r#"
|
|
await tools.exec_command({{ cmd: "printf cell-a" }});
|
|
text("session a start");
|
|
yield_control();
|
|
{session_a_wait}
|
|
text("session a done");
|
|
"#
|
|
);
|
|
let session_b_code = format!(
|
|
r#"
|
|
await tools.exec_command({{ cmd: "printf cell-b" }});
|
|
text("session b start");
|
|
yield_control();
|
|
{session_b_wait}
|
|
text("session b done");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &session_a_code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "session a waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session a").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "session a start");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-2", "exec", &session_b_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "session b waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session b").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = custom_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0));
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "session b start");
|
|
assert_ne!(session_a_id, session_b_id);
|
|
|
|
fs::write(&session_a_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_a_id.clone(),
|
|
"yield_time_ms": 5_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "session a done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session a").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "session a done");
|
|
|
|
fs::write(&session_b_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-4",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_b_id.clone(),
|
|
"yield_time_ms": 5_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
let fourth_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "session b done"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session b").await?;
|
|
|
|
let fourth_request = fourth_completion.single_request();
|
|
let fourth_items = function_tool_output_items(&fourth_request, "call-4");
|
|
assert_eq!(fourth_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&fourth_items, /*index*/ 1), "session b done");
|
|
for (output, originating_call_id, expected_command) in [
|
|
(
|
|
first_request.custom_tool_call_output("call-1"),
|
|
"call-1",
|
|
Some("printf cell-a"),
|
|
),
|
|
(
|
|
second_request.custom_tool_call_output("call-2"),
|
|
"call-2",
|
|
Some("printf cell-b"),
|
|
),
|
|
(third_request.function_call_output("call-3"), "call-1", None),
|
|
(
|
|
fourth_request.function_call_output("call-4"),
|
|
"call-2",
|
|
None,
|
|
),
|
|
] {
|
|
let metadata = &output["internal_chat_message_metadata_passthrough"];
|
|
assert_eq!(metadata["cell_id"], serde_json::json!(originating_call_id));
|
|
assert_eq!(
|
|
metadata.get("tool_calls_complete").and_then(Value::as_bool),
|
|
expected_command.is_none().then_some(true),
|
|
);
|
|
assert_eq!(
|
|
metadata
|
|
.get("executed_tool_calls")
|
|
.and_then(Value::as_array)
|
|
.and_then(|calls| {
|
|
calls.iter().find_map(|call| {
|
|
call["arguments"]["cmd"]
|
|
.as_str()
|
|
.filter(|command| command.starts_with("printf cell-"))
|
|
})
|
|
}),
|
|
expected_command,
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_concurrent_cells_merge_only_the_stored_values_they_write() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let first_gate = test.workspace_path("code-mode-first-store.ready");
|
|
let first_wait = wait_for_file_source(&first_gate)?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-init",
|
|
"exec",
|
|
r#"
|
|
store("a", 1);
|
|
store("b", 2);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "initialized"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("initialize stored values").await?;
|
|
|
|
let first_code = format!(
|
|
r#"
|
|
store("a", 3);
|
|
yield_control();
|
|
{first_wait}
|
|
"#
|
|
);
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-first", "exec", &first_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_started = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "first pending"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start first store").await?;
|
|
|
|
let first_request = first_started.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-first");
|
|
let first_cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
ev_custom_tool_call("call-second", "exec", r#"store("b", 4);"#),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "second complete"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("write the second key").await?;
|
|
|
|
fs::write(&first_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-wait",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": first_cell_id,
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "first completed"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("complete the first store").await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-9"),
|
|
ev_custom_tool_call(
|
|
"call-check",
|
|
"exec",
|
|
r#"text(JSON.stringify({ a: load("a"), b: load("b") }));"#,
|
|
),
|
|
ev_completed("resp-9"),
|
|
]),
|
|
)
|
|
.await;
|
|
let check_response = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-5", "checked"),
|
|
ev_completed("resp-10"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("check merged stored values").await?;
|
|
|
|
let check_request = check_response.single_request();
|
|
let stored_values: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&check_request, "call-check")
|
|
.expect("checking stored values should emit JSON"),
|
|
)?;
|
|
assert_eq!(stored_values, serde_json::json!({ "a": 3, "b": 4 }));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_can_terminate_and_continue() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let termination_gate = test.workspace_path("code-mode-terminate.ready");
|
|
let termination_wait = wait_for_file_source(&termination_gate)?;
|
|
|
|
let code = format!(
|
|
r#"
|
|
text("phase 1");
|
|
yield_control();
|
|
{termination_wait}
|
|
text("phase 2");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "terminated"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate it").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
ev_custom_tool_call(
|
|
"call-3",
|
|
"exec",
|
|
r#"
|
|
text("after terminate");
|
|
"#,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("run another exec").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = custom_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "after terminate");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_returns_error_for_unknown_session() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
responses::ev_function_call(
|
|
"call-1",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": "999999",
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait on an unknown exec cell").await?;
|
|
|
|
let request = completion.single_request();
|
|
let (_, success) = request
|
|
.function_call_output_content_and_success("call-1")
|
|
.expect("function tool output should be present");
|
|
assert_ne!(success, Some(true));
|
|
|
|
let items = function_tool_output_items(&request, "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
text_item(&items, /*index*/ 1),
|
|
"Script error:\nexec cell 999999 not found"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_terminate_returns_completed_session_if_it_finished_after_yield_control()
|
|
-> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let session_a_gate = test.workspace_path("code-mode-session-a-finished.ready");
|
|
let session_b_gate = test.workspace_path("code-mode-session-b-blocked.ready");
|
|
let session_a_done_marker = test.workspace_path("code-mode-session-a-done.txt");
|
|
let session_a_wait = wait_for_file_source(&session_a_gate)?;
|
|
let session_b_wait = wait_for_file_source(&session_b_gate)?;
|
|
let session_a_done_marker_quoted =
|
|
shlex::try_join([session_a_done_marker.to_string_lossy().as_ref()])?;
|
|
let session_a_done_command = format!("printf done > {session_a_done_marker_quoted}");
|
|
|
|
let session_a_code = format!(
|
|
r#"
|
|
text("session a start");
|
|
yield_control();
|
|
{session_a_wait}
|
|
text("session a done");
|
|
await tools.exec_command({{ cmd: {session_a_done_command:?} }});
|
|
"#
|
|
);
|
|
let session_b_code = format!(
|
|
r#"
|
|
text("session b start");
|
|
yield_control();
|
|
{session_b_wait}
|
|
text("session b done");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &session_a_code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "session a waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session a").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "session a start");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-2", "exec", &session_b_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "session b waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session b").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = custom_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0));
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "session b start");
|
|
|
|
fs::write(&session_a_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_b_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "session b still waiting"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session b").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
extract_running_cell_id(text_item(&third_items, /*index*/ 0)),
|
|
session_b_id
|
|
);
|
|
|
|
for _ in 0..100 {
|
|
if session_a_done_marker.exists() {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
assert!(session_a_done_marker.exists());
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-4",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_a_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
let fourth_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "session a already done"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate session a").await?;
|
|
|
|
let fourth_request = fourth_completion.single_request();
|
|
let fourth_items = function_tool_output_items(&fourth_request, "call-4");
|
|
match fourth_items.len() {
|
|
1 => {
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
}
|
|
2 => {
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script (?:completed|terminated)\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&fourth_items, /*index*/ 1), "session a done");
|
|
}
|
|
other => panic!("unexpected number of content items: {other}"),
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_background_keeps_running_on_later_turn_without_wait() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let resumed_file = test.workspace_path("code-mode-yield-resumed.txt");
|
|
let resumed_file_quoted = shlex::try_join([resumed_file.to_string_lossy().as_ref()])?;
|
|
let write_file_command = format!("printf resumed > {resumed_file_quoted}");
|
|
let wait_for_file_command =
|
|
format!("while [ ! -f {resumed_file_quoted} ]; do sleep 0.01; done; printf ready");
|
|
let code = format!(
|
|
r#"
|
|
text("before yield");
|
|
yield_control();
|
|
await tools.exec_command({{ cmd: {write_file_command:?} }});
|
|
text("after yield");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "exec yielded"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start yielded exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "before yield");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"exec_command",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cmd": wait_for_file_command,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "file appeared"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait for resumed file").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
assert!(
|
|
second_request
|
|
.function_call_output_text("call-2")
|
|
.is_some_and(|output| output.ends_with("ready"))
|
|
);
|
|
assert_eq!(fs::read_to_string(&resumed_file)?, "resumed");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
struct InterruptedNestedToolObserver {
|
|
started: Mutex<Option<oneshot::Sender<String>>>,
|
|
finished: Mutex<Option<oneshot::Sender<ToolCallOutcome>>>,
|
|
}
|
|
|
|
impl ToolLifecycleContributor for InterruptedNestedToolObserver {
|
|
fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> {
|
|
Box::pin(async move {
|
|
let codex_extension_api::ToolCallSource::CodeMode { cell_id, .. } = input.source else {
|
|
return;
|
|
};
|
|
if input.tool_name.name != "test_sync_tool" {
|
|
return;
|
|
}
|
|
if let Some(started) = self
|
|
.started
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.take()
|
|
{
|
|
let _ = started.send(cell_id);
|
|
}
|
|
})
|
|
}
|
|
|
|
fn on_tool_finish<'a>(&'a self, input: ToolFinishInput<'a>) -> ToolLifecycleFuture<'a> {
|
|
Box::pin(async move {
|
|
if input.tool_name.name != "test_sync_tool"
|
|
|| !matches!(
|
|
input.source,
|
|
codex_extension_api::ToolCallSource::CodeMode { .. }
|
|
)
|
|
{
|
|
return;
|
|
}
|
|
if let Some(finished) = self
|
|
.finished
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.take()
|
|
{
|
|
let _ = finished.send(input.outcome);
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_interrupt_terminates_active_cells_and_nested_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let (started_tx, started_rx) = oneshot::channel();
|
|
let (finished_tx, finished_rx) = oneshot::channel();
|
|
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
|
|
extensions.tool_lifecycle_contributor(Arc::new(InterruptedNestedToolObserver {
|
|
started: Mutex::new(Some(started_tx)),
|
|
finished: Mutex::new(Some(finished_tx)),
|
|
}));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_extensions(Arc::new(extensions.build()))
|
|
.with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
let _ = config.features.enable(Feature::CodeModeInterrupt);
|
|
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
|
|
let setup = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-store"),
|
|
ev_custom_tool_call("call-store", "exec", r#"store("persisted", "preserved");"#),
|
|
ev_completed("resp-store"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-store", "stored"),
|
|
ev_completed("resp-store-complete"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-background"),
|
|
ev_custom_tool_call(
|
|
"call-background",
|
|
"exec",
|
|
"yield_control(); await new Promise(() => {});",
|
|
),
|
|
ev_completed("resp-background"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-background", "running"),
|
|
ev_completed("resp-background-complete"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("store a value in the reusable code-mode session")
|
|
.await?;
|
|
test.submit_turn("start a background code-mode cell")
|
|
.await?;
|
|
let background_response = setup
|
|
.last_request()
|
|
.expect("background cell should be returned to the model");
|
|
let background_items = custom_tool_output_items(&background_response, "call-background");
|
|
assert!(
|
|
text_item(&background_items, /*index*/ 0).starts_with("Script running with cell ID "),
|
|
"background cell should remain active: {background_items:?}"
|
|
);
|
|
let background_cell_id =
|
|
extract_running_cell_id(text_item(&background_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-interrupted"),
|
|
ev_custom_tool_call(
|
|
"call-interrupted",
|
|
"exec",
|
|
"await tools.test_sync_tool({ sleep_after_ms: 60_000 });",
|
|
),
|
|
ev_completed("resp-interrupted"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.codex
|
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
|
text: "start a long-running nested tool".to_string(),
|
|
text_elements: Vec::new(),
|
|
}]))
|
|
.await?;
|
|
let active_cell_id = tokio::time::timeout(Duration::from_secs(10), started_rx).await??;
|
|
|
|
test.codex.submit(Op::Interrupt).await?;
|
|
wait_for_event(&test.codex, |event| {
|
|
matches!(event, EventMsg::TurnAborted(_))
|
|
})
|
|
.await;
|
|
let nested_outcome = tokio::time::timeout(Duration::from_secs(10), finished_rx).await??;
|
|
assert_eq!(nested_outcome, ToolCallOutcome::Aborted);
|
|
|
|
let recovery = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-wait-background"),
|
|
responses::ev_function_call(
|
|
"call-wait-background",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": background_cell_id,
|
|
"yield_time_ms": 1,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-wait-background"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-wait-active"),
|
|
responses::ev_function_call(
|
|
"call-wait-active",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": active_cell_id,
|
|
"yield_time_ms": 1,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-wait-active"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-recovery"),
|
|
ev_custom_tool_call("call-recovery", "exec", r#"text(load("persisted"));"#),
|
|
ev_completed("resp-recovery"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-recovery", "recovered"),
|
|
ev_completed("resp-recovery-complete"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("verify interrupted cells and reuse their session")
|
|
.await?;
|
|
let requests = recovery.requests();
|
|
let metadata = &requests[0].custom_tool_call_output("call-interrupted")["internal_chat_message_metadata_passthrough"];
|
|
assert_eq!(
|
|
metadata["executed_tool_calls"],
|
|
serde_json::json!([{"name": "test_sync_tool", "arguments": {"sleep_after_ms": 60_000}}]),
|
|
);
|
|
assert_eq!(metadata["tool_calls_complete"], Value::Bool(true));
|
|
let background_output = function_tool_output_items(&requests[1], "call-wait-background");
|
|
assert!(text_item(&background_output, /*index*/ 1).contains("not found"));
|
|
let active_output = function_tool_output_items(&requests[2], "call-wait-active");
|
|
assert!(text_item(&active_output, /*index*/ 1).contains("not found"));
|
|
let recovery_items = custom_tool_output_items(&requests[3], "call-recovery");
|
|
assert_eq!(text_item(&recovery_items, /*index*/ 1), "preserved");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_uses_its_own_max_tokens_budget() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let completion_gate = test.workspace_path("code-mode-max-tokens.ready");
|
|
let completion_wait = wait_for_file_source(&completion_gate)?;
|
|
|
|
let code = format!(
|
|
r#"// @exec: {{"max_output_tokens": 100}}
|
|
text("phase 1");
|
|
yield_control();
|
|
{completion_wait}
|
|
text("token one token two token three token four token five token six token seven");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
fs::write(&completion_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
"max_tokens": 6,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "done"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait for completion").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
let expected_pattern = r#"(?sx)
|
|
\A
|
|
Warning:\ truncated\ output\ \(original\ token\ count:\ \d+\)\n
|
|
Total\ output\ lines:\ 1\n
|
|
\n
|
|
.*…\d+\ tokens\ truncated….*
|
|
\z
|
|
"#;
|
|
assert_regex_match(expected_pattern, text_item(&second_items, /*index*/ 1));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_resume_after_set_timeout() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to wait for a timeout",
|
|
r#"
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
text("timer done");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec setTimeout call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "timer done");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_notify_injects_additional_exec_tool_output_into_active_context() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec notify helper",
|
|
r#"
|
|
notify("code_mode_notify_marker");
|
|
text("done");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let has_notify_output = req
|
|
.inputs_of_type("custom_tool_call_output")
|
|
.iter()
|
|
.any(|item| {
|
|
item.get("call_id").and_then(serde_json::Value::as_str) == Some("call-1")
|
|
&& item
|
|
.get("output")
|
|
.and_then(serde_json::Value::as_str)
|
|
.is_some_and(|text| text.contains("code_mode_notify_marker"))
|
|
&& item.get("name").and_then(serde_json::Value::as_str) == Some("exec")
|
|
});
|
|
assert!(
|
|
has_notify_output,
|
|
"expected notify marker in custom_tool_call_output item: {:?}",
|
|
req.input()
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exit_stops_script_immediately() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to stop script early with exit helper",
|
|
r#"
|
|
text("before");
|
|
exit();
|
|
text("after");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec exit helper call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "before");
|
|
assert_eq!(output, "before");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_omits_short_audio_and_preserves_other_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let [short_audio_url, valid_audio_url] = [120_u32, 600].map(|frames| {
|
|
let sample_rate = 24_000_u32;
|
|
let data_size = frames * 2;
|
|
let mut wav = b"RIFF".to_vec();
|
|
wav.extend_from_slice(&(36 + data_size).to_le_bytes());
|
|
wav.extend_from_slice(b"WAVEfmt ");
|
|
wav.extend_from_slice(&16_u32.to_le_bytes());
|
|
wav.extend_from_slice(&1_u16.to_le_bytes());
|
|
wav.extend_from_slice(&1_u16.to_le_bytes());
|
|
wav.extend_from_slice(&sample_rate.to_le_bytes());
|
|
wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());
|
|
wav.extend_from_slice(&2_u16.to_le_bytes());
|
|
wav.extend_from_slice(&16_u16.to_le_bytes());
|
|
wav.extend_from_slice(b"data");
|
|
wav.extend_from_slice(&data_size.to_le_bytes());
|
|
wav.resize(44 + data_size as usize, /*value*/ 0);
|
|
format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav))
|
|
});
|
|
let code = format!(
|
|
r#"
|
|
text("before");
|
|
audio({short_audio_url:?});
|
|
text("between");
|
|
audio({valid_audio_url:?});
|
|
text("after");
|
|
"#,
|
|
);
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", |model| {
|
|
model.input_modalities.push(InputModality::Audio);
|
|
})
|
|
.with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to return short and valid audio with text")
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(success, Some(false));
|
|
assert_eq!(
|
|
&items[1..],
|
|
&[
|
|
serde_json::json!({ "type": "input_text", "text": "before" }),
|
|
serde_json::json!({
|
|
"type": "input_text",
|
|
"text": "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."
|
|
}),
|
|
serde_json::json!({ "type": "input_text", "text": "between" }),
|
|
serde_json::json!({ "type": "input_audio", "audio_url": valid_audio_url }),
|
|
serde_json::json!({ "type": "input_text", "text": "after" }),
|
|
]
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_replaces_malformed_image() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return an image",
|
|
r#"image("data:image/png;base64,AAA");"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(success, Some(false));
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(
|
|
items[1],
|
|
serde_json::json!({
|
|
"type": "input_text",
|
|
"text": "image content omitted because it could not be processed"
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_resizes_explicit_original_image() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let original_dimensions = (6401, 100);
|
|
let image = ImageBuffer::from_pixel(
|
|
original_dimensions.0,
|
|
original_dimensions.1,
|
|
Rgba([20, 40, 60, 255]),
|
|
);
|
|
let mut encoded = Cursor::new(Vec::new());
|
|
DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
|
|
let image_data_url = format!(
|
|
"data:image/png;base64,{}",
|
|
BASE64_STANDARD.encode(encoded.into_inner())
|
|
);
|
|
let code = format!(
|
|
"image({}, \"original\");",
|
|
serde_json::to_string(&image_data_url)?
|
|
);
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec to return a large original-detail image",
|
|
&code,
|
|
"gpt-5.4",
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(success, Some(false));
|
|
let resized_url = items[1]["image_url"]
|
|
.as_str()
|
|
.expect("code mode image output should contain a data URL");
|
|
assert_eq!(items[1]["detail"], "original");
|
|
let (_, resized_base64) = resized_url
|
|
.split_once(',')
|
|
.expect("resized image should contain a data URL prefix");
|
|
let resized_bytes = BASE64_STANDARD.decode(resized_base64)?;
|
|
let resized = image::load_from_memory(&resized_bytes)?;
|
|
let resized_dimensions = resized.dimensions();
|
|
assert_eq!(resized_dimensions, (6000, 94));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let image = ImageBuffer::from_pixel(
|
|
/*width*/ 2304,
|
|
/*height*/ 864,
|
|
Rgba([20, 40, 60, 255]),
|
|
);
|
|
let mut encoded = Cursor::new(Vec::new());
|
|
DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
|
|
let image_data = BASE64_STANDARD.encode(encoded.into_inner());
|
|
let code = format!(
|
|
r#"
|
|
const data = {};
|
|
const imageUrl = `data:image/png;base64,${{data}}`;
|
|
image(imageUrl);
|
|
image(imageUrl, "auto");
|
|
image(imageUrl, "high");
|
|
image({{ image_url: imageUrl, detail: "low" }});
|
|
image({{
|
|
type: "image",
|
|
data,
|
|
mimeType: "image/png",
|
|
_meta: {{ "codex/imageDetail": "original" }}
|
|
}});
|
|
"#,
|
|
serde_json::to_string(&image_data)?
|
|
);
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"emit images with legacy detail arguments and MCP metadata",
|
|
&code,
|
|
"gpt-5.4",
|
|
|config| {
|
|
let _ = config.features.enable(Feature::UnifiedImageBudget);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let request = second_mock.single_request();
|
|
let items = custom_tool_output_items(&request, "call-1");
|
|
assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}");
|
|
for item in &items[1..] {
|
|
assert_eq!(item["type"], "input_image");
|
|
assert_eq!(item["detail"], "original");
|
|
let image_url = item["image_url"]
|
|
.as_str()
|
|
.expect("image helper should emit a data URL");
|
|
let (_, payload) = image_url
|
|
.split_once(',')
|
|
.expect("image data URL should have a base64 payload");
|
|
let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?;
|
|
assert_eq!(image.dimensions(), (2304, 864));
|
|
}
|
|
|
|
let body = request.body_json();
|
|
let exec_description = body["tools"]
|
|
.as_array()
|
|
.and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
|
|
.and_then(|tool| tool["description"].as_str())
|
|
.expect("the model request should contain the code-mode exec tool");
|
|
assert!(
|
|
exec_description
|
|
.contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`")
|
|
);
|
|
assert!(!exec_description.contains("codex/imageDetail"));
|
|
assert!(!exec_description.contains("detail?:"));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model()
|
|
-> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"emit an image on a legacy model",
|
|
r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#,
|
|
"gpt-5.2",
|
|
|config| {
|
|
let _ = config.features.enable(Feature::UnifiedImageBudget);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let request = second_mock.single_request();
|
|
let items = custom_tool_output_items(&request, "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(items[1]["detail"], "high");
|
|
|
|
let body = request.body_json();
|
|
let exec_description = body["tools"]
|
|
.as_array()
|
|
.and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
|
|
.and_then(|tool| tool["description"].as_str())
|
|
.expect("the model request should contain the code-mode exec tool");
|
|
assert!(exec_description.contains("codex/imageDetail"));
|
|
assert!(exec_description.contains("detail?:"));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_view_image_rejects_invalid_file_without_exposing_contents() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
const INVALID_IMAGE_CONTENTS: &str = "private-file-contents-must-not-be-exposed";
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let builder = test_codex()
|
|
.with_model("gpt-5.4")
|
|
.with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
})
|
|
.with_workspace_setup(|cwd, _fs| async move {
|
|
fs::write(
|
|
cwd.join("not-an-image.txt").as_path(),
|
|
INVALID_IMAGE_CONTENTS,
|
|
)?;
|
|
Ok(())
|
|
});
|
|
let (_test, second_mock) = run_code_mode_turn_with_builder(
|
|
&server,
|
|
"use exec to call view_image on a non-image file",
|
|
r#"await tools.view_image({ path: "not-an-image.txt" });"#,
|
|
builder,
|
|
)
|
|
.await?;
|
|
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(true),
|
|
"code-mode view_image unexpectedly accepted a non-image file"
|
|
);
|
|
assert!(
|
|
output.contains("unable to process image: invalid or unsupported image data"),
|
|
"unexpected code-mode failure: {output}"
|
|
);
|
|
let model_visible_output = serde_json::to_string(&request.custom_tool_call_output("call-1"))?;
|
|
assert!(
|
|
!model_visible_output.contains(INVALID_IMAGE_CONTENTS),
|
|
"invalid file contents leaked into model-visible output: {model_visible_output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test_case(false; "legacy detail")]
|
|
#[test_case(true; "unified image budget")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_use_view_image_result_with_image_helper(
|
|
unified_image_budget: bool,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("gpt-5.4")
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
if unified_image_budget {
|
|
let _ = config.features.enable(Feature::UnifiedImageBudget);
|
|
}
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
let image = ImageBuffer::from_pixel(
|
|
/*width*/ 2,
|
|
/*height*/ 1,
|
|
Rgba([255u8, 0, 0, 255]),
|
|
);
|
|
let rotate_90_exif = vec![
|
|
0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01,
|
|
0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
];
|
|
let mut image_bytes = Vec::new();
|
|
let mut encoder = PngEncoder::new(&mut image_bytes);
|
|
encoder.set_exif_metadata(rotate_90_exif.clone())?;
|
|
encoder.write_image(
|
|
image.as_raw(),
|
|
image.width(),
|
|
image.height(),
|
|
image::ColorType::Rgba8.into(),
|
|
)?;
|
|
let image_path = test.cwd_path().join("code_mode_view_image.png");
|
|
fs::write(&image_path, &image_bytes)?;
|
|
|
|
let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;
|
|
let expected_output_keys = if unified_image_budget {
|
|
r#"["image_url"]"#
|
|
} else {
|
|
r#"["detail","image_url"]"#
|
|
};
|
|
let code = format!(
|
|
r#"
|
|
const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }});
|
|
const keys = Object.keys(out).sort();
|
|
if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{
|
|
throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`);
|
|
}}
|
|
image(out);
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to call view_image and emit its image output")
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode view_image call failed unexpectedly"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
|
|
assert_eq!(
|
|
items[1].get("type").and_then(Value::as_str),
|
|
Some("input_image")
|
|
);
|
|
|
|
let emitted_image_url = items[1]
|
|
.get("image_url")
|
|
.and_then(Value::as_str)
|
|
.expect("image helper should emit an input_image item with image_url");
|
|
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
|
|
let (_, emitted_image_base64) = emitted_image_url
|
|
.split_once(',')
|
|
.expect("emitted image should contain a data URL prefix");
|
|
let emitted_image_bytes = BASE64_STANDARD.decode(emitted_image_base64)?;
|
|
assert_eq!(emitted_image_bytes, image_bytes);
|
|
let mut decoder = ImageReader::with_format(Cursor::new(&emitted_image_bytes), ImageFormat::Png)
|
|
.into_decoder()?;
|
|
assert_eq!(
|
|
(
|
|
decoder.dimensions(),
|
|
decoder.orientation()?,
|
|
decoder.exif_metadata()?
|
|
),
|
|
((2, 1), Orientation::Rotate90, Some(rotate_90_exif))
|
|
);
|
|
assert_eq!(
|
|
items[1].get("detail").and_then(Value::as_str),
|
|
Some("original")
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const out = await tools.mcp__rmcp__image_scenario({
|
|
scenario: "image_only_original_detail",
|
|
});
|
|
const imageItem = out.content.find((item) => item.type === "image");
|
|
image(imageItem);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_model(
|
|
&server,
|
|
"use exec to call the rmcp image scenario tool and emit its image output",
|
|
code,
|
|
"gpt-5.4",
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode mcp image scenario call failed unexpectedly"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
|
|
assert_eq!(
|
|
items[1].get("type").and_then(Value::as_str),
|
|
Some("input_image")
|
|
);
|
|
|
|
let emitted_image_url = items[1]
|
|
.get("image_url")
|
|
.and_then(Value::as_str)
|
|
.expect("image helper should emit an input_image item with image_url");
|
|
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
|
|
assert_eq!(
|
|
items[1].get("detail").and_then(Value::as_str),
|
|
Some("original")
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let file_name = "code_mode_apply_patch.txt";
|
|
let patch = format!(
|
|
"*** Begin Patch\n*** Add File: {file_name}\n+hello from code_mode\n*** End Patch\n"
|
|
);
|
|
let code = format!("text(await tools.apply_patch({patch:?}));\n");
|
|
|
|
let (test, second_mock) =
|
|
run_code_mode_turn(&server, "use exec to run apply_patch", &code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = req
|
|
.custom_tool_call_output_content_and_success("call-1")
|
|
.expect("custom tool output should be present");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec apply_patch call failed unexpectedly: {items:?}"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "{}");
|
|
|
|
let file_path = test.cwd_path().join(file_name);
|
|
assert_eq!(fs::read_to_string(&file_path)?, "hello from code_mode\n");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_structured_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({
|
|
message: "ping",
|
|
});
|
|
text(
|
|
`echo=${structuredContent?.echo ?? "missing"}\n` +
|
|
`env=${structuredContent?.env ?? "missing"}\n` +
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to run the rmcp echo tool", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp echo call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"echo=ECHOING: ping
|
|
env=propagated-env
|
|
isError=false
|
|
contentLength=0"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
#[test_case("node_repl"; "node_repl")]
|
|
#[test_case("cua_repl"; "cua_repl")]
|
|
async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transcript_flags(
|
|
repl_server: &'static str,
|
|
) -> Result<()> {
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
const SCREENSHOT: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
|
|
let server = responses::start_mock_server().await;
|
|
let mcp_server_bin = remote_aware_stdio_server_bin()?;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("enable Code Mode");
|
|
let mcp = serde_json::from_value(serde_json::json!({
|
|
"command": mcp_server_bin,
|
|
"environment_id": remote_aware_environment_id(),
|
|
"cwd": config.cwd,
|
|
"env": { "MCP_TEST_ENABLE_NODE_REPL_JS": "1" },
|
|
"omit_tools_from": ["deferred"],
|
|
}))
|
|
.expect("valid REPL MCP server config");
|
|
config
|
|
.mcp_servers
|
|
.set(HashMap::from([(repl_server.to_owned(), mcp)]))
|
|
.expect("configure REPL MCP server");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
core_test_support::wait_for_mcp_server(&test.codex, repl_server).await?;
|
|
let evidence = test
|
|
.codex
|
|
.thread_extension_data()
|
|
.get_or_init(NodeReplReviewEvidence::default);
|
|
evidence.enable_image_capture();
|
|
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_custom_tool_call(
|
|
"node-repl-image",
|
|
"exec",
|
|
&r#"for (let index = 0; index < 2; index++) await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });"#
|
|
.replace("node_repl", repl_server),
|
|
),
|
|
ev_completed("response-node-repl"),
|
|
]),
|
|
sse(vec![ev_completed("response-done")]),
|
|
],
|
|
)
|
|
.await;
|
|
test.submit_text_turn("capture a computer-use screenshot")
|
|
.await?;
|
|
|
|
assert_eq!(
|
|
evidence.images(),
|
|
vec![ContentItem::InputImage {
|
|
image_url: SCREENSHOT.to_owned(),
|
|
detail: Some(ImageDetail::Low),
|
|
}]
|
|
);
|
|
let parent_request = response_mock
|
|
.requests()
|
|
.pop()
|
|
.expect("parent turn should complete");
|
|
assert!(!serde_json::to_string(&parent_request.input())?.contains(SCREENSHOT));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
#[test_case("node_repl"; "node_repl")]
|
|
#[test_case("cua_repl"; "cua_repl")]
|
|
async fn code_mode_node_repl_image_flag_without_enhanced_stays_disabled(
|
|
repl_server: &'static str,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
skip_if_sandbox!(Ok(()));
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"Guardian approval actions require host-native paths"
|
|
);
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mcp_server_bin = remote_aware_stdio_server_bin()?;
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.5", |model| {
|
|
model.node_repl_auto_review_required = false;
|
|
})
|
|
.with_config(move |config| {
|
|
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
|
|
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("enable Code Mode");
|
|
config
|
|
.features
|
|
.set_enabled(
|
|
Feature::GuardianEnhancedNodeReplTranscripts,
|
|
/*enabled*/ false,
|
|
)
|
|
.expect("disable enhanced Guardian transcripts");
|
|
config
|
|
.features
|
|
.enable(Feature::GuardianNodeReplTranscriptImages)
|
|
.expect("enable Guardian transcript images");
|
|
let mcp = serde_json::from_value(serde_json::json!({
|
|
"command": mcp_server_bin,
|
|
"environment_id": remote_aware_environment_id(),
|
|
"cwd": config.cwd,
|
|
"env": { "MCP_TEST_ENABLE_NODE_REPL_JS": "1" },
|
|
"omit_tools_from": ["deferred"],
|
|
}))
|
|
.expect("valid REPL MCP server config");
|
|
config
|
|
.mcp_servers
|
|
.set(HashMap::from([(repl_server.to_owned(), mcp)]))
|
|
.expect("configure REPL MCP server");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, repl_server).await?;
|
|
|
|
let code = r#"
|
|
await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });
|
|
await tools.exec_command({ cmd: "true", sandbox_permissions: "require_escalated", justification: "review" });
|
|
"#
|
|
.replace("node_repl", repl_server);
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_custom_tool_call("code-mode-call", "exec", &code),
|
|
ev_completed("resp-parent"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("guardian", r#"{"outcome":"allow"}"#),
|
|
ev_completed("resp-guardian"),
|
|
]),
|
|
sse(vec![ev_completed("resp-done")]),
|
|
],
|
|
)
|
|
.await;
|
|
test.submit_text_turn(&format!("review a nested {repl_server} screenshot"))
|
|
.await?;
|
|
|
|
let requests = response_mock.requests();
|
|
assert_eq!(requests.len(), 3);
|
|
let guardian_request = requests
|
|
.iter()
|
|
.find(|request| {
|
|
request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian")
|
|
})
|
|
.expect("the shell command should trigger Guardian review");
|
|
assert!(guardian_request.message_input_image_urls("user").is_empty());
|
|
let guardian_text = guardian_request.message_input_texts("user").concat();
|
|
let parent_input = serde_json::to_string(&requests.last().unwrap().input())?;
|
|
for marker in [
|
|
"guardian-visible-before-image",
|
|
"guardian-visible-after-image",
|
|
"data:image/png;base64,",
|
|
] {
|
|
assert!(!guardian_text.contains(marker));
|
|
assert!(!parent_input.contains(marker));
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
#[test_case("node_repl", false, false, false, None; "disabled")]
|
|
#[test_case("node_repl", true, false, false, None; "manually_enabled_text_only")]
|
|
#[test_case("node_repl", true, true, false, None; "manually_enabled_multimodal")]
|
|
#[test_case("node_repl", false, false, true, None; "required_model_forces_multimodal")]
|
|
#[test_case("node_repl", true, true, false, Some("unsupported"); "text_fallback_without_image_support")]
|
|
#[test_case("node_repl", true, true, false, Some("unbounded"); "text_fallback_without_context_bound")]
|
|
#[test_case("node_repl", true, true, false, Some("small"); "text_fallback_with_insufficient_context")]
|
|
#[test_case("node_repl", true, true, false, Some("large_prompt"); "images_resume_after_prompt_pressure")]
|
|
#[test_case("node_repl", true, true, false, Some("buffered"); "multimodal_reviewer_preserves_model_owned_fallback_buffer")]
|
|
#[test_case("node_repl", true, true, false, Some("rollover"); "multimodal_reviewer_rollover_replays_evidence")]
|
|
#[test_case("cua_repl", false, false, false, None; "cua_disabled")]
|
|
#[test_case("cua_repl", true, false, false, None; "cua_manually_enabled_text_only")]
|
|
#[test_case("cua_repl", true, true, false, None; "cua_manually_enabled_multimodal")]
|
|
#[test_case("cua_repl", false, false, true, None; "cua_required_model_forces_multimodal")]
|
|
#[test_case("cua_repl", true, true, false, Some("unsupported"); "cua_text_fallback_without_image_support")]
|
|
#[test_case("cua_repl", true, true, false, Some("unbounded"); "cua_text_fallback_without_context_bound")]
|
|
#[test_case("cua_repl", true, true, false, Some("small"); "cua_text_fallback_with_insufficient_context")]
|
|
#[test_case("cua_repl", true, true, false, Some("large_prompt"); "cua_images_resume_after_prompt_pressure")]
|
|
async fn code_mode_node_repl_text_evidence_is_visible_only_to_guardian(
|
|
repl_server: &'static str,
|
|
enhanced_transcripts: bool,
|
|
transcript_images: bool,
|
|
auto_review_required: bool,
|
|
reviewer_constraint: Option<&'static str>,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
skip_if_sandbox!(Ok(()));
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"Guardian approval actions require host-native paths"
|
|
);
|
|
const NODE_REPL_DOM_MIDDLE: &str = "guardian-visible-dom-middle";
|
|
const DIRECT_NODE_REPL_MIDDLE: &str = "direct-node-repl-visible-middle";
|
|
const DIRECT_UNRELATED_MIDDLE: &str = "direct-unrelated-hidden-middle";
|
|
const OTHER_NODE_REPL_RESULT: &str = "ECHOING: guardian-visible-other-tool-result";
|
|
const UNRELATED_RESULT: &str = "ECHOING: guardian-hidden-unrelated-result";
|
|
const PRIVATE_IMAGE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
|
|
let server = responses::start_mock_server().await;
|
|
let mcp_server_bin = remote_aware_stdio_server_bin()?;
|
|
let reviewer_rollover = reviewer_constraint == Some("rollover");
|
|
let reviewer_token_budget = matches!(reviewer_constraint, Some("buffered" | "rollover"));
|
|
let check_detail = enhanced_transcripts && transcript_images && reviewer_constraint.is_none();
|
|
let mut large_image = Cursor::new(Vec::new());
|
|
if check_detail {
|
|
DynamicImage::new_rgba8(/*w*/ 2049, /*h*/ 32)
|
|
.write_to(&mut large_image, ImageFormat::Png)?;
|
|
}
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.5", move |model| {
|
|
model.node_repl_auto_review_required = auto_review_required
|
|
})
|
|
.with_config(move |config| {
|
|
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
|
|
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
|
if reviewer_constraint.is_some_and(|value| value != "large_prompt") || check_detail {
|
|
let reviewer = config
|
|
.model_catalog
|
|
.as_mut()
|
|
.expect("bundled model catalog")
|
|
.models
|
|
.iter_mut()
|
|
.find(|model| model.slug == "gpt-5.6-luna")
|
|
.expect("API-key Guardian reviewer");
|
|
if reviewer_token_budget {
|
|
reviewer.context_window = Some(100_000);
|
|
reviewer.max_context_window = Some(100_000);
|
|
reviewer.auto_compact_token_limit = Some(50_000);
|
|
} else if check_detail {
|
|
reviewer.use_responses_lite = false;
|
|
} else if reviewer_constraint == Some("unsupported") {
|
|
reviewer.input_modalities =
|
|
vec![codex_protocol::openai_models::InputModality::Text];
|
|
} else {
|
|
reviewer.context_window =
|
|
(reviewer_constraint == Some("small")).then_some(10_000);
|
|
reviewer.max_context_window = reviewer.context_window;
|
|
}
|
|
}
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("enable Code Mode");
|
|
if reviewer_token_budget {
|
|
config
|
|
.features
|
|
.enable(Feature::TokenBudget)
|
|
.expect("enable token budget");
|
|
}
|
|
config
|
|
.features
|
|
.set_enabled(
|
|
Feature::GuardianEnhancedNodeReplTranscripts,
|
|
enhanced_transcripts,
|
|
)
|
|
.expect("configure enhanced transcripts");
|
|
config
|
|
.features
|
|
.set_enabled(Feature::GuardianNodeReplTranscriptImages, transcript_images)
|
|
.expect("configure Guardian transcript images");
|
|
let mcp: McpServerConfig = serde_json::from_value(serde_json::json!({
|
|
"command": mcp_server_bin,
|
|
"environment_id": remote_aware_environment_id(),
|
|
"cwd": config.cwd,
|
|
"env": {
|
|
"MCP_TEST_ENABLE_NODE_REPL_JS": "1",
|
|
"MCP_TEST_IMAGE_DATA_URL": format!("data:image/png;base64,{}", BASE64_STANDARD.encode(large_image.into_inner())),
|
|
"MCP_TEST_OVERSIZED_INVALID_IMAGE": u8::from(!core_test_support::is_remote_test_environment()).to_string()
|
|
},
|
|
"omit_tools_from": ["deferred"],
|
|
}))
|
|
.expect("valid MCP server config");
|
|
config
|
|
.mcp_servers
|
|
.set(
|
|
[repl_server.to_owned(), format!("{repl_server}_")]
|
|
.into_iter()
|
|
.map(|name| (name, mcp.clone()))
|
|
.collect(),
|
|
)
|
|
.expect("configure MCP servers");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, repl_server).await?;
|
|
let images_enabled = auto_review_required || (enhanced_transcripts && transcript_images);
|
|
let reviewer_images =
|
|
images_enabled && (reviewer_constraint.is_none() || reviewer_token_budget);
|
|
let snapshot_padding = if images_enabled && reviewer_constraint != Some("large_prompt") {
|
|
2_500
|
|
} else {
|
|
10_000
|
|
};
|
|
let padding = "a".repeat(snapshot_padding);
|
|
let snapshot_args = |marker| {
|
|
let padding = if check_detail && marker == DIRECT_NODE_REPL_MIDDLE {
|
|
""
|
|
} else {
|
|
padding.as_str()
|
|
};
|
|
serde_json::json!({
|
|
"message": format!("{padding}{marker}{padding}"),
|
|
})
|
|
.to_string()
|
|
};
|
|
|
|
let code = r#"
|
|
await tools.mcp__node_repl_echo({ message: "guardian-hidden-unrelated-result" });
|
|
await tools.mcp__node_repl__js({ code: 'nodeRepl.fail()' });
|
|
await tools.mcp__node_repl__js({ code: `nodeRepl.write(${JSON.stringify("a".repeat(SNAPSHOT_PADDING) + ["guardian-visible", "dom-middle"].join("-") + "b".repeat(SNAPSHOT_PADDING))})` });
|
|
await tools.mcp__node_repl__echo({ message: ["guardian-visible-other-", "tool-result"].join("") });
|
|
await tools.mcp__node_repl__encrypted_output({});
|
|
await tools.mcp__node_repl__js({ code: 'nodeRepl.empty()' });
|
|
await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });
|
|
if (LARGE_IMAGE) await tools.mcp__node_repl__image_scenario({ scenario: "invalid_image_bytes_then_image" });
|
|
await tools.exec_command({ cmd: "true", sandbox_permissions: "require_escalated", justification: "review" });
|
|
if (!REVIEWER_ROLLOVER) await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });
|
|
if (LARGE_IMAGE) await tools.mcp__node_repl__image({});
|
|
await tools.exec_command({ cmd: "printf second", sandbox_permissions: "require_escalated", justification: "review again" });
|
|
"#
|
|
.replace("node_repl", repl_server)
|
|
.replace("SNAPSHOT_PADDING", &snapshot_padding.to_string())
|
|
.replace("REVIEWER_ROLLOVER", &reviewer_rollover.to_string())
|
|
.replace("LARGE_IMAGE", &check_detail.to_string());
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
responses::ev_function_call_with_namespace(
|
|
"node-repl-call",
|
|
&format!("mcp__{repl_server}"),
|
|
"echo",
|
|
&snapshot_args(DIRECT_NODE_REPL_MIDDLE),
|
|
),
|
|
ev_completed("resp-node-repl"),
|
|
]),
|
|
sse(vec![
|
|
responses::ev_function_call_with_namespace(
|
|
"unrelated-call",
|
|
&format!("mcp__{repl_server}_"),
|
|
"echo",
|
|
&snapshot_args(DIRECT_UNRELATED_MIDDLE),
|
|
),
|
|
ev_completed("resp-unrelated"),
|
|
]),
|
|
sse(vec![
|
|
ev_custom_tool_call("code-mode-call", "exec", &code),
|
|
ev_completed("resp-parent"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("guardian", r#"{"outcome":"allow"}"#),
|
|
if reviewer_token_budget {
|
|
responses::ev_completed_with_tokens(
|
|
"resp-guardian",
|
|
/*total_tokens*/ if reviewer_rollover { 70_000 } else { 60_000 },
|
|
)
|
|
} else {
|
|
ev_completed("resp-guardian")
|
|
},
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("guardian-again", r#"{"outcome":"allow"}"#),
|
|
ev_completed("resp-guardian-again"),
|
|
]),
|
|
sse(vec![ev_completed("resp-done")]),
|
|
],
|
|
)
|
|
.await;
|
|
test.submit_text_turn(&format!("review a nested {repl_server} tool response"))
|
|
.await?;
|
|
let requests = response_mock.requests();
|
|
let guardian_requests = requests
|
|
.iter()
|
|
.filter(|request| {
|
|
request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian")
|
|
})
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(guardian_requests.len(), 2);
|
|
let guardian_text = guardian_requests[0].message_input_texts("user").concat();
|
|
let evidence_enabled = enhanced_transcripts || auto_review_required;
|
|
assert_eq!(
|
|
guardian_text.contains(&format!("tool={repl_server}.js")),
|
|
evidence_enabled,
|
|
"private evidence must identify its originating REPL server"
|
|
);
|
|
for included in [
|
|
DIRECT_NODE_REPL_MIDDLE,
|
|
NODE_REPL_DOM_MIDDLE,
|
|
OTHER_NODE_REPL_RESULT,
|
|
"Lookup completed",
|
|
"<completed without visible text>",
|
|
"guardian-visible-before-image",
|
|
"guardian-visible-after-image",
|
|
] {
|
|
assert_eq!(
|
|
guardian_text.contains(included),
|
|
evidence_enabled,
|
|
"unexpected Guardian evidence visibility for {included}"
|
|
);
|
|
}
|
|
let guardian_request = guardian_requests[0];
|
|
let reviewer_image_urls = guardian_request.message_input_image_urls("user");
|
|
let expected_images = usize::from(reviewer_images) + usize::from(check_detail);
|
|
assert_eq!(reviewer_image_urls.len(), expected_images);
|
|
if reviewer_images {
|
|
assert_eq!(reviewer_image_urls[0], PRIVATE_IMAGE);
|
|
let reviewer_user_content = guardian_request
|
|
.inputs_of_type("message")
|
|
.into_iter()
|
|
.filter(|item| item.get("role").and_then(Value::as_str) == Some("user"))
|
|
.filter_map(|item| item.get("content").and_then(Value::as_array).cloned())
|
|
.flatten()
|
|
.collect::<Vec<_>>();
|
|
let image_index = reviewer_user_content
|
|
.iter()
|
|
.position(|item| item.get("image_url").and_then(Value::as_str) == Some(PRIVATE_IMAGE))
|
|
.expect("private reviewer image should be present");
|
|
if check_detail {
|
|
assert_eq!(reviewer_user_content[image_index]["detail"], "high");
|
|
let payload = reviewer_image_urls[1].split_once(',').unwrap().1;
|
|
let dimensions =
|
|
image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?.dimensions();
|
|
assert_eq!(dimensions, (2048, 32));
|
|
}
|
|
for (index, marker) in [(image_index - 1, "before"), (image_index + 1, "after")] {
|
|
let text = reviewer_user_content[index]["text"].as_str().unwrap();
|
|
assert!(text.contains(marker));
|
|
}
|
|
}
|
|
for excluded in [
|
|
DIRECT_UNRELATED_MIDDLE,
|
|
UNRELATED_RESULT,
|
|
"guardian-hidden-failed-result",
|
|
"gAAAA-test",
|
|
"guardian-hidden-ui-preview",
|
|
"guardian-hidden-structured-override",
|
|
] {
|
|
assert!(
|
|
!guardian_text.contains(excluded),
|
|
"protected, failed, unrelated, or duplicate evidence leaked: {excluded}"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
guardian_requests[1]
|
|
.message_input_texts("user")
|
|
.concat()
|
|
.matches(NODE_REPL_DOM_MIDDLE)
|
|
.count(),
|
|
usize::from(evidence_enabled),
|
|
"a reused Guardian session must not append the same evidence twice"
|
|
);
|
|
assert_eq!(
|
|
guardian_requests[1].message_input_image_urls("user"),
|
|
if reviewer_constraint == Some("large_prompt") {
|
|
vec![PRIVATE_IMAGE.to_string()]
|
|
} else {
|
|
reviewer_image_urls
|
|
}
|
|
);
|
|
if reviewer_rollover {
|
|
let second_request = guardian_requests[1];
|
|
let second_prompt = second_request
|
|
.message_input_text_groups("user")
|
|
.last()
|
|
.expect("post-rollover Guardian review prompt")
|
|
.join("");
|
|
assert!(second_prompt.contains(">>> TRANSCRIPT START\n"));
|
|
assert!(!second_prompt.contains(">>> TRANSCRIPT DELTA START\n"));
|
|
assert!(second_prompt.contains(NODE_REPL_DOM_MIDDLE));
|
|
assert_eq!(
|
|
second_request.message_input_image_urls("user"),
|
|
vec![PRIVATE_IMAGE.to_string()],
|
|
"browser screenshots must be replayed into the fresh reviewer window"
|
|
);
|
|
assert!(
|
|
second_request
|
|
.message_input_texts("developer")
|
|
.iter()
|
|
.any(|text| text.contains("Previous context window id:")),
|
|
"Guardian should roll over before rebuilding browser evidence"
|
|
);
|
|
} else if reviewer_token_budget {
|
|
let second_request = guardian_requests[1];
|
|
let second_prompt = second_request
|
|
.message_input_text_groups("user")
|
|
.last()
|
|
.expect("buffered Guardian review prompt")
|
|
.join("");
|
|
assert!(second_prompt.contains(">>> TRANSCRIPT DELTA START\n"));
|
|
assert!(
|
|
second_request
|
|
.message_input_texts("developer")
|
|
.iter()
|
|
.all(|text| !text.contains("Previous context window id:")),
|
|
"Guardian should preserve the reviewer model's fallback buffer before rolling over"
|
|
);
|
|
}
|
|
let parent_request = requests.last().expect("parent turn should complete");
|
|
let parent_input = serde_json::to_string(&parent_request.input())?;
|
|
assert!(!parent_input.contains("data:image/png;base64,"));
|
|
assert!(!parent_input.contains("guardian-visible-before-image"));
|
|
assert!(!parent_input.contains("guardian-visible-after-image"));
|
|
assert!(
|
|
!parent_input.contains(NODE_REPL_DOM_MIDDLE)
|
|
&& !parent_input.contains(OTHER_NODE_REPL_RESULT)
|
|
&& !parent_input.contains(UNRELATED_RESULT),
|
|
"nested MCP responses leaked into parent model history: {parent_input}"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_can_call_mcp_tool() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.mcp__rmcp__echo({ message: "ping" });
|
|
text(`echo=${result.structuredContent?.echo ?? "missing"}`);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode(
|
|
&server,
|
|
"use exec to run the rmcp echo tool in code mode only",
|
|
code,
|
|
/*code_mode_only*/ true,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only rmcp tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "echo=ECHOING: ping");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({
|
|
message: "ping",
|
|
});
|
|
text(
|
|
`hasEcho=${String(Object.keys(tools).includes("mcp__rmcp__echo"))}\n` +
|
|
`echoType=${typeof tools.mcp__rmcp__echo}\n` +
|
|
`echo=${structuredContent?.echo ?? "missing"}\n` +
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec global rmcp access failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"hasEcho=true
|
|
echoType=function
|
|
echo=ECHOING: ping
|
|
isError=false
|
|
contentLength=0"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_uses_non_prefixed_mcp_tool_names_when_feature_enabled() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.rmcp__echo({ message: "ping" });
|
|
text(JSON.stringify({
|
|
hasNonPrefixedEcho: typeof tools.rmcp__echo === "function",
|
|
hasPrefixedEcho: typeof tools.mcp__rmcp__echo === "function",
|
|
echo: result.structuredContent?.echo ?? "missing",
|
|
}));
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_config(
|
|
&server,
|
|
"use exec to inspect non-prefixed MCP names",
|
|
code,
|
|
"test-gpt-5.1-codex",
|
|
/*code_mode_only*/ false,
|
|
/*non_prefixed_mcp_tool_names*/ true,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec non-prefixed rmcp access failed unexpectedly: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"hasNonPrefixedEcho": true,
|
|
"hasPrefixedEcho": false,
|
|
"echo": "ECHOING: ping",
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
struct NamespacedCustomTool {
|
|
generation: usize,
|
|
generations: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl ToolContributor for NamespacedCustomTool {
|
|
fn tools(
|
|
&self,
|
|
_session_store: &ExtensionData,
|
|
_thread_store: &ExtensionData,
|
|
) -> Vec<Arc<dyn for<'call> ToolExecutor<ToolCall<'call>>>> {
|
|
Vec::new()
|
|
}
|
|
|
|
fn tools_for_step(
|
|
&self,
|
|
_session_store: &ExtensionData,
|
|
_thread_store: &ExtensionData,
|
|
_step_store: &ExtensionData,
|
|
) -> Vec<Arc<dyn for<'call> ToolExecutor<ToolCall<'call>>>> {
|
|
vec![Arc::new(Self {
|
|
generation: self.generations.fetch_add(1, Ordering::Relaxed) + 1,
|
|
generations: Arc::clone(&self.generations),
|
|
})]
|
|
}
|
|
}
|
|
|
|
impl<'call> ToolExecutor<ToolCall<'call>> for NamespacedCustomTool {
|
|
fn tool_name(&self) -> ToolName {
|
|
ToolName::namespaced("editor", "apply_patch")
|
|
}
|
|
|
|
fn spec(&self) -> ToolSpec {
|
|
ToolSpec::Namespace(ResponsesApiNamespace {
|
|
name: "editor".to_string(),
|
|
description: "Editing tools.".to_string(),
|
|
tools: vec![ResponsesApiNamespaceTool::Custom(FreeformTool {
|
|
name: "apply_patch".to_string(),
|
|
description: format!("Apply a raw editor patch (step {}).", self.generation),
|
|
defer_loading: None,
|
|
format: FreeformToolFormat {
|
|
r#type: "grammar".to_string(),
|
|
syntax: "lark".to_string(),
|
|
definition: "start: /.+/".to_string(),
|
|
},
|
|
})],
|
|
})
|
|
}
|
|
|
|
fn handle<'a>(&'a self, call: ToolCall<'call>) -> ToolExecutorFuture<'a>
|
|
where
|
|
'call: 'a,
|
|
{
|
|
Box::pin(async move {
|
|
let ToolPayload::Custom { input } = call.payload else {
|
|
return Err(FunctionCallError::Fatal(
|
|
"expected custom tool payload".to_string(),
|
|
));
|
|
};
|
|
Ok(Box::new(JsonToolOutput::new(serde_json::json!({
|
|
"namespace": call.tool_name.namespace,
|
|
"name": call.tool_name.name,
|
|
"input": input,
|
|
"generation": self.generation,
|
|
}))) as Box<dyn ToolOutput>)
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_and_dispatches_namespaced_custom_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
|
|
extensions.tool_contributor(Arc::new(NamespacedCustomTool {
|
|
generation: 0,
|
|
generations: Arc::new(AtomicUsize::new(0)),
|
|
}));
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_extensions(Arc::new(extensions.build()))
|
|
.with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(({ name }) => name === "editor__apply_patch");
|
|
const result = await tools.editor__apply_patch("nested patch");
|
|
text(JSON.stringify({
|
|
name: tool?.name ?? null,
|
|
description: tool?.description ?? null,
|
|
result,
|
|
}));
|
|
"#;
|
|
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
responses::ev_custom_tool_call_with_namespace(
|
|
"call-direct",
|
|
"editor",
|
|
"apply_patch",
|
|
"direct patch",
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-2"),
|
|
ev_custom_tool_call("call-exec", "exec", code),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("call the namespaced custom editor tool directly and through exec")
|
|
.await?;
|
|
|
|
let requests = response_mock.requests();
|
|
assert_eq!(requests.len(), 3);
|
|
|
|
let declaration =
|
|
"declare const tools: { editor__apply_patch(input: string): Promise<unknown>; };";
|
|
let description = |generation| {
|
|
format!(
|
|
"Apply a raw editor patch (step {generation}).\n\nexec tool declaration:\n```ts\n{declaration}\n```"
|
|
)
|
|
};
|
|
let first_body = requests[0].body_json();
|
|
let namespaced_custom_tool = namespace_child_tool(&first_body, "editor", "apply_patch")
|
|
.expect("namespaced custom tool should be included in the model request");
|
|
assert_eq!(
|
|
namespaced_custom_tool,
|
|
&serde_json::json!({
|
|
"type": "custom",
|
|
"name": "apply_patch",
|
|
"description": description(1),
|
|
"format": {
|
|
"type": "grammar",
|
|
"syntax": "lark",
|
|
"definition": "start: /.+/",
|
|
},
|
|
})
|
|
);
|
|
let second_body = requests[1].body_json();
|
|
let second_description = description(2);
|
|
assert_eq!(
|
|
namespace_child_tool(&second_body, "editor", "apply_patch")
|
|
.and_then(|tool| tool.get("description"))
|
|
.and_then(Value::as_str),
|
|
Some(second_description.as_str())
|
|
);
|
|
|
|
let (direct_output, direct_success) =
|
|
custom_tool_output_body_and_success(&requests[1], "call-direct");
|
|
assert_ne!(direct_success, Some(false));
|
|
let direct_output = serde_json::from_str::<Value>(&direct_output).unwrap_or_else(|error| {
|
|
panic!("invalid direct custom tool output `{direct_output}`: {error}")
|
|
});
|
|
assert_eq!(
|
|
direct_output,
|
|
serde_json::json!({
|
|
"namespace": "editor",
|
|
"name": "apply_patch",
|
|
"input": "direct patch",
|
|
"generation": 1,
|
|
})
|
|
);
|
|
|
|
let (exec_output, exec_success) =
|
|
custom_tool_output_body_and_success(&requests[2], "call-exec");
|
|
assert_ne!(exec_success, Some(false));
|
|
let exec_output = serde_json::from_str::<Value>(&exec_output).unwrap_or_else(|error| {
|
|
panic!("invalid code mode custom tool output `{exec_output}`: {error}")
|
|
});
|
|
assert_eq!(
|
|
exec_output,
|
|
serde_json::json!({
|
|
"name": "editor__apply_patch",
|
|
"description": format!("Editing tools.\n\n{second_description}"),
|
|
"result": {
|
|
"namespace": "editor",
|
|
"name": "apply_patch",
|
|
"input": "nested patch",
|
|
"generation": 2,
|
|
},
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_namespaced_mcp_tools_on_global_tools_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
text(JSON.stringify({
|
|
hasExecCommand: typeof tools.exec_command === "function",
|
|
hasNamespacedEcho: typeof tools.mcp__rmcp__echo === "function",
|
|
}));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec global tools inspection failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"hasExecCommand": true,
|
|
"hasNamespacedEcho": true,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_normalized_illegal_mcp_tool_names() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.mcp__rmcp__echo_tool({ message: "ping" });
|
|
text(`echo=${result.structuredContent.echo}`);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp(
|
|
&server,
|
|
"use exec to call a normalized rmcp tool name",
|
|
code,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec normalized rmcp tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "echo=ECHOING: ping");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exports_all_tools_metadata_for_builtin_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(({ name }) => name === "view_image");
|
|
text(JSON.stringify(tool));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn(&server, "use exec to inspect ALL_TOOLS", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec ALL_TOOLS lookup failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec ALL_TOOLS lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"name": "view_image",
|
|
"description": "View a local image file from the filesystem when visual inspection is needed. Use this for images already available on disk.\n\nexec tool declaration:\n```ts\ndeclare const tools: { view_image(args: {\n // Local filesystem path to an image file.\n path: string;\n}): Promise<{\n // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved.\n detail: \"high\" | \"original\";\n // Data URL for the loaded image.\n image_url: string;\n}>; };\n```",
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exports_all_tools_metadata_for_namespaced_mcp_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(
|
|
({ name }) => name === "mcp__rmcp__echo"
|
|
);
|
|
text(JSON.stringify(tool));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect ALL_TOOLS", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec ALL_TOOLS MCP lookup failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec ALL_TOOLS MCP lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"name": "mcp__rmcp__echo",
|
|
"description": concat!(
|
|
"Use these tools to exercise the rmcp test server.\n\n",
|
|
"Echo back the provided message and include environment data.\n\n",
|
|
"exec tool declaration:\n",
|
|
"```ts\n",
|
|
"declare const tools: { mcp__rmcp__echo(args: { env_var?: string; message: string; }): ",
|
|
"Promise<CallToolResult<{ echo: string; env: string | null; }>>; };\n",
|
|
"```",
|
|
),
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_uses_the_first_dynamic_tool_for_a_normalized_name() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
for use_responses_lite in [false, true] {
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.5", move |model_info| {
|
|
model_info.use_responses_lite = use_responses_lite;
|
|
model_info.tool_mode = Some(ToolMode::CodeMode);
|
|
})
|
|
.with_config(|config| {
|
|
config.tool_registry.turn_metadata_includes_tool_info = true;
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
});
|
|
let base_test = builder.build_with_auto_env(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread(StartThreadOptions {
|
|
dynamic_tools: [
|
|
("foo-bar", "First normalized dynamic tool."),
|
|
("foo_bar", "Shadowed normalized dynamic tool."),
|
|
]
|
|
.into_iter()
|
|
.map(|(name, description)| {
|
|
DynamicToolSpec::Function(DynamicToolFunctionSpec {
|
|
name: name.to_string(),
|
|
description: description.to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {},
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: false,
|
|
})
|
|
})
|
|
.collect(),
|
|
..StartThreadOptions::new(base_test.config.clone())
|
|
})
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
let first_response = if use_responses_lite {
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const matches = ALL_TOOLS.filter(({ name }) => name === "foo_bar");
|
|
const output = await tools.foo_bar({});
|
|
text(JSON.stringify({
|
|
count: matches.length,
|
|
name: matches[0]?.name ?? null,
|
|
description: matches[0]?.description ?? null,
|
|
output,
|
|
}));
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
])
|
|
} else {
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
])
|
|
};
|
|
let first_mock = responses::mount_sse_once(&server, first_response).await;
|
|
let second_mock = if use_responses_lite {
|
|
Some(
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await,
|
|
)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let cwd = test.config.cwd.clone();
|
|
let (sandbox_policy, permission_profile) =
|
|
turn_permission_fields(PermissionProfile::Disabled, cwd.as_path());
|
|
test.codex
|
|
.start_or_steer_turn(
|
|
TurnInputRequest::user_input(vec![UserInput::Text {
|
|
text: "inspect and call normalized dynamic tools".to_string(),
|
|
text_elements: Vec::new(),
|
|
}])
|
|
.with_thread_settings(ThreadSettingsOverrides {
|
|
environments: Some(codex_protocol::protocol::TurnEnvironmentSelections::new(
|
|
cwd,
|
|
Vec::new(),
|
|
)),
|
|
approval_policy: Some(AskForApproval::Never),
|
|
sandbox_policy: Some(sandbox_policy),
|
|
permission_profile,
|
|
collaboration_mode: Some(CollaborationMode {
|
|
mode: ModeKind::Default,
|
|
settings: Settings {
|
|
model: test.session_configured.model.clone(),
|
|
reasoning_effort: None,
|
|
developer_instructions: None,
|
|
},
|
|
}),
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await?;
|
|
|
|
let turn_id = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::TurnStarted(event) => Some(event.turn_id.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
if use_responses_lite {
|
|
let request = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::DynamicToolCallRequest(request) => Some(request.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
assert_eq!(request.namespace, None);
|
|
assert_eq!(request.tool, "foo-bar");
|
|
assert_eq!(request.arguments, serde_json::json!({}));
|
|
test.codex
|
|
.submit(Op::DynamicToolResponse {
|
|
id: request.call_id,
|
|
response: DynamicToolResponse {
|
|
content_items: vec![DynamicToolCallOutputContentItem::InputText {
|
|
text: "first-winner".to_string(),
|
|
}],
|
|
success: true,
|
|
},
|
|
})
|
|
.await?;
|
|
}
|
|
wait_for_event(&test.codex, |event| match event {
|
|
EventMsg::TurnComplete(event) => event.turn_id == turn_id,
|
|
_ => false,
|
|
})
|
|
.await;
|
|
|
|
let first_body = first_mock.single_request().body_json();
|
|
let model_tools = if use_responses_lite {
|
|
first_body["input"]
|
|
.as_array()
|
|
.and_then(|input| {
|
|
input.iter().find(|item| {
|
|
item.get("type").and_then(Value::as_str) == Some("additional_tools")
|
|
})
|
|
})
|
|
.and_then(|item| item["tools"].as_array())
|
|
.and_then(|tools| {
|
|
tools.iter().find(|tool| {
|
|
tool.get("type").and_then(Value::as_str) == Some("namespace")
|
|
&& tool.get("name").and_then(Value::as_str) == Some("functions")
|
|
})
|
|
})
|
|
.and_then(|namespace| namespace["tools"].as_array())
|
|
.expect("the Responses Lite request should contain its default-namespace tools")
|
|
} else {
|
|
first_body["tools"]
|
|
.as_array()
|
|
.expect("the Responses request should contain its visible tools")
|
|
};
|
|
let visible_dynamic_tools = model_tools
|
|
.iter()
|
|
.filter(|tool| matches!(tool["name"].as_str(), Some("foo-bar" | "foo_bar")))
|
|
.map(|tool| {
|
|
(
|
|
tool["name"]
|
|
.as_str()
|
|
.expect("dynamic tools should have a name"),
|
|
tool["description"]
|
|
.as_str()
|
|
.expect("dynamic tools should have a description"),
|
|
)
|
|
})
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(
|
|
visible_dynamic_tools,
|
|
[
|
|
(
|
|
"foo-bar",
|
|
concat!(
|
|
"First normalized dynamic tool.\n\n",
|
|
"exec tool declaration:\n",
|
|
"```ts\n",
|
|
"declare const tools: { foo_bar(args: {}): Promise<unknown>; };\n",
|
|
"```",
|
|
),
|
|
),
|
|
("foo_bar", "Shadowed normalized dynamic tool."),
|
|
]
|
|
);
|
|
|
|
if use_responses_lite {
|
|
let metadata: Value = serde_json::from_str(
|
|
first_body["client_metadata"]["x-codex-turn-metadata"]
|
|
.as_str()
|
|
.expect("Responses Lite should contain serialized turn metadata"),
|
|
)?;
|
|
let functions = &metadata["tool_namespaces_info"]["functions"]["functions"];
|
|
assert_eq!(functions["foo-bar"]["code_mode_name"], "foo_bar");
|
|
assert!(functions["foo_bar"]["code_mode_name"].is_null());
|
|
}
|
|
|
|
let exec_description = model_tools
|
|
.iter()
|
|
.find(|tool| tool["name"] == "exec")
|
|
.and_then(|tool| tool["description"].as_str())
|
|
.expect("the model request should contain the code-mode exec tool");
|
|
assert!(!exec_description.contains("First normalized dynamic tool."));
|
|
assert!(!exec_description.contains("Shadowed normalized dynamic tool."));
|
|
|
|
if let Some(second_mock) = second_mock {
|
|
let request = second_mock.single_request();
|
|
let output = custom_tool_output_last_non_empty_text(&request, "call-1")
|
|
.expect("code mode should return normalized tool metadata");
|
|
let result: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(result["count"], serde_json::json!(1));
|
|
assert_eq!(result["name"], serde_json::json!("foo_bar"));
|
|
assert_eq!(result["output"], serde_json::json!("first-winner"));
|
|
let description = result["description"]
|
|
.as_str()
|
|
.expect("the winning tool should have a description");
|
|
assert!(description.contains("First normalized dynamic tool."));
|
|
assert!(!description.contains("Shadowed normalized dynamic tool."));
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_renders_local_refs_in_outbound_exec_description() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let response = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let mut builder = test_codex().with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("code mode only should be enabled");
|
|
});
|
|
let base_test = builder.build_with_auto_env(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread(StartThreadOptions {
|
|
dynamic_tools: vec![DynamicToolSpec::Function(DynamicToolFunctionSpec {
|
|
name: "boolean_search".to_string(),
|
|
description: "Search with nested Boolean clauses.".to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {
|
|
"clauses": {
|
|
"type": "array",
|
|
"items": { "$ref": "#/$defs/Clause" }
|
|
}
|
|
},
|
|
"$defs": {
|
|
"Clause": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {
|
|
"oneOf": [
|
|
{ "type": "string" },
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"clauses": {
|
|
"type": "array",
|
|
"items": { "$ref": "#/$defs/Clause" }
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}),
|
|
defer_loading: false,
|
|
})],
|
|
..StartThreadOptions::new(base_test.config.clone())
|
|
})
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
test.submit_turn("inspect the tool schema").await?;
|
|
|
|
let body = response.single_request().body_json();
|
|
let exec_description = body["tools"]
|
|
.as_array()
|
|
.expect("request should contain tools")
|
|
.iter()
|
|
.find_map(|tool| {
|
|
(tool["name"].as_str() == Some("exec"))
|
|
.then(|| tool["description"].as_str())
|
|
.flatten()
|
|
})
|
|
.expect("Code Mode exec should remain available");
|
|
assert!(exec_description.contains("query?: string | { clauses?: Array<"));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_hidden_dynamic_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let base_test = builder.build(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread(StartThreadOptions {
|
|
dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
|
name: "codex_app".to_string(),
|
|
description: "Codex app tools.".to_string(),
|
|
tools: vec![DynamicToolNamespaceTool::Function(
|
|
DynamicToolFunctionSpec {
|
|
name: "hidden_dynamic_tool".to_string(),
|
|
description: "A hidden dynamic tool.".to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {
|
|
"city": { "type": "string" }
|
|
},
|
|
"required": ["city"],
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: true,
|
|
},
|
|
)],
|
|
})],
|
|
..StartThreadOptions::new(base_test.config.clone())
|
|
})
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(({ name }) => name === "codex_app__hidden_dynamic_tool");
|
|
const out = await tools.codex_app__hidden_dynamic_tool({ city: "Paris" });
|
|
text(
|
|
JSON.stringify({
|
|
name: tool?.name ?? null,
|
|
description: tool?.description ?? null,
|
|
out,
|
|
})
|
|
);
|
|
"#;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let cwd = test.config.cwd.clone();
|
|
let (sandbox_policy, permission_profile) =
|
|
turn_permission_fields(PermissionProfile::Disabled, cwd.as_path());
|
|
|
|
test.codex
|
|
.start_or_steer_turn(
|
|
TurnInputRequest::user_input(vec![UserInput::Text {
|
|
text: "use exec to inspect and call hidden tools".into(),
|
|
text_elements: Vec::new(),
|
|
}])
|
|
.with_thread_settings(ThreadSettingsOverrides {
|
|
environments: Some(codex_protocol::protocol::TurnEnvironmentSelections::new(
|
|
cwd,
|
|
Vec::new(),
|
|
)),
|
|
approval_policy: Some(AskForApproval::Never),
|
|
sandbox_policy: Some(sandbox_policy),
|
|
permission_profile,
|
|
collaboration_mode: Some(CollaborationMode {
|
|
mode: ModeKind::Default,
|
|
settings: Settings {
|
|
model: test.session_configured.model.clone(),
|
|
reasoning_effort: None,
|
|
developer_instructions: None,
|
|
},
|
|
}),
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await?;
|
|
|
|
let turn_id = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::TurnStarted(event) => Some(event.turn_id.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
let request = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::DynamicToolCallRequest(request) => Some(request.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
assert_eq!(request.namespace.as_deref(), Some("codex_app"));
|
|
assert_eq!(request.tool, "hidden_dynamic_tool");
|
|
assert_eq!(request.arguments, serde_json::json!({ "city": "Paris" }));
|
|
test.codex
|
|
.submit(Op::DynamicToolResponse {
|
|
id: request.call_id,
|
|
response: DynamicToolResponse {
|
|
content_items: vec![DynamicToolCallOutputContentItem::InputText {
|
|
text: "hidden-ok".to_string(),
|
|
}],
|
|
success: true,
|
|
},
|
|
})
|
|
.await?;
|
|
wait_for_event(&test.codex, |event| match event {
|
|
EventMsg::TurnComplete(event) => event.turn_id == turn_id,
|
|
_ => false,
|
|
})
|
|
.await;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec hidden dynamic tool call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec hidden dynamic tool lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed.get("name"),
|
|
Some(&Value::String("codex_app__hidden_dynamic_tool".to_string()))
|
|
);
|
|
assert_eq!(
|
|
parsed.get("out"),
|
|
Some(&Value::String("hidden-ok".to_string()))
|
|
);
|
|
assert!(
|
|
parsed
|
|
.get("description")
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|description| {
|
|
description.contains("Codex app tools.")
|
|
&& description.contains("A hidden dynamic tool.")
|
|
&& description.contains("declare const tools:")
|
|
&& description.contains("codex_app__hidden_dynamic_tool(args:")
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_excludes_configured_nested_tool_namespaces() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(|config| {
|
|
config.update_plan_enabled = true;
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
config.code_mode.excluded_tool_namespaces = vec!["excluded".to_string()];
|
|
});
|
|
let base_test = builder.build(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread(StartThreadOptions {
|
|
dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
|
name: "excluded".to_string(),
|
|
description: "Excluded tools.".to_string(),
|
|
tools: vec![DynamicToolNamespaceTool::Function(
|
|
DynamicToolFunctionSpec {
|
|
name: "lookup".to_string(),
|
|
description: "An excluded dynamic tool.".to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {},
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: false,
|
|
},
|
|
)],
|
|
})],
|
|
..StartThreadOptions::new(base_test.config.clone())
|
|
})
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
text(JSON.stringify({
|
|
excludedType: typeof tools.excluded__lookup,
|
|
excludedMetadata: ALL_TOOLS.some(({ name }) => name === "excluded__lookup"),
|
|
allowedType: typeof tools.update_plan,
|
|
allowedMetadata: ALL_TOOLS.some(({ name }) => name === "update_plan"),
|
|
}));
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to inspect nested tool namespaces")
|
|
.await?;
|
|
|
|
assert!(
|
|
tool_names(&first_mock.single_request().body_json()).contains(&"excluded".to_string()),
|
|
"excluded namespace should remain directly exposed in mixed code mode"
|
|
);
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec configured namespace exclusion failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
serde_json::from_str::<Value>(&output)?,
|
|
serde_json::json!({
|
|
"excludedType": "undefined",
|
|
"excludedMetadata": false,
|
|
"allowedType": "function",
|
|
"allowedMetadata": true,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_omits_configured_mcp_server_tools() -> Result<()> {
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
|
let environment_id = remote_aware_environment_id();
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", |model| {
|
|
model.supports_search_tool = false;
|
|
})
|
|
.with_config(move |config| {
|
|
config.update_plan_enabled = true;
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
serde_json::from_value(serde_json::json!({
|
|
"command": rmcp_test_server_bin,
|
|
"environment_id": environment_id,
|
|
"cwd": config.cwd,
|
|
"omit_tools_from": ["code_mode"],
|
|
}))
|
|
.expect("test MCP server config should be valid"),
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test config should allow MCP servers");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
text(JSON.stringify({
|
|
excludedType: typeof tools.mcp__rmcp__echo,
|
|
excludedMetadata: ALL_TOOLS.some(({ name }) => name === "mcp__rmcp__echo"),
|
|
allowedType: typeof tools.update_plan,
|
|
allowedMetadata: ALL_TOOLS.some(({ name }) => name === "update_plan"),
|
|
}));
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to inspect nested tool namespaces")
|
|
.await?;
|
|
|
|
assert!(
|
|
tool_names(&first_mock.single_request().body_json()).contains(&"mcp__rmcp".to_string()),
|
|
"omitted MCP server should remain directly exposed in mixed code mode"
|
|
);
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec configured MCP omission failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
serde_json::from_str::<Value>(&output)?,
|
|
serde_json::json!({
|
|
"excludedType": "undefined",
|
|
"excludedMetadata": false,
|
|
"allowedType": "function",
|
|
"allowedMetadata": true,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_keeps_mcp_tools_direct_when_nested_exposure_is_omitted() -> Result<()> {
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
|
let environment_id = remote_aware_environment_id();
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", |model| {
|
|
model.supports_search_tool = true;
|
|
})
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
serde_json::from_value(serde_json::json!({
|
|
"command": rmcp_test_server_bin,
|
|
"environment_id": environment_id,
|
|
"cwd": config.cwd,
|
|
"omit_tools_from": ["code_mode"],
|
|
}))
|
|
.expect("test MCP server config should be valid"),
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test config should allow MCP servers");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
responses::ev_function_call_with_namespace(
|
|
"call-1",
|
|
"mcp__rmcp",
|
|
"echo",
|
|
r#"{"message":"ping"}"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("call the directly exposed MCP echo tool")
|
|
.await?;
|
|
|
|
let first_request = first_mock.single_request().body_json();
|
|
assert!(
|
|
namespace_child_tool(&first_request, "mcp__rmcp", "echo").is_some(),
|
|
"an MCP tool omitted only from Code Mode must remain directly callable"
|
|
);
|
|
let exec_description = first_request["tools"]
|
|
.as_array()
|
|
.expect("request should contain tools")
|
|
.iter()
|
|
.find_map(|tool| {
|
|
(tool["name"].as_str() == Some("exec"))
|
|
.then(|| tool["description"].as_str())
|
|
.flatten()
|
|
})
|
|
.expect("Code Mode exec should remain available");
|
|
assert!(
|
|
!exec_description.contains("mcp__rmcp__echo"),
|
|
"an MCP tool omitted from Code Mode must not appear in the exec declaration"
|
|
);
|
|
|
|
let output = second_mock.single_request().function_call_output("call-1");
|
|
assert!(
|
|
output["output"]
|
|
.as_str()
|
|
.is_some_and(|output| output.contains("ECHOING: ping")),
|
|
"the directly exposed MCP tool must execute successfully: {output:?}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_can_call_mcp_tools_hidden_from_direct_and_deferred_exposure() -> Result<()>
|
|
{
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"requires a Windows test_stdio_server in the Wine-exec environment"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
|
let environment_id = remote_aware_environment_id();
|
|
let mut builder = test_codex()
|
|
.with_model_info_override("gpt-5.4", |model| {
|
|
model.supports_search_tool = true;
|
|
})
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
serde_json::from_value(serde_json::json!({
|
|
"command": rmcp_test_server_bin,
|
|
"environment_id": environment_id,
|
|
"cwd": config.cwd,
|
|
"omit_tools_from": ["direct", "deferred"],
|
|
"supports_parallel_tool_calls": true,
|
|
}))
|
|
.expect("test MCP server config should be valid"),
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test config should allow MCP servers");
|
|
});
|
|
let test = builder.build_with_auto_env(&server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const results = await Promise.all([
|
|
tools.mcp__rmcp__echo({ message: "ping" }),
|
|
tools.mcp__rmcp__echo({ message: "pong" }),
|
|
]);
|
|
text(results.map(({ structuredContent }) => structuredContent?.echo ?? "missing").join(","));
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("call the MCP echo tool from Code Mode")
|
|
.await?;
|
|
|
|
let first_request = first_mock.single_request().body_json();
|
|
assert!(
|
|
namespace_child_tool(&first_request, "mcp__rmcp", "echo").is_none(),
|
|
"MCP tool omitted from direct and deferred exposure must remain hidden at the top level"
|
|
);
|
|
let exec_description = first_request["tools"]
|
|
.as_array()
|
|
.expect("request should contain tools")
|
|
.iter()
|
|
.find_map(|tool| {
|
|
(tool["name"].as_str() == Some("exec"))
|
|
.then(|| tool["description"].as_str())
|
|
.flatten()
|
|
})
|
|
.expect("Code Mode exec should remain available");
|
|
assert!(
|
|
exec_description.contains("mcp__rmcp__echo"),
|
|
"hidden MCP tool must remain available inside Code Mode"
|
|
);
|
|
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"hidden Code Mode MCP tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "ECHOING: ping,ECHOING: pong");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_content_only_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__image_scenario({
|
|
scenario: "text_only",
|
|
caption: "caption from mcp",
|
|
});
|
|
text(
|
|
`firstType=${content[0]?.type ?? "missing"}\n` +
|
|
`firstText=${content[0]?.text ?? "missing"}\n` +
|
|
`structuredContent=${String(structuredContent ?? null)}\n` +
|
|
`isError=${String(isError)}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp(
|
|
&server,
|
|
"use exec to run the rmcp image scenario tool",
|
|
code,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp image scenario call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"firstType=text
|
|
firstText=caption from mcp
|
|
structuredContent=null
|
|
isError=false"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_error_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({});
|
|
const firstText = content[0]?.text ?? "";
|
|
const mentionsMissingMessage =
|
|
firstText.includes("missing field") && firstText.includes("message");
|
|
text(
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}\n` +
|
|
`mentionsMissingMessage=${String(mentionsMissingMessage)}\n` +
|
|
`structuredContent=${String(structuredContent ?? null)}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to call rmcp echo badly", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp error call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"isError=true
|
|
contentLength=1
|
|
mentionsMissingMessage=true
|
|
structuredContent=null"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_store_and_load_values_across_turns() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
store("nb", { title: "Notebook", items: [1, true, null] });
|
|
text("stored");
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_follow_up = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "stored"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("store value for later").await?;
|
|
|
|
let first_request = first_follow_up.single_request();
|
|
let (first_output, first_success) =
|
|
custom_tool_output_body_and_success(&first_request, "call-1");
|
|
assert_ne!(
|
|
first_success,
|
|
Some(false),
|
|
"exec store call failed unexpectedly: {first_output}"
|
|
);
|
|
assert_eq!(first_output, "stored");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call(
|
|
"call-2",
|
|
"exec",
|
|
r#"
|
|
text(JSON.stringify(load("nb")));
|
|
"#,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_follow_up = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "loaded"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("load the stored value").await?;
|
|
|
|
let second_request = second_follow_up.single_request();
|
|
let (second_output, second_success) =
|
|
custom_tool_output_body_and_success(&second_request, "call-2");
|
|
assert_ne!(
|
|
second_success,
|
|
Some(false),
|
|
"exec load call failed unexpectedly: {second_output}"
|
|
);
|
|
let loaded: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&second_request, "call-2")
|
|
.expect("exec load call should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
loaded,
|
|
serde_json::json!({ "title": "Notebook", "items": [1, true, null] })
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_compare_elapsed_time_around_set_timeout() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"measure elapsed time around setTimeout",
|
|
r#"
|
|
const start_ms = Date.now();
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
const end_ms = Date.now();
|
|
text(JSON.stringify({
|
|
start_ms,
|
|
end_ms,
|
|
elapsed_ms: end_ms - start_ms,
|
|
waited_long_enough: end_ms - start_ms >= 100,
|
|
}));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let second_request = second_mock.single_request();
|
|
let (second_output, second_success) =
|
|
custom_tool_output_body_and_success(&second_request, "call-1");
|
|
assert_ne!(
|
|
second_success,
|
|
Some(false),
|
|
"exec compare time call failed unexpectedly: {second_output}"
|
|
);
|
|
let compared: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&second_request, "call-1")
|
|
.expect("exec compare time call should emit JSON"),
|
|
)?;
|
|
let elapsed_ms = compared
|
|
.get("elapsed_ms")
|
|
.and_then(Value::as_i64)
|
|
.expect("elapsed_ms should be an integer");
|
|
assert!(
|
|
elapsed_ms >= 100,
|
|
"expected elapsed_ms >= 100, got {elapsed_ms}"
|
|
);
|
|
assert_eq!(compared.get("waited_long_enough"), Some(&Value::Bool(true)));
|
|
|
|
Ok(())
|
|
}
|