Wire remote browser tools into turn/start

Auto-inject the remote browser dynamic tools when the app-server is configured with a remote browser endpoint, route those tool calls internally against the remote browser adapter, and cover the prompt-loop path with integration tests.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Liang-Ting Jiang
2026-03-24 04:27:49 +00:00
co-authored by Codex
parent 4b69a17b9f
commit 34d59e54c3
8 changed files with 720 additions and 15 deletions
@@ -2,8 +2,10 @@ use anyhow::Context;
use anyhow::Result;
use app_test_support::McpProcess;
use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::to_response;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallParams;
use codex_app_server_protocol::DynamicToolCallResponse;
@@ -32,7 +34,11 @@ use std::path::Path;
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
@@ -191,6 +197,266 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu
Ok(())
}
#[tokio::test]
async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()>
{
let server = create_mock_responses_server_repeating_assistant("Done").await;
let remote_browser_server = MockServer::start().await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id,
input: vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let body = bodies
.first()
.context("expected at least one responses request")?;
assert!(find_tool(body, "create_tab").is_some());
assert!(find_tool(body, "tabs_content").is_some());
assert!(find_tool(body, "playwright_screenshot").is_some());
Ok(())
}
#[tokio::test]
async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()> {
let call_id = "dyn-browser-1";
let tool_name = "create_tab";
let tool_args = json!({ "url": "https://news.ycombinator.com/" });
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_123",
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
},
"artifacts": {
"screenshotBase64": "AAA"
}
})))
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let thread_id = thread.id.clone();
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread_id.clone(),
input: vec![V2UserInput::Text {
text: "open Hacker News".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
let turn_id = turn.id.clone();
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
assert_eq!(started.thread_id, thread_id);
assert_eq!(started.turn_id, turn_id.clone());
assert!(
timeout(
Duration::from_millis(500),
mcp.read_stream_until_request_message()
)
.await
.is_err(),
"remote browser tool call should not be forwarded back to the client"
);
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
assert_eq!(browser_update.browser_state.tabs.len(), 1);
assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News");
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
status,
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(status, DynamicToolCallStatus::Completed);
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string_pretty(&json!({
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browser_state": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
}))?
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let payload = bodies
.iter()
.find_map(|body| function_call_output_payload(body, call_id))
.context("expected function_call_output in follow-up request")?;
let FunctionCallOutputBody::ContentItems(content_items) = payload.body else {
panic!("expected content items payload");
};
assert_eq!(
content_items,
vec![
FunctionCallOutputContentItem::InputText {
text: serde_json::to_string_pretty(&json!({
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browser_state": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
}))?,
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
},
]
);
let remote_browser_requests = remote_browser_server
.received_requests()
.await
.context("failed to fetch remote browser requests")?;
assert_eq!(remote_browser_requests.len(), 1);
assert_eq!(
remote_browser_requests[0].body_json::<Value>()?,
json!({
"browserSessionId": null,
"command": "create_tab",
"arguments": {
"url": "https://news.ycombinator.com/"
}
})
);
Ok(())
}
/// Exercises the full dynamic tool call path (server request, client response, model output).
#[tokio::test]
async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
@@ -636,6 +902,20 @@ async fn wait_for_dynamic_tool_completed(
}
}
async fn wait_for_browser_session_updated(
mcp: &mut McpProcess,
) -> Result<BrowserSessionUpdatedNotification> {
let notification: JSONRPCNotification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("browserSession/updated"),
)
.await??;
let params = notification
.params
.context("browserSession/updated should include params")?;
Ok(serde_json::from_value(params)?)
}
fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
std::fs::write(