Wire remote browser tools into turn/start

Auto-inject the remote browser dynamic tools when the app-server is configured with a remote browser endpoint, route those tool calls internally against the remote browser adapter, and cover the prompt-loop path with integration tests.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Liang-Ting Jiang
2026-03-24 04:27:49 +00:00
co-authored by Codex
parent 4b69a17b9f
commit 34d59e54c3
8 changed files with 720 additions and 15 deletions
+119 -13
View File
@@ -6,6 +6,9 @@ use crate::error_code::INTERNAL_ERROR_CODE;
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
use crate::outgoing_message::ClientRequestResult;
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
use crate::remote_browser_api::RemoteBrowserApi;
use crate::remote_browser_tools::build_dynamic_tool_response;
use crate::remote_browser_tools::is_remote_browser_dynamic_tool;
use crate::server_request_error::is_turn_transition_server_request_error;
use crate::thread_state::ThreadListenerCommand;
use crate::thread_state::ThreadState;
@@ -17,6 +20,8 @@ use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermis
use codex_app_server_protocol::AgentMessageDeltaNotification;
use codex_app_server_protocol::ApplyPatchApprovalParams;
use codex_app_server_protocol::ApplyPatchApprovalResponse;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::CodexErrorInfo as V2CodexErrorInfo;
use codex_app_server_protocol::CollabAgentState as V2CollabAgentStatus;
use codex_app_server_protocol::CollabAgentTool;
@@ -192,6 +197,94 @@ async fn resolve_server_request_on_thread_listener(
}
}
fn to_core_dynamic_tool_response(
response: codex_app_server_protocol::DynamicToolCallResponse,
) -> CoreDynamicToolResponse {
CoreDynamicToolResponse {
content_items: response
.content_items
.into_iter()
.map(|item| match item {
DynamicToolCallOutputContentItem::InputText { text } => {
CoreDynamicToolCallOutputContentItem::InputText { text }
}
DynamicToolCallOutputContentItem::InputImage { image_url } => {
CoreDynamicToolCallOutputContentItem::InputImage { image_url }
}
})
.collect(),
success: response.success,
}
}
fn remote_browser_error_response(message: String) -> CoreDynamicToolResponse {
CoreDynamicToolResponse {
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText { text: message }],
success: false,
}
}
async fn submit_dynamic_tool_response(
conversation: Arc<CodexThread>,
call_id: String,
response: CoreDynamicToolResponse,
) {
if let Err(err) = conversation
.submit(Op::DynamicToolResponse {
id: call_id,
response,
})
.await
{
error!("failed to submit DynamicToolResponse: {err}");
}
}
async fn handle_remote_browser_dynamic_tool_request(
call_id: String,
tool: String,
arguments: JsonValue,
conversation: Arc<CodexThread>,
thread_state: Arc<Mutex<ThreadState>>,
outgoing: ThreadScopedOutgoingMessageSender,
remote_browser_api: RemoteBrowserApi,
) {
let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() };
let params = BrowserSessionCommandParams {
browser_session_id,
command: tool,
arguments: match arguments {
JsonValue::Null => None,
value => Some(value),
},
};
let response = match remote_browser_api.command_with_artifacts(params).await {
Ok(outcome) => {
{
let mut state = thread_state.lock().await;
state.remote_browser_session_id = Some(outcome.browser_session_id.clone());
}
if let Some(browser_state) = outcome.browser_state.to_public_state() {
outgoing
.send_server_notification(ServerNotification::BrowserSessionUpdated(
BrowserSessionUpdatedNotification {
browser_session_id: outcome.browser_session_id.clone(),
browser_state,
},
))
.await;
}
to_core_dynamic_tool_response(build_dynamic_tool_response(outcome))
}
Err(err) => remote_browser_error_response(err.message),
};
submit_dynamic_tool_response(conversation, call_id, response).await;
}
fn guardian_auto_approval_review_notification(
conversation_id: &ThreadId,
event_turn_id: &str,
@@ -260,6 +353,7 @@ pub(crate) async fn apply_bespoke_event_handling(
thread_manager: Arc<ThreadManager>,
outgoing: ThreadScopedOutgoingMessageSender,
thread_state: Arc<tokio::sync::Mutex<ThreadState>>,
remote_browser_api: RemoteBrowserApi,
thread_watch_manager: ThreadWatchManager,
api_version: ApiVersion,
fallback_model_provider: String,
@@ -918,19 +1012,31 @@ pub(crate) async fn apply_bespoke_event_handling(
outgoing
.send_server_notification(ServerNotification::ItemStarted(notification))
.await;
let params = DynamicToolCallParams {
thread_id: conversation_id.to_string(),
turn_id: turn_id.clone(),
call_id: call_id.clone(),
tool: tool.clone(),
arguments: arguments.clone(),
};
let (_pending_request_id, rx) = outgoing
.send_request(ServerRequestPayload::DynamicToolCall(params))
.await;
tokio::spawn(async move {
crate::dynamic_tools::on_call_response(call_id, rx, conversation).await;
});
if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) {
tokio::spawn(handle_remote_browser_dynamic_tool_request(
call_id,
tool,
arguments,
conversation,
thread_state.clone(),
outgoing.clone(),
remote_browser_api.clone(),
));
} else {
let params = DynamicToolCallParams {
thread_id: conversation_id.to_string(),
turn_id: turn_id.clone(),
call_id: call_id.clone(),
tool: tool.clone(),
arguments: arguments.clone(),
};
let (_pending_request_id, rx) = outgoing
.send_request(ServerRequestPayload::DynamicToolCall(params))
.await;
tokio::spawn(async move {
crate::dynamic_tools::on_call_response(call_id, rx, conversation).await;
});
}
} else {
error!(
"dynamic tool calls are only supported on api v2 (call_id: {})",
@@ -15,6 +15,8 @@ use crate::outgoing_message::OutgoingMessageSender;
use crate::outgoing_message::OutgoingNotification;
use crate::outgoing_message::RequestContext;
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
use crate::remote_browser_api::RemoteBrowserApi;
use crate::remote_browser_tools::merge_remote_browser_dynamic_tools;
use crate::thread_status::ThreadWatchManager;
use crate::thread_status::resolve_thread_status;
use chrono::DateTime;
@@ -369,6 +371,7 @@ pub(crate) struct CodexMessageProcessor {
auth_manager: Arc<AuthManager>,
thread_manager: Arc<ThreadManager>,
outgoing: Arc<OutgoingMessageSender>,
remote_browser_api: RemoteBrowserApi,
arg0_paths: Arg0DispatchPaths,
config: Arc<Config>,
cli_overrides: Vec<(String, TomlValue)>,
@@ -398,6 +401,7 @@ struct ListenerTaskContext {
thread_manager: Arc<ThreadManager>,
thread_state_manager: ThreadStateManager,
outgoing: Arc<OutgoingMessageSender>,
remote_browser_api: RemoteBrowserApi,
thread_watch_manager: ThreadWatchManager,
fallback_model_provider: String,
codex_home: PathBuf,
@@ -413,6 +417,7 @@ pub(crate) struct CodexMessageProcessorArgs {
pub(crate) auth_manager: Arc<AuthManager>,
pub(crate) thread_manager: Arc<ThreadManager>,
pub(crate) outgoing: Arc<OutgoingMessageSender>,
pub(crate) remote_browser_api: RemoteBrowserApi,
pub(crate) arg0_paths: Arg0DispatchPaths,
pub(crate) config: Arc<Config>,
pub(crate) cli_overrides: Vec<(String, TomlValue)>,
@@ -476,6 +481,7 @@ impl CodexMessageProcessor {
auth_manager,
thread_manager,
outgoing,
remote_browser_api,
arg0_paths,
config,
cli_overrides,
@@ -487,6 +493,7 @@ impl CodexMessageProcessor {
auth_manager,
thread_manager,
outgoing: outgoing.clone(),
remote_browser_api,
arg0_paths,
config,
cli_overrides,
@@ -1879,6 +1886,7 @@ impl CodexMessageProcessor {
thread_manager: Arc::clone(&self.thread_manager),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
remote_browser_api: self.remote_browser_api.clone(),
thread_watch_manager: self.thread_watch_manager.clone(),
fallback_model_provider: self.config.model_provider_id.clone(),
codex_home: self.config.codex_home.clone(),
@@ -1983,7 +1991,11 @@ impl CodexMessageProcessor {
}
};
let dynamic_tools = dynamic_tools.unwrap_or_default();
let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() {
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default())
} else {
dynamic_tools.unwrap_or_default()
};
let core_dynamic_tools = if dynamic_tools.is_empty() {
Vec::new()
} else {
@@ -6694,6 +6706,7 @@ impl CodexMessageProcessor {
thread_manager: Arc::clone(&self.thread_manager),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
remote_browser_api: self.remote_browser_api.clone(),
thread_watch_manager: self.thread_watch_manager.clone(),
fallback_model_provider: self.config.model_provider_id.clone(),
codex_home: self.config.codex_home.clone(),
@@ -6781,6 +6794,7 @@ impl CodexMessageProcessor {
thread_manager: Arc::clone(&self.thread_manager),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
remote_browser_api: self.remote_browser_api.clone(),
thread_watch_manager: self.thread_watch_manager.clone(),
fallback_model_provider: self.config.model_provider_id.clone(),
codex_home: self.config.codex_home.clone(),
@@ -6812,6 +6826,7 @@ impl CodexMessageProcessor {
outgoing,
thread_manager,
thread_state_manager,
remote_browser_api,
thread_watch_manager,
fallback_model_provider,
codex_home,
@@ -6906,6 +6921,7 @@ impl CodexMessageProcessor {
thread_manager.clone(),
thread_outgoing,
thread_state.clone(),
remote_browser_api.clone(),
thread_watch_manager.clone(),
api_version,
fallback_model_provider.clone(),
+1
View File
@@ -72,6 +72,7 @@ mod message_processor;
mod models;
mod outgoing_message;
mod remote_browser_api;
mod remote_browser_tools;
mod server_request_error;
mod thread_state;
mod thread_status;
+2 -1
View File
@@ -239,10 +239,12 @@ impl MessageProcessor {
.set_analytics_events_client(analytics_events_client.clone());
let cloud_requirements = Arc::new(RwLock::new(cloud_requirements));
let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint);
let codex_message_processor = CodexMessageProcessor::new(CodexMessageProcessorArgs {
auth_manager: auth_manager.clone(),
thread_manager: Arc::clone(&thread_manager),
outgoing: outgoing.clone(),
remote_browser_api: remote_browser_api.clone(),
arg0_paths,
config: Arc::clone(&config),
cli_overrides: cli_overrides.clone(),
@@ -265,7 +267,6 @@ impl MessageProcessor {
);
let external_agent_config_api = ExternalAgentConfigApi::new(config.codex_home.clone());
let fs_api = FsApi::default();
let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint);
Self {
outgoing,
@@ -2,7 +2,12 @@ use crate::error_code::INTERNAL_ERROR_CODE;
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionCommandResponse;
use codex_app_server_protocol::BrowserSessionState;
use codex_app_server_protocol::BrowserTabState;
use codex_app_server_protocol::JSONRPCErrorError;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
#[derive(Clone)]
pub(crate) struct RemoteBrowserApi {
@@ -10,6 +15,104 @@ pub(crate) struct RemoteBrowserApi {
client: reqwest::Client,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RemoteBrowserTabState {
pub(crate) id: String,
#[serde(default)]
pub(crate) title: Option<String>,
#[serde(default)]
pub(crate) url: Option<String>,
#[serde(default)]
pub(crate) selected: bool,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RemoteBrowserState {
#[serde(default)]
pub(crate) selected_tab_id: Option<String>,
#[serde(default)]
pub(crate) tabs: Vec<RemoteBrowserTabState>,
}
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct RemoteBrowserArtifacts {
#[serde(default)]
screenshot_base64: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RemoteBrowserCommandOutcome {
pub(crate) browser_session_id: String,
pub(crate) result: JsonValue,
pub(crate) browser_state: RemoteBrowserState,
#[serde(default)]
artifacts: RemoteBrowserArtifacts,
}
impl RemoteBrowserState {
fn selected_tab_id(&self) -> Option<String> {
self.selected_tab_id.clone().or_else(|| {
self.tabs
.iter()
.find(|tab| tab.selected)
.map(|tab| tab.id.clone())
})
}
pub(crate) fn to_public_state(&self) -> Option<BrowserSessionState> {
let selected_tab_id = self.selected_tab_id()?;
Some(BrowserSessionState {
selected_tab_id,
tabs: self
.tabs
.iter()
.map(|tab| BrowserTabState {
id: tab.id.clone(),
title: tab.title.clone().unwrap_or_default(),
url: tab.url.clone().unwrap_or_default(),
selected: tab.selected,
})
.collect(),
})
}
}
impl RemoteBrowserCommandOutcome {
pub(crate) fn to_public_response(
&self,
) -> Result<BrowserSessionCommandResponse, JSONRPCErrorError> {
let browser_state =
self.browser_state
.to_public_state()
.ok_or_else(|| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: "remote browser response did not include a selected tab".to_string(),
data: None,
})?;
Ok(BrowserSessionCommandResponse {
browser_session_id: self.browser_session_id.clone(),
result: self.result.clone(),
browser_state,
})
}
pub(crate) fn screenshot_data_url(&self) -> Option<String> {
self.artifacts
.screenshot_base64
.as_ref()
.map(|encoded| format!("data:image/png;base64,{encoded}"))
}
pub(crate) fn browser_state_json(&self) -> JsonValue {
serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null)
}
}
impl RemoteBrowserApi {
pub(crate) fn new(endpoint: Option<String>) -> Self {
Self {
@@ -18,10 +121,23 @@ impl RemoteBrowserApi {
}
}
pub(crate) fn is_configured(&self) -> bool {
self.endpoint.is_some()
}
pub(crate) async fn command(
&self,
params: BrowserSessionCommandParams,
) -> Result<BrowserSessionCommandResponse, JSONRPCErrorError> {
self.command_with_artifacts(params)
.await?
.to_public_response()
}
pub(crate) async fn command_with_artifacts(
&self,
params: BrowserSessionCommandParams,
) -> Result<RemoteBrowserCommandOutcome, JSONRPCErrorError> {
let Some(endpoint) = &self.endpoint else {
return Err(JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
@@ -0,0 +1,184 @@
use crate::remote_browser_api::RemoteBrowserCommandOutcome;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallResponse;
use codex_app_server_protocol::DynamicToolSpec;
use serde_json::json;
use std::collections::HashSet;
const CREATE_TAB_TOOL: &str = "create_tab";
const LIST_TABS_TOOL: &str = "list_tabs";
const SELECTED_TAB_TOOL: &str = "selected_tab";
const SELECT_TAB_TOOL: &str = "select_tab";
const NAVIGATE_TAB_URL_TOOL: &str = "navigate_tab_url";
const TABS_CONTENT_TOOL: &str = "tabs_content";
const WAIT_FOR_LOAD_STATE_TOOL: &str = "playwright_wait_for_load_state";
const SCREENSHOT_TOOL: &str = "playwright_screenshot";
pub(crate) fn merge_remote_browser_dynamic_tools(
mut tools: Vec<DynamicToolSpec>,
) -> Vec<DynamicToolSpec> {
let existing_names = tools
.iter()
.map(|tool| tool.name.clone())
.collect::<HashSet<_>>();
for tool in remote_browser_dynamic_tools() {
if !existing_names.contains(&tool.name) {
tools.push(tool);
}
}
tools
}
pub(crate) fn is_remote_browser_dynamic_tool(name: &str) -> bool {
matches!(
name,
CREATE_TAB_TOOL
| LIST_TABS_TOOL
| SELECTED_TAB_TOOL
| SELECT_TAB_TOOL
| NAVIGATE_TAB_URL_TOOL
| TABS_CONTENT_TOOL
| WAIT_FOR_LOAD_STATE_TOOL
| SCREENSHOT_TOOL
)
}
pub(crate) fn build_dynamic_tool_response(
outcome: RemoteBrowserCommandOutcome,
) -> DynamicToolCallResponse {
let mut content_items = vec![DynamicToolCallOutputContentItem::InputText {
text: format_remote_browser_text_result(&outcome),
}];
if let Some(image_url) = outcome.screenshot_data_url() {
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
}
DynamicToolCallResponse {
content_items,
success: true,
}
}
fn format_remote_browser_text_result(outcome: &RemoteBrowserCommandOutcome) -> String {
serde_json::to_string_pretty(&json!({
"result": outcome.result,
"browser_state": outcome.browser_state_json(),
}))
.unwrap_or_else(|_| "{\"result\":\"remote browser output unavailable\"}".to_string())
}
fn remote_browser_dynamic_tools() -> Vec<DynamicToolSpec> {
vec![
DynamicToolSpec {
name: CREATE_TAB_TOOL.to_string(),
description: "Create a browser tab in the shared remote browser session. Prefer this for the first navigation by passing a URL.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"url": { "type": "string" },
"select": { "type": "boolean" },
"timeout_ms": { "type": "integer", "minimum": 1 }
},
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: LIST_TABS_TOOL.to_string(),
description: "List the current tabs in the shared remote browser session.".to_string(),
input_schema: json!({
"type": "object",
"properties": {},
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: SELECTED_TAB_TOOL.to_string(),
description: "Get the currently selected tab in the shared remote browser session.".to_string(),
input_schema: json!({
"type": "object",
"properties": {},
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: SELECT_TAB_TOOL.to_string(),
description: "Select an existing browser tab by tab_id.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"tab_id": { "type": "string" }
},
"required": ["tab_id"],
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: NAVIGATE_TAB_URL_TOOL.to_string(),
description: "Navigate an existing tab, or the selected tab, to a URL.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"url": { "type": "string" },
"tab_id": { "type": "string" },
"timeout_ms": { "type": "integer", "minimum": 1 }
},
"required": ["url"],
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: TABS_CONTENT_TOOL.to_string(),
description: "Read sanitized text content from one or more tabs. Use this to inspect a page before summarizing it.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"tab_ids": {
"type": "array",
"items": { "type": "string" }
},
"max_chars_per_tab": { "type": "integer", "minimum": 1 }
},
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: WAIT_FOR_LOAD_STATE_TOOL.to_string(),
description: "Wait for the selected tab, or a specific tab, to reach a Playwright load state.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"tab_id": { "type": "string" },
"state": {
"type": "string",
"enum": ["domcontentloaded", "load", "networkidle"]
},
"timeout_ms": { "type": "integer", "minimum": 1 }
},
"additionalProperties": false
}),
defer_loading: false,
},
DynamicToolSpec {
name: SCREENSHOT_TOOL.to_string(),
description: "Capture a screenshot from the selected tab, or a specific tab, and return it to the model.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"tab_id": { "type": "string" },
"full_page": { "type": "boolean" }
},
"additionalProperties": false
}),
defer_loading: false,
},
]
}
+1
View File
@@ -54,6 +54,7 @@ pub(crate) struct ThreadState {
pub(crate) pending_interrupts: PendingInterruptQueue,
pub(crate) pending_rollbacks: Option<ConnectionRequestId>,
pub(crate) turn_summary: TurnSummary,
pub(crate) remote_browser_session_id: Option<String>,
pub(crate) cancel_tx: Option<oneshot::Sender<()>>,
pub(crate) experimental_raw_events: bool,
pub(crate) listener_generation: u64,
@@ -2,8 +2,10 @@ use anyhow::Context;
use anyhow::Result;
use app_test_support::McpProcess;
use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::to_response;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallParams;
use codex_app_server_protocol::DynamicToolCallResponse;
@@ -32,7 +34,11 @@ use std::path::Path;
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
@@ -191,6 +197,266 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu
Ok(())
}
#[tokio::test]
async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()>
{
let server = create_mock_responses_server_repeating_assistant("Done").await;
let remote_browser_server = MockServer::start().await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id,
input: vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let body = bodies
.first()
.context("expected at least one responses request")?;
assert!(find_tool(body, "create_tab").is_some());
assert!(find_tool(body, "tabs_content").is_some());
assert!(find_tool(body, "playwright_screenshot").is_some());
Ok(())
}
#[tokio::test]
async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()> {
let call_id = "dyn-browser-1";
let tool_name = "create_tab";
let tool_args = json!({ "url": "https://news.ycombinator.com/" });
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_123",
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
},
"artifacts": {
"screenshotBase64": "AAA"
}
})))
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let thread_id = thread.id.clone();
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread_id.clone(),
input: vec![V2UserInput::Text {
text: "open Hacker News".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
let turn_id = turn.id.clone();
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
assert_eq!(started.thread_id, thread_id);
assert_eq!(started.turn_id, turn_id.clone());
assert!(
timeout(
Duration::from_millis(500),
mcp.read_stream_until_request_message()
)
.await
.is_err(),
"remote browser tool call should not be forwarded back to the client"
);
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
assert_eq!(browser_update.browser_state.tabs.len(), 1);
assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News");
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
status,
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(status, DynamicToolCallStatus::Completed);
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string_pretty(&json!({
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browser_state": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
}))?
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let payload = bodies
.iter()
.find_map(|body| function_call_output_payload(body, call_id))
.context("expected function_call_output in follow-up request")?;
let FunctionCallOutputBody::ContentItems(content_items) = payload.body else {
panic!("expected content items payload");
};
assert_eq!(
content_items,
vec![
FunctionCallOutputContentItem::InputText {
text: serde_json::to_string_pretty(&json!({
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browser_state": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
}))?,
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
},
]
);
let remote_browser_requests = remote_browser_server
.received_requests()
.await
.context("failed to fetch remote browser requests")?;
assert_eq!(remote_browser_requests.len(), 1);
assert_eq!(
remote_browser_requests[0].body_json::<Value>()?,
json!({
"browserSessionId": null,
"command": "create_tab",
"arguments": {
"url": "https://news.ycombinator.com/"
}
})
);
Ok(())
}
/// Exercises the full dynamic tool call path (server request, client response, model output).
#[tokio::test]
async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
@@ -636,6 +902,20 @@ async fn wait_for_dynamic_tool_completed(
}
}
async fn wait_for_browser_session_updated(
mcp: &mut McpProcess,
) -> Result<BrowserSessionUpdatedNotification> {
let notification: JSONRPCNotification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("browserSession/updated"),
)
.await??;
let params = notification
.params
.context("browserSession/updated should include params")?;
Ok(serde_json::from_value(params)?)
}
fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
std::fs::write(