diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 34640a50cf..8af1d5a43e 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -6,6 +6,9 @@ use crate::error_code::INTERNAL_ERROR_CODE; use crate::error_code::INVALID_REQUEST_ERROR_CODE; use crate::outgoing_message::ClientRequestResult; use crate::outgoing_message::ThreadScopedOutgoingMessageSender; +use crate::remote_browser_api::RemoteBrowserApi; +use crate::remote_browser_tools::build_dynamic_tool_response; +use crate::remote_browser_tools::is_remote_browser_dynamic_tool; use crate::server_request_error::is_turn_transition_server_request_error; use crate::thread_state::ThreadListenerCommand; use crate::thread_state::ThreadState; @@ -17,6 +20,8 @@ use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermis use codex_app_server_protocol::AgentMessageDeltaNotification; use codex_app_server_protocol::ApplyPatchApprovalParams; use codex_app_server_protocol::ApplyPatchApprovalResponse; +use codex_app_server_protocol::BrowserSessionCommandParams; +use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::CodexErrorInfo as V2CodexErrorInfo; use codex_app_server_protocol::CollabAgentState as V2CollabAgentStatus; use codex_app_server_protocol::CollabAgentTool; @@ -192,6 +197,94 @@ async fn resolve_server_request_on_thread_listener( } } +fn to_core_dynamic_tool_response( + response: codex_app_server_protocol::DynamicToolCallResponse, +) -> CoreDynamicToolResponse { + CoreDynamicToolResponse { + content_items: response + .content_items + .into_iter() + .map(|item| match item { + DynamicToolCallOutputContentItem::InputText { text } => { + CoreDynamicToolCallOutputContentItem::InputText { text } + } + DynamicToolCallOutputContentItem::InputImage { image_url } => { + CoreDynamicToolCallOutputContentItem::InputImage { image_url } + } + }) + .collect(), + success: response.success, + } +} + +fn remote_browser_error_response(message: String) -> CoreDynamicToolResponse { + CoreDynamicToolResponse { + content_items: vec![CoreDynamicToolCallOutputContentItem::InputText { text: message }], + success: false, + } +} + +async fn submit_dynamic_tool_response( + conversation: Arc, + call_id: String, + response: CoreDynamicToolResponse, +) { + if let Err(err) = conversation + .submit(Op::DynamicToolResponse { + id: call_id, + response, + }) + .await + { + error!("failed to submit DynamicToolResponse: {err}"); + } +} + +async fn handle_remote_browser_dynamic_tool_request( + call_id: String, + tool: String, + arguments: JsonValue, + conversation: Arc, + thread_state: Arc>, + outgoing: ThreadScopedOutgoingMessageSender, + remote_browser_api: RemoteBrowserApi, +) { + let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() }; + let params = BrowserSessionCommandParams { + browser_session_id, + command: tool, + arguments: match arguments { + JsonValue::Null => None, + value => Some(value), + }, + }; + + let response = match remote_browser_api.command_with_artifacts(params).await { + Ok(outcome) => { + { + let mut state = thread_state.lock().await; + state.remote_browser_session_id = Some(outcome.browser_session_id.clone()); + } + + if let Some(browser_state) = outcome.browser_state.to_public_state() { + outgoing + .send_server_notification(ServerNotification::BrowserSessionUpdated( + BrowserSessionUpdatedNotification { + browser_session_id: outcome.browser_session_id.clone(), + browser_state, + }, + )) + .await; + } + + to_core_dynamic_tool_response(build_dynamic_tool_response(outcome)) + } + Err(err) => remote_browser_error_response(err.message), + }; + + submit_dynamic_tool_response(conversation, call_id, response).await; +} + fn guardian_auto_approval_review_notification( conversation_id: &ThreadId, event_turn_id: &str, @@ -260,6 +353,7 @@ pub(crate) async fn apply_bespoke_event_handling( thread_manager: Arc, outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, + remote_browser_api: RemoteBrowserApi, thread_watch_manager: ThreadWatchManager, api_version: ApiVersion, fallback_model_provider: String, @@ -918,19 +1012,31 @@ pub(crate) async fn apply_bespoke_event_handling( outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; - let params = DynamicToolCallParams { - thread_id: conversation_id.to_string(), - turn_id: turn_id.clone(), - call_id: call_id.clone(), - tool: tool.clone(), - arguments: arguments.clone(), - }; - let (_pending_request_id, rx) = outgoing - .send_request(ServerRequestPayload::DynamicToolCall(params)) - .await; - tokio::spawn(async move { - crate::dynamic_tools::on_call_response(call_id, rx, conversation).await; - }); + if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) { + tokio::spawn(handle_remote_browser_dynamic_tool_request( + call_id, + tool, + arguments, + conversation, + thread_state.clone(), + outgoing.clone(), + remote_browser_api.clone(), + )); + } else { + let params = DynamicToolCallParams { + thread_id: conversation_id.to_string(), + turn_id: turn_id.clone(), + call_id: call_id.clone(), + tool: tool.clone(), + arguments: arguments.clone(), + }; + let (_pending_request_id, rx) = outgoing + .send_request(ServerRequestPayload::DynamicToolCall(params)) + .await; + tokio::spawn(async move { + crate::dynamic_tools::on_call_response(call_id, rx, conversation).await; + }); + } } else { error!( "dynamic tool calls are only supported on api v2 (call_id: {})", diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 354b74434d..bcf9a542db 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -15,6 +15,8 @@ use crate::outgoing_message::OutgoingMessageSender; use crate::outgoing_message::OutgoingNotification; use crate::outgoing_message::RequestContext; use crate::outgoing_message::ThreadScopedOutgoingMessageSender; +use crate::remote_browser_api::RemoteBrowserApi; +use crate::remote_browser_tools::merge_remote_browser_dynamic_tools; use crate::thread_status::ThreadWatchManager; use crate::thread_status::resolve_thread_status; use chrono::DateTime; @@ -369,6 +371,7 @@ pub(crate) struct CodexMessageProcessor { auth_manager: Arc, thread_manager: Arc, outgoing: Arc, + remote_browser_api: RemoteBrowserApi, arg0_paths: Arg0DispatchPaths, config: Arc, cli_overrides: Vec<(String, TomlValue)>, @@ -398,6 +401,7 @@ struct ListenerTaskContext { thread_manager: Arc, thread_state_manager: ThreadStateManager, outgoing: Arc, + remote_browser_api: RemoteBrowserApi, thread_watch_manager: ThreadWatchManager, fallback_model_provider: String, codex_home: PathBuf, @@ -413,6 +417,7 @@ pub(crate) struct CodexMessageProcessorArgs { pub(crate) auth_manager: Arc, pub(crate) thread_manager: Arc, pub(crate) outgoing: Arc, + pub(crate) remote_browser_api: RemoteBrowserApi, pub(crate) arg0_paths: Arg0DispatchPaths, pub(crate) config: Arc, pub(crate) cli_overrides: Vec<(String, TomlValue)>, @@ -476,6 +481,7 @@ impl CodexMessageProcessor { auth_manager, thread_manager, outgoing, + remote_browser_api, arg0_paths, config, cli_overrides, @@ -487,6 +493,7 @@ impl CodexMessageProcessor { auth_manager, thread_manager, outgoing: outgoing.clone(), + remote_browser_api, arg0_paths, config, cli_overrides, @@ -1879,6 +1886,7 @@ impl CodexMessageProcessor { thread_manager: Arc::clone(&self.thread_manager), thread_state_manager: self.thread_state_manager.clone(), outgoing: Arc::clone(&self.outgoing), + remote_browser_api: self.remote_browser_api.clone(), thread_watch_manager: self.thread_watch_manager.clone(), fallback_model_provider: self.config.model_provider_id.clone(), codex_home: self.config.codex_home.clone(), @@ -1983,7 +1991,11 @@ impl CodexMessageProcessor { } }; - let dynamic_tools = dynamic_tools.unwrap_or_default(); + let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() { + merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default()) + } else { + dynamic_tools.unwrap_or_default() + }; let core_dynamic_tools = if dynamic_tools.is_empty() { Vec::new() } else { @@ -6694,6 +6706,7 @@ impl CodexMessageProcessor { thread_manager: Arc::clone(&self.thread_manager), thread_state_manager: self.thread_state_manager.clone(), outgoing: Arc::clone(&self.outgoing), + remote_browser_api: self.remote_browser_api.clone(), thread_watch_manager: self.thread_watch_manager.clone(), fallback_model_provider: self.config.model_provider_id.clone(), codex_home: self.config.codex_home.clone(), @@ -6781,6 +6794,7 @@ impl CodexMessageProcessor { thread_manager: Arc::clone(&self.thread_manager), thread_state_manager: self.thread_state_manager.clone(), outgoing: Arc::clone(&self.outgoing), + remote_browser_api: self.remote_browser_api.clone(), thread_watch_manager: self.thread_watch_manager.clone(), fallback_model_provider: self.config.model_provider_id.clone(), codex_home: self.config.codex_home.clone(), @@ -6812,6 +6826,7 @@ impl CodexMessageProcessor { outgoing, thread_manager, thread_state_manager, + remote_browser_api, thread_watch_manager, fallback_model_provider, codex_home, @@ -6906,6 +6921,7 @@ impl CodexMessageProcessor { thread_manager.clone(), thread_outgoing, thread_state.clone(), + remote_browser_api.clone(), thread_watch_manager.clone(), api_version, fallback_model_provider.clone(), diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index b558e37373..b9bf5a8e5c 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -72,6 +72,7 @@ mod message_processor; mod models; mod outgoing_message; mod remote_browser_api; +mod remote_browser_tools; mod server_request_error; mod thread_state; mod thread_status; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 61c407c398..614b6519b3 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -239,10 +239,12 @@ impl MessageProcessor { .set_analytics_events_client(analytics_events_client.clone()); let cloud_requirements = Arc::new(RwLock::new(cloud_requirements)); + let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint); let codex_message_processor = CodexMessageProcessor::new(CodexMessageProcessorArgs { auth_manager: auth_manager.clone(), thread_manager: Arc::clone(&thread_manager), outgoing: outgoing.clone(), + remote_browser_api: remote_browser_api.clone(), arg0_paths, config: Arc::clone(&config), cli_overrides: cli_overrides.clone(), @@ -265,7 +267,6 @@ impl MessageProcessor { ); let external_agent_config_api = ExternalAgentConfigApi::new(config.codex_home.clone()); let fs_api = FsApi::default(); - let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint); Self { outgoing, diff --git a/codex-rs/app-server/src/remote_browser_api.rs b/codex-rs/app-server/src/remote_browser_api.rs index 4ad7cd3757..eae81665e4 100644 --- a/codex-rs/app-server/src/remote_browser_api.rs +++ b/codex-rs/app-server/src/remote_browser_api.rs @@ -2,7 +2,12 @@ use crate::error_code::INTERNAL_ERROR_CODE; use crate::error_code::INVALID_REQUEST_ERROR_CODE; use codex_app_server_protocol::BrowserSessionCommandParams; use codex_app_server_protocol::BrowserSessionCommandResponse; +use codex_app_server_protocol::BrowserSessionState; +use codex_app_server_protocol::BrowserTabState; use codex_app_server_protocol::JSONRPCErrorError; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value as JsonValue; #[derive(Clone)] pub(crate) struct RemoteBrowserApi { @@ -10,6 +15,104 @@ pub(crate) struct RemoteBrowserApi { client: reqwest::Client, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RemoteBrowserTabState { + pub(crate) id: String, + #[serde(default)] + pub(crate) title: Option, + #[serde(default)] + pub(crate) url: Option, + #[serde(default)] + pub(crate) selected: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RemoteBrowserState { + #[serde(default)] + pub(crate) selected_tab_id: Option, + #[serde(default)] + pub(crate) tabs: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct RemoteBrowserArtifacts { + #[serde(default)] + screenshot_base64: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RemoteBrowserCommandOutcome { + pub(crate) browser_session_id: String, + pub(crate) result: JsonValue, + pub(crate) browser_state: RemoteBrowserState, + #[serde(default)] + artifacts: RemoteBrowserArtifacts, +} + +impl RemoteBrowserState { + fn selected_tab_id(&self) -> Option { + self.selected_tab_id.clone().or_else(|| { + self.tabs + .iter() + .find(|tab| tab.selected) + .map(|tab| tab.id.clone()) + }) + } + + pub(crate) fn to_public_state(&self) -> Option { + let selected_tab_id = self.selected_tab_id()?; + Some(BrowserSessionState { + selected_tab_id, + tabs: self + .tabs + .iter() + .map(|tab| BrowserTabState { + id: tab.id.clone(), + title: tab.title.clone().unwrap_or_default(), + url: tab.url.clone().unwrap_or_default(), + selected: tab.selected, + }) + .collect(), + }) + } +} + +impl RemoteBrowserCommandOutcome { + pub(crate) fn to_public_response( + &self, + ) -> Result { + let browser_state = + self.browser_state + .to_public_state() + .ok_or_else(|| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: "remote browser response did not include a selected tab".to_string(), + data: None, + })?; + + Ok(BrowserSessionCommandResponse { + browser_session_id: self.browser_session_id.clone(), + result: self.result.clone(), + browser_state, + }) + } + + pub(crate) fn screenshot_data_url(&self) -> Option { + self.artifacts + .screenshot_base64 + .as_ref() + .map(|encoded| format!("data:image/png;base64,{encoded}")) + } + + pub(crate) fn browser_state_json(&self) -> JsonValue { + serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null) + } +} + impl RemoteBrowserApi { pub(crate) fn new(endpoint: Option) -> Self { Self { @@ -18,10 +121,23 @@ impl RemoteBrowserApi { } } + pub(crate) fn is_configured(&self) -> bool { + self.endpoint.is_some() + } + pub(crate) async fn command( &self, params: BrowserSessionCommandParams, ) -> Result { + self.command_with_artifacts(params) + .await? + .to_public_response() + } + + pub(crate) async fn command_with_artifacts( + &self, + params: BrowserSessionCommandParams, + ) -> Result { let Some(endpoint) = &self.endpoint else { return Err(JSONRPCErrorError { code: INVALID_REQUEST_ERROR_CODE, diff --git a/codex-rs/app-server/src/remote_browser_tools.rs b/codex-rs/app-server/src/remote_browser_tools.rs new file mode 100644 index 0000000000..6da9865518 --- /dev/null +++ b/codex-rs/app-server/src/remote_browser_tools.rs @@ -0,0 +1,184 @@ +use crate::remote_browser_api::RemoteBrowserCommandOutcome; +use codex_app_server_protocol::DynamicToolCallOutputContentItem; +use codex_app_server_protocol::DynamicToolCallResponse; +use codex_app_server_protocol::DynamicToolSpec; +use serde_json::json; +use std::collections::HashSet; + +const CREATE_TAB_TOOL: &str = "create_tab"; +const LIST_TABS_TOOL: &str = "list_tabs"; +const SELECTED_TAB_TOOL: &str = "selected_tab"; +const SELECT_TAB_TOOL: &str = "select_tab"; +const NAVIGATE_TAB_URL_TOOL: &str = "navigate_tab_url"; +const TABS_CONTENT_TOOL: &str = "tabs_content"; +const WAIT_FOR_LOAD_STATE_TOOL: &str = "playwright_wait_for_load_state"; +const SCREENSHOT_TOOL: &str = "playwright_screenshot"; + +pub(crate) fn merge_remote_browser_dynamic_tools( + mut tools: Vec, +) -> Vec { + let existing_names = tools + .iter() + .map(|tool| tool.name.clone()) + .collect::>(); + + for tool in remote_browser_dynamic_tools() { + if !existing_names.contains(&tool.name) { + tools.push(tool); + } + } + + tools +} + +pub(crate) fn is_remote_browser_dynamic_tool(name: &str) -> bool { + matches!( + name, + CREATE_TAB_TOOL + | LIST_TABS_TOOL + | SELECTED_TAB_TOOL + | SELECT_TAB_TOOL + | NAVIGATE_TAB_URL_TOOL + | TABS_CONTENT_TOOL + | WAIT_FOR_LOAD_STATE_TOOL + | SCREENSHOT_TOOL + ) +} + +pub(crate) fn build_dynamic_tool_response( + outcome: RemoteBrowserCommandOutcome, +) -> DynamicToolCallResponse { + let mut content_items = vec![DynamicToolCallOutputContentItem::InputText { + text: format_remote_browser_text_result(&outcome), + }]; + + if let Some(image_url) = outcome.screenshot_data_url() { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } + + DynamicToolCallResponse { + content_items, + success: true, + } +} + +fn format_remote_browser_text_result(outcome: &RemoteBrowserCommandOutcome) -> String { + serde_json::to_string_pretty(&json!({ + "result": outcome.result, + "browser_state": outcome.browser_state_json(), + })) + .unwrap_or_else(|_| "{\"result\":\"remote browser output unavailable\"}".to_string()) +} + +fn remote_browser_dynamic_tools() -> Vec { + vec![ + DynamicToolSpec { + name: CREATE_TAB_TOOL.to_string(), + description: "Create a browser tab in the shared remote browser session. Prefer this for the first navigation by passing a URL.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "url": { "type": "string" }, + "select": { "type": "boolean" }, + "timeout_ms": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: LIST_TABS_TOOL.to_string(), + description: "List the current tabs in the shared remote browser session.".to_string(), + input_schema: json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: SELECTED_TAB_TOOL.to_string(), + description: "Get the currently selected tab in the shared remote browser session.".to_string(), + input_schema: json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: SELECT_TAB_TOOL.to_string(), + description: "Select an existing browser tab by tab_id.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "tab_id": { "type": "string" } + }, + "required": ["tab_id"], + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: NAVIGATE_TAB_URL_TOOL.to_string(), + description: "Navigate an existing tab, or the selected tab, to a URL.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "url": { "type": "string" }, + "tab_id": { "type": "string" }, + "timeout_ms": { "type": "integer", "minimum": 1 } + }, + "required": ["url"], + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: TABS_CONTENT_TOOL.to_string(), + description: "Read sanitized text content from one or more tabs. Use this to inspect a page before summarizing it.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "tab_ids": { + "type": "array", + "items": { "type": "string" } + }, + "max_chars_per_tab": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: WAIT_FOR_LOAD_STATE_TOOL.to_string(), + description: "Wait for the selected tab, or a specific tab, to reach a Playwright load state.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "tab_id": { "type": "string" }, + "state": { + "type": "string", + "enum": ["domcontentloaded", "load", "networkidle"] + }, + "timeout_ms": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": false + }), + defer_loading: false, + }, + DynamicToolSpec { + name: SCREENSHOT_TOOL.to_string(), + description: "Capture a screenshot from the selected tab, or a specific tab, and return it to the model.".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "tab_id": { "type": "string" }, + "full_page": { "type": "boolean" } + }, + "additionalProperties": false + }), + defer_loading: false, + }, + ] +} diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index be5478dd51..5499c28706 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -54,6 +54,7 @@ pub(crate) struct ThreadState { pub(crate) pending_interrupts: PendingInterruptQueue, pub(crate) pending_rollbacks: Option, pub(crate) turn_summary: TurnSummary, + pub(crate) remote_browser_session_id: Option, pub(crate) cancel_tx: Option>, pub(crate) experimental_raw_events: bool, pub(crate) listener_generation: u64, diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 0ab3f47235..aa9c856d2e 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -2,8 +2,10 @@ use anyhow::Context; use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_final_assistant_message_sse_response; +use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; +use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::DynamicToolCallOutputContentItem; use codex_app_server_protocol::DynamicToolCallParams; use codex_app_server_protocol::DynamicToolCallResponse; @@ -32,7 +34,11 @@ use std::path::Path; use std::time::Duration; use tempfile::TempDir; use tokio::time::timeout; +use wiremock::Mock; use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); @@ -191,6 +197,266 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu Ok(()) } +#[tokio::test] +async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()> +{ + let server = create_mock_responses_server_repeating_assistant("Done").await; + let remote_browser_server = MockServer::start().await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "Hello".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _turn: TurnStartResponse = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let body = bodies + .first() + .context("expected at least one responses request")?; + assert!(find_tool(body, "create_tab").is_some()); + assert!(find_tool(body, "tabs_content").is_some()); + assert!(find_tool(body, "playwright_screenshot").is_some()); + + Ok(()) +} + +#[tokio::test] +async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()> { + let call_id = "dyn-browser-1"; + let tool_name = "create_tab"; + let tool_args = json!({ "url": "https://news.ycombinator.com/" }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/remote_browser")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_123", + "result": { + "tab_id": "tab_123", + "selected_tab_id": "tab_123" + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + }, + "artifacts": { + "screenshotBase64": "AAA" + } + }))) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + let thread_id = thread.id.clone(); + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.clone(), + input: vec![V2UserInput::Text { + text: "open Hacker News".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(started.turn_id, turn_id.clone()); + + assert!( + timeout( + Duration::from_millis(500), + mcp.read_stream_until_request_message() + ) + .await + .is_err(), + "remote browser tool call should not be forwarded back to the client" + ); + + let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.browser_session_id, "sess_123"); + assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); + assert_eq!(browser_update.browser_state.tabs.len(), 1); + assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News"); + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + let ThreadItem::DynamicToolCall { + status, + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(status, DynamicToolCallStatus::Completed); + assert_eq!(success, Some(true)); + assert_eq!( + content_items, + Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string_pretty(&json!({ + "result": { + "tab_id": "tab_123", + "selected_tab_id": "tab_123" + }, + "browser_state": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + } + }))? + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + ]) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let payload = bodies + .iter() + .find_map(|body| function_call_output_payload(body, call_id)) + .context("expected function_call_output in follow-up request")?; + let FunctionCallOutputBody::ContentItems(content_items) = payload.body else { + panic!("expected content items payload"); + }; + assert_eq!( + content_items, + vec![ + FunctionCallOutputContentItem::InputText { + text: serde_json::to_string_pretty(&json!({ + "result": { + "tab_id": "tab_123", + "selected_tab_id": "tab_123" + }, + "browser_state": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + } + }))?, + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: None, + }, + ] + ); + + let remote_browser_requests = remote_browser_server + .received_requests() + .await + .context("failed to fetch remote browser requests")?; + assert_eq!(remote_browser_requests.len(), 1); + assert_eq!( + remote_browser_requests[0].body_json::()?, + json!({ + "browserSessionId": null, + "command": "create_tab", + "arguments": { + "url": "https://news.ycombinator.com/" + } + }) + ); + + Ok(()) +} + /// Exercises the full dynamic tool call path (server request, client response, model output). #[tokio::test] async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> { @@ -636,6 +902,20 @@ async fn wait_for_dynamic_tool_completed( } } +async fn wait_for_browser_session_updated( + mcp: &mut McpProcess, +) -> Result { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("browserSession/updated"), + ) + .await??; + let params = notification + .params + .context("browserSession/updated should include params")?; + Ok(serde_json::from_value(params)?) +} + fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { let config_toml = codex_home.join("config.toml"); std::fs::write(