mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Wire remote browser tools into turn/start
Auto-inject the remote browser dynamic tools when the app-server is configured with a remote browser endpoint, route those tool calls internally against the remote browser adapter, and cover the prompt-loop path with integration tests. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
4b69a17b9f
commit
34d59e54c3
@@ -6,6 +6,9 @@ use crate::error_code::INTERNAL_ERROR_CODE;
|
||||
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
|
||||
use crate::outgoing_message::ClientRequestResult;
|
||||
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
|
||||
use crate::remote_browser_api::RemoteBrowserApi;
|
||||
use crate::remote_browser_tools::build_dynamic_tool_response;
|
||||
use crate::remote_browser_tools::is_remote_browser_dynamic_tool;
|
||||
use crate::server_request_error::is_turn_transition_server_request_error;
|
||||
use crate::thread_state::ThreadListenerCommand;
|
||||
use crate::thread_state::ThreadState;
|
||||
@@ -17,6 +20,8 @@ use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermis
|
||||
use codex_app_server_protocol::AgentMessageDeltaNotification;
|
||||
use codex_app_server_protocol::ApplyPatchApprovalParams;
|
||||
use codex_app_server_protocol::ApplyPatchApprovalResponse;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::CodexErrorInfo as V2CodexErrorInfo;
|
||||
use codex_app_server_protocol::CollabAgentState as V2CollabAgentStatus;
|
||||
use codex_app_server_protocol::CollabAgentTool;
|
||||
@@ -192,6 +197,94 @@ async fn resolve_server_request_on_thread_listener(
|
||||
}
|
||||
}
|
||||
|
||||
fn to_core_dynamic_tool_response(
|
||||
response: codex_app_server_protocol::DynamicToolCallResponse,
|
||||
) -> CoreDynamicToolResponse {
|
||||
CoreDynamicToolResponse {
|
||||
content_items: response
|
||||
.content_items
|
||||
.into_iter()
|
||||
.map(|item| match item {
|
||||
DynamicToolCallOutputContentItem::InputText { text } => {
|
||||
CoreDynamicToolCallOutputContentItem::InputText { text }
|
||||
}
|
||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||
CoreDynamicToolCallOutputContentItem::InputImage { image_url }
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
success: response.success,
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_browser_error_response(message: String) -> CoreDynamicToolResponse {
|
||||
CoreDynamicToolResponse {
|
||||
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText { text: message }],
|
||||
success: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_dynamic_tool_response(
|
||||
conversation: Arc<CodexThread>,
|
||||
call_id: String,
|
||||
response: CoreDynamicToolResponse,
|
||||
) {
|
||||
if let Err(err) = conversation
|
||||
.submit(Op::DynamicToolResponse {
|
||||
id: call_id,
|
||||
response,
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("failed to submit DynamicToolResponse: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_remote_browser_dynamic_tool_request(
|
||||
call_id: String,
|
||||
tool: String,
|
||||
arguments: JsonValue,
|
||||
conversation: Arc<CodexThread>,
|
||||
thread_state: Arc<Mutex<ThreadState>>,
|
||||
outgoing: ThreadScopedOutgoingMessageSender,
|
||||
remote_browser_api: RemoteBrowserApi,
|
||||
) {
|
||||
let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() };
|
||||
let params = BrowserSessionCommandParams {
|
||||
browser_session_id,
|
||||
command: tool,
|
||||
arguments: match arguments {
|
||||
JsonValue::Null => None,
|
||||
value => Some(value),
|
||||
},
|
||||
};
|
||||
|
||||
let response = match remote_browser_api.command_with_artifacts(params).await {
|
||||
Ok(outcome) => {
|
||||
{
|
||||
let mut state = thread_state.lock().await;
|
||||
state.remote_browser_session_id = Some(outcome.browser_session_id.clone());
|
||||
}
|
||||
|
||||
if let Some(browser_state) = outcome.browser_state.to_public_state() {
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::BrowserSessionUpdated(
|
||||
BrowserSessionUpdatedNotification {
|
||||
browser_session_id: outcome.browser_session_id.clone(),
|
||||
browser_state,
|
||||
},
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
to_core_dynamic_tool_response(build_dynamic_tool_response(outcome))
|
||||
}
|
||||
Err(err) => remote_browser_error_response(err.message),
|
||||
};
|
||||
|
||||
submit_dynamic_tool_response(conversation, call_id, response).await;
|
||||
}
|
||||
|
||||
fn guardian_auto_approval_review_notification(
|
||||
conversation_id: &ThreadId,
|
||||
event_turn_id: &str,
|
||||
@@ -260,6 +353,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
thread_manager: Arc<ThreadManager>,
|
||||
outgoing: ThreadScopedOutgoingMessageSender,
|
||||
thread_state: Arc<tokio::sync::Mutex<ThreadState>>,
|
||||
remote_browser_api: RemoteBrowserApi,
|
||||
thread_watch_manager: ThreadWatchManager,
|
||||
api_version: ApiVersion,
|
||||
fallback_model_provider: String,
|
||||
@@ -918,19 +1012,31 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::ItemStarted(notification))
|
||||
.await;
|
||||
let params = DynamicToolCallParams {
|
||||
thread_id: conversation_id.to_string(),
|
||||
turn_id: turn_id.clone(),
|
||||
call_id: call_id.clone(),
|
||||
tool: tool.clone(),
|
||||
arguments: arguments.clone(),
|
||||
};
|
||||
let (_pending_request_id, rx) = outgoing
|
||||
.send_request(ServerRequestPayload::DynamicToolCall(params))
|
||||
.await;
|
||||
tokio::spawn(async move {
|
||||
crate::dynamic_tools::on_call_response(call_id, rx, conversation).await;
|
||||
});
|
||||
if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) {
|
||||
tokio::spawn(handle_remote_browser_dynamic_tool_request(
|
||||
call_id,
|
||||
tool,
|
||||
arguments,
|
||||
conversation,
|
||||
thread_state.clone(),
|
||||
outgoing.clone(),
|
||||
remote_browser_api.clone(),
|
||||
));
|
||||
} else {
|
||||
let params = DynamicToolCallParams {
|
||||
thread_id: conversation_id.to_string(),
|
||||
turn_id: turn_id.clone(),
|
||||
call_id: call_id.clone(),
|
||||
tool: tool.clone(),
|
||||
arguments: arguments.clone(),
|
||||
};
|
||||
let (_pending_request_id, rx) = outgoing
|
||||
.send_request(ServerRequestPayload::DynamicToolCall(params))
|
||||
.await;
|
||||
tokio::spawn(async move {
|
||||
crate::dynamic_tools::on_call_response(call_id, rx, conversation).await;
|
||||
});
|
||||
}
|
||||
} else {
|
||||
error!(
|
||||
"dynamic tool calls are only supported on api v2 (call_id: {})",
|
||||
|
||||
@@ -15,6 +15,8 @@ use crate::outgoing_message::OutgoingMessageSender;
|
||||
use crate::outgoing_message::OutgoingNotification;
|
||||
use crate::outgoing_message::RequestContext;
|
||||
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
|
||||
use crate::remote_browser_api::RemoteBrowserApi;
|
||||
use crate::remote_browser_tools::merge_remote_browser_dynamic_tools;
|
||||
use crate::thread_status::ThreadWatchManager;
|
||||
use crate::thread_status::resolve_thread_status;
|
||||
use chrono::DateTime;
|
||||
@@ -369,6 +371,7 @@ pub(crate) struct CodexMessageProcessor {
|
||||
auth_manager: Arc<AuthManager>,
|
||||
thread_manager: Arc<ThreadManager>,
|
||||
outgoing: Arc<OutgoingMessageSender>,
|
||||
remote_browser_api: RemoteBrowserApi,
|
||||
arg0_paths: Arg0DispatchPaths,
|
||||
config: Arc<Config>,
|
||||
cli_overrides: Vec<(String, TomlValue)>,
|
||||
@@ -398,6 +401,7 @@ struct ListenerTaskContext {
|
||||
thread_manager: Arc<ThreadManager>,
|
||||
thread_state_manager: ThreadStateManager,
|
||||
outgoing: Arc<OutgoingMessageSender>,
|
||||
remote_browser_api: RemoteBrowserApi,
|
||||
thread_watch_manager: ThreadWatchManager,
|
||||
fallback_model_provider: String,
|
||||
codex_home: PathBuf,
|
||||
@@ -413,6 +417,7 @@ pub(crate) struct CodexMessageProcessorArgs {
|
||||
pub(crate) auth_manager: Arc<AuthManager>,
|
||||
pub(crate) thread_manager: Arc<ThreadManager>,
|
||||
pub(crate) outgoing: Arc<OutgoingMessageSender>,
|
||||
pub(crate) remote_browser_api: RemoteBrowserApi,
|
||||
pub(crate) arg0_paths: Arg0DispatchPaths,
|
||||
pub(crate) config: Arc<Config>,
|
||||
pub(crate) cli_overrides: Vec<(String, TomlValue)>,
|
||||
@@ -476,6 +481,7 @@ impl CodexMessageProcessor {
|
||||
auth_manager,
|
||||
thread_manager,
|
||||
outgoing,
|
||||
remote_browser_api,
|
||||
arg0_paths,
|
||||
config,
|
||||
cli_overrides,
|
||||
@@ -487,6 +493,7 @@ impl CodexMessageProcessor {
|
||||
auth_manager,
|
||||
thread_manager,
|
||||
outgoing: outgoing.clone(),
|
||||
remote_browser_api,
|
||||
arg0_paths,
|
||||
config,
|
||||
cli_overrides,
|
||||
@@ -1879,6 +1886,7 @@ impl CodexMessageProcessor {
|
||||
thread_manager: Arc::clone(&self.thread_manager),
|
||||
thread_state_manager: self.thread_state_manager.clone(),
|
||||
outgoing: Arc::clone(&self.outgoing),
|
||||
remote_browser_api: self.remote_browser_api.clone(),
|
||||
thread_watch_manager: self.thread_watch_manager.clone(),
|
||||
fallback_model_provider: self.config.model_provider_id.clone(),
|
||||
codex_home: self.config.codex_home.clone(),
|
||||
@@ -1983,7 +1991,11 @@ impl CodexMessageProcessor {
|
||||
}
|
||||
};
|
||||
|
||||
let dynamic_tools = dynamic_tools.unwrap_or_default();
|
||||
let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() {
|
||||
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default())
|
||||
} else {
|
||||
dynamic_tools.unwrap_or_default()
|
||||
};
|
||||
let core_dynamic_tools = if dynamic_tools.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
@@ -6694,6 +6706,7 @@ impl CodexMessageProcessor {
|
||||
thread_manager: Arc::clone(&self.thread_manager),
|
||||
thread_state_manager: self.thread_state_manager.clone(),
|
||||
outgoing: Arc::clone(&self.outgoing),
|
||||
remote_browser_api: self.remote_browser_api.clone(),
|
||||
thread_watch_manager: self.thread_watch_manager.clone(),
|
||||
fallback_model_provider: self.config.model_provider_id.clone(),
|
||||
codex_home: self.config.codex_home.clone(),
|
||||
@@ -6781,6 +6794,7 @@ impl CodexMessageProcessor {
|
||||
thread_manager: Arc::clone(&self.thread_manager),
|
||||
thread_state_manager: self.thread_state_manager.clone(),
|
||||
outgoing: Arc::clone(&self.outgoing),
|
||||
remote_browser_api: self.remote_browser_api.clone(),
|
||||
thread_watch_manager: self.thread_watch_manager.clone(),
|
||||
fallback_model_provider: self.config.model_provider_id.clone(),
|
||||
codex_home: self.config.codex_home.clone(),
|
||||
@@ -6812,6 +6826,7 @@ impl CodexMessageProcessor {
|
||||
outgoing,
|
||||
thread_manager,
|
||||
thread_state_manager,
|
||||
remote_browser_api,
|
||||
thread_watch_manager,
|
||||
fallback_model_provider,
|
||||
codex_home,
|
||||
@@ -6906,6 +6921,7 @@ impl CodexMessageProcessor {
|
||||
thread_manager.clone(),
|
||||
thread_outgoing,
|
||||
thread_state.clone(),
|
||||
remote_browser_api.clone(),
|
||||
thread_watch_manager.clone(),
|
||||
api_version,
|
||||
fallback_model_provider.clone(),
|
||||
|
||||
@@ -72,6 +72,7 @@ mod message_processor;
|
||||
mod models;
|
||||
mod outgoing_message;
|
||||
mod remote_browser_api;
|
||||
mod remote_browser_tools;
|
||||
mod server_request_error;
|
||||
mod thread_state;
|
||||
mod thread_status;
|
||||
|
||||
@@ -239,10 +239,12 @@ impl MessageProcessor {
|
||||
.set_analytics_events_client(analytics_events_client.clone());
|
||||
|
||||
let cloud_requirements = Arc::new(RwLock::new(cloud_requirements));
|
||||
let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint);
|
||||
let codex_message_processor = CodexMessageProcessor::new(CodexMessageProcessorArgs {
|
||||
auth_manager: auth_manager.clone(),
|
||||
thread_manager: Arc::clone(&thread_manager),
|
||||
outgoing: outgoing.clone(),
|
||||
remote_browser_api: remote_browser_api.clone(),
|
||||
arg0_paths,
|
||||
config: Arc::clone(&config),
|
||||
cli_overrides: cli_overrides.clone(),
|
||||
@@ -265,7 +267,6 @@ impl MessageProcessor {
|
||||
);
|
||||
let external_agent_config_api = ExternalAgentConfigApi::new(config.codex_home.clone());
|
||||
let fs_api = FsApi::default();
|
||||
let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint);
|
||||
|
||||
Self {
|
||||
outgoing,
|
||||
|
||||
@@ -2,7 +2,12 @@ use crate::error_code::INTERNAL_ERROR_CODE;
|
||||
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionCommandResponse;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
use codex_app_server_protocol::BrowserTabState;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RemoteBrowserApi {
|
||||
@@ -10,6 +15,104 @@ pub(crate) struct RemoteBrowserApi {
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct RemoteBrowserTabState {
|
||||
pub(crate) id: String,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) url: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) selected: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct RemoteBrowserState {
|
||||
#[serde(default)]
|
||||
pub(crate) selected_tab_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) tabs: Vec<RemoteBrowserTabState>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RemoteBrowserArtifacts {
|
||||
#[serde(default)]
|
||||
screenshot_base64: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct RemoteBrowserCommandOutcome {
|
||||
pub(crate) browser_session_id: String,
|
||||
pub(crate) result: JsonValue,
|
||||
pub(crate) browser_state: RemoteBrowserState,
|
||||
#[serde(default)]
|
||||
artifacts: RemoteBrowserArtifacts,
|
||||
}
|
||||
|
||||
impl RemoteBrowserState {
|
||||
fn selected_tab_id(&self) -> Option<String> {
|
||||
self.selected_tab_id.clone().or_else(|| {
|
||||
self.tabs
|
||||
.iter()
|
||||
.find(|tab| tab.selected)
|
||||
.map(|tab| tab.id.clone())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn to_public_state(&self) -> Option<BrowserSessionState> {
|
||||
let selected_tab_id = self.selected_tab_id()?;
|
||||
Some(BrowserSessionState {
|
||||
selected_tab_id,
|
||||
tabs: self
|
||||
.tabs
|
||||
.iter()
|
||||
.map(|tab| BrowserTabState {
|
||||
id: tab.id.clone(),
|
||||
title: tab.title.clone().unwrap_or_default(),
|
||||
url: tab.url.clone().unwrap_or_default(),
|
||||
selected: tab.selected,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteBrowserCommandOutcome {
|
||||
pub(crate) fn to_public_response(
|
||||
&self,
|
||||
) -> Result<BrowserSessionCommandResponse, JSONRPCErrorError> {
|
||||
let browser_state =
|
||||
self.browser_state
|
||||
.to_public_state()
|
||||
.ok_or_else(|| JSONRPCErrorError {
|
||||
code: INTERNAL_ERROR_CODE,
|
||||
message: "remote browser response did not include a selected tab".to_string(),
|
||||
data: None,
|
||||
})?;
|
||||
|
||||
Ok(BrowserSessionCommandResponse {
|
||||
browser_session_id: self.browser_session_id.clone(),
|
||||
result: self.result.clone(),
|
||||
browser_state,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn screenshot_data_url(&self) -> Option<String> {
|
||||
self.artifacts
|
||||
.screenshot_base64
|
||||
.as_ref()
|
||||
.map(|encoded| format!("data:image/png;base64,{encoded}"))
|
||||
}
|
||||
|
||||
pub(crate) fn browser_state_json(&self) -> JsonValue {
|
||||
serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null)
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteBrowserApi {
|
||||
pub(crate) fn new(endpoint: Option<String>) -> Self {
|
||||
Self {
|
||||
@@ -18,10 +121,23 @@ impl RemoteBrowserApi {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_configured(&self) -> bool {
|
||||
self.endpoint.is_some()
|
||||
}
|
||||
|
||||
pub(crate) async fn command(
|
||||
&self,
|
||||
params: BrowserSessionCommandParams,
|
||||
) -> Result<BrowserSessionCommandResponse, JSONRPCErrorError> {
|
||||
self.command_with_artifacts(params)
|
||||
.await?
|
||||
.to_public_response()
|
||||
}
|
||||
|
||||
pub(crate) async fn command_with_artifacts(
|
||||
&self,
|
||||
params: BrowserSessionCommandParams,
|
||||
) -> Result<RemoteBrowserCommandOutcome, JSONRPCErrorError> {
|
||||
let Some(endpoint) = &self.endpoint else {
|
||||
return Err(JSONRPCErrorError {
|
||||
code: INVALID_REQUEST_ERROR_CODE,
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
use crate::remote_browser_api::RemoteBrowserCommandOutcome;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallResponse;
|
||||
use codex_app_server_protocol::DynamicToolSpec;
|
||||
use serde_json::json;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const CREATE_TAB_TOOL: &str = "create_tab";
|
||||
const LIST_TABS_TOOL: &str = "list_tabs";
|
||||
const SELECTED_TAB_TOOL: &str = "selected_tab";
|
||||
const SELECT_TAB_TOOL: &str = "select_tab";
|
||||
const NAVIGATE_TAB_URL_TOOL: &str = "navigate_tab_url";
|
||||
const TABS_CONTENT_TOOL: &str = "tabs_content";
|
||||
const WAIT_FOR_LOAD_STATE_TOOL: &str = "playwright_wait_for_load_state";
|
||||
const SCREENSHOT_TOOL: &str = "playwright_screenshot";
|
||||
|
||||
pub(crate) fn merge_remote_browser_dynamic_tools(
|
||||
mut tools: Vec<DynamicToolSpec>,
|
||||
) -> Vec<DynamicToolSpec> {
|
||||
let existing_names = tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
for tool in remote_browser_dynamic_tools() {
|
||||
if !existing_names.contains(&tool.name) {
|
||||
tools.push(tool);
|
||||
}
|
||||
}
|
||||
|
||||
tools
|
||||
}
|
||||
|
||||
pub(crate) fn is_remote_browser_dynamic_tool(name: &str) -> bool {
|
||||
matches!(
|
||||
name,
|
||||
CREATE_TAB_TOOL
|
||||
| LIST_TABS_TOOL
|
||||
| SELECTED_TAB_TOOL
|
||||
| SELECT_TAB_TOOL
|
||||
| NAVIGATE_TAB_URL_TOOL
|
||||
| TABS_CONTENT_TOOL
|
||||
| WAIT_FOR_LOAD_STATE_TOOL
|
||||
| SCREENSHOT_TOOL
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_dynamic_tool_response(
|
||||
outcome: RemoteBrowserCommandOutcome,
|
||||
) -> DynamicToolCallResponse {
|
||||
let mut content_items = vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: format_remote_browser_text_result(&outcome),
|
||||
}];
|
||||
|
||||
if let Some(image_url) = outcome.screenshot_data_url() {
|
||||
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
|
||||
}
|
||||
|
||||
DynamicToolCallResponse {
|
||||
content_items,
|
||||
success: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_remote_browser_text_result(outcome: &RemoteBrowserCommandOutcome) -> String {
|
||||
serde_json::to_string_pretty(&json!({
|
||||
"result": outcome.result,
|
||||
"browser_state": outcome.browser_state_json(),
|
||||
}))
|
||||
.unwrap_or_else(|_| "{\"result\":\"remote browser output unavailable\"}".to_string())
|
||||
}
|
||||
|
||||
fn remote_browser_dynamic_tools() -> Vec<DynamicToolSpec> {
|
||||
vec![
|
||||
DynamicToolSpec {
|
||||
name: CREATE_TAB_TOOL.to_string(),
|
||||
description: "Create a browser tab in the shared remote browser session. Prefer this for the first navigation by passing a URL.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": { "type": "string" },
|
||||
"select": { "type": "boolean" },
|
||||
"timeout_ms": { "type": "integer", "minimum": 1 }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: LIST_TABS_TOOL.to_string(),
|
||||
description: "List the current tabs in the shared remote browser session.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: SELECTED_TAB_TOOL.to_string(),
|
||||
description: "Get the currently selected tab in the shared remote browser session.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: SELECT_TAB_TOOL.to_string(),
|
||||
description: "Select an existing browser tab by tab_id.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tab_id": { "type": "string" }
|
||||
},
|
||||
"required": ["tab_id"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: NAVIGATE_TAB_URL_TOOL.to_string(),
|
||||
description: "Navigate an existing tab, or the selected tab, to a URL.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": { "type": "string" },
|
||||
"tab_id": { "type": "string" },
|
||||
"timeout_ms": { "type": "integer", "minimum": 1 }
|
||||
},
|
||||
"required": ["url"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: TABS_CONTENT_TOOL.to_string(),
|
||||
description: "Read sanitized text content from one or more tabs. Use this to inspect a page before summarizing it.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tab_ids": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"max_chars_per_tab": { "type": "integer", "minimum": 1 }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: WAIT_FOR_LOAD_STATE_TOOL.to_string(),
|
||||
description: "Wait for the selected tab, or a specific tab, to reach a Playwright load state.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tab_id": { "type": "string" },
|
||||
"state": {
|
||||
"type": "string",
|
||||
"enum": ["domcontentloaded", "load", "networkidle"]
|
||||
},
|
||||
"timeout_ms": { "type": "integer", "minimum": 1 }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
DynamicToolSpec {
|
||||
name: SCREENSHOT_TOOL.to_string(),
|
||||
description: "Capture a screenshot from the selected tab, or a specific tab, and return it to the model.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tab_id": { "type": "string" },
|
||||
"full_page": { "type": "boolean" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -54,6 +54,7 @@ pub(crate) struct ThreadState {
|
||||
pub(crate) pending_interrupts: PendingInterruptQueue,
|
||||
pub(crate) pending_rollbacks: Option<ConnectionRequestId>,
|
||||
pub(crate) turn_summary: TurnSummary,
|
||||
pub(crate) remote_browser_session_id: Option<String>,
|
||||
pub(crate) cancel_tx: Option<oneshot::Sender<()>>,
|
||||
pub(crate) experimental_raw_events: bool,
|
||||
pub(crate) listener_generation: u64,
|
||||
|
||||
@@ -2,8 +2,10 @@ use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use app_test_support::McpProcess;
|
||||
use app_test_support::create_final_assistant_message_sse_response;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||
use app_test_support::to_response;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallParams;
|
||||
use codex_app_server_protocol::DynamicToolCallResponse;
|
||||
@@ -32,7 +34,11 @@ use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
use tokio::time::timeout;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
@@ -191,6 +197,266 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()>
|
||||
{
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id,
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "Hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let bodies = responses_bodies(&server).await?;
|
||||
let body = bodies
|
||||
.first()
|
||||
.context("expected at least one responses request")?;
|
||||
assert!(find_tool(body, "create_tab").is_some());
|
||||
assert!(find_tool(body, "tabs_content").is_some());
|
||||
assert!(find_tool(body, "playwright_screenshot").is_some());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()> {
|
||||
let call_id = "dyn-browser-1";
|
||||
let tool_name = "create_tab";
|
||||
let tool_args = json!({ "url": "https://news.ycombinator.com/" });
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_123",
|
||||
"result": {
|
||||
"tab_id": "tab_123",
|
||||
"selected_tab_id": "tab_123"
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
},
|
||||
"artifacts": {
|
||||
"screenshotBase64": "AAA"
|
||||
}
|
||||
})))
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
let thread_id = thread.id.clone();
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread_id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "open Hacker News".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
let turn_id = turn.id.clone();
|
||||
|
||||
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
|
||||
assert_eq!(started.thread_id, thread_id);
|
||||
assert_eq!(started.turn_id, turn_id.clone());
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
Duration::from_millis(500),
|
||||
mcp.read_stream_until_request_message()
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"remote browser tool call should not be forwarded back to the client"
|
||||
);
|
||||
|
||||
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
|
||||
assert_eq!(browser_update.browser_session_id, "sess_123");
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
|
||||
assert_eq!(browser_update.browser_state.tabs.len(), 1);
|
||||
assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News");
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
status,
|
||||
content_items,
|
||||
success,
|
||||
..
|
||||
} = completed.item
|
||||
else {
|
||||
panic!("expected dynamic tool call item");
|
||||
};
|
||||
assert_eq!(status, DynamicToolCallStatus::Completed);
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
content_items,
|
||||
Some(vec![
|
||||
DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string_pretty(&json!({
|
||||
"result": {
|
||||
"tab_id": "tab_123",
|
||||
"selected_tab_id": "tab_123"
|
||||
},
|
||||
"browser_state": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
}))?
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
])
|
||||
);
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let bodies = responses_bodies(&server).await?;
|
||||
let payload = bodies
|
||||
.iter()
|
||||
.find_map(|body| function_call_output_payload(body, call_id))
|
||||
.context("expected function_call_output in follow-up request")?;
|
||||
let FunctionCallOutputBody::ContentItems(content_items) = payload.body else {
|
||||
panic!("expected content items payload");
|
||||
};
|
||||
assert_eq!(
|
||||
content_items,
|
||||
vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string_pretty(&json!({
|
||||
"result": {
|
||||
"tab_id": "tab_123",
|
||||
"selected_tab_id": "tab_123"
|
||||
},
|
||||
"browser_state": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
}))?,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
let remote_browser_requests = remote_browser_server
|
||||
.received_requests()
|
||||
.await
|
||||
.context("failed to fetch remote browser requests")?;
|
||||
assert_eq!(remote_browser_requests.len(), 1);
|
||||
assert_eq!(
|
||||
remote_browser_requests[0].body_json::<Value>()?,
|
||||
json!({
|
||||
"browserSessionId": null,
|
||||
"command": "create_tab",
|
||||
"arguments": {
|
||||
"url": "https://news.ycombinator.com/"
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Exercises the full dynamic tool call path (server request, client response, model output).
|
||||
#[tokio::test]
|
||||
async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
|
||||
@@ -636,6 +902,20 @@ async fn wait_for_dynamic_tool_completed(
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_browser_session_updated(
|
||||
mcp: &mut McpProcess,
|
||||
) -> Result<BrowserSessionUpdatedNotification> {
|
||||
let notification: JSONRPCNotification = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("browserSession/updated"),
|
||||
)
|
||||
.await??;
|
||||
let params = notification
|
||||
.params
|
||||
.context("browserSession/updated should include params")?;
|
||||
Ok(serde_json::from_value(params)?)
|
||||
}
|
||||
|
||||
fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> {
|
||||
let config_toml = codex_home.join("config.toml");
|
||||
std::fs::write(
|
||||
|
||||
Reference in New Issue
Block a user