From 509a7435019bd9c1a82a307484651317f7ca02ef Mon Sep 17 00:00:00 2001 From: Liang-Ting Jiang Date: Wed, 25 Mar 2026 01:53:56 +0000 Subject: [PATCH] Keep remote browsing on atlas command path Only auto-inject the hidden atlas bridge when remote browsing is configured, include thread ids in browser session notifications, and persist Atlas browser state plus image artifacts so replay items can be rebuilt from history. Co-authored-by: Codex --- .../schema/json/ServerNotification.json | 8 +- .../codex_app_server_protocol.schemas.json | 8 +- .../codex_app_server_protocol.v2.schemas.json | 8 +- .../v2/BrowserSessionUpdatedNotification.json | 8 +- .../v2/BrowserSessionUpdatedNotification.ts | 2 +- .../src/protocol/common.rs | 3 + .../app-server-protocol/src/protocol/v2.rs | 3 + codex-rs/app-server/README.md | 2 + codex-rs/app-server/src/atlas_command.rs | 37 +++++-- .../app-server/src/bespoke_event_handling.rs | 2 + codex-rs/app-server/src/browser_replay.rs | 96 ++++++++++++++++++- .../app-server/src/codex_message_processor.rs | 5 +- codex-rs/app-server/src/message_processor.rs | 1 + codex-rs/app-server/src/outgoing_message.rs | 4 + .../tests/suite/v2/dynamic_tools.rs | 14 ++- 15 files changed, 178 insertions(+), 23 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 0a608471fd..5b0d114c1e 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -524,6 +524,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -4994,4 +5000,4 @@ } ], "title": "ServerNotification" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index cb14d466c7..b73e28b1bf 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5656,6 +5656,12 @@ }, "browserState": { "$ref": "#/definitions/v2/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -14921,4 +14927,4 @@ }, "title": "CodexAppServerProtocol", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 76168a31ce..9b8ade7756 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -856,6 +856,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -12680,4 +12686,4 @@ }, "title": "CodexAppServerProtocolV2", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json index e26d365e0a..5eefde5541 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json @@ -92,6 +92,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -100,4 +106,4 @@ ], "title": "BrowserSessionUpdatedNotification", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts index f84b109547..32cb34b8d7 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts @@ -4,4 +4,4 @@ import type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts"; import type { BrowserSessionState } from "./BrowserSessionState"; -export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, }; +export type BrowserSessionUpdatedNotification = { threadId?: string | null, browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, }; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index d37ed143bf..48f3a9b122 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1651,6 +1651,7 @@ mod tests { fn serialize_browser_session_updated_notification() -> Result<()> { let notification = ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + thread_id: Some("thread-1".to_string()), browser_session_id: "sess_123".to_string(), browser_state: v2::BrowserSessionState { selected_tab_id: "tab_123".to_string(), @@ -1672,6 +1673,7 @@ mod tests { json!({ "method": "browserSession/updated", "params": { + "threadId": "thread-1", "browserSessionId": "sess_123", "browserState": { "selectedTabId": "tab_123", @@ -1725,6 +1727,7 @@ mod tests { fn browser_session_updated_is_marked_experimental() { let notification = ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + thread_id: Some("thread-1".to_string()), browser_session_id: "sess_123".to_string(), browser_state: v2::BrowserSessionState { selected_tab_id: String::new(), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 9e95006353..6d329d7c7b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -2162,6 +2162,9 @@ pub struct BrowserSessionCommandResponse { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct BrowserSessionUpdatedNotification { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub thread_id: Option, pub browser_session_id: String, pub browser_state: BrowserSessionState, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index ace7f2bd43..aeb438b57b 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -1027,6 +1027,8 @@ If the session approval policy uses `Granular` with `request_permissions: false` Each dynamic tool may set `deferLoading`. When omitted, it defaults to `false`. Set it to `true` to keep the tool registered and callable by runtime features such as `js_repl`, while excluding it from the model-facing tool list sent on ordinary turns. +When the app-server is started with `--remote-browser-endpoint`, it auto-registers a hidden `atlas_command` dynamic tool with `deferLoading: true`. This bridge is intended for AgentLib/Atlas-style calls issued from `js_repl` via `codex.tool(...)`, so it remains callable at runtime without teaching the model a separate public remote-browsing tool surface on ordinary turns. + When a dynamic tool is invoked during a turn, the server sends an `item/tool/call` JSON-RPC request to the client: ```json diff --git a/codex-rs/app-server/src/atlas_command.rs b/codex-rs/app-server/src/atlas_command.rs index ff4be9eecf..a10c0d534f 100644 --- a/codex-rs/app-server/src/atlas_command.rs +++ b/codex-rs/app-server/src/atlas_command.rs @@ -65,7 +65,7 @@ pub(crate) async fn execute_atlas_command( response: atlas_json_response(json!({ "type": "codex_repl", "error": message, - })), + }), None), }, } } @@ -349,24 +349,43 @@ async fn execute_atlas_command_impl( } }; + let response = atlas_json_response( + json!({ + "type": "codex_repl", + "command_output": command_output, + "browser_state": last_browser_state.clone(), + "artifacts": last_artifacts.clone(), + }), + last_artifacts.as_ref(), + ); + Ok(AtlasCommandExecution { browser_session_id: session_id, browser_state: last_browser_state, artifacts: last_artifacts, - response: atlas_json_response(json!({ - "type": "codex_repl", - "command_output": command_output, - })), + response, }) } -fn atlas_json_response(payload: JsonValue) -> DynamicToolCallResponse { - DynamicToolCallResponse { - content_items: vec![DynamicToolCallOutputContentItem::InputText { +fn atlas_json_response( + payload: JsonValue, + artifacts: Option<&BrowserSessionArtifacts>, +) -> DynamicToolCallResponse { + let mut content_items = vec![DynamicToolCallOutputContentItem::InputText { text: serde_json::to_string(&payload).unwrap_or_else(|_| { "{\"type\":\"codex_repl\",\"error\":\"serialization failure\"}".to_string() }), - }], + }]; + + if let Some(image_url) = artifacts.and_then(|value| value.screenshot_image_url.clone()) { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } + if let Some(image_url) = artifacts.and_then(|value| value.replay_gif_image_url.clone()) { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } + + DynamicToolCallResponse { + content_items, success: true, } } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index ea3d705a38..1503117945 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -272,6 +272,7 @@ async fn handle_remote_browser_dynamic_tool_request( outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: Some(outgoing.thread_id_string()), browser_session_id: outcome.browser_session_id.clone(), browser_state, artifacts: outcome.to_public_artifacts(), @@ -308,6 +309,7 @@ async fn handle_atlas_command_dynamic_tool_request( outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: Some(outgoing.thread_id_string()), browser_session_id: outcome.browser_session_id.unwrap_or_default(), browser_state, artifacts: outcome.artifacts, diff --git a/codex-rs/app-server/src/browser_replay.rs b/codex-rs/app-server/src/browser_replay.rs index c1472507d7..bfa201a87a 100644 --- a/codex-rs/app-server/src/browser_replay.rs +++ b/codex-rs/app-server/src/browser_replay.rs @@ -1,3 +1,4 @@ +use crate::atlas_command::is_atlas_command_dynamic_tool; use crate::remote_browser_tools::is_remote_browser_dynamic_tool; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; @@ -49,7 +50,7 @@ fn build_browser_replay_item(turn_id: &str, items: &[ThreadItem]) -> Option Option bool { + is_remote_browser_dynamic_tool(name) || is_atlas_command_dynamic_tool(name) +} + #[derive(Default)] struct FrameContent { image_url: Option, @@ -122,6 +127,14 @@ struct RemoteBrowserToolTextPayload { browser_state: Option, } +#[derive(Debug, Deserialize)] +struct AtlasCommandTextPayload { + #[serde(default)] + command_output: JsonValue, + #[serde(default)] + browser_state: Option, +} + fn inspect_content_items(items: &[DynamicToolCallOutputContentItem]) -> FrameContent { let mut content = FrameContent::default(); @@ -162,6 +175,11 @@ fn build_text_snapshot(text: &str) -> TextSnapshot { serde_json::from_str::(text) .ok() .and_then(|payload| render_result_snapshot(&payload.result)) + .or_else(|| { + serde_json::from_str::(text) + .ok() + .and_then(|payload| render_result_snapshot(&payload.command_output)) + }) .or_else(|| { truncate_snapshot(text).map(|value| TextSnapshot { text: Some(value), @@ -175,6 +193,11 @@ fn parse_browser_state(text: &str) -> Option { serde_json::from_str::(text) .ok() .and_then(|payload| payload.browser_state) + .or_else(|| { + serde_json::from_str::(text) + .ok() + .and_then(|payload| payload.browser_state) + }) } fn render_result_snapshot(result: &JsonValue) -> Option { @@ -495,4 +518,75 @@ mod tests { assert_eq!(animation_image_url, None); assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS)); } + + #[test] + fn builds_replay_from_atlas_command_tool_calls() { + let items = vec![ThreadItem::DynamicToolCall { + id: "tool-1".to_string(), + tool: "atlas_command".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "type": "codex_repl", + "command_output": { + "results": [{ + "title": "Atlas search", + "content": "result one\nresult two" + }] + }, + "browser_state": { + "selectedTabId": "tab_1", + "tabs": [{ + "id": "tab_1", + "title": "Atlas search", + "url": "https://example.com/search", + "selected": true + }] + } + }) + .to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: png_data_url([0, 255, 0, 255], 14, 9), + }, + ]), + success: Some(true), + duration_ms: Some(10), + }]; + + let replay = build_browser_replay_item("turn-atlas", &items).expect("replay item"); + let ThreadItem::BrowserReplay { + frames, + render_mode, + animation_image_url, + frame_duration_ms, + .. + } = replay + else { + panic!("expected browser replay item"); + }; + + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].tool, "atlas_command"); + assert_eq!(render_mode, BrowserReplayRenderMode::Frames); + assert_eq!( + frames[0].browser_state.as_ref().map(|state| state.selected_tab_id.as_str()), + Some("tab_1") + ); + let snapshot = frames[0] + .text_snapshot + .as_deref() + .expect("atlas command snapshot"); + assert!(snapshot.contains("\"title\": \"Atlas search\"")); + assert!(snapshot.contains("\"content\": \"result one\\nresult two\"")); + assert_eq!( + frames[0].text_snapshot_format, + Some(BrowserReplayTextSnapshotFormat::Json) + ); + assert!(frames[0].image_url.is_some()); + assert_eq!(animation_image_url, None); + assert_eq!(frame_duration_ms, None); + } } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 0c421c9f26..be5d06e3dc 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -18,7 +18,6 @@ use crate::outgoing_message::OutgoingNotification; use crate::outgoing_message::RequestContext; use crate::outgoing_message::ThreadScopedOutgoingMessageSender; use crate::remote_browser_api::RemoteBrowserApi; -use crate::remote_browser_tools::merge_remote_browser_dynamic_tools; use crate::thread_status::ThreadWatchManager; use crate::thread_status::resolve_thread_status; use chrono::DateTime; @@ -1994,9 +1993,7 @@ impl CodexMessageProcessor { }; let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() { - let dynamic_tools = - merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default()); - merge_atlas_command_dynamic_tool(dynamic_tools) + merge_atlas_command_dynamic_tool(dynamic_tools.unwrap_or_default()) } else { dynamic_tools.unwrap_or_default() }; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index e88576eaee..20b97ba063 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -950,6 +950,7 @@ impl MessageProcessor { self.outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: None, browser_session_id: response.browser_session_id.clone(), browser_state: response.browser_state.clone(), artifacts: response.artifacts.clone(), diff --git a/codex-rs/app-server/src/outgoing_message.rs b/codex-rs/app-server/src/outgoing_message.rs index 67761525b3..32880079d6 100644 --- a/codex-rs/app-server/src/outgoing_message.rs +++ b/codex-rs/app-server/src/outgoing_message.rs @@ -155,6 +155,10 @@ impl ThreadScopedOutgoingMessageSender { self.outgoing.send_server_notification(notification).await; } + pub(crate) fn thread_id_string(&self) -> String { + self.thread_id.to_string() + } + pub(crate) async fn abort_pending_server_requests(&self) { self.outgoing .cancel_requests_for_thread( diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index ec73a9a783..384cee0571 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -204,7 +204,8 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu } #[tokio::test] -async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()> +async fn thread_start_auto_injects_only_atlas_command_bridge_when_endpoint_is_configured( +) -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let remote_browser_server = MockServer::start().await; @@ -257,9 +258,9 @@ async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configu let body = bodies .first() .context("expected at least one responses request")?; - assert!(find_tool(body, "create_tab").is_some()); - assert!(find_tool(body, "tabs_content").is_some()); - assert!(find_tool(body, "playwright_screenshot").is_some()); + assert!(find_tool(body, "create_tab").is_none()); + assert!(find_tool(body, "tabs_content").is_none()); + assert!(find_tool(body, "playwright_screenshot").is_none()); assert!(find_tool(body, "atlas_command").is_none()); Ok(()) @@ -370,6 +371,7 @@ async fn atlas_command_dynamic_tool_calls_are_handled_internally() -> Result<()> ); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!(browser_update.browser_state.tabs.len(), 1); @@ -566,6 +568,7 @@ async fn atlas_command_tabs_content_uses_background_tabs() -> Result<()> { let _started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread.id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_tabs_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_main"); @@ -702,6 +705,7 @@ async fn atlas_command_locator_count_passes_through_to_remote_browser() -> Resul assert_eq!(started.turn_id, turn_id.clone()); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_count_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); @@ -858,6 +862,7 @@ async fn atlas_command_locator_click_passes_through_to_remote_browser() -> Resul assert_eq!(started.turn_id, turn_id.clone()); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_click_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!( @@ -1012,6 +1017,7 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() ); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!(browser_update.browser_state.tabs.len(), 1);