diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 0a608471fd..5b0d114c1e 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -524,6 +524,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -4994,4 +5000,4 @@ } ], "title": "ServerNotification" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index cb14d466c7..b73e28b1bf 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5656,6 +5656,12 @@ }, "browserState": { "$ref": "#/definitions/v2/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -14921,4 +14927,4 @@ }, "title": "CodexAppServerProtocol", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 76168a31ce..9b8ade7756 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -856,6 +856,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -12680,4 +12686,4 @@ }, "title": "CodexAppServerProtocolV2", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json index e26d365e0a..5eefde5541 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json @@ -92,6 +92,12 @@ }, "browserState": { "$ref": "#/definitions/BrowserSessionState" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -100,4 +106,4 @@ ], "title": "BrowserSessionUpdatedNotification", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts index f84b109547..32cb34b8d7 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts @@ -4,4 +4,4 @@ import type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts"; import type { BrowserSessionState } from "./BrowserSessionState"; -export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, }; +export type BrowserSessionUpdatedNotification = { threadId?: string | null, browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, }; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index d37ed143bf..48f3a9b122 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1651,6 +1651,7 @@ mod tests { fn serialize_browser_session_updated_notification() -> Result<()> { let notification = ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + thread_id: Some("thread-1".to_string()), browser_session_id: "sess_123".to_string(), browser_state: v2::BrowserSessionState { selected_tab_id: "tab_123".to_string(), @@ -1672,6 +1673,7 @@ mod tests { json!({ "method": "browserSession/updated", "params": { + "threadId": "thread-1", "browserSessionId": "sess_123", "browserState": { "selectedTabId": "tab_123", @@ -1725,6 +1727,7 @@ mod tests { fn browser_session_updated_is_marked_experimental() { let notification = ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + thread_id: Some("thread-1".to_string()), browser_session_id: "sess_123".to_string(), browser_state: v2::BrowserSessionState { selected_tab_id: String::new(), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 9e95006353..6d329d7c7b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -2162,6 +2162,9 @@ pub struct BrowserSessionCommandResponse { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct BrowserSessionUpdatedNotification { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub thread_id: Option, pub browser_session_id: String, pub browser_state: BrowserSessionState, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index ace7f2bd43..aeb438b57b 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -1027,6 +1027,8 @@ If the session approval policy uses `Granular` with `request_permissions: false` Each dynamic tool may set `deferLoading`. When omitted, it defaults to `false`. Set it to `true` to keep the tool registered and callable by runtime features such as `js_repl`, while excluding it from the model-facing tool list sent on ordinary turns. +When the app-server is started with `--remote-browser-endpoint`, it auto-registers a hidden `atlas_command` dynamic tool with `deferLoading: true`. This bridge is intended for AgentLib/Atlas-style calls issued from `js_repl` via `codex.tool(...)`, so it remains callable at runtime without teaching the model a separate public remote-browsing tool surface on ordinary turns. + When a dynamic tool is invoked during a turn, the server sends an `item/tool/call` JSON-RPC request to the client: ```json diff --git a/codex-rs/app-server/src/atlas_command.rs b/codex-rs/app-server/src/atlas_command.rs index ff4be9eecf..a10c0d534f 100644 --- a/codex-rs/app-server/src/atlas_command.rs +++ b/codex-rs/app-server/src/atlas_command.rs @@ -65,7 +65,7 @@ pub(crate) async fn execute_atlas_command( response: atlas_json_response(json!({ "type": "codex_repl", "error": message, - })), + }), None), }, } } @@ -349,24 +349,43 @@ async fn execute_atlas_command_impl( } }; + let response = atlas_json_response( + json!({ + "type": "codex_repl", + "command_output": command_output, + "browser_state": last_browser_state.clone(), + "artifacts": last_artifacts.clone(), + }), + last_artifacts.as_ref(), + ); + Ok(AtlasCommandExecution { browser_session_id: session_id, browser_state: last_browser_state, artifacts: last_artifacts, - response: atlas_json_response(json!({ - "type": "codex_repl", - "command_output": command_output, - })), + response, }) } -fn atlas_json_response(payload: JsonValue) -> DynamicToolCallResponse { - DynamicToolCallResponse { - content_items: vec![DynamicToolCallOutputContentItem::InputText { +fn atlas_json_response( + payload: JsonValue, + artifacts: Option<&BrowserSessionArtifacts>, +) -> DynamicToolCallResponse { + let mut content_items = vec![DynamicToolCallOutputContentItem::InputText { text: serde_json::to_string(&payload).unwrap_or_else(|_| { "{\"type\":\"codex_repl\",\"error\":\"serialization failure\"}".to_string() }), - }], + }]; + + if let Some(image_url) = artifacts.and_then(|value| value.screenshot_image_url.clone()) { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } + if let Some(image_url) = artifacts.and_then(|value| value.replay_gif_image_url.clone()) { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } + + DynamicToolCallResponse { + content_items, success: true, } } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index ea3d705a38..1503117945 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -272,6 +272,7 @@ async fn handle_remote_browser_dynamic_tool_request( outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: Some(outgoing.thread_id_string()), browser_session_id: outcome.browser_session_id.clone(), browser_state, artifacts: outcome.to_public_artifacts(), @@ -308,6 +309,7 @@ async fn handle_atlas_command_dynamic_tool_request( outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: Some(outgoing.thread_id_string()), browser_session_id: outcome.browser_session_id.unwrap_or_default(), browser_state, artifacts: outcome.artifacts, diff --git a/codex-rs/app-server/src/browser_replay.rs b/codex-rs/app-server/src/browser_replay.rs index c1472507d7..bfa201a87a 100644 --- a/codex-rs/app-server/src/browser_replay.rs +++ b/codex-rs/app-server/src/browser_replay.rs @@ -1,3 +1,4 @@ +use crate::atlas_command::is_atlas_command_dynamic_tool; use crate::remote_browser_tools::is_remote_browser_dynamic_tool; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; @@ -49,7 +50,7 @@ fn build_browser_replay_item(turn_id: &str, items: &[ThreadItem]) -> Option Option bool { + is_remote_browser_dynamic_tool(name) || is_atlas_command_dynamic_tool(name) +} + #[derive(Default)] struct FrameContent { image_url: Option, @@ -122,6 +127,14 @@ struct RemoteBrowserToolTextPayload { browser_state: Option, } +#[derive(Debug, Deserialize)] +struct AtlasCommandTextPayload { + #[serde(default)] + command_output: JsonValue, + #[serde(default)] + browser_state: Option, +} + fn inspect_content_items(items: &[DynamicToolCallOutputContentItem]) -> FrameContent { let mut content = FrameContent::default(); @@ -162,6 +175,11 @@ fn build_text_snapshot(text: &str) -> TextSnapshot { serde_json::from_str::(text) .ok() .and_then(|payload| render_result_snapshot(&payload.result)) + .or_else(|| { + serde_json::from_str::(text) + .ok() + .and_then(|payload| render_result_snapshot(&payload.command_output)) + }) .or_else(|| { truncate_snapshot(text).map(|value| TextSnapshot { text: Some(value), @@ -175,6 +193,11 @@ fn parse_browser_state(text: &str) -> Option { serde_json::from_str::(text) .ok() .and_then(|payload| payload.browser_state) + .or_else(|| { + serde_json::from_str::(text) + .ok() + .and_then(|payload| payload.browser_state) + }) } fn render_result_snapshot(result: &JsonValue) -> Option { @@ -495,4 +518,75 @@ mod tests { assert_eq!(animation_image_url, None); assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS)); } + + #[test] + fn builds_replay_from_atlas_command_tool_calls() { + let items = vec![ThreadItem::DynamicToolCall { + id: "tool-1".to_string(), + tool: "atlas_command".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "type": "codex_repl", + "command_output": { + "results": [{ + "title": "Atlas search", + "content": "result one\nresult two" + }] + }, + "browser_state": { + "selectedTabId": "tab_1", + "tabs": [{ + "id": "tab_1", + "title": "Atlas search", + "url": "https://example.com/search", + "selected": true + }] + } + }) + .to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: png_data_url([0, 255, 0, 255], 14, 9), + }, + ]), + success: Some(true), + duration_ms: Some(10), + }]; + + let replay = build_browser_replay_item("turn-atlas", &items).expect("replay item"); + let ThreadItem::BrowserReplay { + frames, + render_mode, + animation_image_url, + frame_duration_ms, + .. + } = replay + else { + panic!("expected browser replay item"); + }; + + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].tool, "atlas_command"); + assert_eq!(render_mode, BrowserReplayRenderMode::Frames); + assert_eq!( + frames[0].browser_state.as_ref().map(|state| state.selected_tab_id.as_str()), + Some("tab_1") + ); + let snapshot = frames[0] + .text_snapshot + .as_deref() + .expect("atlas command snapshot"); + assert!(snapshot.contains("\"title\": \"Atlas search\"")); + assert!(snapshot.contains("\"content\": \"result one\\nresult two\"")); + assert_eq!( + frames[0].text_snapshot_format, + Some(BrowserReplayTextSnapshotFormat::Json) + ); + assert!(frames[0].image_url.is_some()); + assert_eq!(animation_image_url, None); + assert_eq!(frame_duration_ms, None); + } } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 0c421c9f26..be5d06e3dc 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -18,7 +18,6 @@ use crate::outgoing_message::OutgoingNotification; use crate::outgoing_message::RequestContext; use crate::outgoing_message::ThreadScopedOutgoingMessageSender; use crate::remote_browser_api::RemoteBrowserApi; -use crate::remote_browser_tools::merge_remote_browser_dynamic_tools; use crate::thread_status::ThreadWatchManager; use crate::thread_status::resolve_thread_status; use chrono::DateTime; @@ -1994,9 +1993,7 @@ impl CodexMessageProcessor { }; let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() { - let dynamic_tools = - merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default()); - merge_atlas_command_dynamic_tool(dynamic_tools) + merge_atlas_command_dynamic_tool(dynamic_tools.unwrap_or_default()) } else { dynamic_tools.unwrap_or_default() }; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index e88576eaee..20b97ba063 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -950,6 +950,7 @@ impl MessageProcessor { self.outgoing .send_server_notification(ServerNotification::BrowserSessionUpdated( BrowserSessionUpdatedNotification { + thread_id: None, browser_session_id: response.browser_session_id.clone(), browser_state: response.browser_state.clone(), artifacts: response.artifacts.clone(), diff --git a/codex-rs/app-server/src/outgoing_message.rs b/codex-rs/app-server/src/outgoing_message.rs index 67761525b3..32880079d6 100644 --- a/codex-rs/app-server/src/outgoing_message.rs +++ b/codex-rs/app-server/src/outgoing_message.rs @@ -155,6 +155,10 @@ impl ThreadScopedOutgoingMessageSender { self.outgoing.send_server_notification(notification).await; } + pub(crate) fn thread_id_string(&self) -> String { + self.thread_id.to_string() + } + pub(crate) async fn abort_pending_server_requests(&self) { self.outgoing .cancel_requests_for_thread( diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index ec73a9a783..384cee0571 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -204,7 +204,8 @@ async fn thread_start_keeps_hidden_dynamic_tools_out_of_model_requests() -> Resu } #[tokio::test] -async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configured() -> Result<()> +async fn thread_start_auto_injects_only_atlas_command_bridge_when_endpoint_is_configured( +) -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let remote_browser_server = MockServer::start().await; @@ -257,9 +258,9 @@ async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configu let body = bodies .first() .context("expected at least one responses request")?; - assert!(find_tool(body, "create_tab").is_some()); - assert!(find_tool(body, "tabs_content").is_some()); - assert!(find_tool(body, "playwright_screenshot").is_some()); + assert!(find_tool(body, "create_tab").is_none()); + assert!(find_tool(body, "tabs_content").is_none()); + assert!(find_tool(body, "playwright_screenshot").is_none()); assert!(find_tool(body, "atlas_command").is_none()); Ok(()) @@ -370,6 +371,7 @@ async fn atlas_command_dynamic_tool_calls_are_handled_internally() -> Result<()> ); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!(browser_update.browser_state.tabs.len(), 1); @@ -566,6 +568,7 @@ async fn atlas_command_tabs_content_uses_background_tabs() -> Result<()> { let _started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread.id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_tabs_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_main"); @@ -702,6 +705,7 @@ async fn atlas_command_locator_count_passes_through_to_remote_browser() -> Resul assert_eq!(started.turn_id, turn_id.clone()); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_count_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); @@ -858,6 +862,7 @@ async fn atlas_command_locator_click_passes_through_to_remote_browser() -> Resul assert_eq!(started.turn_id, turn_id.clone()); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_atlas_click_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!( @@ -1012,6 +1017,7 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() ); let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.thread_id.as_deref(), Some(thread_id.as_str())); assert_eq!(browser_update.browser_session_id, "sess_123"); assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!(browser_update.browser_state.tabs.len(), 1);