From ba0d3b467bbb5b2508364ca6be15ac992dcd111d Mon Sep 17 00:00:00 2001 From: Liang-Ting Jiang Date: Tue, 24 Mar 2026 04:59:03 +0000 Subject: [PATCH] Surface remote browser replay artifacts Co-authored-by: Codex --- .../src/protocol/common.rs | 13 +++++++ .../app-server-protocol/src/protocol/v2.rs | 24 ++++++++++++ .../app-server/src/bespoke_event_handling.rs | 1 + codex-rs/app-server/src/message_processor.rs | 1 + .../src/message_processor/tracing_tests.rs | 17 +++++++++ codex-rs/app-server/src/remote_browser_api.rs | 37 +++++++++++++++++++ .../app-server/src/remote_browser_tools.rs | 3 ++ .../tests/suite/v2/dynamic_tools.rs | 22 ++++++++++- 8 files changed, 117 insertions(+), 1 deletion(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 4f0c966a70..d37ed143bf 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1661,6 +1661,12 @@ mod tests { selected: true, }], }, + artifacts: Some(v2::BrowserSessionArtifacts { + screenshot_image_url: Some("data:image/png;base64,AAA".to_string()), + replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()), + replay_frame_count: Some(2), + replay_frame_duration_ms: Some(700), + }), }); assert_eq!( json!({ @@ -1677,6 +1683,12 @@ mod tests { "selected": true } ] + }, + "artifacts": { + "screenshotImageUrl": "data:image/png;base64,AAA", + "replayGifImageUrl": "data:image/gif;base64,R0lGODlh", + "replayFrameCount": 2, + "replayFrameDurationMs": 700 } } }), @@ -1718,6 +1730,7 @@ mod tests { selected_tab_id: String::new(), tabs: Vec::new(), }, + artifacts: None, }); let reason = crate::experimental_api::ExperimentalApi::experimental_reason(¬ification); assert_eq!(reason, Some("browserSession/updated")); diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 22ee3a0dde..e61e8a23ac 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -2117,6 +2117,24 @@ pub struct BrowserSessionState { pub tabs: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserSessionArtifacts { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub screenshot_image_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub replay_gif_image_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub replay_frame_count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub replay_frame_duration_ms: Option, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -2135,6 +2153,9 @@ pub struct BrowserSessionCommandResponse { pub browser_session_id: String, pub result: JsonValue, pub browser_state: BrowserSessionState, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub artifacts: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -2143,6 +2164,9 @@ pub struct BrowserSessionCommandResponse { pub struct BrowserSessionUpdatedNotification { pub browser_session_id: String, pub browser_state: BrowserSessionState, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub artifacts: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 8af1d5a43e..9179ae536f 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -272,6 +272,7 @@ async fn handle_remote_browser_dynamic_tool_request( BrowserSessionUpdatedNotification { browser_session_id: outcome.browser_session_id.clone(), browser_state, + artifacts: outcome.to_public_artifacts(), }, )) .await; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 614b6519b3..e88576eaee 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -952,6 +952,7 @@ impl MessageProcessor { BrowserSessionUpdatedNotification { browser_session_id: response.browser_session_id.clone(), browser_state: response.browser_state.clone(), + artifacts: response.artifacts.clone(), }, )) .await; diff --git a/codex-rs/app-server/src/message_processor/tracing_tests.rs b/codex-rs/app-server/src/message_processor/tracing_tests.rs index 90a4abcf44..4f7baf285e 100644 --- a/codex-rs/app-server/src/message_processor/tracing_tests.rs +++ b/codex-rs/app-server/src/message_processor/tracing_tests.rs @@ -9,6 +9,7 @@ use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::write_mock_responses_config_toml; use codex_app_server_protocol::BrowserSessionCommandParams; use codex_app_server_protocol::BrowserSessionCommandResponse; +use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionState; use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::BrowserTabState; @@ -695,6 +696,12 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result< "url": "https://example.com", "selected": true, }] + }, + "artifacts": { + "screenshotBase64": "AAA", + "replayGifBase64": "R0lGODlh", + "replayFrameCount": 2, + "replayFrameDurationMs": 700 } }); @@ -732,10 +739,20 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result< assert_eq!(response.browser_session_id, "sess_123"); assert_eq!(response.result, serde_json::json!({"ok": true})); assert_eq!(response.browser_state, browser_state); + assert_eq!( + response.artifacts, + Some(BrowserSessionArtifacts { + screenshot_image_url: Some("data:image/png;base64,AAA".to_string()), + replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()), + replay_frame_count: Some(2), + replay_frame_duration_ms: Some(700), + }) + ); let notification = read_browser_session_updated_notification(&mut harness.outgoing_rx).await; assert_eq!(notification.browser_session_id, "sess_123"); assert_eq!(notification.browser_state, browser_state); + assert_eq!(notification.artifacts, response.artifacts); harness.shutdown().await; remote_browser.verify().await; diff --git a/codex-rs/app-server/src/remote_browser_api.rs b/codex-rs/app-server/src/remote_browser_api.rs index eae81665e4..68780e3d15 100644 --- a/codex-rs/app-server/src/remote_browser_api.rs +++ b/codex-rs/app-server/src/remote_browser_api.rs @@ -1,5 +1,6 @@ use crate::error_code::INTERNAL_ERROR_CODE; use crate::error_code::INVALID_REQUEST_ERROR_CODE; +use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionCommandParams; use codex_app_server_protocol::BrowserSessionCommandResponse; use codex_app_server_protocol::BrowserSessionState; @@ -41,6 +42,12 @@ pub(crate) struct RemoteBrowserState { struct RemoteBrowserArtifacts { #[serde(default)] screenshot_base64: Option, + #[serde(default)] + replay_gif_base64: Option, + #[serde(default)] + replay_frame_count: Option, + #[serde(default)] + replay_frame_duration_ms: Option, } #[derive(Clone, Debug, Deserialize)] @@ -98,6 +105,7 @@ impl RemoteBrowserCommandOutcome { browser_session_id: self.browser_session_id.clone(), result: self.result.clone(), browser_state, + artifacts: self.to_public_artifacts(), }) } @@ -108,6 +116,35 @@ impl RemoteBrowserCommandOutcome { .map(|encoded| format!("data:image/png;base64,{encoded}")) } + pub(crate) fn replay_gif_data_url(&self) -> Option { + self.artifacts + .replay_gif_base64 + .as_ref() + .map(|encoded| format!("data:image/gif;base64,{encoded}")) + } + + pub(crate) fn to_public_artifacts(&self) -> Option { + let screenshot_image_url = self.screenshot_data_url(); + let replay_gif_image_url = self.replay_gif_data_url(); + let replay_frame_count = self.artifacts.replay_frame_count; + let replay_frame_duration_ms = self.artifacts.replay_frame_duration_ms; + + if screenshot_image_url.is_none() + && replay_gif_image_url.is_none() + && replay_frame_count.is_none() + && replay_frame_duration_ms.is_none() + { + return None; + } + + Some(BrowserSessionArtifacts { + screenshot_image_url, + replay_gif_image_url, + replay_frame_count, + replay_frame_duration_ms, + }) + } + pub(crate) fn browser_state_json(&self) -> JsonValue { serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null) } diff --git a/codex-rs/app-server/src/remote_browser_tools.rs b/codex-rs/app-server/src/remote_browser_tools.rs index 6da9865518..fa10fc78ac 100644 --- a/codex-rs/app-server/src/remote_browser_tools.rs +++ b/codex-rs/app-server/src/remote_browser_tools.rs @@ -55,6 +55,9 @@ pub(crate) fn build_dynamic_tool_response( if let Some(image_url) = outcome.screenshot_data_url() { content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); } + if let Some(image_url) = outcome.replay_gif_data_url() { + content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url }); + } DynamicToolCallResponse { content_items, diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index aa9c856d2e..5c72d134b4 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -5,6 +5,7 @@ use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; +use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::DynamicToolCallOutputContentItem; use codex_app_server_protocol::DynamicToolCallParams; @@ -293,7 +294,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() }] }, "artifacts": { - "screenshotBase64": "AAA" + "screenshotBase64": "AAA", + "replayGifBase64": "R0lGODlh", + "replayFrameCount": 2, + "replayFrameDurationMs": 700 } }))) .mount(&remote_browser_server) @@ -358,6 +362,15 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); assert_eq!(browser_update.browser_state.tabs.len(), 1); assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News"); + assert_eq!( + browser_update.artifacts, + Some(BrowserSessionArtifacts { + screenshot_image_url: Some("data:image/png;base64,AAA".to_string()), + replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()), + replay_frame_count: Some(2), + replay_frame_duration_ms: Some(700), + }) + ); let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; let ThreadItem::DynamicToolCall { @@ -394,6 +407,9 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() DynamicToolCallOutputContentItem::InputImage { image_url: "data:image/png;base64,AAA".to_string(), }, + DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/gif;base64,R0lGODlh".to_string(), + }, ]) ); @@ -435,6 +451,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() image_url: "data:image/png;base64,AAA".to_string(), detail: None, }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/gif;base64,R0lGODlh".to_string(), + detail: None, + }, ] );