Surface remote browser replay artifacts

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Liang-Ting Jiang
2026-03-24 04:59:03 +00:00
co-authored by Codex
parent 34d59e54c3
commit ba0d3b467b
8 changed files with 117 additions and 1 deletions
@@ -1661,6 +1661,12 @@ mod tests {
selected: true,
}],
},
artifacts: Some(v2::BrowserSessionArtifacts {
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
replay_frame_count: Some(2),
replay_frame_duration_ms: Some(700),
}),
});
assert_eq!(
json!({
@@ -1677,6 +1683,12 @@ mod tests {
"selected": true
}
]
},
"artifacts": {
"screenshotImageUrl": "data:image/png;base64,AAA",
"replayGifImageUrl": "data:image/gif;base64,R0lGODlh",
"replayFrameCount": 2,
"replayFrameDurationMs": 700
}
}
}),
@@ -1718,6 +1730,7 @@ mod tests {
selected_tab_id: String::new(),
tabs: Vec::new(),
},
artifacts: None,
});
let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&notification);
assert_eq!(reason, Some("browserSession/updated"));
@@ -2117,6 +2117,24 @@ pub struct BrowserSessionState {
pub tabs: Vec<BrowserTabState>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct BrowserSessionArtifacts {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub screenshot_image_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub replay_gif_image_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub replay_frame_count: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub replay_frame_duration_ms: Option<u32>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
@@ -2135,6 +2153,9 @@ pub struct BrowserSessionCommandResponse {
pub browser_session_id: String,
pub result: JsonValue,
pub browser_state: BrowserSessionState,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub artifacts: Option<BrowserSessionArtifacts>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
@@ -2143,6 +2164,9 @@ pub struct BrowserSessionCommandResponse {
pub struct BrowserSessionUpdatedNotification {
pub browser_session_id: String,
pub browser_state: BrowserSessionState,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub artifacts: Option<BrowserSessionArtifacts>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
@@ -272,6 +272,7 @@ async fn handle_remote_browser_dynamic_tool_request(
BrowserSessionUpdatedNotification {
browser_session_id: outcome.browser_session_id.clone(),
browser_state,
artifacts: outcome.to_public_artifacts(),
},
))
.await;
@@ -952,6 +952,7 @@ impl MessageProcessor {
BrowserSessionUpdatedNotification {
browser_session_id: response.browser_session_id.clone(),
browser_state: response.browser_state.clone(),
artifacts: response.artifacts.clone(),
},
))
.await;
@@ -9,6 +9,7 @@ use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::write_mock_responses_config_toml;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionCommandResponse;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionState;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::BrowserTabState;
@@ -695,6 +696,12 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result<
"url": "https://example.com",
"selected": true,
}]
},
"artifacts": {
"screenshotBase64": "AAA",
"replayGifBase64": "R0lGODlh",
"replayFrameCount": 2,
"replayFrameDurationMs": 700
}
});
@@ -732,10 +739,20 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result<
assert_eq!(response.browser_session_id, "sess_123");
assert_eq!(response.result, serde_json::json!({"ok": true}));
assert_eq!(response.browser_state, browser_state);
assert_eq!(
response.artifacts,
Some(BrowserSessionArtifacts {
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
replay_frame_count: Some(2),
replay_frame_duration_ms: Some(700),
})
);
let notification = read_browser_session_updated_notification(&mut harness.outgoing_rx).await;
assert_eq!(notification.browser_session_id, "sess_123");
assert_eq!(notification.browser_state, browser_state);
assert_eq!(notification.artifacts, response.artifacts);
harness.shutdown().await;
remote_browser.verify().await;
@@ -1,5 +1,6 @@
use crate::error_code::INTERNAL_ERROR_CODE;
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionCommandResponse;
use codex_app_server_protocol::BrowserSessionState;
@@ -41,6 +42,12 @@ pub(crate) struct RemoteBrowserState {
struct RemoteBrowserArtifacts {
#[serde(default)]
screenshot_base64: Option<String>,
#[serde(default)]
replay_gif_base64: Option<String>,
#[serde(default)]
replay_frame_count: Option<u32>,
#[serde(default)]
replay_frame_duration_ms: Option<u32>,
}
#[derive(Clone, Debug, Deserialize)]
@@ -98,6 +105,7 @@ impl RemoteBrowserCommandOutcome {
browser_session_id: self.browser_session_id.clone(),
result: self.result.clone(),
browser_state,
artifacts: self.to_public_artifacts(),
})
}
@@ -108,6 +116,35 @@ impl RemoteBrowserCommandOutcome {
.map(|encoded| format!("data:image/png;base64,{encoded}"))
}
pub(crate) fn replay_gif_data_url(&self) -> Option<String> {
self.artifacts
.replay_gif_base64
.as_ref()
.map(|encoded| format!("data:image/gif;base64,{encoded}"))
}
pub(crate) fn to_public_artifacts(&self) -> Option<BrowserSessionArtifacts> {
let screenshot_image_url = self.screenshot_data_url();
let replay_gif_image_url = self.replay_gif_data_url();
let replay_frame_count = self.artifacts.replay_frame_count;
let replay_frame_duration_ms = self.artifacts.replay_frame_duration_ms;
if screenshot_image_url.is_none()
&& replay_gif_image_url.is_none()
&& replay_frame_count.is_none()
&& replay_frame_duration_ms.is_none()
{
return None;
}
Some(BrowserSessionArtifacts {
screenshot_image_url,
replay_gif_image_url,
replay_frame_count,
replay_frame_duration_ms,
})
}
pub(crate) fn browser_state_json(&self) -> JsonValue {
serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null)
}
@@ -55,6 +55,9 @@ pub(crate) fn build_dynamic_tool_response(
if let Some(image_url) = outcome.screenshot_data_url() {
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
}
if let Some(image_url) = outcome.replay_gif_data_url() {
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
}
DynamicToolCallResponse {
content_items,
@@ -5,6 +5,7 @@ use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::to_response;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallParams;
@@ -293,7 +294,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
}]
},
"artifacts": {
"screenshotBase64": "AAA"
"screenshotBase64": "AAA",
"replayGifBase64": "R0lGODlh",
"replayFrameCount": 2,
"replayFrameDurationMs": 700
}
})))
.mount(&remote_browser_server)
@@ -358,6 +362,15 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
assert_eq!(browser_update.browser_state.tabs.len(), 1);
assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News");
assert_eq!(
browser_update.artifacts,
Some(BrowserSessionArtifacts {
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
replay_frame_count: Some(2),
replay_frame_duration_ms: Some(700),
})
);
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
@@ -394,6 +407,9 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/gif;base64,R0lGODlh".to_string(),
},
])
);
@@ -435,6 +451,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/gif;base64,R0lGODlh".to_string(),
detail: None,
},
]
);