mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Surface remote browser replay artifacts
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
34d59e54c3
commit
ba0d3b467b
@@ -272,6 +272,7 @@ async fn handle_remote_browser_dynamic_tool_request(
|
||||
BrowserSessionUpdatedNotification {
|
||||
browser_session_id: outcome.browser_session_id.clone(),
|
||||
browser_state,
|
||||
artifacts: outcome.to_public_artifacts(),
|
||||
},
|
||||
))
|
||||
.await;
|
||||
|
||||
@@ -952,6 +952,7 @@ impl MessageProcessor {
|
||||
BrowserSessionUpdatedNotification {
|
||||
browser_session_id: response.browser_session_id.clone(),
|
||||
browser_state: response.browser_state.clone(),
|
||||
artifacts: response.artifacts.clone(),
|
||||
},
|
||||
))
|
||||
.await;
|
||||
|
||||
@@ -9,6 +9,7 @@ use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::write_mock_responses_config_toml;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionCommandResponse;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::BrowserTabState;
|
||||
@@ -695,6 +696,12 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result<
|
||||
"url": "https://example.com",
|
||||
"selected": true,
|
||||
}]
|
||||
},
|
||||
"artifacts": {
|
||||
"screenshotBase64": "AAA",
|
||||
"replayGifBase64": "R0lGODlh",
|
||||
"replayFrameCount": 2,
|
||||
"replayFrameDurationMs": 700
|
||||
}
|
||||
});
|
||||
|
||||
@@ -732,10 +739,20 @@ async fn browser_session_command_posts_and_emits_state_notification() -> Result<
|
||||
assert_eq!(response.browser_session_id, "sess_123");
|
||||
assert_eq!(response.result, serde_json::json!({"ok": true}));
|
||||
assert_eq!(response.browser_state, browser_state);
|
||||
assert_eq!(
|
||||
response.artifacts,
|
||||
Some(BrowserSessionArtifacts {
|
||||
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
|
||||
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
|
||||
replay_frame_count: Some(2),
|
||||
replay_frame_duration_ms: Some(700),
|
||||
})
|
||||
);
|
||||
|
||||
let notification = read_browser_session_updated_notification(&mut harness.outgoing_rx).await;
|
||||
assert_eq!(notification.browser_session_id, "sess_123");
|
||||
assert_eq!(notification.browser_state, browser_state);
|
||||
assert_eq!(notification.artifacts, response.artifacts);
|
||||
harness.shutdown().await;
|
||||
|
||||
remote_browser.verify().await;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::error_code::INTERNAL_ERROR_CODE;
|
||||
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionCommandResponse;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
@@ -41,6 +42,12 @@ pub(crate) struct RemoteBrowserState {
|
||||
struct RemoteBrowserArtifacts {
|
||||
#[serde(default)]
|
||||
screenshot_base64: Option<String>,
|
||||
#[serde(default)]
|
||||
replay_gif_base64: Option<String>,
|
||||
#[serde(default)]
|
||||
replay_frame_count: Option<u32>,
|
||||
#[serde(default)]
|
||||
replay_frame_duration_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
@@ -98,6 +105,7 @@ impl RemoteBrowserCommandOutcome {
|
||||
browser_session_id: self.browser_session_id.clone(),
|
||||
result: self.result.clone(),
|
||||
browser_state,
|
||||
artifacts: self.to_public_artifacts(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -108,6 +116,35 @@ impl RemoteBrowserCommandOutcome {
|
||||
.map(|encoded| format!("data:image/png;base64,{encoded}"))
|
||||
}
|
||||
|
||||
pub(crate) fn replay_gif_data_url(&self) -> Option<String> {
|
||||
self.artifacts
|
||||
.replay_gif_base64
|
||||
.as_ref()
|
||||
.map(|encoded| format!("data:image/gif;base64,{encoded}"))
|
||||
}
|
||||
|
||||
pub(crate) fn to_public_artifacts(&self) -> Option<BrowserSessionArtifacts> {
|
||||
let screenshot_image_url = self.screenshot_data_url();
|
||||
let replay_gif_image_url = self.replay_gif_data_url();
|
||||
let replay_frame_count = self.artifacts.replay_frame_count;
|
||||
let replay_frame_duration_ms = self.artifacts.replay_frame_duration_ms;
|
||||
|
||||
if screenshot_image_url.is_none()
|
||||
&& replay_gif_image_url.is_none()
|
||||
&& replay_frame_count.is_none()
|
||||
&& replay_frame_duration_ms.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(BrowserSessionArtifacts {
|
||||
screenshot_image_url,
|
||||
replay_gif_image_url,
|
||||
replay_frame_count,
|
||||
replay_frame_duration_ms,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn browser_state_json(&self) -> JsonValue {
|
||||
serde_json::to_value(&self.browser_state).unwrap_or(JsonValue::Null)
|
||||
}
|
||||
|
||||
@@ -55,6 +55,9 @@ pub(crate) fn build_dynamic_tool_response(
|
||||
if let Some(image_url) = outcome.screenshot_data_url() {
|
||||
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
|
||||
}
|
||||
if let Some(image_url) = outcome.replay_gif_data_url() {
|
||||
content_items.push(DynamicToolCallOutputContentItem::InputImage { image_url });
|
||||
}
|
||||
|
||||
DynamicToolCallResponse {
|
||||
content_items,
|
||||
|
||||
@@ -5,6 +5,7 @@ use app_test_support::create_final_assistant_message_sse_response;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||
use app_test_support::to_response;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallParams;
|
||||
@@ -293,7 +294,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
|
||||
}]
|
||||
},
|
||||
"artifacts": {
|
||||
"screenshotBase64": "AAA"
|
||||
"screenshotBase64": "AAA",
|
||||
"replayGifBase64": "R0lGODlh",
|
||||
"replayFrameCount": 2,
|
||||
"replayFrameDurationMs": 700
|
||||
}
|
||||
})))
|
||||
.mount(&remote_browser_server)
|
||||
@@ -358,6 +362,15 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
|
||||
assert_eq!(browser_update.browser_state.tabs.len(), 1);
|
||||
assert_eq!(browser_update.browser_state.tabs[0].title, "Hacker News");
|
||||
assert_eq!(
|
||||
browser_update.artifacts,
|
||||
Some(BrowserSessionArtifacts {
|
||||
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
|
||||
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
|
||||
replay_frame_count: Some(2),
|
||||
replay_frame_duration_ms: Some(700),
|
||||
})
|
||||
);
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
@@ -394,6 +407,9 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/gif;base64,R0lGODlh".to_string(),
|
||||
},
|
||||
])
|
||||
);
|
||||
|
||||
@@ -435,6 +451,10 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/gif;base64,R0lGODlh".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user