diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index c73b5593c6..e892507291 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -7,6 +7,8 @@ use app_test_support::ChatGptAuthFixture; use app_test_support::TestAppServer; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; @@ -148,7 +150,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul #[tokio::test] async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> { - let edit_request = run_image_edit_test(|codex_home| { + let edit_request = run_image_edit_test(|codex_home, _server_uri| { let image_path = codex_home.join("attached.png"); std::fs::write(&image_path, TINY_PNG_BYTES)?; Ok(( @@ -180,9 +182,9 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> } #[tokio::test] -async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> { - let image_url = "https://example.com/reference.png"; - let edit_request = run_image_edit_test(|_| { +async fn standalone_image_edit_uses_materialized_recent_pathless_image() -> Result<()> { + let edit_request = run_image_edit_test(|_, server_uri| { + let image_url = format!("{server_uri}/reference.png"); Ok(( json!({ "prompt": "add a red hat", @@ -194,7 +196,7 @@ async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> { text_elements: Vec::new(), }, V2UserInput::Image { - url: image_url.to_string(), + url: image_url, detail: None, }, ], @@ -202,7 +204,13 @@ async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> { }) .await?; assert_eq!(edit_request["prompt"], "add a red hat"); - assert_eq!(edit_request["images"][0]["image_url"], image_url); + assert_eq!( + edit_request["images"][0]["image_url"], + format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(TINY_PNG_BYTES) + ) + ); Ok(()) } @@ -338,14 +346,19 @@ async fn start_image_generation_turn(mcp: &mut TestAppServer) -> Result<()> { } async fn run_image_edit_test( - input: impl FnOnce(&Path) -> Result<(serde_json::Value, Vec)>, + input: impl FnOnce(&Path, &str) -> Result<(serde_json::Value, Vec)>, ) -> Result { let call_id = "image-edit-1"; let server = responses::start_mock_server().await; mount_image_edit_response(&server).await; + Mock::given(method("GET")) + .and(path("/reference.png")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(TINY_PNG_BYTES)) + .mount(&server) + .await; let codex_home = TempDir::new()?; - let (arguments, input) = input(codex_home.path())?; + let (arguments, input) = input(codex_home.path(), &server.uri())?; let response_mock = responses::mount_sse_sequence( &server, vec![ diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 5080937997..22552d9c74 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -872,7 +872,7 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { thread_id: thread.id.clone(), client_user_message_id: None, input: vec![V2UserInput::Image { - url: "https://example.com/a.png".to_string(), + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=".to_string(), detail: None, }], responsesapi_client_metadata: Some(HashMap::from([( diff --git a/codex-rs/core/src/image_url_materializer.rs b/codex-rs/core/src/image_url_materializer.rs new file mode 100644 index 0000000000..3a3c5381a7 --- /dev/null +++ b/codex-rs/core/src/image_url_materializer.rs @@ -0,0 +1,172 @@ +use std::borrow::Cow; +use std::path::Path; +use std::sync::LazyLock; + +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ResponseItem; +use codex_utils_image::ImageProcessingError; +use codex_utils_image::PromptImageMode; +use codex_utils_image::load_for_prompt_bytes; +use reqwest::Client; +use thiserror::Error; +use tracing::warn; +use url::Url; + +const IMAGE_MATERIALIZATION_ERROR_PLACEHOLDER: &str = + "image content omitted because it could not be downloaded or processed"; + +static IMAGE_URL_CLIENT: LazyLock = LazyLock::new(Client::new); + +#[derive(Debug, Error)] +pub(crate) enum ImageUrlMaterializationError { + #[error("failed to download image: {0}")] + Download(#[from] reqwest::Error), + #[error("failed to process downloaded image: {0}")] + Processing(#[from] ImageProcessingError), +} + +/// Downloads an HTTP(S) image and returns it as an inline data URL. +/// +/// Non-HTTP(S) values, including existing data URLs, are returned unchanged. +pub(crate) async fn materialize_http_image_url<'a>( + client: &Client, + image_url: &'a str, +) -> Result, ImageUrlMaterializationError> { + let Ok(url) = Url::parse(image_url) else { + return Ok(Cow::Borrowed(image_url)); + }; + if !matches!(url.scheme(), "http" | "https") { + return Ok(Cow::Borrowed(image_url)); + } + + let response = client.get(url.as_str()).send().await?.error_for_status()?; + let bytes = response.bytes().await?; + let image = load_for_prompt_bytes( + Path::new(url.path()), + bytes.to_vec(), + PromptImageMode::Original, + )?; + + Ok(Cow::Owned(image.into_data_url())) +} + +/// Materializes HTTP(S) image URLs in a newly recorded batch. +/// +/// Batches without remote images remain borrowed, avoiding copies of existing +/// inline image data. +pub(crate) async fn materialize_conversation_item_images<'a>( + items: &'a [ResponseItem], +) -> Cow<'a, [ResponseItem]> { + if !items.iter().any(response_item_has_http_image) { + return Cow::Borrowed(items); + } + + let mut items = items.to_vec(); + for item in &mut items { + match item { + ResponseItem::Message { content, .. } => materialize_content_items(content).await, + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content_items) = output.content_items_mut() { + materialize_function_call_output_items(content_items).await; + } + } + ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => {} + } + } + + Cow::Owned(items) +} + +fn response_item_has_http_image(item: &ResponseItem) -> bool { + match item { + ResponseItem::Message { content, .. } => content.iter().any(|item| match item { + ContentItem::InputImage { image_url, .. } => is_http_url(image_url), + ContentItem::InputText { .. } | ContentItem::OutputText { .. } => false, + }), + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => output + .content_items() + .is_some_and(|items| items.iter().any(function_call_output_item_has_http_image)), + ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => false, + } +} + +fn function_call_output_item_has_http_image(item: &FunctionCallOutputContentItem) -> bool { + match item { + FunctionCallOutputContentItem::InputImage { image_url, .. } => is_http_url(image_url), + FunctionCallOutputContentItem::InputText { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => false, + } +} + +fn is_http_url(value: &str) -> bool { + Url::parse(value).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) +} + +async fn materialize_content_items(items: &mut [ContentItem]) { + for item in items { + let result = match item { + ContentItem::InputImage { image_url, .. } => materialize_image_url(image_url).await, + ContentItem::InputText { .. } | ContentItem::OutputText { .. } => continue, + }; + if let Err(err) = result { + warn!(error = %err, "failed to materialize remote message image"); + *item = ContentItem::InputText { + text: IMAGE_MATERIALIZATION_ERROR_PLACEHOLDER.to_string(), + }; + } + } +} + +async fn materialize_function_call_output_items(items: &mut [FunctionCallOutputContentItem]) { + for item in items { + let result = match item { + FunctionCallOutputContentItem::InputImage { image_url, .. } => { + materialize_image_url(image_url).await + } + FunctionCallOutputContentItem::InputText { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => continue, + }; + if let Err(err) = result { + warn!(error = %err, "failed to materialize remote tool output image"); + *item = FunctionCallOutputContentItem::InputText { + text: IMAGE_MATERIALIZATION_ERROR_PLACEHOLDER.to_string(), + }; + } + } +} + +async fn materialize_image_url(image_url: &mut String) -> Result<(), ImageUrlMaterializationError> { + if let Cow::Owned(materialized_url) = + materialize_http_image_url(&IMAGE_URL_CLIENT, image_url).await? + { + *image_url = materialized_url; + } + Ok(()) +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 400c803680..461ccdc789 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -41,6 +41,7 @@ mod exec_policy; mod git_info_tests; mod guardian; mod hook_runtime; +mod image_url_materializer; mod installation_id; pub(crate) mod landlock; pub use landlock::spawn_command_under_linux_sandbox; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 23dbe1f45e..094745d622 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2588,6 +2588,9 @@ impl Session { turn_context: &TurnContext, items: &[ResponseItem], ) { + let items = + crate::image_url_materializer::materialize_conversation_item_images(items).await; + let items = items.as_ref(); { let mut state = self.state.lock().await; state.record_items(items.iter(), turn_context.truncation_policy); diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index e1b02292ef..33a926f892 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2486,19 +2486,29 @@ text(circular); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn code_mode_can_output_images_via_global_helper() -> Result<()> { +async fn code_mode_materializes_http_images_before_responses_request() -> Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; - let (_test, second_mock) = run_code_mode_turn( - &server, - "use exec to return images", + let image_bytes = BASE64_STANDARD.decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + )?; + Mock::given(method("GET")) + .and(path("/image.png")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(image_bytes.clone())) + .expect(1) + .mount(&server) + .await; + let image_url = format!("{}/image.png", server.uri()); + let image_url_json = serde_json::to_string(&image_url)?; + let code = format!( r#" -image("https://example.com/image.jpg"); +image({image_url_json}); image("data:image/png;base64,AAA"); -"#, - ) - .await?; +"# + ); + let (_test, second_mock) = + run_code_mode_turn(&server, "use exec to return images", &code).await?; let req = second_mock.single_request(); let items = custom_tool_output_items(&req, "call-1"); @@ -2520,7 +2530,10 @@ image("data:image/png;base64,AAA"); items[1], serde_json::json!({ "type": "input_image", - "image_url": "https://example.com/image.jpg", + "image_url": format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(image_bytes) + ), "detail": "high" }), ); diff --git a/sdk/python/tests/app_server_harness.py b/sdk/python/tests/app_server_harness.py index cf0c847394..77e0ac1f1b 100644 --- a/sdk/python/tests/app_server_harness.py +++ b/sdk/python/tests/app_server_harness.py @@ -106,6 +106,7 @@ class MockResponsesServer: self._responses: queue.Queue[MockSseResponse] = queue.Queue() self._requests: list[CapturedResponsesRequest] = [] self._requests_lock = threading.Lock() + self._get_responses: dict[str, tuple[bytes, str]] = {} self._server = _ResponsesHttpServer(("127.0.0.1", 0), _ResponsesHandler, self) self._thread = threading.Thread( target=self._server.serve_forever, @@ -158,6 +159,13 @@ class MockResponsesServer: ) ) + def serve_bytes(self, path: str, body: bytes, *, content_type: str) -> str: + """Serve bytes from a GET path and return its absolute URL.""" + if not path.startswith("/"): + raise ValueError("GET response path must start with '/'") + self._get_responses[path] = (body, content_type) + return f"{self.url}{path}" + def requests(self) -> list[CapturedResponsesRequest]: """Return all recorded Responses API requests.""" with self._requests_lock: @@ -202,6 +210,9 @@ class MockResponsesServer: """Return the next queued SSE response or fail the HTTP request.""" return self._responses.get_nowait() + def _get_response(self, path: str) -> tuple[bytes, str] | None: + return self._get_responses.get(path) + class AppServerHarness: """Test fixture that points a pinned runtime app-server at MockResponsesServer.""" @@ -283,6 +294,14 @@ class _ResponsesHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: """Serve a minimal `/v1/models` response if app-server asks for models.""" + if response := self.server.mock._get_response(self.path): + body, content_type = response + self.send_response(200) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return if self.path.endswith("/v1/models") or self.path.endswith("/models"): self._send_json( { diff --git a/sdk/python/tests/test_app_server_inputs.py b/sdk/python/tests/test_app_server_inputs.py index 56505e4817..42b63c5319 100644 --- a/sdk/python/tests/test_app_server_inputs.py +++ b/sdk/python/tests/test_app_server_inputs.py @@ -1,18 +1,24 @@ from __future__ import annotations +import base64 + from app_server_harness import AppServerHarness from app_server_helpers import TINY_PNG_BYTES from openai_codex import Codex, ImageInput, LocalImageInput, SkillInput, TextInput -def test_remote_image_input_reaches_responses_api( +def test_remote_image_input_is_materialized_before_responses_api( tmp_path, ) -> None: - """Remote image inputs should survive the SDK and app-server boundary.""" - remote_image_url = "https://example.com/codex.png" + """Remote image inputs should be inlined by the app-server.""" with AppServerHarness(tmp_path) as harness: + remote_image_url = harness.responses.serve_bytes( + "/codex.png", + TINY_PNG_BYTES, + content_type="image/png", + ) harness.responses.enqueue_assistant_message( "remote image received", response_id="remote-image", @@ -34,7 +40,7 @@ def test_remote_image_input_reaches_responses_api( } == { "final_response": "remote image received", "contains_user_prompt": True, - "image_urls": [remote_image_url], + "image_urls": [f"data:image/png;base64,{base64.b64encode(TINY_PNG_BYTES).decode('ascii')}"], }