diff --git a/codex-rs/codex-api/src/endpoint/images.rs b/codex-rs/codex-api/src/endpoint/images.rs index ce52f4ccbe..2217ce8961 100644 --- a/codex-rs/codex-api/src/endpoint/images.rs +++ b/codex-rs/codex-api/src/endpoint/images.rs @@ -122,13 +122,13 @@ mod tests { use crate::images::ImageBackground; use crate::images::ImageData; use crate::images::ImageQuality; - use crate::images::ImageUrl; use crate::provider::RetryConfig; use codex_client::Request; use codex_client::RequestBody; use codex_client::Response; use codex_client::StreamResponse; use codex_client::TransportError; + use codex_protocol::models::ImageReference; use http::StatusCode; use pretty_assertions::assert_eq; use serde_json::json; @@ -339,9 +339,14 @@ mod tests { let (response, imagegen_request_id) = client .edit( &ImageEditRequest { - images: vec![ImageUrl { - image_url: "data:image/png;base64,Zm9v".to_string(), - }], + images: vec![ + ImageReference::Inline { + image_url: "data:image/png;base64,Zm9v".to_string(), + }, + ImageReference::File { + file_id: "file-image".to_string(), + }, + ], prompt: "add a red hat".to_string(), background: None, model: "gpt-image-1.5".to_string(), @@ -362,7 +367,10 @@ mod tests { assert_eq!( request.body.as_ref().and_then(RequestBody::json), Some(&json!({ - "images": [{"image_url": "data:image/png;base64,Zm9v"}], + "images": [ + {"image_url": "data:image/png;base64,Zm9v"}, + {"file_id": "file-image"}, + ], "prompt": "add a red hat", "model": "gpt-image-1.5", })) diff --git a/codex-rs/codex-api/src/images.rs b/codex-rs/codex-api/src/images.rs index 96c7c37543..3f14d17334 100644 --- a/codex-rs/codex-api/src/images.rs +++ b/codex-rs/codex-api/src/images.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::ImageReference; use serde::Deserialize; use serde::Serialize; @@ -17,7 +18,7 @@ pub struct ImageGenerationRequest { #[derive(Debug, Clone, Serialize, PartialEq)] pub struct ImageEditRequest { - pub images: Vec, + pub images: Vec, pub prompt: String, #[serde(skip_serializing_if = "Option::is_none")] pub background: Option, @@ -30,11 +31,6 @@ pub struct ImageEditRequest { pub size: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ImageUrl { - pub image_url: String, -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ImageBackground { diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index 12b55ca1d2..05df1c2708 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -83,7 +83,6 @@ pub use crate::images::ImageEditRequest; pub use crate::images::ImageGenerationRequest; pub use crate::images::ImageQuality; pub use crate::images::ImageResponse; -pub use crate::images::ImageUrl; pub use crate::provider::Provider; pub use crate::provider::RetryConfig; pub use crate::provider::is_azure_responses_provider; diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index 0e6b9b278c..a2008d4e52 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -2,7 +2,6 @@ use codex_api::ImageBackground; use codex_api::ImageEditRequest; use codex_api::ImageGenerationRequest; use codex_api::ImageQuality; -use codex_api::ImageUrl; use codex_extension_api::ToolOutput; use codex_extension_api::ToolPayload; use codex_extension_api::ToolSpec; @@ -168,10 +167,9 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() { ); } -/// A file-backed image in the requested window must not cause an older inline image to be edited. #[tokio::test] -async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() { - let error = request_for_call_args( +async fn recent_image_fallback_passes_file_backed_image_to_edit_request() { + let request = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), transparent_background: false, @@ -196,19 +194,28 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() { &[], ) .await - .expect_err("a selected file-backed image should fail"); + .expect("a selected file-backed image should be passed through"); assert_eq!( - error.to_string(), - "requested the last 1 conversation images, but that window includes a file-backed image \ - that cannot be used for editing" + request, + ImageRequest::Edit(ImageEditRequest { + images: vec![ImageReference::File { + file_id: "newer-file-backed-image".to_string(), + }], + prompt: "change the lighting".to_string(), + background: Some(ImageBackground::Opaque), + model: "gpt-image-2".to_string(), + n: None, + quality: Some(ImageQuality::Auto), + size: Some("auto".to_string()), + }) ); } /// Tool-output file references must count toward the window instead of exposing an older image. #[tokio::test] -async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_window() { - let error = request_for_call_args( +async fn recent_image_fallback_passes_file_backed_tool_output_to_edit_request() { + let request = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), transparent_background: false, @@ -242,12 +249,21 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_wind &[], ) .await - .expect_err("a selected file-backed tool output should fail"); + .expect("a selected file-backed tool output should be passed through"); assert_eq!( - error.to_string(), - "requested the last 1 conversation images, but that window includes a file-backed image \ - that cannot be used for editing" + request, + ImageRequest::Edit(ImageEditRequest { + images: vec![ImageReference::File { + file_id: "newer-file-backed-image".to_string(), + }], + prompt: "change the lighting".to_string(), + background: Some(ImageBackground::Opaque), + model: "gpt-image-2".to_string(), + n: None, + quality: Some(ImageQuality::Auto), + size: Some("auto".to_string()), + }) ); } @@ -432,7 +448,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest { ImageEditRequest { images: images .iter() - .map(|image| ImageUrl { + .map(|image| ImageReference::Inline { image_url: format!("data:image/png;base64,{image}"), }) .collect(), diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index fb54cb3595..f31a130ce3 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -6,7 +6,6 @@ use codex_api::ImageBackground; use codex_api::ImageEditRequest; use codex_api::ImageGenerationRequest; use codex_api::ImageQuality; -use codex_api::ImageUrl; use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::LOCAL_FS; use codex_extension_api::ExtensionTurnItem; @@ -500,16 +499,14 @@ async fn request_for_call_args( fn recent_images( history: &[ResponseItem], count: usize, -) -> Result, FunctionCallError> { +) -> Result, FunctionCallError> { let mut images = Vec::with_capacity(count); 'history: for item in history.iter().rev() { - let mut image_urls = Vec::new(); + let mut image_references = Vec::new(); match item { ResponseItem::Message { content, .. } => { - image_urls.extend(content.iter().rev().filter_map(|item| match item { - ContentItem::InputImage { image, .. } => { - Some(inline_image_url(image).map(str::to_owned)) - } + image_references.extend(content.iter().rev().filter_map(|item| match item { + ContentItem::InputImage { image, .. } => Some(image.clone()), ContentItem::InputText { .. } | ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => None, @@ -517,12 +514,12 @@ fn recent_images( } ResponseItem::FunctionCallOutput { output, .. } | ResponseItem::CustomToolCallOutput { output, .. } => { - image_urls.extend( - output_images(output).map(|image| inline_image_url(image).map(str::to_owned)), - ); + image_references.extend(output_images(output).cloned()); } ResponseItem::ImageGenerationCall { result, .. } if !result.is_empty() => { - image_urls.push(Some(format!("data:image/png;base64,{result}"))); + image_references.push(ImageReference::Inline { + image_url: format!("data:image/png;base64,{result}"), + }); } ResponseItem::AdditionalTools { .. } | ResponseItem::Reasoning { .. } @@ -540,8 +537,8 @@ fn recent_images( | ResponseItem::ContextCompaction { .. } | ResponseItem::Other => {} } - for image_url in image_urls { - images.push(image_url.map(|image_url| ImageUrl { image_url })); + for image in image_references { + images.push(image); if images.len() == count { break 'history; } @@ -553,18 +550,8 @@ fn recent_images( images.len() ))); } - let mut inline_images = Vec::with_capacity(count); - for image in images { - let Some(image) = image else { - return Err(FunctionCallError::RespondToModel(format!( - "requested the last {count} conversation images, but that window includes a \ - file-backed image that cannot be used for editing" - ))); - }; - inline_images.push(image); - } - inline_images.reverse(); - Ok(inline_images) + images.reverse(); + Ok(images) } /// Extracts image references from a tool output in newest-first order. @@ -582,18 +569,10 @@ fn output_images(output: &FunctionCallOutputPayload) -> impl Iterator Option<&str> { - match image { - ImageReference::Inline { image_url } => Some(image_url), - // TODO(kc) Image generation and the Images API only accept inline image URLs. - ImageReference::File { .. } => None, - } -} - async fn image_url( path: &AbsolutePathBuf, environment: &ToolEnvironment<'_>, -) -> Result { +) -> Result { let path_uri = PathUri::from_abs_path(path); let sandbox = environment.file_system_sandbox_context.clone(); let bytes = environment @@ -614,7 +593,7 @@ async fn image_url( )) }, )?; - Ok(ImageUrl { + Ok(ImageReference::Inline { image_url: image.into_data_url(), }) }