From c7bab107498b8d5cc81f578b3ee7c147c153cfec Mon Sep 17 00:00:00 2001 From: Rasmus Rygaard Date: Fri, 19 Sep 2025 14:57:17 -0700 Subject: [PATCH] Resize local images before upload --- codex-rs/Cargo.lock | 1 + codex-rs/protocol/Cargo.toml | 4 + codex-rs/protocol/src/models.rs | 154 ++++++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 7 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e3930b507a..1e3f71bf95 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -874,6 +874,7 @@ dependencies = [ "base64", "icu_decimal", "icu_locale_core", + "image", "mcp-types", "mime_guess", "pretty_assertions", diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index bbe2ed3f65..c67994a5c6 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -16,6 +16,10 @@ icu_decimal = "2.0.0" icu_locale_core = "2.0.0" mcp-types = { path = "../mcp-types" } mime_guess = "2.0.5" +image = { version = "^0.25.8", default-features = false, features = [ + "jpeg", + "png", +] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_with = { version = "3.14.0", features = ["macros", "base64"] } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 08cd64f600..5bc2d9f9c4 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1,6 +1,11 @@ use std::collections::HashMap; +use std::io::Cursor; +use std::path::Path; use base64::Engine; +use image::ImageFormat; +use image::ImageReader; +use image::imageops::FilterType; use mcp_types::CallToolResult; use serde::Deserialize; use serde::Deserializer; @@ -214,13 +219,9 @@ impl From> for ResponseInputItem { .filter_map(|c| match c { InputItem::Text { text } => Some(ContentItem::InputText { text }), InputItem::Image { image_url } => Some(ContentItem::InputImage { image_url }), - InputItem::LocalImage { path } => match std::fs::read(&path) { - Ok(bytes) => { - let mime = mime_guess::from_path(&path) - .first() - .map(|m| m.essence_str().to_owned()) - .unwrap_or_else(|| "image".to_string()); - let encoded = base64::engine::general_purpose::STANDARD.encode(bytes); + InputItem::LocalImage { path } => match load_local_image(&path) { + Ok((bytes, mime)) => { + let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); Some(ContentItem::InputImage { image_url: format!("data:{mime};base64,{encoded}"), }) @@ -240,6 +241,107 @@ impl From> for ResponseInputItem { } } +const MAX_IMAGE_WIDTH: u32 = 2048; +const MAX_IMAGE_HEIGHT: u32 = 768; + +fn load_local_image(path: &Path) -> Result<(Vec, String), std::io::Error> { + let mut bytes = std::fs::read(path)?; + let mut mime = guess_mime(path); + + if let Some(resized) = resize_image_if_needed(&bytes) { + if let Some(mime_override) = resized.mime_override { + mime = mime_override; + } + bytes = resized.bytes; + } + + Ok((bytes, mime)) +} + +#[derive(Debug)] +struct ResizedImage { + bytes: Vec, + mime_override: Option, +} + +fn resize_image_if_needed(bytes: &[u8]) -> Option { + let reader = ImageReader::new(Cursor::new(bytes)) + .with_guessed_format() + .ok()?; + let format = reader.format(); + let image = reader.decode().ok()?; + + let width = image.width(); + let height = image.height(); + if width <= MAX_IMAGE_WIDTH && height <= MAX_IMAGE_HEIGHT { + return None; + } + + let scale = f64::min( + MAX_IMAGE_WIDTH as f64 / width as f64, + MAX_IMAGE_HEIGHT as f64 / height as f64, + ); + let new_width = (width as f64 * scale).floor().max(1.0) as u32; + let new_height = (height as f64 * scale).floor().max(1.0) as u32; + + if new_width == width && new_height == height { + return None; + } + + tracing::debug!( + original_width = width, + original_height = height, + resized_width = new_width, + resized_height = new_height, + "resizing local image before upload", + ); + + let resized = image.resize(new_width, new_height, FilterType::Lanczos3); + let mut cursor = Cursor::new(Vec::new()); + let mut mime_override = None; + + match format { + Some(fmt) => { + if resized.write_to(&mut cursor, fmt).is_err() { + cursor = Cursor::new(Vec::new()); + if resized.write_to(&mut cursor, ImageFormat::Png).is_err() { + return None; + } + mime_override = Some("image/png".to_string()); + } else if let Some(mime) = mime_from_image_format(fmt) { + mime_override = Some(mime); + } + } + None => { + if resized.write_to(&mut cursor, ImageFormat::Png).is_err() { + return None; + } + mime_override = Some("image/png".to_string()); + } + } + + let bytes = cursor.into_inner(); + Some(ResizedImage { + bytes, + mime_override, + }) +} + +fn guess_mime(path: &Path) -> String { + mime_guess::from_path(path) + .first() + .map(|m| m.essence_str().to_owned()) + .unwrap_or_else(|| "image".to_string()) +} + +fn mime_from_image_format(format: ImageFormat) -> Option { + format + .extensions_str() + .iter() + .find_map(|ext| mime_guess::from_ext(ext).first()) + .map(|mime| mime.essence_str().to_owned()) +} + /// If the `name` of a `ResponseItem::FunctionCall` is either `container.exec` /// or shell`, the `arguments` field should deserialize to this struct. #[derive(Deserialize, Debug, Clone, PartialEq, TS)] @@ -318,6 +420,12 @@ impl std::ops::Deref for FunctionCallOutputPayload { #[cfg(test)] mod tests { use super::*; + use image::DynamicImage; + use image::ImageFormat; + use image::ImageReader as TestImageReader; + use image::Rgba; + use image::RgbaImage; + use std::io::Cursor as TestCursor; #[test] fn serializes_success_as_plain_string() { @@ -372,4 +480,36 @@ mod tests { params ); } + + #[test] + fn resize_image_if_needed_skips_small_images() { + let image = RgbaImage::from_pixel(1024, 512, Rgba([0, 0, 0, 255])); + let dynamic = DynamicImage::ImageRgba8(image); + let mut cursor = TestCursor::new(Vec::new()); + dynamic.write_to(&mut cursor, ImageFormat::Png).unwrap(); + let bytes = cursor.into_inner(); + + assert!(resize_image_if_needed(&bytes).is_none()); + } + + #[test] + fn resize_image_if_needed_constrains_to_bounding_box() { + let image = RgbaImage::from_pixel(4096, 2048, Rgba([255, 0, 0, 255])); + let dynamic = DynamicImage::ImageRgba8(image); + let mut cursor = TestCursor::new(Vec::new()); + dynamic.write_to(&mut cursor, ImageFormat::Png).unwrap(); + let bytes = cursor.into_inner(); + + let resized = resize_image_if_needed(&bytes).expect("image should be resized"); + assert_eq!(resized.mime_override.as_deref(), Some("image/png")); + + let decoded = TestImageReader::new(TestCursor::new(&resized.bytes)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + assert_eq!(decoded.width(), 1536); + assert_eq!(decoded.height(), 768); + } }