mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add opt-in prompt image sanitization (#61341)
## What changed - Add the disabled-by-default `safe_image_decoder` feature to decode and re-encode prompt images without source metadata, with byte and decoded-pixel limits. - Apply JPEG orientation and supported ICC color conversion to sRGB, emitting PNG for PNG/GIF inputs and JPEG for JPEG/WebP inputs. - Use the selected decoder for image preparation, `view_image` validation, and MCP image validation. Include the decoder in image cache keys. - Preserve original inline images in history metadata so resume and fork can regenerate prepared images without repeated JPEG encoding changing their bytes. ## Testing Add regression tests for identical JPEG replay after history serialization and for preventing legacy cache entries from bypassing sanitization. GitOrigin-RevId: 718cd83d0a233e2eb8efd1a64c14d8944e44c350
This commit is contained in:
@@ -27,6 +27,8 @@ pub enum ImageProcessingError {
|
||||
UnsupportedImageFormat { mime: String },
|
||||
#[error("invalid image data URL: {reason}")]
|
||||
InvalidDataUrl { reason: String },
|
||||
#[error("image has too many decoded pixels ({pixels}; max {max})")]
|
||||
ImageTooManyPixels { pixels: u64, max: u64 },
|
||||
#[error("image {representation} is too large ({size} bytes; max {max} bytes)")]
|
||||
ImageTooLarge {
|
||||
representation: &'static str,
|
||||
|
||||
@@ -236,8 +236,12 @@ async fn data_url_processing_preserves_supported_source_bytes() {
|
||||
.replacen("data:", "DATA:", 1)
|
||||
.replacen(";base64,", ";BASE64,", 1);
|
||||
|
||||
let processed = load_data_url_for_prompt_uncached(&image_url, PromptImageMode::ResizeToFit)
|
||||
.expect("process data URL image");
|
||||
let processed = load_data_url_for_prompt_uncached(
|
||||
&image_url,
|
||||
PromptImageMode::ResizeToFit,
|
||||
PromptImageDecoder::Legacy,
|
||||
)
|
||||
.expect("process data URL image");
|
||||
|
||||
assert_eq!(processed.width, 64);
|
||||
assert_eq!(processed.height, 32);
|
||||
@@ -251,8 +255,12 @@ async fn data_url_processing_converts_gif_to_png() {
|
||||
let gif_bytes = image_bytes(&image, ImageFormat::Gif);
|
||||
let image_url = data_url_from_bytes("image/gif", &gif_bytes);
|
||||
|
||||
let processed = load_data_url_for_prompt(&image_url, PromptImageMode::ResizeToFit)
|
||||
.expect("process GIF data URL");
|
||||
let processed = load_data_url_for_prompt(
|
||||
&image_url,
|
||||
PromptImageMode::ResizeToFit,
|
||||
PromptImageDecoder::Legacy,
|
||||
)
|
||||
.expect("process GIF data URL");
|
||||
|
||||
assert_eq!(processed.mime, "image/png");
|
||||
assert_eq!(
|
||||
@@ -270,7 +278,11 @@ fn data_url_processing_rejects_malformed_input() {
|
||||
"data:image/png;base64,not base64",
|
||||
] {
|
||||
assert!(matches!(
|
||||
load_data_url_for_prompt(image_url, PromptImageMode::ResizeToFit),
|
||||
load_data_url_for_prompt(
|
||||
image_url,
|
||||
PromptImageMode::ResizeToFit,
|
||||
PromptImageDecoder::Legacy,
|
||||
),
|
||||
Err(ImageProcessingError::InvalidDataUrl { .. })
|
||||
));
|
||||
}
|
||||
@@ -382,12 +394,29 @@ async fn reprocesses_updated_file_contents() {
|
||||
assert_ne!(second.bytes, first.bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_cache_entry_cannot_bypass_sanitization() {
|
||||
let image = ImageBuffer::from_pixel(7, 11, Rgba([42, 64, 128, 255]));
|
||||
let bytes = image_bytes_with_metadata(&image, ImageFormat::Jpeg, TEST_RGB_ICC_PROFILE);
|
||||
let path = Path::new("oriented.jpg");
|
||||
let mode = PromptImageMode::Original;
|
||||
|
||||
let legacy = load_for_prompt_bytes(path, bytes.clone(), mode).expect("load legacy image");
|
||||
let sanitized =
|
||||
load_for_prompt_bytes_with_decoder(path, bytes, mode, PromptImageDecoder::Sanitizer)
|
||||
.expect("sanitize image after warming the legacy cache");
|
||||
|
||||
assert_eq!((legacy.width, legacy.height), (7, 11));
|
||||
assert_eq!((sanitized.width, sanitized.height), (11, 7));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn bounds_cache_by_encoded_byte_size() {
|
||||
let cache = ImageCache::new(NonZeroUsize::new(4).expect("non-zero cache capacity"));
|
||||
let key = |digest_byte| ImageCacheKey {
|
||||
digest: [digest_byte; 20],
|
||||
mode: PromptImageMode::Original,
|
||||
decoder: PromptImageDecoder::Legacy,
|
||||
};
|
||||
let image = |size| EncodedImage {
|
||||
bytes: vec![0; size].into(),
|
||||
|
||||
@@ -29,9 +29,13 @@ pub const MAX_DIMENSION: u32 = 2048;
|
||||
/// This is a high sanity guard against pathological inputs, not a protocol
|
||||
/// requirement or target upload size.
|
||||
pub const MAX_PROMPT_IMAGE_INPUT_BYTES: usize = 1024 * 1024 * 1024;
|
||||
/// Use the same byte limit for input and output so a saved image fits on replay.
|
||||
/// At the pixel limit, 16-bit RGBA uses 81,920,000 bytes. Round up for PNG overhead.
|
||||
pub const MAX_SANITIZED_IMAGE_BYTES: usize = 79 * 1024 * 1024;
|
||||
const MAX_IMAGE_CACHE_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
pub mod error;
|
||||
mod sanitizer;
|
||||
|
||||
pub use crate::error::ImageProcessingError;
|
||||
|
||||
@@ -64,6 +68,13 @@ pub enum PromptImageMode {
|
||||
ResizeWithLimits(PromptImageResizeLimits),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum PromptImageDecoder {
|
||||
#[default]
|
||||
Legacy,
|
||||
Sanitizer,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct PromptImageResizeLimits {
|
||||
pub max_dimension: u32,
|
||||
@@ -92,6 +103,7 @@ struct ImageMetadata {
|
||||
struct ImageCacheKey {
|
||||
digest: [u8; 20],
|
||||
mode: PromptImageMode,
|
||||
decoder: PromptImageDecoder,
|
||||
}
|
||||
|
||||
type ImageCache = BlockingLruCache<ImageCacheKey, EncodedImage>;
|
||||
@@ -104,18 +116,26 @@ pub fn load_for_prompt_bytes(
|
||||
file_bytes: Vec<u8>,
|
||||
mode: PromptImageMode,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
let path_buf = path.to_path_buf();
|
||||
load_for_prompt_bytes_with_decoder(path, file_bytes, mode, PromptImageDecoder::Legacy)
|
||||
}
|
||||
|
||||
fn load_for_prompt_bytes_with_decoder(
|
||||
path: &Path,
|
||||
file_bytes: Vec<u8>,
|
||||
mode: PromptImageMode,
|
||||
decoder: PromptImageDecoder,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
let key = ImageCacheKey {
|
||||
digest: sha1_digest(&file_bytes),
|
||||
mode,
|
||||
decoder,
|
||||
};
|
||||
|
||||
if let Some(image) = IMAGE_CACHE.get(&key) {
|
||||
return Ok(image);
|
||||
}
|
||||
|
||||
let image = load_for_prompt_bytes_uncached(&path_buf, file_bytes, mode)?;
|
||||
let image = load_for_prompt_bytes_uncached(path, file_bytes, mode, decoder)?;
|
||||
cache_image(&IMAGE_CACHE, key, image.clone(), MAX_IMAGE_CACHE_BYTES);
|
||||
Ok(image)
|
||||
}
|
||||
@@ -124,7 +144,11 @@ fn load_for_prompt_bytes_uncached(
|
||||
path: &Path,
|
||||
file_bytes: Vec<u8>,
|
||||
mode: PromptImageMode,
|
||||
decoder: PromptImageDecoder,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
if decoder == PromptImageDecoder::Sanitizer {
|
||||
return sanitizer::sanitize(path, &file_bytes, mode);
|
||||
}
|
||||
let path_buf = path.to_path_buf();
|
||||
(move || {
|
||||
let guessed_format = image::guess_format(&file_bytes)
|
||||
@@ -159,26 +183,7 @@ fn load_for_prompt_bytes_uncached(
|
||||
|
||||
let (width, height) = dynamic.dimensions();
|
||||
|
||||
let target_dimensions = match mode {
|
||||
PromptImageMode::ResizeToFit if width > MAX_DIMENSION || height > MAX_DIMENSION => {
|
||||
let resized = dynamic.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle);
|
||||
Some((resized.width(), resized.height(), resized))
|
||||
}
|
||||
PromptImageMode::ResizeWithLimits(limits) => {
|
||||
let (target_width, target_height) =
|
||||
prompt_image_output_dimensions_for_limits(width, height, limits);
|
||||
if (target_width, target_height) == (width, height) {
|
||||
None
|
||||
} else {
|
||||
let resized =
|
||||
dynamic.resize_exact(target_width, target_height, FilterType::Triangle);
|
||||
Some((target_width, target_height, resized))
|
||||
}
|
||||
}
|
||||
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
|
||||
};
|
||||
|
||||
let encoded = if let Some((prepared_width, prepared_height, resized)) = target_dimensions {
|
||||
let encoded = if let Some(resized) = resize_for_prompt(&dynamic, mode) {
|
||||
let target_format = format
|
||||
.filter(|format| can_preserve_source_bytes(*format))
|
||||
.unwrap_or(ImageFormat::Png);
|
||||
@@ -189,8 +194,8 @@ fn load_for_prompt_bytes_uncached(
|
||||
mime,
|
||||
source_width: width,
|
||||
source_height: height,
|
||||
width: prepared_width,
|
||||
height: prepared_height,
|
||||
width: resized.width(),
|
||||
height: resized.height(),
|
||||
}
|
||||
} else {
|
||||
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) {
|
||||
@@ -244,22 +249,25 @@ fn cache_image(cache: &ImageCache, key: ImageCacheKey, image: EncodedImage, byte
|
||||
pub fn load_data_url_for_prompt(
|
||||
image_url: &str,
|
||||
mode: PromptImageMode,
|
||||
decoder: PromptImageDecoder,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
load_data_url_for_prompt_with(image_url, mode, load_for_prompt_bytes)
|
||||
let bytes = decode_data_url(image_url, decoder)?;
|
||||
load_for_prompt_bytes_with_decoder(Path::new("<data-url-image>"), bytes, mode, decoder)
|
||||
}
|
||||
|
||||
pub fn load_data_url_for_prompt_uncached(
|
||||
image_url: &str,
|
||||
mode: PromptImageMode,
|
||||
decoder: PromptImageDecoder,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
load_data_url_for_prompt_with(image_url, mode, load_for_prompt_bytes_uncached)
|
||||
let bytes = decode_data_url(image_url, decoder)?;
|
||||
load_for_prompt_bytes_uncached(Path::new("<data-url-image>"), bytes, mode, decoder)
|
||||
}
|
||||
|
||||
fn load_data_url_for_prompt_with(
|
||||
fn decode_data_url(
|
||||
image_url: &str,
|
||||
mode: PromptImageMode,
|
||||
load: impl FnOnce(&Path, Vec<u8>, PromptImageMode) -> Result<EncodedImage, ImageProcessingError>,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
decoder: PromptImageDecoder,
|
||||
) -> Result<Vec<u8>, ImageProcessingError> {
|
||||
let rest = image_url
|
||||
.get(..DATA_URL_PREFIX.len())
|
||||
.filter(|prefix| prefix.eq_ignore_ascii_case(DATA_URL_PREFIX))
|
||||
@@ -281,11 +289,18 @@ fn load_data_url_for_prompt_with(
|
||||
});
|
||||
}
|
||||
|
||||
if encoded.len() > MAX_PROMPT_IMAGE_INPUT_BYTES {
|
||||
let (max_bytes, max_base64_bytes) = match decoder {
|
||||
PromptImageDecoder::Legacy => (MAX_PROMPT_IMAGE_INPUT_BYTES, MAX_PROMPT_IMAGE_INPUT_BYTES),
|
||||
PromptImageDecoder::Sanitizer => (
|
||||
MAX_SANITIZED_IMAGE_BYTES,
|
||||
MAX_SANITIZED_IMAGE_BYTES.div_ceil(3) * 4,
|
||||
),
|
||||
};
|
||||
if encoded.len() > max_base64_bytes {
|
||||
return Err(ImageProcessingError::ImageTooLarge {
|
||||
representation: "base64 payload",
|
||||
size: encoded.len(),
|
||||
max: MAX_PROMPT_IMAGE_INPUT_BYTES,
|
||||
max: max_base64_bytes,
|
||||
});
|
||||
}
|
||||
let file_bytes =
|
||||
@@ -294,15 +309,33 @@ fn load_data_url_for_prompt_with(
|
||||
.map_err(|source| ImageProcessingError::InvalidDataUrl {
|
||||
reason: format!("invalid base64 payload: {source}"),
|
||||
})?;
|
||||
if file_bytes.len() > MAX_PROMPT_IMAGE_INPUT_BYTES {
|
||||
if file_bytes.len() > max_bytes {
|
||||
return Err(ImageProcessingError::ImageTooLarge {
|
||||
representation: "decoded input",
|
||||
size: file_bytes.len(),
|
||||
max: MAX_PROMPT_IMAGE_INPUT_BYTES,
|
||||
max: max_bytes,
|
||||
});
|
||||
}
|
||||
|
||||
load(Path::new("<data-url-image>"), file_bytes, mode)
|
||||
Ok(file_bytes)
|
||||
}
|
||||
|
||||
fn resize_for_prompt(image: &DynamicImage, mode: PromptImageMode) -> Option<DynamicImage> {
|
||||
let (width, height) = image.dimensions();
|
||||
match mode {
|
||||
PromptImageMode::ResizeToFit if width > MAX_DIMENSION || height > MAX_DIMENSION => {
|
||||
Some(image.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle))
|
||||
}
|
||||
PromptImageMode::ResizeWithLimits(limits) => {
|
||||
let (target_width, target_height) =
|
||||
prompt_image_output_dimensions_for_limits(width, height, limits);
|
||||
if (target_width, target_height) == (width, height) {
|
||||
return None;
|
||||
}
|
||||
Some(image.resize_exact(target_width, target_height, FilterType::Triangle))
|
||||
}
|
||||
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt_image_output_dimensions_for_limits(
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Decode supported prompt images and write fresh PNG or JPEG bytes without the source metadata.
|
||||
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
|
||||
use image::DynamicImage;
|
||||
use image::ImageDecoder;
|
||||
use image::ImageFormat;
|
||||
use image::ImageReader;
|
||||
use image::codecs::jpeg::JpegEncoder;
|
||||
use image::codecs::png::CompressionType;
|
||||
use image::codecs::png::FilterType;
|
||||
use image::codecs::png::PngEncoder;
|
||||
use moxcms::ColorProfile;
|
||||
|
||||
use crate::EncodedImage;
|
||||
use crate::ImageProcessingError;
|
||||
use crate::MAX_SANITIZED_IMAGE_BYTES;
|
||||
use crate::PromptImageMode;
|
||||
use crate::resize_for_prompt;
|
||||
|
||||
// Original detail allows 10,000 patches of 32 × 32 pixels each.
|
||||
const MAX_DECODED_PIXELS: u64 = 10_240_000;
|
||||
|
||||
pub(super) fn sanitize(
|
||||
path: &Path,
|
||||
bytes: &[u8],
|
||||
mode: PromptImageMode,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
if bytes.len() > MAX_SANITIZED_IMAGE_BYTES {
|
||||
return Err(ImageProcessingError::ImageTooLarge {
|
||||
representation: "input",
|
||||
size: bytes.len(),
|
||||
max: MAX_SANITIZED_IMAGE_BYTES,
|
||||
});
|
||||
}
|
||||
|
||||
let decode_error = |source| ImageProcessingError::decode_error(path, source);
|
||||
let reader = ImageReader::new(Cursor::new(bytes))
|
||||
.with_guessed_format()
|
||||
.map_err(|error| decode_error(error.into()))?;
|
||||
let (format, output_format) = match reader.format() {
|
||||
Some(format @ (ImageFormat::Png | ImageFormat::Gif)) => (format, ImageFormat::Png),
|
||||
Some(format @ (ImageFormat::Jpeg | ImageFormat::WebP)) => (format, ImageFormat::Jpeg),
|
||||
format => {
|
||||
return Err(ImageProcessingError::UnsupportedImageFormat {
|
||||
mime: format
|
||||
.as_ref()
|
||||
.map_or("unknown", ImageFormat::to_mime_type)
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let (mut img, orientation, icc_profile) = std::panic::catch_unwind(|| {
|
||||
let mut decoder = reader.into_decoder().map_err(decode_error)?;
|
||||
let (width, height) = decoder.dimensions();
|
||||
let pixels = u64::from(width) * u64::from(height);
|
||||
if pixels > MAX_DECODED_PIXELS {
|
||||
return Err(ImageProcessingError::ImageTooManyPixels {
|
||||
pixels,
|
||||
max: MAX_DECODED_PIXELS,
|
||||
});
|
||||
}
|
||||
let icc_profile = decoder.icc_profile().ok().flatten();
|
||||
// Read EXIF orientation only for JPEG.
|
||||
let orientation = if format == ImageFormat::Jpeg {
|
||||
decoder.orientation().ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let img = DynamicImage::from_decoder(decoder).map_err(decode_error)?;
|
||||
Ok((img, orientation, icc_profile))
|
||||
})
|
||||
.map_err(|_| {
|
||||
decode_error(image::ImageError::Decoding(
|
||||
image::error::DecodingError::new(
|
||||
format.into(),
|
||||
"image decoder rejected malformed data",
|
||||
),
|
||||
))
|
||||
})??;
|
||||
|
||||
if let Some(orientation) = orientation {
|
||||
img.apply_orientation(orientation);
|
||||
}
|
||||
let (source_width, source_height) = (img.width(), img.height());
|
||||
if format == ImageFormat::WebP {
|
||||
// Drop WebP alpha before resizing because the JPEG output has no alpha channel.
|
||||
img = img.into_rgb8().into();
|
||||
}
|
||||
|
||||
if let Some(resized) = resize_for_prompt(&img, mode) {
|
||||
img = resized;
|
||||
}
|
||||
|
||||
if let Some(icc_profile) = icc_profile
|
||||
&& let Some(new_img) = convert_color_profile(&img, &icc_profile)
|
||||
{
|
||||
img = new_img;
|
||||
}
|
||||
|
||||
let mut output = Vec::new();
|
||||
let encoded = if output_format == ImageFormat::Png {
|
||||
img.write_with_encoder(PngEncoder::new_with_quality(
|
||||
&mut output,
|
||||
CompressionType::Fast,
|
||||
FilterType::Adaptive,
|
||||
))
|
||||
} else {
|
||||
img.write_with_encoder(JpegEncoder::new_with_quality(
|
||||
&mut output,
|
||||
/*quality*/ 90,
|
||||
))
|
||||
};
|
||||
encoded.map_err(|source| ImageProcessingError::Encode {
|
||||
format: output_format,
|
||||
source,
|
||||
})?;
|
||||
if output.len() > MAX_SANITIZED_IMAGE_BYTES {
|
||||
return Err(ImageProcessingError::ImageTooLarge {
|
||||
representation: "sanitized output",
|
||||
size: output.len(),
|
||||
max: MAX_SANITIZED_IMAGE_BYTES,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(EncodedImage {
|
||||
bytes: output.into(),
|
||||
mime: output_format.to_mime_type().to_string(),
|
||||
source_width,
|
||||
source_height,
|
||||
width: img.width(),
|
||||
height: img.height(),
|
||||
})
|
||||
}
|
||||
|
||||
fn convert_color_profile(img: &DynamicImage, icc_profile: &[u8]) -> Option<DynamicImage> {
|
||||
// Convert color profiles only for 8-bit RGB and RGBA images.
|
||||
let layout = match img.color() {
|
||||
image::ColorType::Rgba8 => moxcms::Layout::Rgba,
|
||||
image::ColorType::Rgb8 => moxcms::Layout::Rgb,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let parsed_icc_profile = ColorProfile::new_from_slice(icc_profile).ok()?;
|
||||
let dest_profile = ColorProfile::new_srgb();
|
||||
|
||||
let transform_opts = moxcms::TransformOptions {
|
||||
// Use floating point for more precise color conversion.
|
||||
prefer_fixed_point: false,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let transform = parsed_icc_profile
|
||||
.create_transform_8bit(layout, &dest_profile, layout, transform_opts)
|
||||
.ok()?;
|
||||
|
||||
let mut output = vec![0; img.as_bytes().len()];
|
||||
transform.transform(img.as_bytes(), &mut output).ok()?;
|
||||
let (width, height) = (img.width(), img.height());
|
||||
if layout == moxcms::Layout::Rgba {
|
||||
image::RgbaImage::from_raw(width, height, output).map(DynamicImage::ImageRgba8)
|
||||
} else {
|
||||
image::RgbImage::from_raw(width, height, output).map(DynamicImage::ImageRgb8)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user