Add opt-in prompt image sanitization (#61341)

## What changed

- Add the disabled-by-default `safe_image_decoder` feature to decode and re-encode prompt images without source metadata, with byte and decoded-pixel limits.
- Apply JPEG orientation and supported ICC color conversion to sRGB, emitting PNG for PNG/GIF inputs and JPEG for JPEG/WebP inputs.
- Use the selected decoder for image preparation, `view_image` validation, and MCP image validation. Include the decoder in image cache keys.
- Preserve original inline images in history metadata so resume and fork can regenerate prepared images without repeated JPEG encoding changing their bytes.

## Testing

Add regression tests for identical JPEG replay after history serialization and for preventing legacy cache entries from bypassing sanitization.

GitOrigin-RevId: 718cd83d0a233e2eb8efd1a64c14d8944e44c350
This commit is contained in:
Vivian Fang
2026-11-13 18:44:31 +00:00
committed by copyberry
parent a4b69a0ae9
commit 7844815bd7
21 changed files with 632 additions and 216 deletions
+2
View File
@@ -27,6 +27,8 @@ pub enum ImageProcessingError {
UnsupportedImageFormat { mime: String },
#[error("invalid image data URL: {reason}")]
InvalidDataUrl { reason: String },
#[error("image has too many decoded pixels ({pixels}; max {max})")]
ImageTooManyPixels { pixels: u64, max: u64 },
#[error("image {representation} is too large ({size} bytes; max {max} bytes)")]
ImageTooLarge {
representation: &'static str,
+34 -5
View File
@@ -236,8 +236,12 @@ async fn data_url_processing_preserves_supported_source_bytes() {
.replacen("data:", "DATA:", 1)
.replacen(";base64,", ";BASE64,", 1);
let processed = load_data_url_for_prompt_uncached(&image_url, PromptImageMode::ResizeToFit)
.expect("process data URL image");
let processed = load_data_url_for_prompt_uncached(
&image_url,
PromptImageMode::ResizeToFit,
PromptImageDecoder::Legacy,
)
.expect("process data URL image");
assert_eq!(processed.width, 64);
assert_eq!(processed.height, 32);
@@ -251,8 +255,12 @@ async fn data_url_processing_converts_gif_to_png() {
let gif_bytes = image_bytes(&image, ImageFormat::Gif);
let image_url = data_url_from_bytes("image/gif", &gif_bytes);
let processed = load_data_url_for_prompt(&image_url, PromptImageMode::ResizeToFit)
.expect("process GIF data URL");
let processed = load_data_url_for_prompt(
&image_url,
PromptImageMode::ResizeToFit,
PromptImageDecoder::Legacy,
)
.expect("process GIF data URL");
assert_eq!(processed.mime, "image/png");
assert_eq!(
@@ -270,7 +278,11 @@ fn data_url_processing_rejects_malformed_input() {
"data:image/png;base64,not base64",
] {
assert!(matches!(
load_data_url_for_prompt(image_url, PromptImageMode::ResizeToFit),
load_data_url_for_prompt(
image_url,
PromptImageMode::ResizeToFit,
PromptImageDecoder::Legacy,
),
Err(ImageProcessingError::InvalidDataUrl { .. })
));
}
@@ -382,12 +394,29 @@ async fn reprocesses_updated_file_contents() {
assert_ne!(second.bytes, first.bytes);
}
#[test]
fn legacy_cache_entry_cannot_bypass_sanitization() {
let image = ImageBuffer::from_pixel(7, 11, Rgba([42, 64, 128, 255]));
let bytes = image_bytes_with_metadata(&image, ImageFormat::Jpeg, TEST_RGB_ICC_PROFILE);
let path = Path::new("oriented.jpg");
let mode = PromptImageMode::Original;
let legacy = load_for_prompt_bytes(path, bytes.clone(), mode).expect("load legacy image");
let sanitized =
load_for_prompt_bytes_with_decoder(path, bytes, mode, PromptImageDecoder::Sanitizer)
.expect("sanitize image after warming the legacy cache");
assert_eq!((legacy.width, legacy.height), (7, 11));
assert_eq!((sanitized.width, sanitized.height), (11, 7));
}
#[tokio::test(flavor = "multi_thread")]
async fn bounds_cache_by_encoded_byte_size() {
let cache = ImageCache::new(NonZeroUsize::new(4).expect("non-zero cache capacity"));
let key = |digest_byte| ImageCacheKey {
digest: [digest_byte; 20],
mode: PromptImageMode::Original,
decoder: PromptImageDecoder::Legacy,
};
let image = |size| EncodedImage {
bytes: vec![0; size].into(),
+68 -35
View File
@@ -29,9 +29,13 @@ pub const MAX_DIMENSION: u32 = 2048;
/// This is a high sanity guard against pathological inputs, not a protocol
/// requirement or target upload size.
pub const MAX_PROMPT_IMAGE_INPUT_BYTES: usize = 1024 * 1024 * 1024;
/// Use the same byte limit for input and output so a saved image fits on replay.
/// At the pixel limit, 16-bit RGBA uses 81,920,000 bytes. Round up for PNG overhead.
pub const MAX_SANITIZED_IMAGE_BYTES: usize = 79 * 1024 * 1024;
const MAX_IMAGE_CACHE_BYTES: usize = 64 * 1024 * 1024;
pub mod error;
mod sanitizer;
pub use crate::error::ImageProcessingError;
@@ -64,6 +68,13 @@ pub enum PromptImageMode {
ResizeWithLimits(PromptImageResizeLimits),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum PromptImageDecoder {
#[default]
Legacy,
Sanitizer,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PromptImageResizeLimits {
pub max_dimension: u32,
@@ -92,6 +103,7 @@ struct ImageMetadata {
struct ImageCacheKey {
digest: [u8; 20],
mode: PromptImageMode,
decoder: PromptImageDecoder,
}
type ImageCache = BlockingLruCache<ImageCacheKey, EncodedImage>;
@@ -104,18 +116,26 @@ pub fn load_for_prompt_bytes(
file_bytes: Vec<u8>,
mode: PromptImageMode,
) -> Result<EncodedImage, ImageProcessingError> {
let path_buf = path.to_path_buf();
load_for_prompt_bytes_with_decoder(path, file_bytes, mode, PromptImageDecoder::Legacy)
}
fn load_for_prompt_bytes_with_decoder(
path: &Path,
file_bytes: Vec<u8>,
mode: PromptImageMode,
decoder: PromptImageDecoder,
) -> Result<EncodedImage, ImageProcessingError> {
let key = ImageCacheKey {
digest: sha1_digest(&file_bytes),
mode,
decoder,
};
if let Some(image) = IMAGE_CACHE.get(&key) {
return Ok(image);
}
let image = load_for_prompt_bytes_uncached(&path_buf, file_bytes, mode)?;
let image = load_for_prompt_bytes_uncached(path, file_bytes, mode, decoder)?;
cache_image(&IMAGE_CACHE, key, image.clone(), MAX_IMAGE_CACHE_BYTES);
Ok(image)
}
@@ -124,7 +144,11 @@ fn load_for_prompt_bytes_uncached(
path: &Path,
file_bytes: Vec<u8>,
mode: PromptImageMode,
decoder: PromptImageDecoder,
) -> Result<EncodedImage, ImageProcessingError> {
if decoder == PromptImageDecoder::Sanitizer {
return sanitizer::sanitize(path, &file_bytes, mode);
}
let path_buf = path.to_path_buf();
(move || {
let guessed_format = image::guess_format(&file_bytes)
@@ -159,26 +183,7 @@ fn load_for_prompt_bytes_uncached(
let (width, height) = dynamic.dimensions();
let target_dimensions = match mode {
PromptImageMode::ResizeToFit if width > MAX_DIMENSION || height > MAX_DIMENSION => {
let resized = dynamic.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle);
Some((resized.width(), resized.height(), resized))
}
PromptImageMode::ResizeWithLimits(limits) => {
let (target_width, target_height) =
prompt_image_output_dimensions_for_limits(width, height, limits);
if (target_width, target_height) == (width, height) {
None
} else {
let resized =
dynamic.resize_exact(target_width, target_height, FilterType::Triangle);
Some((target_width, target_height, resized))
}
}
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
};
let encoded = if let Some((prepared_width, prepared_height, resized)) = target_dimensions {
let encoded = if let Some(resized) = resize_for_prompt(&dynamic, mode) {
let target_format = format
.filter(|format| can_preserve_source_bytes(*format))
.unwrap_or(ImageFormat::Png);
@@ -189,8 +194,8 @@ fn load_for_prompt_bytes_uncached(
mime,
source_width: width,
source_height: height,
width: prepared_width,
height: prepared_height,
width: resized.width(),
height: resized.height(),
}
} else {
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) {
@@ -244,22 +249,25 @@ fn cache_image(cache: &ImageCache, key: ImageCacheKey, image: EncodedImage, byte
pub fn load_data_url_for_prompt(
image_url: &str,
mode: PromptImageMode,
decoder: PromptImageDecoder,
) -> Result<EncodedImage, ImageProcessingError> {
load_data_url_for_prompt_with(image_url, mode, load_for_prompt_bytes)
let bytes = decode_data_url(image_url, decoder)?;
load_for_prompt_bytes_with_decoder(Path::new("<data-url-image>"), bytes, mode, decoder)
}
pub fn load_data_url_for_prompt_uncached(
image_url: &str,
mode: PromptImageMode,
decoder: PromptImageDecoder,
) -> Result<EncodedImage, ImageProcessingError> {
load_data_url_for_prompt_with(image_url, mode, load_for_prompt_bytes_uncached)
let bytes = decode_data_url(image_url, decoder)?;
load_for_prompt_bytes_uncached(Path::new("<data-url-image>"), bytes, mode, decoder)
}
fn load_data_url_for_prompt_with(
fn decode_data_url(
image_url: &str,
mode: PromptImageMode,
load: impl FnOnce(&Path, Vec<u8>, PromptImageMode) -> Result<EncodedImage, ImageProcessingError>,
) -> Result<EncodedImage, ImageProcessingError> {
decoder: PromptImageDecoder,
) -> Result<Vec<u8>, ImageProcessingError> {
let rest = image_url
.get(..DATA_URL_PREFIX.len())
.filter(|prefix| prefix.eq_ignore_ascii_case(DATA_URL_PREFIX))
@@ -281,11 +289,18 @@ fn load_data_url_for_prompt_with(
});
}
if encoded.len() > MAX_PROMPT_IMAGE_INPUT_BYTES {
let (max_bytes, max_base64_bytes) = match decoder {
PromptImageDecoder::Legacy => (MAX_PROMPT_IMAGE_INPUT_BYTES, MAX_PROMPT_IMAGE_INPUT_BYTES),
PromptImageDecoder::Sanitizer => (
MAX_SANITIZED_IMAGE_BYTES,
MAX_SANITIZED_IMAGE_BYTES.div_ceil(3) * 4,
),
};
if encoded.len() > max_base64_bytes {
return Err(ImageProcessingError::ImageTooLarge {
representation: "base64 payload",
size: encoded.len(),
max: MAX_PROMPT_IMAGE_INPUT_BYTES,
max: max_base64_bytes,
});
}
let file_bytes =
@@ -294,15 +309,33 @@ fn load_data_url_for_prompt_with(
.map_err(|source| ImageProcessingError::InvalidDataUrl {
reason: format!("invalid base64 payload: {source}"),
})?;
if file_bytes.len() > MAX_PROMPT_IMAGE_INPUT_BYTES {
if file_bytes.len() > max_bytes {
return Err(ImageProcessingError::ImageTooLarge {
representation: "decoded input",
size: file_bytes.len(),
max: MAX_PROMPT_IMAGE_INPUT_BYTES,
max: max_bytes,
});
}
load(Path::new("<data-url-image>"), file_bytes, mode)
Ok(file_bytes)
}
fn resize_for_prompt(image: &DynamicImage, mode: PromptImageMode) -> Option<DynamicImage> {
let (width, height) = image.dimensions();
match mode {
PromptImageMode::ResizeToFit if width > MAX_DIMENSION || height > MAX_DIMENSION => {
Some(image.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle))
}
PromptImageMode::ResizeWithLimits(limits) => {
let (target_width, target_height) =
prompt_image_output_dimensions_for_limits(width, height, limits);
if (target_width, target_height) == (width, height) {
return None;
}
Some(image.resize_exact(target_width, target_height, FilterType::Triangle))
}
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
}
}
fn prompt_image_output_dimensions_for_limits(
+167
View File
@@ -0,0 +1,167 @@
//! Decode supported prompt images and write fresh PNG or JPEG bytes without the source metadata.
use std::io::Cursor;
use std::path::Path;
use image::DynamicImage;
use image::ImageDecoder;
use image::ImageFormat;
use image::ImageReader;
use image::codecs::jpeg::JpegEncoder;
use image::codecs::png::CompressionType;
use image::codecs::png::FilterType;
use image::codecs::png::PngEncoder;
use moxcms::ColorProfile;
use crate::EncodedImage;
use crate::ImageProcessingError;
use crate::MAX_SANITIZED_IMAGE_BYTES;
use crate::PromptImageMode;
use crate::resize_for_prompt;
// Original detail allows 10,000 patches of 32 × 32 pixels each.
const MAX_DECODED_PIXELS: u64 = 10_240_000;
pub(super) fn sanitize(
path: &Path,
bytes: &[u8],
mode: PromptImageMode,
) -> Result<EncodedImage, ImageProcessingError> {
if bytes.len() > MAX_SANITIZED_IMAGE_BYTES {
return Err(ImageProcessingError::ImageTooLarge {
representation: "input",
size: bytes.len(),
max: MAX_SANITIZED_IMAGE_BYTES,
});
}
let decode_error = |source| ImageProcessingError::decode_error(path, source);
let reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format()
.map_err(|error| decode_error(error.into()))?;
let (format, output_format) = match reader.format() {
Some(format @ (ImageFormat::Png | ImageFormat::Gif)) => (format, ImageFormat::Png),
Some(format @ (ImageFormat::Jpeg | ImageFormat::WebP)) => (format, ImageFormat::Jpeg),
format => {
return Err(ImageProcessingError::UnsupportedImageFormat {
mime: format
.as_ref()
.map_or("unknown", ImageFormat::to_mime_type)
.to_string(),
});
}
};
let (mut img, orientation, icc_profile) = std::panic::catch_unwind(|| {
let mut decoder = reader.into_decoder().map_err(decode_error)?;
let (width, height) = decoder.dimensions();
let pixels = u64::from(width) * u64::from(height);
if pixels > MAX_DECODED_PIXELS {
return Err(ImageProcessingError::ImageTooManyPixels {
pixels,
max: MAX_DECODED_PIXELS,
});
}
let icc_profile = decoder.icc_profile().ok().flatten();
// Read EXIF orientation only for JPEG.
let orientation = if format == ImageFormat::Jpeg {
decoder.orientation().ok()
} else {
None
};
let img = DynamicImage::from_decoder(decoder).map_err(decode_error)?;
Ok((img, orientation, icc_profile))
})
.map_err(|_| {
decode_error(image::ImageError::Decoding(
image::error::DecodingError::new(
format.into(),
"image decoder rejected malformed data",
),
))
})??;
if let Some(orientation) = orientation {
img.apply_orientation(orientation);
}
let (source_width, source_height) = (img.width(), img.height());
if format == ImageFormat::WebP {
// Drop WebP alpha before resizing because the JPEG output has no alpha channel.
img = img.into_rgb8().into();
}
if let Some(resized) = resize_for_prompt(&img, mode) {
img = resized;
}
if let Some(icc_profile) = icc_profile
&& let Some(new_img) = convert_color_profile(&img, &icc_profile)
{
img = new_img;
}
let mut output = Vec::new();
let encoded = if output_format == ImageFormat::Png {
img.write_with_encoder(PngEncoder::new_with_quality(
&mut output,
CompressionType::Fast,
FilterType::Adaptive,
))
} else {
img.write_with_encoder(JpegEncoder::new_with_quality(
&mut output,
/*quality*/ 90,
))
};
encoded.map_err(|source| ImageProcessingError::Encode {
format: output_format,
source,
})?;
if output.len() > MAX_SANITIZED_IMAGE_BYTES {
return Err(ImageProcessingError::ImageTooLarge {
representation: "sanitized output",
size: output.len(),
max: MAX_SANITIZED_IMAGE_BYTES,
});
}
Ok(EncodedImage {
bytes: output.into(),
mime: output_format.to_mime_type().to_string(),
source_width,
source_height,
width: img.width(),
height: img.height(),
})
}
fn convert_color_profile(img: &DynamicImage, icc_profile: &[u8]) -> Option<DynamicImage> {
// Convert color profiles only for 8-bit RGB and RGBA images.
let layout = match img.color() {
image::ColorType::Rgba8 => moxcms::Layout::Rgba,
image::ColorType::Rgb8 => moxcms::Layout::Rgb,
_ => return None,
};
let parsed_icc_profile = ColorProfile::new_from_slice(icc_profile).ok()?;
let dest_profile = ColorProfile::new_srgb();
let transform_opts = moxcms::TransformOptions {
// Use floating point for more precise color conversion.
prefer_fixed_point: false,
..Default::default()
};
let transform = parsed_icc_profile
.create_transform_8bit(layout, &dest_profile, layout, transform_opts)
.ok()?;
let mut output = vec![0; img.as_bytes().len()];
transform.transform(img.as_bytes(), &mut output).ok()?;
let (width, height) = (img.width(), img.height());
if layout == moxcms::Layout::Rgba {
image::RgbaImage::from_raw(width, height, output).map(DynamicImage::ImageRgba8)
} else {
image::RgbImage::from_raw(width, height, output).map(DynamicImage::ImageRgb8)
}
}