Validate images before returning view_image output (#37939)

## What changed

- Reject invalid or unsupported image data before producing tool output, so
  non-image file contents are not exposed through code mode.
- Preserve valid image bytes and metadata while leaving image preparation and
  resizing to the history insertion path.

## Testing

- Cover invalid images in both code mode and standard `view_image` calls.
- Verify that code mode preserves PNG bytes, dimensions, EXIF orientation, and
  metadata.

GitOrigin-RevId: 07343ed10b781d0ce0ffb5b24432658be4c046e3
This commit is contained in:
rka-oai
2026-08-11 05:32:28 +00:00
committed by copyberry
parent 2cc9dbb984
commit 41ece455b7
3 changed files with 163 additions and 12 deletions
+86 -3
View File
@@ -76,7 +76,13 @@ use core_test_support::wait_for_mcp_server;
use image::DynamicImage;
use image::GenericImageView;
use image::ImageBuffer;
use image::ImageDecoder;
use image::ImageEncoder;
use image::ImageFormat;
use image::ImageReader;
use image::Rgba;
use image::codecs::png::PngEncoder;
use image::metadata::Orientation;
use pretty_assertions::assert_eq;
use serde_json::Value;
use std::collections::HashMap;
@@ -4038,6 +4044,53 @@ image(s.trim(), "original");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_view_image_rejects_invalid_file_without_exposing_contents() -> Result<()> {
skip_if_no_network!(Ok(()));
const INVALID_IMAGE_CONTENTS: &str = "private-file-contents-must-not-be-exposed";
let server = responses::start_mock_server().await;
let builder = test_codex()
.with_model("gpt-5.4")
.with_config(|config| {
let _ = config.features.enable(Feature::CodeMode);
})
.with_workspace_setup(|cwd, _fs| async move {
fs::write(
cwd.join("not-an-image.txt").as_path(),
INVALID_IMAGE_CONTENTS,
)?;
Ok(())
});
let (_test, second_mock) = run_code_mode_turn_with_builder(
&server,
"use exec to call view_image on a non-image file",
r#"await tools.view_image({ path: "not-an-image.txt" });"#,
builder,
)
.await?;
let request = second_mock.single_request();
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
assert_ne!(
success,
Some(true),
"code-mode view_image unexpectedly accepted a non-image file"
);
assert!(
output.contains("unable to process image: invalid or unsupported image data"),
"unexpected code-mode failure: {output}"
);
let model_visible_output = serde_json::to_string(&request.custom_tool_call_output("call-1"))?;
assert!(
!model_visible_output.contains(INVALID_IMAGE_CONTENTS),
"invalid file contents leaked into model-visible output: {model_visible_output}"
);
Ok(())
}
#[test_case(false; "legacy detail")]
#[test_case(true; "unified image budget")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -4057,11 +4110,26 @@ async fn code_mode_can_use_view_image_result_with_image_helper(
});
let test = builder.build(&server).await?;
let image_bytes = BASE64_STANDARD.decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
let image = ImageBuffer::from_pixel(
/*width*/ 2,
/*height*/ 1,
Rgba([255u8, 0, 0, 255]),
);
let rotate_90_exif = vec![
0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01,
0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
];
let mut image_bytes = Vec::new();
let mut encoder = PngEncoder::new(&mut image_bytes);
encoder.set_exif_metadata(rotate_90_exif.clone())?;
encoder.write_image(
image.as_raw(),
image.width(),
image.height(),
image::ColorType::Rgba8.into(),
)?;
let image_path = test.cwd_path().join("code_mode_view_image.png");
fs::write(&image_path, image_bytes)?;
fs::write(&image_path, &image_bytes)?;
let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;
let expected_output_keys = if unified_image_budget {
@@ -4129,6 +4197,21 @@ image(out);
.and_then(Value::as_str)
.expect("image helper should emit an input_image item with image_url");
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
let (_, emitted_image_base64) = emitted_image_url
.split_once(',')
.expect("emitted image should contain a data URL prefix");
let emitted_image_bytes = BASE64_STANDARD.decode(emitted_image_base64)?;
assert_eq!(emitted_image_bytes, image_bytes);
let mut decoder = ImageReader::with_format(Cursor::new(&emitted_image_bytes), ImageFormat::Png)
.into_decoder()?;
assert_eq!(
(
decoder.dimensions(),
decoder.orientation()?,
decoder.exif_metadata()?
),
((2, 1), Orientation::Rotate90, Some(rotate_90_exif))
);
assert_eq!(
items[1].get("detail").and_then(Value::as_str),
Some("original")
+7 -6
View File
@@ -1443,7 +1443,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_tool_turns_invalid_image_into_placeholder() -> anyhow::Result<()> {
async fn view_image_tool_rejects_invalid_image_before_tool_output() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -1496,12 +1496,13 @@ async fn view_image_tool_turns_invalid_image_into_placeholder() -> anyhow::Resul
.await;
let request = second_mock.single_request();
let output_text = request
.function_call_output_content_and_success(call_id)
.and_then(|(content, _)| content)
.context("invalid view_image error text present")?;
assert_eq!(
request.function_call_output(call_id).get("output"),
Some(&serde_json::json!([{
"type": "input_text",
"text": "image content omitted because it could not be processed"
}]))
output_text,
"unable to process image: invalid or unsupported image data"
);
Ok(())
}