Add opt-in prompt image sanitization (#61341)

## What changed

- Add the disabled-by-default `safe_image_decoder` feature to decode and re-encode prompt images without source metadata, with byte and decoded-pixel limits.
- Apply JPEG orientation and supported ICC color conversion to sRGB, emitting PNG for PNG/GIF inputs and JPEG for JPEG/WebP inputs.
- Use the selected decoder for image preparation, `view_image` validation, and MCP image validation. Include the decoder in image cache keys.
- Preserve original inline images in history metadata so resume and fork can regenerate prepared images without repeated JPEG encoding changing their bytes.

## Testing

Add regression tests for identical JPEG replay after history serialization and for preventing legacy cache entries from bypassing sanitization.

GitOrigin-RevId: 718cd83d0a233e2eb8efd1a64c14d8944e44c350
This commit is contained in:
Vivian Fang
2026-11-13 18:44:31 +00:00
committed by copyberry
parent a4b69a0ae9
commit 7844815bd7
21 changed files with 632 additions and 216 deletions
+1 -1
View File
@@ -148,7 +148,7 @@ pub(crate) async fn prepare_prompt(
})?;
}
restored = session
.prepare_annotated_conversation_items_for_history(step.turn.as_ref(), model, messages)
.prepare_conversation_items_for_history(step.turn.as_ref(), model, messages)
.await
.0;
prompt
@@ -22,7 +22,6 @@ use super::GuardianReviewSessionParams;
use crate::context_manager::ContextManager;
use crate::image_preparation::ImagePreparationMode;
use crate::image_preparation::resize_image;
use crate::image_preparation::unified_image_budget_enabled;
pub(super) async fn build_guardian_prompt_items(
params: &GuardianReviewSessionParams,
@@ -71,11 +70,7 @@ pub(super) async fn build_guardian_prompt_items(
});
let context_window =
context_window.filter(|limit| *limit > 0 && model_info.origin != ModelOrigin::Synthetic);
let image_mode = if unified_image_budget_enabled(&params.spawn_config.features, model_info) {
ImagePreparationMode::UnifiedBudget
} else {
ImagePreparationMode::DetailBased
};
let image_mode = ImagePreparationMode::for_model(&params.spawn_config.features, model_info);
let history_tokens = if context_window.is_some() && collected.sections.has_images() {
let base_instructions = reviewer.session.get_base_instructions().await;
reviewer_history
@@ -121,7 +116,7 @@ impl codex_guardian_context::SyncImagePreparer for ReviewImagePreparer<'_> {
detail: &mut Option<ImageDetail>,
) -> Option<Cow<'image, ImageReference>> {
*detail = match normalize_output_image_detail(self.model_info, *detail) {
_ if self.mode == ImagePreparationMode::UnifiedBudget => Some(ImageDetail::Original),
_ if self.mode.unified_budget => Some(ImageDetail::Original),
Some(ImageDetail::Low) => Some(ImageDetail::High),
detail => detail,
};
+157 -53
View File
@@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::config::ManagedFeatures;
use crate::context::ContextualUserFragment;
@@ -15,6 +16,9 @@ use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_features::Feature;
use codex_history::CodexHarnessMetadata;
use codex_history::ImageReplaySource;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputContentItem;
@@ -23,8 +27,10 @@ use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_utils_cache::sha1_digest;
use codex_utils_image::EncodedImage;
use codex_utils_image::ImageProcessingError;
use codex_utils_image::PromptImageDecoder;
use codex_utils_image::PromptImageMode;
use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_data_url_for_prompt;
@@ -38,10 +44,37 @@ const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because
const REMOTE_IMAGE_URL_PLACEHOLDER: &str =
"image content omitted because remote image URLs are not supported";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ImagePreparationMode {
DetailBased,
UnifiedBudget,
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct ImagePreparationMode {
pub(crate) unified_budget: bool,
pub(crate) decoder: PromptImageDecoder,
replay: bool,
}
impl ImagePreparationMode {
pub(crate) fn for_replay(features: &ManagedFeatures) -> Self {
Self {
unified_budget: false,
decoder: prompt_image_decoder(features),
replay: true,
}
}
pub(crate) fn for_model(features: &ManagedFeatures, model_info: &ModelInfo) -> Self {
Self {
unified_budget: unified_image_budget_enabled(features, model_info),
decoder: prompt_image_decoder(features),
replay: false,
}
}
}
pub(crate) fn prompt_image_decoder(features: &ManagedFeatures) -> PromptImageDecoder {
if features.enabled(Feature::SafeImageDecoder) {
PromptImageDecoder::Sanitizer
} else {
PromptImageDecoder::Legacy
}
}
pub(crate) fn unified_image_budget_enabled(
@@ -74,6 +107,15 @@ struct PreparedImageResize {
prepared_height: u32,
}
struct ImagePreparation<'a> {
mode: ImagePreparationMode,
image_store: &'a dyn AttachmentStore,
analytics: &'a mut Vec<ImagePreparationMetadata>,
history: &'a mut Option<CodexHarnessMetadata>,
}
const REPLAY_IMAGE_DATA_URL_PREFIX: &str = "data:application/octet-stream;base64,";
pub(crate) struct PreparedInlineImage {
encoded: EncodedImage,
effective_detail: ImageDetailSetting,
@@ -115,9 +157,10 @@ impl ImagePreparationError {
match self {
ImagePreparationError::RemoteUrlUnsupported => REMOTE_IMAGE_URL_PLACEHOLDER,
ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER,
ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => {
IMAGE_TOO_LARGE_PLACEHOLDER
}
ImagePreparationError::Processing(
ImageProcessingError::ImageTooLarge { .. }
| ImageProcessingError::ImageTooManyPixels { .. },
) => IMAGE_TOO_LARGE_PLACEHOLDER,
ImagePreparationError::Processing(_) => IMAGE_PROCESSING_ERROR_PLACEHOLDER,
}
}
@@ -126,16 +169,23 @@ impl ImagePreparationError {
pub(crate) async fn prepare_response_items(
thread_id: &str,
ephemeral: bool,
items: &mut Vec<ResponseItem>,
items: &mut Vec<ResponseItemEnvelope>,
mode: ImagePreparationMode,
resize_notice_mode: ImageResizeNoticeMode,
image_store: &dyn AttachmentStore,
) -> Vec<ImagePreparationMetadata> {
let mut metadata = Vec::new();
let mut image_preparations = Vec::new();
let mut prepared_items = Vec::with_capacity(items.len());
for mut item in std::mem::take(items) {
let mut annotated_content = to_annotated_content(&mut item);
let resize_notice = match &mut item {
for mut envelope in std::mem::take(items) {
let mut preparation = ImagePreparation {
mode,
image_store,
analytics: &mut image_preparations,
history: &mut envelope.metadata,
};
let item = &mut envelope.item;
let mut annotated_content = to_annotated_content(item);
let resize_notice = match item {
ResponseItem::Message { role, .. } => {
let Some(mut content) = annotated_content.take() else {
continue;
@@ -153,12 +203,10 @@ pub(crate) async fn prepare_response_items(
} else {
ImageResizeNoticeMode::Disabled
},
&mut metadata,
mode,
image_store,
&mut preparation,
)
.await;
let _ = set_annotated_content(&mut item, content);
let _ = set_annotated_content(item, content);
(!resized_images.is_empty()).then(|| {
ImageResizeNotice::new(ImageResizeNoticeSource::UserMessage, resized_images)
})
@@ -172,9 +220,7 @@ pub(crate) async fn prepare_response_items(
output,
call_id.as_deref(),
resize_notice_mode,
&mut metadata,
mode,
image_store,
&mut preparation,
)
.await
}
@@ -187,9 +233,7 @@ pub(crate) async fn prepare_response_items(
output,
Some(call_id.as_str()),
resize_notice_mode,
&mut metadata,
mode,
image_store,
&mut preparation,
)
.await
}
@@ -209,25 +253,57 @@ pub(crate) async fn prepare_response_items(
| ResponseItem::ContextCompaction { .. }
| ResponseItem::Other => None,
};
prepared_items.push(item);
prepared_items.push(envelope);
if let Some(resize_notice) = resize_notice {
prepared_items.push(ContextualUserFragment::into(resize_notice));
prepared_items.push(ResponseItemEnvelope::new(ContextualUserFragment::into(
resize_notice,
)));
}
}
*items = prepared_items;
metadata
image_preparations
}
fn replay_image_url<'a>(
image_url: &str,
detail: Option<ImageDetail>,
sources: &'a [ImageReplaySource],
mode: ImagePreparationMode,
) -> Option<&'a str> {
if sources.is_empty() {
return None;
}
let digest = sha1_digest(image_url.as_bytes());
sources
.iter()
.find(|source| {
if source.detail != detail
|| !source
.original_data_url
.starts_with(REPLAY_IMAGE_DATA_URL_PREFIX)
{
return false;
}
if source.prepared_url_digest.len() == 32 {
// Older builds wrote a 32-byte digest. Match their original by reproducing the recorded URL.
let mut original_detail = detail;
return resize_image(&source.original_data_url, &mut original_detail, mode)
.ok()
.flatten()
.is_some_and(|prepared| prepared.into_data_url() == image_url);
}
source.prepared_url_digest == digest
})
.map(|source| source.original_data_url.as_str())
}
#[allow(clippy::too_many_arguments)]
async fn prepare_tool_output(
thread_id: &str,
ephemeral: bool,
output: &mut FunctionCallOutputPayload,
item_id: Option<&str>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
image_store: &dyn AttachmentStore,
preparation: &mut ImagePreparation<'_>,
) -> Option<ImageResizeNotice> {
let content = output.content_items_mut()?;
let resized_images = prepare_tool_output_content(
@@ -239,9 +315,7 @@ async fn prepare_tool_output(
item_id,
},
resize_notice_mode,
metadata,
mode,
image_store,
preparation,
)
.await;
if resized_images.is_empty() {
@@ -258,9 +332,7 @@ async fn prepare_message_content(
items: &mut [AnnotatedContent],
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
image_store: &dyn AttachmentStore,
preparation: &mut ImagePreparation<'_>,
) -> Vec<ResizedImage> {
// File-backed images are already prepared and pass through while retaining their positions in
// resize notices.
@@ -273,7 +345,7 @@ async fn prepare_message_content(
for item in items {
if let ContentItem::InputImage { image, detail } = item.content_mut() {
image_number += 1;
match prepare_image(image, detail, origin, metadata, mode, image_store).await {
match prepare_image(image, detail, origin, preparation).await {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -303,9 +375,7 @@ async fn prepare_tool_output_content(
items: &mut [FunctionCallOutputContentItem],
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
image_store: &dyn AttachmentStore,
preparation: &mut ImagePreparation<'_>,
) -> Vec<ResizedImage> {
let image_count = items
.iter()
@@ -316,7 +386,7 @@ async fn prepare_tool_output_content(
for item in items {
if let FunctionCallOutputContentItem::InputImage { image, detail } = item {
image_number += 1;
match prepare_image(image, detail, origin, metadata, mode, image_store).await {
match prepare_image(image, detail, origin, preparation).await {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -356,17 +426,25 @@ async fn prepare_image(
image: &mut ImageReference,
detail: &mut Option<ImageDetail>,
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
image_store: &dyn AttachmentStore,
preparation: &mut ImagePreparation<'_>,
) -> Result<Option<PreparedImageResize>, ImagePreparationError> {
let ImageReference::Inline { image_url } = image else {
return Ok(None);
};
let Some(prepared) = resize_image(image_url, detail, mode)? else {
let mode = preparation.mode;
// On resume or fork, sanitize the saved original. If there is no match, sanitize
// the inline image from history instead.
let replay_url = if mode.replay && mode.decoder == PromptImageDecoder::Sanitizer {
preparation.history.as_ref().and_then(|history| {
replay_image_url(image_url, *detail, &history.image_replay_sources, mode)
})
} else {
None
};
let Some(prepared) = resize_image(replay_url.unwrap_or(image_url), detail, mode)? else {
return Ok(None);
};
metadata.push(prepared.metadata(origin));
preparation.analytics.push(prepared.metadata(origin));
let resize = prepared.resize;
let request = UploadRequest {
thread_id: origin.thread_id.to_owned(),
@@ -375,7 +453,7 @@ async fn prepare_image(
tags: HashMap::new(),
data: prepared.encoded.bytes.to_vec(),
};
*image = match image_store.upload(request).await {
let prepared_image = match preparation.image_store.upload(request).await {
Ok(UploadResult::Inline { bytes }) => ImageReference::Inline {
image_url: data_url_from_bytes(&prepared.encoded.mime, &bytes),
},
@@ -387,6 +465,31 @@ async fn prepare_image(
}
}
};
// Keep the original so replay can reproduce the image the model saw. Encoding an
// already prepared JPEG again can change its bytes. If only the MIME label changed,
// the saved image already has the original payload.
if !mode.replay
&& mode.decoder == PromptImageDecoder::Sanitizer
&& let ImageReference::Inline {
image_url: prepared_url,
} = &prepared_image
&& let Some((_, source_payload)) = image_url.split_once(',')
&& let Some((_, prepared_payload)) = prepared_url.split_once(',')
&& source_payload != prepared_payload
{
preparation
.history
.get_or_insert_default()
.image_replay_sources
.push(ImageReplaySource {
prepared_url_digest: sha1_digest(prepared_url.as_bytes()).to_vec(),
detail: *detail,
original_data_url: Arc::new(format!(
"{REPLAY_IMAGE_DATA_URL_PREFIX}{source_payload}"
)),
});
}
*image = prepared_image;
Ok(resize)
}
@@ -402,12 +505,13 @@ pub(crate) fn resize_image(
return Ok(None);
}
let (effective_detail, image_mode) = match mode {
ImagePreparationMode::UnifiedBudget => (
let (effective_detail, image_mode) = if mode.unified_budget {
(
ImageDetailSetting::Original,
PromptImageMode::ORIGINAL_DETAIL,
),
ImagePreparationMode::DetailBased => match detail {
)
} else {
match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => {
(ImageDetailSetting::High, PromptImageMode::HIGH_DETAIL)
}
@@ -416,9 +520,9 @@ pub(crate) fn resize_image(
PromptImageMode::ORIGINAL_DETAIL,
),
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
},
}
};
let encoded = load_data_url_for_prompt(image_url, image_mode)?;
let encoded = load_data_url_for_prompt(image_url, image_mode, mode.decoder)?;
let resize = ((encoded.source_width, encoded.source_height) != (encoded.width, encoded.height))
.then_some(PreparedImageResize {
source_width: encoded.source_width,
@@ -426,7 +530,7 @@ pub(crate) fn resize_image(
prepared_width: encoded.width,
prepared_height: encoded.height,
});
if mode == ImagePreparationMode::UnifiedBudget {
if mode.unified_budget {
// Preserve accurate context-window accounting while older transports still require an
// image detail field. Responses Lite removes this compatibility hint before sending.
*detail = Some(ImageDetail::Original);
+82 -6
View File
@@ -9,6 +9,7 @@ use codex_attachment_store::IsResolvableFuture;
use codex_attachment_store::ResolveFuture;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadFuture;
use codex_history::RolloutItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
@@ -26,6 +27,28 @@ use serde_json::json;
use super::*;
async fn prepare_response_items(
thread_id: &str,
ephemeral: bool,
items: &mut Vec<ResponseItem>,
mode: ImagePreparationMode,
resize_notice_mode: ImageResizeNoticeMode,
image_store: &dyn AttachmentStore,
) -> Vec<ImagePreparationMetadata> {
let mut envelopes = items.drain(..).map(ResponseItemEnvelope::new).collect();
let metadata = super::prepare_response_items(
thread_id,
ephemeral,
&mut envelopes,
mode,
resize_notice_mode,
image_store,
)
.await;
items.extend(envelopes.into_iter().map(ResponseItemEnvelope::into_item));
metadata
}
pub(crate) struct FailingAttachmentStore;
impl AttachmentStore for FailingAttachmentStore {
@@ -70,6 +93,59 @@ fn decoded_image(image_url: &str) -> (Vec<u8>, DynamicImage) {
(bytes, image)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn replay_regenerates_the_recorded_jpeg_from_its_original() {
let pixels = image::RgbImage::from_fn(81, 57, |x, y| {
image::Rgb([(19 * x + y) as u8, (11 * y + x) as u8, (x * y) as u8])
});
let mut bytes = Cursor::new(Vec::new());
DynamicImage::ImageRgb8(pixels)
.write_to(&mut bytes, ImageFormat::Jpeg)
.expect("encode original JPEG");
let source = data_url_from_bytes("image/jpeg", &bytes.into_inner());
let message = ResponseItem::Message {
status: None,
encrypted_content: None,
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image: ImageReference::Inline { image_url: source },
detail: Some(ImageDetail::High),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let mut items = vec![ResponseItemEnvelope::new(message)];
let prepare = async |items: &mut Vec<ResponseItemEnvelope>, replay| {
super::prepare_response_items(
"image-preparation-thread",
/*ephemeral*/ false,
items,
ImagePreparationMode {
decoder: PromptImageDecoder::Sanitizer,
replay,
..Default::default()
},
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
)
.await;
};
prepare(&mut items, /*replay*/ false).await;
let recorded = items.pop().expect("prepared item");
assert!(recorded.metadata.is_some());
let json =
serde_json::to_string(&RolloutItem::ResponseItem(recorded.clone())).expect("persist item");
let RolloutItem::ResponseItem(reloaded) =
serde_json::from_str(&json).expect("read persisted item")
else {
panic!("expected persisted response item");
};
let mut replayed = vec![reloaded];
prepare(&mut replayed, /*replay*/ true).await;
assert_eq!(replayed, vec![recorded]);
}
#[tokio::test(flavor = "multi_thread")]
async fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32);
@@ -114,7 +190,7 @@ async fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
)
@@ -206,7 +282,7 @@ async fn detail_policies_apply_the_expected_budgets() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
)
@@ -261,7 +337,7 @@ async fn preparation_reports_tool_output_item_id() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
)
@@ -301,7 +377,7 @@ async fn upload_failure_keeps_resized_image_inline() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Disabled,
&FailingAttachmentStore,
)
@@ -405,7 +481,7 @@ async fn resize_notices_count_file_backed_images_and_skip_failed_images() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Enabled,
&InlineAttachmentStore,
)
@@ -602,7 +678,7 @@ async fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
"image-preparation-thread",
/*ephemeral*/ false,
&mut items,
ImagePreparationMode::DetailBased,
ImagePreparationMode::default(),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
)
+7 -39
View File
@@ -1,7 +1,6 @@
use super::TurnInput as PendingTurnInput;
use super::session::Session;
use super::turn_context::TurnContext;
use codex_analytics::ImagePreparationMetadata;
use codex_features::Feature;
use codex_history::ResponseItemEnvelope;
use codex_protocol::error::CodexErr;
@@ -126,24 +125,20 @@ impl Session {
envelope
}
#[tracing::instrument(
name = "record_conversation_items",
level = "trace",
skip_all,
fields(item_count = items.len())
)]
pub(crate) async fn record_annotated_conversation_items(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: Vec<ResponseItemEnvelope>,
) {
if items.iter().all(|item| item.metadata.is_none()) {
let items = items
.into_iter()
.map(ResponseItemEnvelope::into_item)
.collect::<Vec<_>>();
self.record_conversation_items(turn_context, model_info, &items)
.await;
return;
}
let (annotated_items, image_preparations) = self
.prepare_annotated_conversation_items_for_history(turn_context, model_info, items)
.prepare_conversation_items_for_history(turn_context, model_info, items)
.await;
self.record_prepared_conversation_items(
turn_context,
@@ -154,33 +149,6 @@ impl Session {
.await;
}
pub(crate) async fn prepare_annotated_conversation_items_for_history(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: Vec<ResponseItemEnvelope>,
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
let mut annotated_items = Vec::with_capacity(items.len());
let mut image_preparations = Vec::new();
for envelope in items {
let (prepared_items, prepared_images) = self
.prepare_conversation_items_for_history(
turn_context,
model_info,
std::slice::from_ref(&envelope.item),
)
.await;
image_preparations.extend(prepared_images);
let mut metadata = envelope.metadata;
annotated_items.extend(prepared_items.into_iter().map(|item| ResponseItemEnvelope {
item,
metadata: metadata.take(),
}));
}
(annotated_items, image_preparations)
}
/// Injects items into active work, or records them without starting a turn.
pub(crate) async fn inject_no_new_turn(
&self,
+31 -53
View File
@@ -49,7 +49,6 @@ use crate::exec_policy::default_policy_path;
use crate::image_preparation::ImagePreparationMode;
use crate::image_preparation::ImageResizeNoticeMode;
use crate::image_preparation::prepare_response_items as prepare_image_response_items;
use crate::image_preparation::unified_image_budget_enabled;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::realtime_history::RealtimeEventOrder;
@@ -1999,38 +1998,22 @@ impl Session {
.and_then(|context| context.direct_only_tool_namespaces.clone())
.map(|namespaces| (turn.turn_id.clone(), namespaces))
});
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy media is processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// Replay disables image-resize notices, so media preparation remains one-to-one. Keep
// the prior batch behavior and carry history-only metadata in a positional sidecar.
let (mut prepared_history, metadata): (Vec<_>, Vec<_>) = history
.into_iter()
.map(|envelope| (envelope.item, envelope.metadata))
.unzip();
// Replay must not upload or migrate recorded history. The inline store returns prepared
// inline bytes, while existing file references bypass preparation and remain unchanged.
// Prepare inline images in memory on resume or fork. Keep file references as they are.
// The inline store avoids uploads, and replay leaves the saved history untouched.
// Skip resize notices here so replay doesn't add messages to the conversation.
// Bound replay future size now that image preparation can await storage.
let _ = Box::pin(prepare_image_response_items(
&self.thread_id.to_string(),
turn_context.config.ephemeral,
&mut prepared_history,
ImagePreparationMode::DetailBased,
&mut history,
ImagePreparationMode::for_replay(&turn_context.config.features),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
))
.await;
prepare_audio_response_items(&mut prepared_history);
assert_eq!(
prepared_history.len(),
metadata.len(),
"replay media preparation must remain one-to-one when resize notices are disabled"
);
history = prepared_history
.into_iter()
.zip(metadata)
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
.collect();
for envelope in &mut history {
prepare_audio_response_items(std::slice::from_mut(&mut envelope.item));
}
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
// Ordinary replay needs no reviewer refresh. Only an opaque
// checkpoint can make reconstruction depend on reviewer compatibility.
@@ -3946,21 +3929,16 @@ impl Session {
item.set_create_time_if_missing(Self::response_item_create_time());
}
/// Prepares media using the originating model and preserves existing item identity.
/// Prepares media for the model that received these items and keeps their existing IDs.
/// Each saved original stays with the item that contains the prepared image.
pub(crate) async fn prepare_conversation_items_for_history(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: &[ResponseItem],
) -> (Vec<ResponseItem>, Vec<ImagePreparationMetadata>) {
let mut items = items.to_vec();
let image_preparation_mode =
if unified_image_budget_enabled(&turn_context.config.features, model_info) {
ImagePreparationMode::UnifiedBudget
} else {
ImagePreparationMode::DetailBased
};
let image_resize_notice_mode = if turn_context
mut items: Vec<ResponseItemEnvelope>,
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
let image_mode = ImagePreparationMode::for_model(&turn_context.config.features, model_info);
let resize_notices = if turn_context
.config
.features
.enabled(Feature::ImageResizeNotice)
@@ -3974,14 +3952,15 @@ impl Session {
&self.thread_id.to_string(),
turn_context.config.ephemeral,
&mut items,
image_preparation_mode,
image_resize_notice_mode,
image_mode,
resize_notices,
self.services.image_store.as_ref(),
))
.await;
prepare_audio_response_items(&mut items);
// Most response items get their passthrough turn ID at the durable history boundary.
for item in &mut items {
for envelope in &mut items {
let item = &mut envelope.item;
prepare_audio_response_items(std::slice::from_mut(item));
// Most response items get their passthrough turn ID at the durable history boundary.
Self::stamp_response_item_for_history(item, &turn_context.sub_id);
self.assign_missing_response_item_id(item);
}
@@ -4039,22 +4018,20 @@ impl Session {
/// Appends to history, persists the prepared items, then notifies raw-item observers.
/// Execution callers supply their captured model; standalone callers need no tool runtime.
#[tracing::instrument(level = "trace", skip_all, fields(item_count = items.len()))]
pub(crate) async fn record_conversation_items(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: &[ResponseItem],
) {
let (items, image_preparations) = self
.prepare_conversation_items_for_history(turn_context, model_info, items)
.await;
let items = items.into_iter().map(ResponseItemEnvelope::new).collect();
self.record_prepared_conversation_items(
self.record_annotated_conversation_items(
turn_context,
model_info,
items,
image_preparations,
items
.iter()
.cloned()
.map(ResponseItemEnvelope::new)
.collect(),
)
.await;
}
@@ -4696,14 +4673,15 @@ impl Session {
) {
let communication = communication.into();
let response_item = Self::inter_agent_history_item(&communication, &turn_context.config);
let (items, _) = self
let (mut prepared, _) = self
.prepare_conversation_items_for_history(
turn_context,
model_info,
std::slice::from_ref(&response_item),
vec![ResponseItemEnvelope::new(response_item)],
)
.await;
let mut response_item = ResponseItemEnvelope::new(items[0].clone());
let mut response_item = prepared.remove(0);
let items = [response_item.item.clone()];
// A send confirmed after the pending snapshot must not reach the rollout
// before this boundary; older readers assign deliveries by physical order.
let boundary = self
@@ -5978,7 +5956,7 @@ impl Session {
)]);
}
let (prepared_items, image_preparations) = self
.prepare_annotated_conversation_items_for_history(
.prepare_conversation_items_for_history(
turn_context,
model_info,
vec![ResponseItemEnvelope {
+6 -3
View File
@@ -390,14 +390,17 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
role: "developer".to_string(),
tools: Arc::from([]),
},
];
]
.into_iter()
.map(ResponseItemEnvelope::new)
.collect();
let (items, _) = session
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), &items)
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), items)
.await;
assert_eq!(items.len(), 3);
for (item, prefix) in items.iter().zip(["msg_", "amsg_", "at_"]) {
let suffix = item.id().unwrap().strip_prefix(prefix).unwrap();
let suffix = item.item.id().unwrap().strip_prefix(prefix).unwrap();
assert_eq!(Uuid::parse_str(suffix).unwrap().get_version_num(), 7);
}
}
+8 -2
View File
@@ -6,6 +6,7 @@ use crate::context::NodeReplReviewEvidence;
use crate::context::NodeReplReviewEvidenceMode;
use crate::context::node_repl_review_evidence_mode;
use crate::function_tool::FunctionCallError;
use crate::image_preparation::prompt_image_decoder;
use crate::mcp_tool_call::handle_mcp_tool_call;
use crate::original_image_detail::can_request_original_image_detail;
use crate::session::session::Session;
@@ -415,6 +416,7 @@ impl CoreToolRuntime for McpHandler {
.and_then(Value::as_bool)
== Some(true)
};
let decoder = prompt_image_decoder(&invocation.turn.config.features);
let mut captured_image_bytes = 0_usize;
let mut items = content
.iter()
@@ -458,8 +460,12 @@ impl CoreToolRuntime for McpHandler {
.and_then(|detail| serde_json::from_value(detail.clone()).ok());
let image_url =
format!("data:{};base64,{payload}", mime_type.to_ascii_lowercase());
load_data_url_for_prompt_uncached(&image_url, PromptImageMode::Original)
.ok()?;
load_data_url_for_prompt_uncached(
&image_url,
PromptImageMode::Original,
decoder,
)
.ok()?;
captured_image_bytes = next_image_bytes;
Some(UserInput::Image {
image: codex_protocol::models::ImageReference::Inline { image_url },
+25 -6
View File
@@ -10,10 +10,14 @@ use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::openai_models::InputModality;
use codex_utils_image::MAX_SANITIZED_IMAGE_BYTES;
use codex_utils_image::PromptImageDecoder;
use codex_utils_image::data_url_from_bytes;
use serde::Deserialize;
use crate::function_tool::FunctionCallError;
use crate::image_preparation::ImagePreparationMode;
use crate::image_preparation::resize_image;
use crate::original_image_detail::can_request_original_image_detail;
use crate::tools::context::ToolEnvironments;
use crate::tools::context::ToolInvocation;
@@ -165,6 +169,16 @@ impl ViewImageHandler {
"image path `{model_visible_path}` is not a file"
)));
}
let mode = ImagePreparationMode::for_model(
&turn.config.features,
&step_context.settings.model_info,
);
let sanitize = mode.decoder == PromptImageDecoder::Sanitizer;
let invalid_image =
|| FunctionCallError::RespondToModel(VIEW_IMAGE_INVALID_MESSAGE.to_string());
if sanitize && metadata.size > MAX_SANITIZED_IMAGE_BYTES as u64 {
return Err(invalid_image());
}
let file_bytes = fs
.read_file(&path_uri, ReadFileOptions::default(), Some(&sandbox))
.await
@@ -173,11 +187,12 @@ impl ViewImageHandler {
"unable to read image at `{model_visible_path}`: {error}"
))
})?;
// Reject non-images before their bytes can reach code mode without changing
// valid image bytes, metadata, or centralized image preparation.
image::load_from_memory(&file_bytes).map_err(|_| {
FunctionCallError::RespondToModel(VIEW_IMAGE_INVALID_MESSAGE.to_string())
})?;
// Validate before passing the image to Code Mode or history.
if !sanitize {
image::load_from_memory(&file_bytes).map_err(|_| invalid_image())?;
} else if file_bytes.len() > MAX_SANITIZED_IMAGE_BYTES {
return Err(invalid_image());
}
let can_request_original_detail =
can_request_original_image_detail(&step_context.settings.model_info);
@@ -189,8 +204,12 @@ impl ViewImageHandler {
DEFAULT_IMAGE_DETAIL
};
// The history insertion path owns image preparation and resizing.
// Return the original so history can resize it and report the change to the model.
let image_url = data_url_from_bytes("application/octet-stream", &file_bytes);
if sanitize {
// Check the image now so view_image can report an error before returning it.
resize_image(&image_url, &mut Some(image_detail), mode).map_err(|_| invalid_image())?;
}
let item = TurnItem::ImageView(ImageViewItem {
id: call_id,