Add opt-in prompt image sanitization (#61341)

## What changed

- Add the disabled-by-default `safe_image_decoder` feature to decode and re-encode prompt images without source metadata, with byte and decoded-pixel limits.
- Apply JPEG orientation and supported ICC color conversion to sRGB, emitting PNG for PNG/GIF inputs and JPEG for JPEG/WebP inputs.
- Use the selected decoder for image preparation, `view_image` validation, and MCP image validation. Include the decoder in image cache keys.
- Preserve original inline images in history metadata so resume and fork can regenerate prepared images without repeated JPEG encoding changing their bytes.

## Testing

Add regression tests for identical JPEG replay after history serialization and for preventing legacy cache entries from bypassing sanitization.

GitOrigin-RevId: 718cd83d0a233e2eb8efd1a64c14d8944e44c350
This commit is contained in:
Vivian Fang
2026-11-13 18:44:31 +00:00
committed by copyberry
parent a4b69a0ae9
commit 7844815bd7
21 changed files with 632 additions and 216 deletions
+7 -39
View File
@@ -1,7 +1,6 @@
use super::TurnInput as PendingTurnInput;
use super::session::Session;
use super::turn_context::TurnContext;
use codex_analytics::ImagePreparationMetadata;
use codex_features::Feature;
use codex_history::ResponseItemEnvelope;
use codex_protocol::error::CodexErr;
@@ -126,24 +125,20 @@ impl Session {
envelope
}
#[tracing::instrument(
name = "record_conversation_items",
level = "trace",
skip_all,
fields(item_count = items.len())
)]
pub(crate) async fn record_annotated_conversation_items(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: Vec<ResponseItemEnvelope>,
) {
if items.iter().all(|item| item.metadata.is_none()) {
let items = items
.into_iter()
.map(ResponseItemEnvelope::into_item)
.collect::<Vec<_>>();
self.record_conversation_items(turn_context, model_info, &items)
.await;
return;
}
let (annotated_items, image_preparations) = self
.prepare_annotated_conversation_items_for_history(turn_context, model_info, items)
.prepare_conversation_items_for_history(turn_context, model_info, items)
.await;
self.record_prepared_conversation_items(
turn_context,
@@ -154,33 +149,6 @@ impl Session {
.await;
}
pub(crate) async fn prepare_annotated_conversation_items_for_history(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: Vec<ResponseItemEnvelope>,
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
let mut annotated_items = Vec::with_capacity(items.len());
let mut image_preparations = Vec::new();
for envelope in items {
let (prepared_items, prepared_images) = self
.prepare_conversation_items_for_history(
turn_context,
model_info,
std::slice::from_ref(&envelope.item),
)
.await;
image_preparations.extend(prepared_images);
let mut metadata = envelope.metadata;
annotated_items.extend(prepared_items.into_iter().map(|item| ResponseItemEnvelope {
item,
metadata: metadata.take(),
}));
}
(annotated_items, image_preparations)
}
/// Injects items into active work, or records them without starting a turn.
pub(crate) async fn inject_no_new_turn(
&self,
+31 -53
View File
@@ -49,7 +49,6 @@ use crate::exec_policy::default_policy_path;
use crate::image_preparation::ImagePreparationMode;
use crate::image_preparation::ImageResizeNoticeMode;
use crate::image_preparation::prepare_response_items as prepare_image_response_items;
use crate::image_preparation::unified_image_budget_enabled;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::realtime_history::RealtimeEventOrder;
@@ -1999,38 +1998,22 @@ impl Session {
.and_then(|context| context.direct_only_tool_namespaces.clone())
.map(|namespaces| (turn.turn_id.clone(), namespaces))
});
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy media is processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// Replay disables image-resize notices, so media preparation remains one-to-one. Keep
// the prior batch behavior and carry history-only metadata in a positional sidecar.
let (mut prepared_history, metadata): (Vec<_>, Vec<_>) = history
.into_iter()
.map(|envelope| (envelope.item, envelope.metadata))
.unzip();
// Replay must not upload or migrate recorded history. The inline store returns prepared
// inline bytes, while existing file references bypass preparation and remain unchanged.
// Prepare inline images in memory on resume or fork. Keep file references as they are.
// The inline store avoids uploads, and replay leaves the saved history untouched.
// Skip resize notices here so replay doesn't add messages to the conversation.
// Bound replay future size now that image preparation can await storage.
let _ = Box::pin(prepare_image_response_items(
&self.thread_id.to_string(),
turn_context.config.ephemeral,
&mut prepared_history,
ImagePreparationMode::DetailBased,
&mut history,
ImagePreparationMode::for_replay(&turn_context.config.features),
ImageResizeNoticeMode::Disabled,
&InlineAttachmentStore,
))
.await;
prepare_audio_response_items(&mut prepared_history);
assert_eq!(
prepared_history.len(),
metadata.len(),
"replay media preparation must remain one-to-one when resize notices are disabled"
);
history = prepared_history
.into_iter()
.zip(metadata)
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
.collect();
for envelope in &mut history {
prepare_audio_response_items(std::slice::from_mut(&mut envelope.item));
}
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
// Ordinary replay needs no reviewer refresh. Only an opaque
// checkpoint can make reconstruction depend on reviewer compatibility.
@@ -3946,21 +3929,16 @@ impl Session {
item.set_create_time_if_missing(Self::response_item_create_time());
}
/// Prepares media using the originating model and preserves existing item identity.
/// Prepares media for the model that received these items and keeps their existing IDs.
/// Each saved original stays with the item that contains the prepared image.
pub(crate) async fn prepare_conversation_items_for_history(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: &[ResponseItem],
) -> (Vec<ResponseItem>, Vec<ImagePreparationMetadata>) {
let mut items = items.to_vec();
let image_preparation_mode =
if unified_image_budget_enabled(&turn_context.config.features, model_info) {
ImagePreparationMode::UnifiedBudget
} else {
ImagePreparationMode::DetailBased
};
let image_resize_notice_mode = if turn_context
mut items: Vec<ResponseItemEnvelope>,
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
let image_mode = ImagePreparationMode::for_model(&turn_context.config.features, model_info);
let resize_notices = if turn_context
.config
.features
.enabled(Feature::ImageResizeNotice)
@@ -3974,14 +3952,15 @@ impl Session {
&self.thread_id.to_string(),
turn_context.config.ephemeral,
&mut items,
image_preparation_mode,
image_resize_notice_mode,
image_mode,
resize_notices,
self.services.image_store.as_ref(),
))
.await;
prepare_audio_response_items(&mut items);
// Most response items get their passthrough turn ID at the durable history boundary.
for item in &mut items {
for envelope in &mut items {
let item = &mut envelope.item;
prepare_audio_response_items(std::slice::from_mut(item));
// Most response items get their passthrough turn ID at the durable history boundary.
Self::stamp_response_item_for_history(item, &turn_context.sub_id);
self.assign_missing_response_item_id(item);
}
@@ -4039,22 +4018,20 @@ impl Session {
/// Appends to history, persists the prepared items, then notifies raw-item observers.
/// Execution callers supply their captured model; standalone callers need no tool runtime.
#[tracing::instrument(level = "trace", skip_all, fields(item_count = items.len()))]
pub(crate) async fn record_conversation_items(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
items: &[ResponseItem],
) {
let (items, image_preparations) = self
.prepare_conversation_items_for_history(turn_context, model_info, items)
.await;
let items = items.into_iter().map(ResponseItemEnvelope::new).collect();
self.record_prepared_conversation_items(
self.record_annotated_conversation_items(
turn_context,
model_info,
items,
image_preparations,
items
.iter()
.cloned()
.map(ResponseItemEnvelope::new)
.collect(),
)
.await;
}
@@ -4696,14 +4673,15 @@ impl Session {
) {
let communication = communication.into();
let response_item = Self::inter_agent_history_item(&communication, &turn_context.config);
let (items, _) = self
let (mut prepared, _) = self
.prepare_conversation_items_for_history(
turn_context,
model_info,
std::slice::from_ref(&response_item),
vec![ResponseItemEnvelope::new(response_item)],
)
.await;
let mut response_item = ResponseItemEnvelope::new(items[0].clone());
let mut response_item = prepared.remove(0);
let items = [response_item.item.clone()];
// A send confirmed after the pending snapshot must not reach the rollout
// before this boundary; older readers assign deliveries by physical order.
let boundary = self
@@ -5978,7 +5956,7 @@ impl Session {
)]);
}
let (prepared_items, image_preparations) = self
.prepare_annotated_conversation_items_for_history(
.prepare_conversation_items_for_history(
turn_context,
model_info,
vec![ResponseItemEnvelope {
+6 -3
View File
@@ -390,14 +390,17 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
role: "developer".to_string(),
tools: Arc::from([]),
},
];
]
.into_iter()
.map(ResponseItemEnvelope::new)
.collect();
let (items, _) = session
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), &items)
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), items)
.await;
assert_eq!(items.len(), 3);
for (item, prefix) in items.iter().zip(["msg_", "amsg_", "at_"]) {
let suffix = item.id().unwrap().strip_prefix(prefix).unwrap();
let suffix = item.item.id().unwrap().strip_prefix(prefix).unwrap();
assert_eq!(Uuid::parse_str(suffix).unwrap().get_version_num(), 7);
}
}