mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add opt-in prompt image sanitization (#61341)
## What changed - Add the disabled-by-default `safe_image_decoder` feature to decode and re-encode prompt images without source metadata, with byte and decoded-pixel limits. - Apply JPEG orientation and supported ICC color conversion to sRGB, emitting PNG for PNG/GIF inputs and JPEG for JPEG/WebP inputs. - Use the selected decoder for image preparation, `view_image` validation, and MCP image validation. Include the decoder in image cache keys. - Preserve original inline images in history metadata so resume and fork can regenerate prepared images without repeated JPEG encoding changing their bytes. ## Testing Add regression tests for identical JPEG replay after history serialization and for preventing legacy cache entries from bypassing sanitization. GitOrigin-RevId: 718cd83d0a233e2eb8efd1a64c14d8944e44c350
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
use super::TurnInput as PendingTurnInput;
|
||||
use super::session::Session;
|
||||
use super::turn_context::TurnContext;
|
||||
use codex_analytics::ImagePreparationMetadata;
|
||||
use codex_features::Feature;
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::error::CodexErr;
|
||||
@@ -126,24 +125,20 @@ impl Session {
|
||||
envelope
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "record_conversation_items",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(item_count = items.len())
|
||||
)]
|
||||
pub(crate) async fn record_annotated_conversation_items(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
items: Vec<ResponseItemEnvelope>,
|
||||
) {
|
||||
if items.iter().all(|item| item.metadata.is_none()) {
|
||||
let items = items
|
||||
.into_iter()
|
||||
.map(ResponseItemEnvelope::into_item)
|
||||
.collect::<Vec<_>>();
|
||||
self.record_conversation_items(turn_context, model_info, &items)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
let (annotated_items, image_preparations) = self
|
||||
.prepare_annotated_conversation_items_for_history(turn_context, model_info, items)
|
||||
.prepare_conversation_items_for_history(turn_context, model_info, items)
|
||||
.await;
|
||||
self.record_prepared_conversation_items(
|
||||
turn_context,
|
||||
@@ -154,33 +149,6 @@ impl Session {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_annotated_conversation_items_for_history(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
items: Vec<ResponseItemEnvelope>,
|
||||
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
|
||||
let mut annotated_items = Vec::with_capacity(items.len());
|
||||
let mut image_preparations = Vec::new();
|
||||
for envelope in items {
|
||||
let (prepared_items, prepared_images) = self
|
||||
.prepare_conversation_items_for_history(
|
||||
turn_context,
|
||||
model_info,
|
||||
std::slice::from_ref(&envelope.item),
|
||||
)
|
||||
.await;
|
||||
image_preparations.extend(prepared_images);
|
||||
|
||||
let mut metadata = envelope.metadata;
|
||||
annotated_items.extend(prepared_items.into_iter().map(|item| ResponseItemEnvelope {
|
||||
item,
|
||||
metadata: metadata.take(),
|
||||
}));
|
||||
}
|
||||
(annotated_items, image_preparations)
|
||||
}
|
||||
|
||||
/// Injects items into active work, or records them without starting a turn.
|
||||
pub(crate) async fn inject_no_new_turn(
|
||||
&self,
|
||||
|
||||
@@ -49,7 +49,6 @@ use crate::exec_policy::default_policy_path;
|
||||
use crate::image_preparation::ImagePreparationMode;
|
||||
use crate::image_preparation::ImageResizeNoticeMode;
|
||||
use crate::image_preparation::prepare_response_items as prepare_image_response_items;
|
||||
use crate::image_preparation::unified_image_budget_enabled;
|
||||
use crate::parse_turn_item;
|
||||
use crate::realtime_conversation::RealtimeConversationManager;
|
||||
use crate::realtime_history::RealtimeEventOrder;
|
||||
@@ -1999,38 +1998,22 @@ impl Session {
|
||||
.and_then(|context| context.direct_only_tool_namespaces.clone())
|
||||
.map(|namespaces| (turn.turn_id.clone(), namespaces))
|
||||
});
|
||||
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
|
||||
// installing it, so legacy media is processed once for this resume or fork and
|
||||
// will be processed again if the rollout is reconstructed in a future session.
|
||||
// Replay disables image-resize notices, so media preparation remains one-to-one. Keep
|
||||
// the prior batch behavior and carry history-only metadata in a positional sidecar.
|
||||
let (mut prepared_history, metadata): (Vec<_>, Vec<_>) = history
|
||||
.into_iter()
|
||||
.map(|envelope| (envelope.item, envelope.metadata))
|
||||
.unzip();
|
||||
// Replay must not upload or migrate recorded history. The inline store returns prepared
|
||||
// inline bytes, while existing file references bypass preparation and remain unchanged.
|
||||
// Prepare inline images in memory on resume or fork. Keep file references as they are.
|
||||
// The inline store avoids uploads, and replay leaves the saved history untouched.
|
||||
// Skip resize notices here so replay doesn't add messages to the conversation.
|
||||
// Bound replay future size now that image preparation can await storage.
|
||||
let _ = Box::pin(prepare_image_response_items(
|
||||
&self.thread_id.to_string(),
|
||||
turn_context.config.ephemeral,
|
||||
&mut prepared_history,
|
||||
ImagePreparationMode::DetailBased,
|
||||
&mut history,
|
||||
ImagePreparationMode::for_replay(&turn_context.config.features),
|
||||
ImageResizeNoticeMode::Disabled,
|
||||
&InlineAttachmentStore,
|
||||
))
|
||||
.await;
|
||||
prepare_audio_response_items(&mut prepared_history);
|
||||
assert_eq!(
|
||||
prepared_history.len(),
|
||||
metadata.len(),
|
||||
"replay media preparation must remain one-to-one when resize notices are disabled"
|
||||
);
|
||||
history = prepared_history
|
||||
.into_iter()
|
||||
.zip(metadata)
|
||||
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
|
||||
.collect();
|
||||
for envelope in &mut history {
|
||||
prepare_audio_response_items(std::slice::from_mut(&mut envelope.item));
|
||||
}
|
||||
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
|
||||
// Ordinary replay needs no reviewer refresh. Only an opaque
|
||||
// checkpoint can make reconstruction depend on reviewer compatibility.
|
||||
@@ -3946,21 +3929,16 @@ impl Session {
|
||||
item.set_create_time_if_missing(Self::response_item_create_time());
|
||||
}
|
||||
|
||||
/// Prepares media using the originating model and preserves existing item identity.
|
||||
/// Prepares media for the model that received these items and keeps their existing IDs.
|
||||
/// Each saved original stays with the item that contains the prepared image.
|
||||
pub(crate) async fn prepare_conversation_items_for_history(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
items: &[ResponseItem],
|
||||
) -> (Vec<ResponseItem>, Vec<ImagePreparationMetadata>) {
|
||||
let mut items = items.to_vec();
|
||||
let image_preparation_mode =
|
||||
if unified_image_budget_enabled(&turn_context.config.features, model_info) {
|
||||
ImagePreparationMode::UnifiedBudget
|
||||
} else {
|
||||
ImagePreparationMode::DetailBased
|
||||
};
|
||||
let image_resize_notice_mode = if turn_context
|
||||
mut items: Vec<ResponseItemEnvelope>,
|
||||
) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {
|
||||
let image_mode = ImagePreparationMode::for_model(&turn_context.config.features, model_info);
|
||||
let resize_notices = if turn_context
|
||||
.config
|
||||
.features
|
||||
.enabled(Feature::ImageResizeNotice)
|
||||
@@ -3974,14 +3952,15 @@ impl Session {
|
||||
&self.thread_id.to_string(),
|
||||
turn_context.config.ephemeral,
|
||||
&mut items,
|
||||
image_preparation_mode,
|
||||
image_resize_notice_mode,
|
||||
image_mode,
|
||||
resize_notices,
|
||||
self.services.image_store.as_ref(),
|
||||
))
|
||||
.await;
|
||||
prepare_audio_response_items(&mut items);
|
||||
// Most response items get their passthrough turn ID at the durable history boundary.
|
||||
for item in &mut items {
|
||||
for envelope in &mut items {
|
||||
let item = &mut envelope.item;
|
||||
prepare_audio_response_items(std::slice::from_mut(item));
|
||||
// Most response items get their passthrough turn ID at the durable history boundary.
|
||||
Self::stamp_response_item_for_history(item, &turn_context.sub_id);
|
||||
self.assign_missing_response_item_id(item);
|
||||
}
|
||||
@@ -4039,22 +4018,20 @@ impl Session {
|
||||
|
||||
/// Appends to history, persists the prepared items, then notifies raw-item observers.
|
||||
/// Execution callers supply their captured model; standalone callers need no tool runtime.
|
||||
#[tracing::instrument(level = "trace", skip_all, fields(item_count = items.len()))]
|
||||
pub(crate) async fn record_conversation_items(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
items: &[ResponseItem],
|
||||
) {
|
||||
let (items, image_preparations) = self
|
||||
.prepare_conversation_items_for_history(turn_context, model_info, items)
|
||||
.await;
|
||||
let items = items.into_iter().map(ResponseItemEnvelope::new).collect();
|
||||
self.record_prepared_conversation_items(
|
||||
self.record_annotated_conversation_items(
|
||||
turn_context,
|
||||
model_info,
|
||||
items,
|
||||
image_preparations,
|
||||
items
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(ResponseItemEnvelope::new)
|
||||
.collect(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -4696,14 +4673,15 @@ impl Session {
|
||||
) {
|
||||
let communication = communication.into();
|
||||
let response_item = Self::inter_agent_history_item(&communication, &turn_context.config);
|
||||
let (items, _) = self
|
||||
let (mut prepared, _) = self
|
||||
.prepare_conversation_items_for_history(
|
||||
turn_context,
|
||||
model_info,
|
||||
std::slice::from_ref(&response_item),
|
||||
vec![ResponseItemEnvelope::new(response_item)],
|
||||
)
|
||||
.await;
|
||||
let mut response_item = ResponseItemEnvelope::new(items[0].clone());
|
||||
let mut response_item = prepared.remove(0);
|
||||
let items = [response_item.item.clone()];
|
||||
// A send confirmed after the pending snapshot must not reach the rollout
|
||||
// before this boundary; older readers assign deliveries by physical order.
|
||||
let boundary = self
|
||||
@@ -5978,7 +5956,7 @@ impl Session {
|
||||
)]);
|
||||
}
|
||||
let (prepared_items, image_preparations) = self
|
||||
.prepare_annotated_conversation_items_for_history(
|
||||
.prepare_conversation_items_for_history(
|
||||
turn_context,
|
||||
model_info,
|
||||
vec![ResponseItemEnvelope {
|
||||
|
||||
@@ -390,14 +390,17 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
|
||||
role: "developer".to_string(),
|
||||
tools: Arc::from([]),
|
||||
},
|
||||
];
|
||||
]
|
||||
.into_iter()
|
||||
.map(ResponseItemEnvelope::new)
|
||||
.collect();
|
||||
let (items, _) = session
|
||||
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), &items)
|
||||
.prepare_conversation_items_for_history(&turn_context, turn_context.model_info(), items)
|
||||
.await;
|
||||
|
||||
assert_eq!(items.len(), 3);
|
||||
for (item, prefix) in items.iter().zip(["msg_", "amsg_", "at_"]) {
|
||||
let suffix = item.id().unwrap().strip_prefix(prefix).unwrap();
|
||||
let suffix = item.item.id().unwrap().strip_prefix(prefix).unwrap();
|
||||
assert_eq!(Uuid::parse_str(suffix).unwrap().get_version_num(), 7);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user