mirror of
https://github.com/openai/codex.git
synced 2026-10-25 18:28:38 +00:00
Move Guardian image selection into shared context sections (#43601)
## What changed Move bounded transcript image collection into `codex-guardian-context` and expose it through an optional `TranscriptImages` section. Update the async scorer to consume this section, preserving source filtering, image detail, omission accounting, and the existing limits of four images and 8 MiB of image URL data. ## Testing Add coverage for source order, tool-output filtering, disabled image collection, oldest-image eviction, oversized images, and omitted-byte accounting. GitOrigin-RevId: 097799ea1be6d73b347088f959de1fd23f02ec2c
This commit is contained in:
@@ -239,6 +239,7 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
|
||||
push_text(headings.intro.to_string());
|
||||
let mut action_items = Vec::new();
|
||||
let mut permission_items = Vec::new();
|
||||
let mut image_items = Vec::new();
|
||||
for section in sections {
|
||||
match section {
|
||||
ContextSection::RootConversation { items }
|
||||
@@ -253,6 +254,14 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
|
||||
| ContextSection::TrustedSkills(_) => {
|
||||
unreachable!("trusted review and tool sections are async-only")
|
||||
}
|
||||
ContextSection::TranscriptImages(images) => {
|
||||
image_items.extend(images.images.into_iter().filter_map(|image| match image {
|
||||
codex_protocol::models::ContentItem::InputImage { image_url, detail } => {
|
||||
Some(UserInput::Image { image_url, detail })
|
||||
}
|
||||
_ => None,
|
||||
}));
|
||||
}
|
||||
ContextSection::ConversationTranscript { .. } => {}
|
||||
ContextSection::PermissionContext { items } => permission_items = items,
|
||||
ContextSection::PlannedAction(action) => {
|
||||
@@ -276,6 +285,7 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
|
||||
for text in permission_items {
|
||||
push_text(text);
|
||||
}
|
||||
items.extend(image_items);
|
||||
let mut node_repl_evidence_sequence = reviewed_node_repl_evidence_sequence;
|
||||
if node_repl_transcripts_enabled
|
||||
&& let Some(fragment) = session
|
||||
@@ -488,6 +498,7 @@ pub(super) fn collect_guardian_context(
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -73,6 +73,8 @@ use super::wrapper_lag::WrapperLag;
|
||||
use codex_core::context::GuardianReviewEvidenceFragment;
|
||||
use codex_guardian_context::PreviousReviews;
|
||||
use codex_guardian_context::ReviewEvidence;
|
||||
use codex_guardian_context::TranscriptImageInput;
|
||||
use codex_guardian_context::TranscriptImages;
|
||||
use codex_guardian_context::render_review_evidence;
|
||||
|
||||
enum ClassificationOutcome {
|
||||
@@ -479,9 +481,6 @@ impl GuardianV2Extension {
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let rendered_images = guardian_config
|
||||
.transcript
|
||||
.images(input.conversation_history.review_items(), node_repl_images);
|
||||
// Capture root evidence before background metadata resolution or model I/O.
|
||||
// Later root changes invalidate this sample through its captured authorization version.
|
||||
let root_snapshot = if context_mode == GuardianContextMode::ThreadOwned {
|
||||
@@ -581,6 +580,14 @@ impl GuardianV2Extension {
|
||||
previous_reviews: Some(&reviews),
|
||||
trusted_tool: trusted_tool_context.as_ref(),
|
||||
trusted_skill_paths: &trusted_skill_paths,
|
||||
images: Some(TranscriptImageInput {
|
||||
enabled: guardian_config.transcript.include_images,
|
||||
include_tool_outputs: guardian_config
|
||||
.transcript
|
||||
.sources
|
||||
.contains(&super::transcript::TranscriptSource::ToolOutputs),
|
||||
node_repl_images: &node_repl_images,
|
||||
}),
|
||||
})
|
||||
});
|
||||
let transcript = match transcript {
|
||||
@@ -602,19 +609,16 @@ impl GuardianV2Extension {
|
||||
}
|
||||
};
|
||||
drop(history);
|
||||
drop(node_repl_images);
|
||||
truncations.extend(transcript.truncations);
|
||||
truncations.record(
|
||||
"transcript_image",
|
||||
rendered_images.omitted_bytes,
|
||||
/*retained_bytes*/ 0,
|
||||
);
|
||||
let images = rendered_images.images;
|
||||
let mut classification_input = Vec::new();
|
||||
let mut trusted_review_evidence = None;
|
||||
let mut trusted_tool_context = None;
|
||||
let mut trusted_skills = None;
|
||||
let mut rendered_images = TranscriptImages::default();
|
||||
for section in transcript.sections {
|
||||
match section {
|
||||
ContextSection::TranscriptImages(images) => rendered_images = images,
|
||||
ContextSection::TrustedSkills(skills) => trusted_skills = Some(skills),
|
||||
ContextSection::TrustedTool(tool) => trusted_tool_context = Some(tool),
|
||||
ContextSection::PreviousReviews(reviews) => {
|
||||
@@ -636,6 +640,12 @@ impl GuardianV2Extension {
|
||||
}
|
||||
}
|
||||
}
|
||||
truncations.record(
|
||||
"transcript_image",
|
||||
rendered_images.omitted_bytes,
|
||||
/*retained_bytes*/ 0,
|
||||
);
|
||||
let images = rendered_images.images;
|
||||
let mut failure_reason = "invalid_output";
|
||||
let mut classification_risk = None;
|
||||
let mut classification_finished_at = None;
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use codex_extension_api::ConversationHistorySnapshot;
|
||||
use codex_extension_api::ResponseItem;
|
||||
pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;
|
||||
@@ -18,13 +16,11 @@ use codex_guardian_context::SectionError;
|
||||
use codex_guardian_context::SectionHistory;
|
||||
use codex_guardian_context::SectionInput;
|
||||
use codex_guardian_context::TranscriptEntryLimits;
|
||||
use codex_guardian_context::TranscriptImageInput;
|
||||
use codex_guardian_context::TranscriptRetentionConfig;
|
||||
use codex_guardian_context::TrustedTool;
|
||||
use codex_guardian_context::default_registry;
|
||||
pub(crate) use codex_guardian_context::truncate_text as truncate_entry;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
|
||||
use self::window::TranscriptWindow;
|
||||
@@ -37,8 +33,6 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
|
||||
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
|
||||
pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
|
||||
pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;
|
||||
const MAX_TRANSCRIPT_IMAGES: usize = 4;
|
||||
const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TranscriptEntryKind {
|
||||
User,
|
||||
@@ -65,6 +59,7 @@ pub(crate) struct ContextInput<'a> {
|
||||
pub(crate) previous_reviews: Option<&'a PreviousReviews>,
|
||||
pub(crate) trusted_tool: Option<&'a TrustedTool>,
|
||||
pub(crate) trusted_skill_paths: &'a [String],
|
||||
pub(crate) images: Option<TranscriptImageInput<'a>>,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedContext {
|
||||
@@ -72,11 +67,6 @@ pub(crate) struct RenderedContext {
|
||||
pub(crate) truncations: Vec<TruncationObservation>,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedImages {
|
||||
pub(crate) images: Vec<ContentItem>,
|
||||
pub(crate) omitted_bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct TranscriptConfig {
|
||||
pub(crate) sources: Vec<TranscriptSource>,
|
||||
@@ -103,84 +93,6 @@ impl Default for TranscriptConfig {
|
||||
}
|
||||
|
||||
impl TranscriptConfig {
|
||||
pub(crate) fn images<'a>(
|
||||
&self,
|
||||
items: impl IntoIterator<Item = &'a ResponseItem>,
|
||||
node_repl_images: impl IntoIterator<Item = ContentItem>,
|
||||
) -> RenderedImages {
|
||||
if !self.include_images {
|
||||
return RenderedImages {
|
||||
images: Vec::new(),
|
||||
omitted_bytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let mut images = VecDeque::new();
|
||||
let mut image_bytes = 0usize;
|
||||
let mut omitted_bytes = 0usize;
|
||||
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
return;
|
||||
}
|
||||
while images.len() >= MAX_TRANSCRIPT_IMAGES
|
||||
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
|
||||
{
|
||||
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
|
||||
break;
|
||||
};
|
||||
image_bytes -= image_url.len();
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
}
|
||||
image_bytes += image_url.len();
|
||||
images.push_back(ContentItem::InputImage {
|
||||
image_url: image_url.to_owned(),
|
||||
detail,
|
||||
});
|
||||
};
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Message { role, content, .. }
|
||||
if matches!(role.as_str(), "user" | "assistant") =>
|
||||
{
|
||||
for item in content {
|
||||
if let ContentItem::InputImage { image_url, detail } = item {
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. }
|
||||
if self.sources.contains(&TranscriptSource::ToolOutputs) =>
|
||||
{
|
||||
if let Some(content) = output.content_items() {
|
||||
for item in content {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
|
||||
item
|
||||
{
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if self.sources.contains(&TranscriptSource::ToolOutputs) {
|
||||
for image in node_repl_images {
|
||||
if let ContentItem::InputImage { image_url, detail } = image {
|
||||
include_image(&image_url, detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RenderedImages {
|
||||
images: images.into_iter().collect(),
|
||||
omitted_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_context(
|
||||
&self,
|
||||
input: ContextInput<'_>,
|
||||
@@ -194,6 +106,7 @@ impl TranscriptConfig {
|
||||
previous_reviews,
|
||||
trusted_tool,
|
||||
trusted_skill_paths,
|
||||
images,
|
||||
} = input;
|
||||
let history = SnapshotHistory(history);
|
||||
let retention = TranscriptRetentionConfig {
|
||||
@@ -224,6 +137,7 @@ impl TranscriptConfig {
|
||||
previous_reviews,
|
||||
trusted_tool,
|
||||
trusted_skill_paths,
|
||||
images,
|
||||
})?;
|
||||
let mut truncations = Vec::new();
|
||||
let sections = context
|
||||
@@ -246,6 +160,9 @@ impl TranscriptConfig {
|
||||
ContextSection::PermissionContext { items } => {
|
||||
ContextSection::PermissionContext { items }
|
||||
}
|
||||
ContextSection::TranscriptImages(images) => {
|
||||
ContextSection::TranscriptImages(images)
|
||||
}
|
||||
ContextSection::TrustedSkills(skills) => ContextSection::TrustedSkills(skills),
|
||||
ContextSection::TrustedTool(tool) => ContextSection::TrustedTool(tool),
|
||||
ContextSection::PreviousReviews(reviews) => {
|
||||
|
||||
@@ -110,6 +110,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -136,6 +137,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("compose authorization and transcript");
|
||||
assert_eq!(
|
||||
@@ -168,6 +170,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -195,6 +198,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -247,6 +251,7 @@ fn transcript_truncates_oversized_entries_without_splitting_characters() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript");
|
||||
let transcript = rendered.transcript_entries();
|
||||
@@ -314,6 +319,7 @@ fn transcript_preserves_first_and_latest_user_messages_and_recent_history() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -379,6 +385,7 @@ fn transcript_preserves_user_restrictions_before_final_assistant_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -418,6 +425,7 @@ fn transcript_preserves_recent_tool_evidence_when_protected_messages_fill_entry_
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript");
|
||||
|
||||
@@ -483,6 +491,7 @@ fn transcript_reserves_five_recent_tool_entries_from_protected_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -538,6 +547,7 @@ fn rejected_commentary_does_not_evict_retained_message_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -578,6 +588,7 @@ fn transcript_evicts_protected_messages_in_cacheable_chunks() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries()
|
||||
@@ -648,6 +659,7 @@ fn transcript_preserves_latest_final_when_reserved_tools_fill_entry_window() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -720,6 +732,7 @@ fn transcript_does_not_protect_legacy_inter_agent_instructions() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -766,6 +779,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -827,6 +841,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -890,6 +905,7 @@ fn transcript_preserves_newest_manual_approval_when_message_budget_overflows() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -949,6 +965,7 @@ fn rejected_message_does_not_evict_retained_tool_entries() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -993,6 +1010,7 @@ fn transcript_evicts_non_user_entries_in_cacheable_chunks() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries()
|
||||
@@ -1065,6 +1083,7 @@ fn transcript_truncates_tool_results_using_standard_budget() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1119,6 +1138,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries(),
|
||||
@@ -1147,6 +1167,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries(),
|
||||
@@ -1193,6 +1214,7 @@ fn configured_reasoning_counts_against_message_budget() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1261,6 +1283,7 @@ fn transcript_keeps_only_manual_approval_developer_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1347,6 +1370,7 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1415,6 +1439,7 @@ fn transcript_omits_encrypted_messages_arguments_and_tool_outputs() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Bounded image selection shared by Guardian consumers.
|
||||
//! Keeps source order and evicts oldest images using the existing count/byte caps.
|
||||
//! Consumers still choose source visibility and model-specific image detail.
|
||||
|
||||
use crate::ContextSection;
|
||||
use crate::SectionContributor;
|
||||
use crate::SectionError;
|
||||
use crate::SectionInput;
|
||||
use crate::SectionScope;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use std::collections::VecDeque;
|
||||
|
||||
const MAX_TRANSCRIPT_IMAGES: usize = 4;
|
||||
const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Request-local image policy and a frozen view of captured REPL screenshots.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct TranscriptImageInput<'a> {
|
||||
pub enabled: bool,
|
||||
pub include_tool_outputs: bool,
|
||||
pub node_repl_images: &'a [ContentItem],
|
||||
}
|
||||
|
||||
/// Selected images and byte omissions for the consumer's existing telemetry.
|
||||
#[derive(Clone, Default, PartialEq)]
|
||||
pub struct TranscriptImages {
|
||||
pub images: Vec<ContentItem>,
|
||||
pub omitted_bytes: usize,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for TranscriptImages {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("TranscriptImages")
|
||||
.field("count", &self.images.len())
|
||||
.field("omitted_bytes", &self.omitted_bytes)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl TranscriptImages {
|
||||
/// Selects images without changing their detail or allocating a second history.
|
||||
pub fn collect<'a>(
|
||||
items: impl IntoIterator<Item = &'a ResponseItem>,
|
||||
input: TranscriptImageInput<'_>,
|
||||
) -> Self {
|
||||
if !input.enabled {
|
||||
return TranscriptImages {
|
||||
images: Vec::new(),
|
||||
omitted_bytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let mut images = VecDeque::new();
|
||||
let mut image_bytes = 0usize;
|
||||
let mut omitted_bytes = 0usize;
|
||||
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
return;
|
||||
}
|
||||
while images.len() >= MAX_TRANSCRIPT_IMAGES
|
||||
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
|
||||
{
|
||||
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
|
||||
break;
|
||||
};
|
||||
image_bytes -= image_url.len();
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
}
|
||||
image_bytes += image_url.len();
|
||||
images.push_back(ContentItem::InputImage {
|
||||
image_url: image_url.to_owned(),
|
||||
detail,
|
||||
});
|
||||
};
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Message { role, content, .. }
|
||||
if matches!(role.as_str(), "user" | "assistant") =>
|
||||
{
|
||||
for item in content {
|
||||
if let ContentItem::InputImage { image_url, detail } = item {
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. }
|
||||
if input.include_tool_outputs =>
|
||||
{
|
||||
if let Some(content) = output.content_items() {
|
||||
for item in content {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
|
||||
item
|
||||
{
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if input.include_tool_outputs {
|
||||
for image in input.node_repl_images {
|
||||
if let ContentItem::InputImage { image_url, detail } = image {
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TranscriptImages {
|
||||
images: images.into_iter().collect(),
|
||||
omitted_bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct TranscriptImagesSection;
|
||||
impl SectionContributor for TranscriptImagesSection {
|
||||
fn scope(&self) -> SectionScope {
|
||||
SectionScope::Shared
|
||||
}
|
||||
fn contribute(&self, input: &SectionInput<'_>) -> Result<Option<ContextSection>, SectionError> {
|
||||
Ok(input.images.map(|images| {
|
||||
ContextSection::TranscriptImages(TranscriptImages::collect(
|
||||
input.history.items(),
|
||||
images,
|
||||
))
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "images_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,95 @@
|
||||
use super::MAX_TRANSCRIPT_IMAGE_BYTES;
|
||||
use super::TranscriptImageInput;
|
||||
use super::TranscriptImages;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
fn image(url: &str) -> ContentItem {
|
||||
ContentItem::InputImage {
|
||||
image_url: url.into(),
|
||||
detail: Some(ImageDetail::High),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_selection_preserves_source_policy_order_and_both_limits() {
|
||||
let history = [
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".into(),
|
||||
content: ["first", "second", "third", "fourth"].map(image).to_vec(),
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
id: None,
|
||||
call_id: Some("screenshot".into()),
|
||||
name: None,
|
||||
namespace: None,
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "tool".into(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
];
|
||||
let repl = [image("repl")];
|
||||
let input = TranscriptImageInput {
|
||||
enabled: true,
|
||||
include_tool_outputs: true,
|
||||
node_repl_images: &repl,
|
||||
};
|
||||
assert_eq!(
|
||||
TranscriptImages::collect(&history, input),
|
||||
TranscriptImages {
|
||||
images: ["third", "fourth", "tool", "repl"].map(image).to_vec(),
|
||||
omitted_bytes: "firstsecond".len(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
TranscriptImages::collect(
|
||||
&history,
|
||||
TranscriptImageInput {
|
||||
include_tool_outputs: false,
|
||||
..input
|
||||
}
|
||||
),
|
||||
TranscriptImages {
|
||||
images: ["first", "second", "third", "fourth"].map(image).to_vec(),
|
||||
omitted_bytes: 0,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
TranscriptImages::collect(
|
||||
&history,
|
||||
TranscriptImageInput {
|
||||
enabled: false,
|
||||
..input
|
||||
}
|
||||
),
|
||||
TranscriptImages::default()
|
||||
);
|
||||
|
||||
let oversized = "x".repeat(MAX_TRANSCRIPT_IMAGE_BYTES + 1);
|
||||
let fits_alone = "y".repeat(MAX_TRANSCRIPT_IMAGE_BYTES);
|
||||
let repl = [image(&oversized), image(&fits_alone), image("recent")];
|
||||
assert_eq!(
|
||||
TranscriptImages::collect(
|
||||
&[],
|
||||
TranscriptImageInput {
|
||||
node_repl_images: &repl,
|
||||
..input
|
||||
}
|
||||
),
|
||||
TranscriptImages {
|
||||
images: vec![image("recent")],
|
||||
omitted_bytes: oversized.len() + fits_alone.len(),
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -50,7 +50,10 @@ mod action;
|
||||
mod authorization;
|
||||
mod entry;
|
||||
mod history;
|
||||
mod images;
|
||||
mod permissions;
|
||||
pub use images::TranscriptImageInput;
|
||||
pub use images::TranscriptImages;
|
||||
mod trusted_skills;
|
||||
mod trusted_tool;
|
||||
pub use trusted_skills::TrustedSkills;
|
||||
@@ -122,6 +125,8 @@ pub struct SectionInput<'a> {
|
||||
pub trusted_tool: Option<&'a TrustedTool>,
|
||||
/// Current-turn and delegated skill paths verified and bounded by the host.
|
||||
pub trusted_skill_paths: &'a [String],
|
||||
/// Optional consumer image policy; no history images are added implicitly.
|
||||
pub images: Option<TranscriptImageInput<'a>>,
|
||||
}
|
||||
|
||||
/// Supplies repeatable, zero-copy access to a host-owned conversation snapshot.
|
||||
@@ -216,6 +221,7 @@ pub fn default_registry() -> &'static SectionRegistry {
|
||||
registry.register(RetainedUserInstructionsSection);
|
||||
registry.register(TrustedUserAnswersSection);
|
||||
registry.register(ConversationTranscriptSection);
|
||||
registry.register(images::TranscriptImagesSection);
|
||||
registry.register(permissions::PermissionContextSection);
|
||||
registry.register(action::PlannedActionSection);
|
||||
registry
|
||||
|
||||
@@ -93,6 +93,7 @@ fn registry_collects_target_specific_sections_in_registration_order() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
});
|
||||
let async_sections = registry.collect(&SectionInput {
|
||||
target: ContextTarget::Async,
|
||||
@@ -105,6 +106,7 @@ fn registry_collects_target_specific_sections_in_registration_order() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
@@ -168,6 +170,7 @@ fn registry_skips_optional_sections_and_stops_on_missing_required_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
}),
|
||||
Err(error.clone())
|
||||
);
|
||||
@@ -232,6 +235,7 @@ fn reused_registry_preserves_section_identity_and_source_roles() {
|
||||
previous_reviews: Some(&reviews),
|
||||
trusted_tool: Some(&tool),
|
||||
trusted_skill_paths: &["debug-secret/SKILL.md".into()],
|
||||
images: None,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!format!("{context:?}").contains("debug-secret"));
|
||||
@@ -288,6 +292,7 @@ fn reused_registry_preserves_section_identity_and_source_roles() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.unwrap(),
|
||||
vec![ContextSection::ConversationTranscript { items: Vec::new() }]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use crate::ConversationTranscriptEntry;
|
||||
use crate::PlannedAction;
|
||||
use crate::PreviousReviews;
|
||||
use crate::TranscriptImages;
|
||||
use crate::TrustedSkills;
|
||||
use crate::TrustedTool;
|
||||
|
||||
@@ -23,5 +24,6 @@ pub enum ContextSection<T = ConversationTranscriptEntry> {
|
||||
PreviousReviews(PreviousReviews),
|
||||
TrustedTool(TrustedTool),
|
||||
TrustedSkills(TrustedSkills),
|
||||
TranscriptImages(TranscriptImages),
|
||||
PermissionContext { items: Vec<String> },
|
||||
}
|
||||
|
||||
@@ -103,6 +103,7 @@ fn registered_transcript_preserves_shared_roles_and_node_repl_tool_attribution()
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("transcript collection should succeed");
|
||||
|
||||
@@ -180,6 +181,7 @@ fn excluded_tool_calls_still_attribute_included_results() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("transcript collection should succeed");
|
||||
|
||||
@@ -276,6 +278,7 @@ fn outputs_with_call_ids_or_explicit_names_are_retained() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("transcript collection should succeed");
|
||||
let mut expected = vec![
|
||||
@@ -326,6 +329,7 @@ fn reused_registry_applies_current_history_sources_and_entry_limits() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("transcript collection should succeed");
|
||||
assert_eq!(
|
||||
@@ -363,6 +367,7 @@ fn reused_registry_applies_current_history_sources_and_entry_limits() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("transcript collection should succeed");
|
||||
let mut expected = vec![ConversationTranscriptEntry {
|
||||
|
||||
Reference in New Issue
Block a user