openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-19 17:28:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Include images in Guardian v2 transcripts (#38987)

Browse Source 
## What changed

- Add the `features.guardianv2.transcript.include_images` opt-in setting.
- Include recent images from user and assistant messages and configured tool outputs in Guardian v2 sampling requests.
- Bound image context to the four most recent images and 8 MiB total, and omit image detail hints when constructing the sampling request.

## Testing

- Cover configuration parsing and image forwarding from messages and tool outputs.

GitOrigin-RevId: b0ae6b6182535fd084ea103ed01dc3c803d7d1f7
This commit is contained in:
jif
2026-08-17 10:56:13 +00:00
committed by copyberry
parent 53aa7bb1aa
commit 37cf6c84c0
9 changed files with 168 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/config.schema.json
+4
View File
@@ -1276,6 +1276,10 @@
"additionalProperties": false,
"description": "Bounds and optional sources for the Guardian v2 conversation transcript.",
"properties": {
"include_images": {
"description": "Include recent screenshots from messages and configured tool outputs.",
"type": "boolean"
},
"max_message_entry_tokens": {
"format": "uint",
"maximum": 100000.0,
codex-rs/ext/guardian-v2/src/config.rs
+3
View File
@@ -144,6 +144,9 @@ impl GuardianV2Config {
.unwrap_or_else(|| {
vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs]
}),
include_images: transcript_config
.and_then(|transcript| transcript.include_images)
.unwrap_or(false),
max_message_entry_tokens,
max_tool_entry_tokens,
max_message_transcript_tokens,
codex-rs/ext/guardian-v2/src/extension.rs
+4
View File
@@ -350,6 +350,9 @@ impl ToolLifecycleContributor for GuardianV2Extension {
let transcript = guardian_config
.transcript
.build(conversation_history.items());
let images = guardian_config
.transcript
.images(conversation_history.items());
drop(conversation_history);
let planned_action = match action.render(guardian_config.max_action_tokens) {
Ok(planned_action) => planned_action,
@@ -418,6 +421,7 @@ impl ToolLifecycleContributor for GuardianV2Extension {
.sample(LunaSamplingRequest {
instructions,
input: classification_input,
images,
parent_compaction,
parent_compaction_hash,
output_schema: json!({
codex-rs/ext/guardian-v2/src/extension_tests.rs
+72
View File
@@ -21,7 +21,9 @@ use codex_login::CodexAuth;
use codex_model_provider_info::ModelProviderInfo;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::protocol::ReviewDecision;
@@ -459,6 +461,76 @@ max_recent_non_user_entries = 8
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn contributor_includes_configured_transcript_images() -> Result<()> {
skip_if_no_network!(Ok(()));
let history = vec![
ResponseItem::Message {
id: None,
role: "user".to_owned(),
content: vec![
ContentItem::InputText {
text: "Review what is shown on screen.".to_owned(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,user-screenshot".to_owned(),
detail: Some(ImageDetail::High),
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
},
ResponseItem::FunctionCallOutput {
id: None,
call_id: "previous-call".to_owned(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "Screenshot captured.".to_owned(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,tool-screenshot".to_owned(),
detail: Some(ImageDetail::Low),
},
]),
internal_chat_message_metadata_passthrough: None,
},
];
let configuration = r#"
[features.guardianv2]
enabled = true
[features.guardianv2.transcript]
include_images = true
"#;
let (request, _test, _registry) = sample_configured_conversation_history(
history,
r#"{"path":"README.md"}"#,
Some(TEST_GUARDIAN_POLICY),
configuration,
)
.await?;
let content = request["input"][2]["content"]
.as_array()
.expect("Luna user content should be an array");
assert_eq!(
content[content.len() - 2..],
[
json!({
"type": "input_image",
"image_url": "data:image/png;base64,user-screenshot",
}),
json!({
"type": "input_image",
"image_url": "data:image/png;base64,tool-screenshot",
}),
]
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<()> {
skip_if_no_network!(Ok(()));
codex-rs/ext/guardian-v2/src/sampler.rs
+8
View File
@@ -74,6 +74,8 @@ pub struct LunaSamplingRequest {
pub instructions: String,
/// Ordered untrusted input entries that the model should classify.
pub input: Vec<String>,
/// Optional bounded screenshots accompanying the transcript.
pub images: Vec<ContentItem>,
/// Opaque parent compaction to reuse only for compatible model configurations.
pub parent_compaction: Option<ResponseItem>,
/// Current parent model's encrypted-compaction compatibility hash.
@@ -317,6 +319,12 @@ impl LunaSampler {
.input
.into_iter()
.map(|text| ContentItem::InputText { text })
.chain(request.images.into_iter().map(|mut image| {
if let ContentItem::InputImage { detail, .. } = &mut image {
*detail = None;
}
image
}))
.collect(),
phase: None,
internal_chat_message_metadata_passthrough: None,
codex-rs/ext/guardian-v2/src/sampler_tests.rs
+4
View File
@@ -75,6 +75,7 @@ fn sample_request(turn_id: &str) -> LunaSamplingRequest {
LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a README summary.".to_owned()],
images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: json!({
@@ -169,6 +170,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ
"The user requested a README summary.".to_owned(),
"The assistant inspected README.md.".to_owned(),
],
images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: schema.clone(),
@@ -192,6 +194,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ
.sample(LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a source review.".to_owned()],
images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: schema,
@@ -342,6 +345,7 @@ async fn sampler_returns_complete_json_before_terminal_response_events() -> Resu
sampler.sample(LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a README summary.".to_owned()],
images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: json!({
codex-rs/ext/guardian-v2/src/transcript.rs
+68
View File
@@ -1,8 +1,11 @@
use std::collections::HashMap;
use std::collections::VecDeque;
use codex_extension_api::ResponseItem;
pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::plaintext_agent_message_content;
@@ -13,6 +16,8 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;
const MAX_TRANSCRIPT_IMAGES: usize = 4;
const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
const MANUAL_APPROVAL_DEVELOPER_PREFIX: &str =
"The user has manually approved a specific action that was previously `Rejected`.";
@@ -32,6 +37,7 @@ struct TranscriptEntry {
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct TranscriptConfig {
pub(crate) sources: Vec<TranscriptSource>,
pub(crate) include_images: bool,
pub(crate) max_message_entry_tokens: usize,
pub(crate) max_tool_entry_tokens: usize,
pub(crate) max_message_transcript_tokens: usize,
@@ -43,6 +49,7 @@ impl Default for TranscriptConfig {
fn default() -> Self {
Self {
sources: vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs],
include_images: false,
max_message_entry_tokens: MAX_MESSAGE_ENTRY_TOKENS,
max_tool_entry_tokens: MAX_TOOL_ENTRY_TOKENS,
max_message_transcript_tokens: MAX_MESSAGE_TRANSCRIPT_TOKENS,
@@ -53,6 +60,67 @@ impl Default for TranscriptConfig {
}
impl TranscriptConfig {
pub(crate) fn images<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
) -> Vec<ContentItem> {
if !self.include_images {
return Vec::new();
}
let mut images = VecDeque::new();
let mut image_bytes = 0usize;
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
return;
}
while images.len() >= MAX_TRANSCRIPT_IMAGES
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
{
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
break;
};
image_bytes -= image_url.len();
}
image_bytes += image_url.len();
images.push_back(ContentItem::InputImage {
image_url: image_url.to_owned(),
detail,
});
};
for item in items {
match item {
ResponseItem::Message { role, content, .. }
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
if let ContentItem::InputImage { image_url, detail } = item {
include_image(image_url, *detail);
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. }
if self.sources.contains(&TranscriptSource::ToolOutputs) =>
{
if let Some(content) = output.content_items() {
for item in content {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
item
{
include_image(image_url, *detail);
}
}
}
}
_ => {}
}
}
images.into_iter().collect()
}
pub(crate) fn build<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
codex-rs/features/src/feature_configs.rs
+3
View File
@@ -88,6 +88,9 @@ pub enum GuardianV2TranscriptSource {
pub struct GuardianV2TranscriptConfigToml {
#[serde(skip_serializing_if = "Option::is_none")]
pub sources: Option<Vec<GuardianV2TranscriptSource>>,
/// Include recent screenshots from messages and configured tool outputs.
#[serde(skip_serializing_if = "Option::is_none")]
pub include_images: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 100, max = 100000))]
pub max_message_entry_tokens: Option<usize>,
codex-rs/features/src/tests.rs
+2
View File
@@ -143,6 +143,7 @@ max_parent_compaction_tokens = 384
[guardianv2.transcript]
sources = ["tool_outputs", "reasoning"]
include_images = true
max_message_entry_tokens = 128
max_tool_entry_tokens = 128
max_message_transcript_tokens = 512
@@ -167,6 +168,7 @@ max_recent_non_user_entries = 12
crate::GuardianV2TranscriptSource::ToolOutputs,
crate::GuardianV2TranscriptSource::Reasoning,
]),
include_images: Some(true),
max_message_entry_tokens: Some(128),
max_tool_entry_tokens: Some(128),
max_message_transcript_tokens: Some(512),