Track image preparation details in turn analytics (#36388)

## What changed

- Record the effective detail setting and source and prepared dimensions for
  successfully decoded images.
- Associate message images with their role and tool-output images with their
  originating call ID.
- Include the collected image preparation metadata in the corresponding turn
  analytics event.

## Testing

- Cover image resize metadata, tool-output attribution, turn aggregation, event
  serialization, and app-server analytics output.

GitOrigin-RevId: dbfb5ba1ce7cb02dd93d9b579c5109817f16aef5
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-07-31 20:34:41 +00:00
committed by copyberry
parent bf7804c254
commit 1bef168976
13 changed files with 315 additions and 29 deletions
@@ -60,6 +60,9 @@ use crate::facts::ExternalAgentConfigImportCompletedInput;
use crate::facts::ExternalAgentConfigImportFailureInput;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::ImageDetailSetting;
use crate::facts::ImagePreparationFact;
use crate::facts::ImagePreparationMetadata;
use crate::facts::InputError;
use crate::facts::InvocationType;
use crate::facts::PluginInstallFailedInput;
@@ -1503,6 +1506,58 @@ fn compaction_event_serializes_expected_shape() {
);
}
#[tokio::test]
async fn image_preparation_fact_is_included_in_turn_event() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut events,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ true,
/*include_token_usage*/ false,
)
.await;
let metadata = ImagePreparationMetadata {
message_role: None,
item_id: Some("call-1".to_string()),
effective_detail: ImageDetailSetting::High,
source_width: 2_048,
source_height: 2_048,
prepared_width: 1_600,
prepared_height: 1_600,
};
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::ImagePreparation(Box::new(
ImagePreparationFact {
turn_id: "turn-2".to_string(),
metadata: metadata.clone(),
},
))),
&mut events,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut events,
)
.await;
let [TrackEventRequest::TurnEvent(event)] = events.as_slice() else {
panic!("expected one turn event");
};
assert_eq!(event.event_params.image_preparations, vec![metadata]);
}
#[test]
fn compaction_implementation_serializes_remote_v2() {
let payload = serde_json::to_value(CompactionImplementation::ResponsesCompactionV2)
@@ -4076,6 +4131,15 @@ fn turn_event_serializes_expected_shape() {
personality: Some("pragmatic".to_string()),
workspace_kind: Some("projectless".to_string()),
num_input_images: 2,
image_preparations: vec![ImagePreparationMetadata {
message_role: Some("user".to_string()),
item_id: None,
effective_detail: ImageDetailSetting::High,
source_width: 2_048,
source_height: 2_048,
prepared_width: 1_600,
prepared_height: 1_600,
}],
is_first_turn: true,
status: Some(TurnStatus::Completed),
explicit_client_interrupt_requested_at_ms: None,
@@ -4151,6 +4215,15 @@ fn turn_event_serializes_expected_shape() {
"personality": "pragmatic",
"workspace_kind": "projectless",
"num_input_images": 2,
"image_preparations": [{
"message_role": "user",
"item_id": null,
"effective_detail": "high",
"source_width": 2048,
"source_height": 2048,
"prepared_width": 1600,
"prepared_height": 1600
}],
"is_first_turn": true,
"status": "completed",
"explicit_client_interrupt_requested_at_ms": null,
+7
View File
@@ -15,6 +15,7 @@ use crate::facts::ExternalAgentConfigImportCompletedInput;
use crate::facts::ExternalAgentConfigImportFailureInput;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::ImagePreparationFact;
use crate::facts::PluginInstallFailedInput;
use crate::facts::PluginInstallRequested;
use crate::facts::PluginInstallRequestedInput;
@@ -393,6 +394,12 @@ impl AnalyticsEventsClient {
))));
}
pub fn track_image_preparation(&self, fact: ImagePreparationFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::ImagePreparation(Box::new(fact)),
));
}
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),
+2
View File
@@ -13,6 +13,7 @@ use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::GoalEventKind;
use crate::facts::HookRunFact;
use crate::facts::ImagePreparationMetadata;
use crate::facts::InvocationType;
use crate::facts::PluginInstallRequested;
use crate::facts::PluginState;
@@ -895,6 +896,7 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) personality: Option<String>,
pub(crate) workspace_kind: Option<String>,
pub(crate) num_input_images: usize,
pub(crate) image_preparations: Vec<ImagePreparationMetadata>,
pub(crate) is_first_turn: bool,
pub(crate) status: Option<TurnStatus>,
/// Client wall-clock time for the first non-startup turn/interrupt request
+30
View File
@@ -59,6 +59,35 @@ pub fn build_track_events_context(
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ImageDetailSetting {
High,
Original,
}
/// Measurements for one successfully decoded image at the point where Codex prepares it for
/// durable conversation history.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct ImagePreparationMetadata {
/// Set for images embedded in message content.
pub message_role: Option<String>,
/// Set to the originating call ID for tool-output images. This joins to the `item_id` on
/// existing tool events for tool type and provenance.
pub item_id: Option<String>,
pub effective_detail: ImageDetailSetting,
pub source_width: u32,
pub source_height: u32,
pub prepared_width: u32,
pub prepared_height: u32,
}
#[derive(Clone)]
pub struct ImagePreparationFact {
pub turn_id: String,
pub metadata: ImagePreparationMetadata,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnSubmissionType {
@@ -424,6 +453,7 @@ pub(crate) enum CustomAnalyticsFact {
TurnTokenUsage(Box<TurnTokenUsageFact>),
TurnProfile(Box<TurnProfileFact>),
TurnCodexError(Box<TurnCodexErrorFact>),
ImagePreparation(Box<ImagePreparationFact>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),
+3
View File
@@ -40,6 +40,9 @@ pub use facts::ExternalAgentConfigImportCompletedInput;
pub use facts::ExternalAgentConfigImportFailureInput;
pub use facts::GoalEventKind;
pub use facts::HookRunFact;
pub use facts::ImageDetailSetting;
pub use facts::ImagePreparationFact;
pub use facts::ImagePreparationMetadata;
pub use facts::InputError;
pub use facts::InvocationType;
pub use facts::PluginInstallRequestSource;
+12
View File
@@ -77,6 +77,8 @@ use crate::facts::CustomAnalyticsFact;
use crate::facts::ExternalAgentConfigImportCompletedInput;
use crate::facts::ExternalAgentConfigImportFailureInput;
use crate::facts::HookRunInput;
use crate::facts::ImagePreparationFact;
use crate::facts::ImagePreparationMetadata;
use crate::facts::PluginInstallFailedInput;
use crate::facts::PluginInstallRequestedInput;
use crate::facts::PluginState;
@@ -369,6 +371,7 @@ struct TurnState {
connection_id: Option<u64>,
thread_id: Option<String>,
num_input_images: Option<usize>,
image_preparations: Vec<ImagePreparationMetadata>,
resolved_config: Option<TurnResolvedConfigFact>,
started_at: Option<u64>,
token_usage: Option<TokenUsage>,
@@ -543,6 +546,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::TurnCodexError(input) => {
self.ingest_turn_codex_error(*input);
}
CustomAnalyticsFact::ImagePreparation(input) => {
self.ingest_image_preparation(*input);
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
@@ -736,6 +742,11 @@ impl AnalyticsReducer {
turn_state.codex_error = Some(error);
}
fn ingest_image_preparation(&mut self, input: ImagePreparationFact) {
let turn_state = self.turns.entry(input.turn_id).or_default();
turn_state.image_preparations.push(input.metadata);
}
async fn ingest_skill_invoked(
&mut self,
input: SkillInvokedInput,
@@ -2729,6 +2740,7 @@ fn codex_turn_event_params(
personality: personality_mode(personality),
workspace_kind,
num_input_images,
image_preparations: turn_state.image_preparations.clone(),
is_first_turn,
status: completed.status,
explicit_client_interrupt_requested_at_ms: turn_state
@@ -911,6 +911,18 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> {
serde_json::Value::Null
);
assert_eq!(event["event_params"]["num_input_images"], 1);
assert_eq!(
event["event_params"]["image_preparations"],
json!([{
"message_role": "user",
"item_id": null,
"effective_detail": "high",
"source_width": 1,
"source_height": 1,
"prepared_width": 1,
"prepared_height": 1,
}])
);
assert_eq!(event["event_params"]["status"], "completed");
assert!(event["event_params"]["started_at"].as_u64().is_some());
assert!(event["event_params"]["completed_at"].as_u64().is_some());
+64 -12
View File
@@ -1,3 +1,5 @@
use codex_analytics::ImageDetailSetting;
use codex_analytics::ImagePreparationMetadata;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
@@ -24,6 +26,13 @@ const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits
max_dimension: 6000,
max_patches: 10_000,
};
#[derive(Clone, Copy, Debug)]
struct ImageOrigin<'a> {
message_role: Option<&'a str>,
item_id: Option<&'a str>,
}
#[derive(Debug, thiserror::Error)]
enum ImagePreparationError {
#[error("remote image URLs are not supported")]
@@ -47,14 +56,35 @@ impl ImagePreparationError {
}
}
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) -> Vec<ImagePreparationMetadata> {
let mut metadata = Vec::new();
for item in items {
match item {
ResponseItem::Message { content, .. } => prepare_message_content(content),
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
ResponseItem::Message { role, content, .. } => {
prepare_message_content(
content,
ImageOrigin {
message_role: Some(role),
item_id: None,
},
&mut metadata,
);
}
ResponseItem::FunctionCallOutput {
call_id, output, ..
}
| ResponseItem::CustomToolCallOutput {
call_id, output, ..
} => {
if let Some(content) = output.content_items_mut() {
prepare_tool_output_content(content);
prepare_tool_output_content(
content,
ImageOrigin {
message_role: None,
item_id: Some(call_id),
},
&mut metadata,
);
}
}
ResponseItem::AdditionalTools { .. }
@@ -73,12 +103,17 @@ pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
| ResponseItem::Other => {}
}
}
metadata
}
fn prepare_message_content(items: &mut [ContentItem]) {
fn prepare_message_content(
items: &mut [ContentItem],
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
) {
for item in items {
if let ContentItem::InputImage { image_url, detail } = item
&& let Err(error) = prepare_image(image_url, *detail)
&& let Err(error) = prepare_image(image_url, *detail, origin, metadata)
{
warn!(%error, "failed to prepare message image");
*item = ContentItem::InputText {
@@ -88,10 +123,14 @@ fn prepare_message_content(items: &mut [ContentItem]) {
}
}
fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
fn prepare_tool_output_content(
items: &mut [FunctionCallOutputContentItem],
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
) {
for item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item
&& let Err(error) = prepare_image(image_url, *detail)
&& let Err(error) = prepare_image(image_url, *detail, origin, metadata)
{
warn!(%error, "failed to prepare tool output image");
*item = FunctionCallOutputContentItem::InputText {
@@ -116,6 +155,8 @@ fn is_data_url(image_url: &str) -> bool {
fn prepare_image(
image_url: &mut String,
detail: Option<ImageDetail>,
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
) -> Result<(), ImagePreparationError> {
if is_remote_image_url(image_url) {
return Err(ImagePreparationError::RemoteUrlUnsupported);
@@ -124,12 +165,23 @@ fn prepare_image(
return Ok(());
}
let limits = match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS,
Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS,
let (effective_detail, limits) = match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => {
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
}
Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS),
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
};
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
metadata.push(ImagePreparationMetadata {
message_role: origin.message_role.map(str::to_string),
item_id: origin.item_id.map(str::to_string),
effective_detail,
source_width: image.source_width,
source_height: image.source_height,
prepared_width: image.width,
prepared_height: image.height,
});
*image_url = image.into_data_url();
Ok(())
}
+70 -7
View File
@@ -69,12 +69,32 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
#[test]
fn detail_policies_apply_the_expected_budgets() {
for (detail, input_dimensions, expected_dimensions) in [
(Some(ImageDetail::High), (2048, 2048), (1600, 1600)),
(Some(ImageDetail::Original), (6401, 100), (6000, 94)),
(Some(ImageDetail::Original), (3201, 3201), (3200, 3200)),
(Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)),
(None, (2048, 2048), (1600, 1600)),
for (detail, effective_detail, input_dimensions, expected_dimensions) in [
(
Some(ImageDetail::High),
ImageDetailSetting::High,
(2048, 2048),
(1600, 1600),
),
(
Some(ImageDetail::Original),
ImageDetailSetting::Original,
(6401, 100),
(6000, 94),
),
(
Some(ImageDetail::Original),
ImageDetailSetting::Original,
(3201, 3201),
(3200, 3200),
),
(
Some(ImageDetail::Auto),
ImageDetailSetting::High,
(2048, 2048),
(1600, 1600),
),
(None, ImageDetailSetting::High, (2048, 2048), (1600, 1600)),
] {
let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1);
let mut items = vec![ResponseItem::Message {
@@ -85,7 +105,7 @@ fn detail_policies_apply_the_expected_budgets() {
internal_chat_message_metadata_passthrough: None,
}];
prepare_response_items(&mut items);
let metadata = prepare_response_items(&mut items);
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
@@ -94,9 +114,52 @@ fn detail_policies_apply_the_expected_budgets() {
panic!("expected image");
};
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
assert_eq!(
metadata,
vec![ImagePreparationMetadata {
message_role: Some("user".to_string()),
item_id: None,
effective_detail,
source_width: input_dimensions.0,
source_height: input_dimensions.1,
prepared_width: expected_dimensions.0,
prepared_height: expected_dimensions.1,
}]
);
}
}
#[test]
fn preparation_reports_tool_output_item_id() {
let call_id = "call-image";
let (image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![ResponseItem::FunctionCallOutput {
id: None,
call_id: call_id.to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
detail: Some(ImageDetail::High),
},
]),
internal_chat_message_metadata_passthrough: None,
}];
let metadata = prepare_response_items(&mut items);
assert_eq!(
metadata,
vec![ImagePreparationMetadata {
message_role: None,
item_id: Some(call_id.to_string()),
effective_detail: ImageDetailSetting::High,
source_width: 64,
source_height: 32,
prepared_width: 64,
prepared_height: 32,
}]
);
}
#[test]
fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
+19 -5
View File
@@ -47,6 +47,8 @@ use async_channel::Sender;
use chrono::Local;
use chrono::Utc;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::ImagePreparationFact;
use codex_analytics::ImagePreparationMetadata;
use codex_analytics::SubAgentThreadStartedInput;
use codex_analytics::TurnCodexErrorFact;
use codex_async_utils::OrCancelExt;
@@ -2907,15 +2909,18 @@ impl Session {
&self,
turn_context: &TurnContext,
items: &'a [ResponseItem],
) -> Cow<'a, [ResponseItem]> {
) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {
let mut items = Cow::Borrowed(items);
prepare_image_response_items(items.to_mut());
let image_preparations = prepare_image_response_items(items.to_mut());
prepare_audio_response_items(items.to_mut());
// Most response items get their passthrough turn ID at the durable history boundary.
for item in items.to_mut() {
item.set_turn_id_if_missing(&turn_context.sub_id);
}
Self::assign_missing_response_item_ids(items)
(
Self::assign_missing_response_item_ids(items),
image_preparations,
)
}
fn assign_missing_response_item_ids(items: Cow<'_, [ResponseItem]>) -> Cow<'_, [ResponseItem]> {
@@ -2963,7 +2968,8 @@ impl Session {
turn_context: &TurnContext,
items: &[ResponseItem],
) {
let items = self.prepare_conversation_items_for_history(turn_context, items);
let (items, image_preparations) =
self.prepare_conversation_items_for_history(turn_context, items);
let items = items.as_ref();
{
let mut state = self.state.lock().await;
@@ -2973,6 +2979,14 @@ impl Session {
turn_context.model_info.truncation_policy.into(),
);
}
for image in image_preparations {
self.services
.analytics_events_client
.track_image_preparation(ImagePreparationFact {
turn_id: turn_context.sub_id.clone(),
metadata: image,
});
}
self.persist_rollout_response_items(items).await;
self.send_raw_response_items(turn_context, items).await;
}
@@ -3124,7 +3138,7 @@ impl Session {
) {
communication.set_turn_id_if_missing(&turn_context.sub_id);
let response_item = communication.to_model_input_item();
let items = self.prepare_conversation_items_for_history(
let (items, _) = self.prepare_conversation_items_for_history(
turn_context,
std::slice::from_ref(&response_item),
);
+1 -1
View File
@@ -285,7 +285,7 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
let (session, turn_context) = make_session_and_context().await;
let response_item = user_message("hello");
let items = session.prepare_conversation_items_for_history(
let (items, _) = session.prepare_conversation_items_for_history(
&turn_context,
std::slice::from_ref(&response_item),
);
+11 -1
View File
@@ -292,7 +292,15 @@ async fn resize_with_limits_respects_dimension_and_patch_budgets() {
)
.expect("process image with explicit limits");
assert_eq!((processed.width, processed.height), (1600, 1600));
assert_eq!(
(
processed.source_width,
processed.source_height,
processed.width,
processed.height,
),
(2048, 2048, 1600, 1600)
);
}
#[tokio::test(flavor = "multi_thread")]
@@ -350,6 +358,8 @@ async fn bounds_cache_by_encoded_byte_size() {
let image = |size| EncodedImage {
bytes: vec![0; size].into(),
mime: "image/png".to_string(),
source_width: 1,
source_height: 1,
width: 1,
height: 1,
};
+11 -3
View File
@@ -39,6 +39,8 @@ pub use crate::error::ImageProcessingError;
pub struct EncodedImage {
pub bytes: Arc<[u8]>,
pub mime: String,
pub source_width: u32,
pub source_height: u32,
pub width: u32,
pub height: u32,
}
@@ -152,7 +154,7 @@ pub fn load_for_prompt_bytes(
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
};
let encoded = if let Some((width, height, resized)) = target_dimensions {
let encoded = if let Some((prepared_width, prepared_height, resized)) = target_dimensions {
let target_format = format
.filter(|format| can_preserve_source_bytes(*format))
.unwrap_or(ImageFormat::Png);
@@ -161,8 +163,10 @@ pub fn load_for_prompt_bytes(
EncodedImage {
bytes: bytes.into(),
mime,
width,
height,
source_width: width,
source_height: height,
width: prepared_width,
height: prepared_height,
}
} else {
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) {
@@ -170,6 +174,8 @@ pub fn load_for_prompt_bytes(
EncodedImage {
bytes: file_bytes.into(),
mime,
source_width: width,
source_height: height,
width,
height,
}
@@ -179,6 +185,8 @@ pub fn load_for_prompt_bytes(
EncodedImage {
bytes: bytes.into(),
mime,
source_width: width,
source_height: height,
width,
height,
}