mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Track image preparation details in turn analytics (#36388)
## What changed - Record the effective detail setting and source and prepared dimensions for successfully decoded images. - Associate message images with their role and tool-output images with their originating call ID. - Include the collected image preparation metadata in the corresponding turn analytics event. ## Testing - Cover image resize metadata, tool-output attribution, turn aggregation, event serialization, and app-server analytics output. GitOrigin-RevId: dbfb5ba1ce7cb02dd93d9b579c5109817f16aef5
This commit is contained in:
committed by
copyberry
parent
bf7804c254
commit
1bef168976
@@ -60,6 +60,9 @@ use crate::facts::ExternalAgentConfigImportCompletedInput;
|
||||
use crate::facts::ExternalAgentConfigImportFailureInput;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::ImageDetailSetting;
|
||||
use crate::facts::ImagePreparationFact;
|
||||
use crate::facts::ImagePreparationMetadata;
|
||||
use crate::facts::InputError;
|
||||
use crate::facts::InvocationType;
|
||||
use crate::facts::PluginInstallFailedInput;
|
||||
@@ -1503,6 +1506,58 @@ fn compaction_event_serializes_expected_shape() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_preparation_fact_is_included_in_turn_event() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut events,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
|
||||
let metadata = ImagePreparationMetadata {
|
||||
message_role: None,
|
||||
item_id: Some("call-1".to_string()),
|
||||
effective_detail: ImageDetailSetting::High,
|
||||
source_width: 2_048,
|
||||
source_height: 2_048,
|
||||
prepared_width: 1_600,
|
||||
prepared_height: 1_600,
|
||||
};
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::ImagePreparation(Box::new(
|
||||
ImagePreparationFact {
|
||||
turn_id: "turn-2".to_string(),
|
||||
metadata: metadata.clone(),
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
let [TrackEventRequest::TurnEvent(event)] = events.as_slice() else {
|
||||
panic!("expected one turn event");
|
||||
};
|
||||
assert_eq!(event.event_params.image_preparations, vec![metadata]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compaction_implementation_serializes_remote_v2() {
|
||||
let payload = serde_json::to_value(CompactionImplementation::ResponsesCompactionV2)
|
||||
@@ -4076,6 +4131,15 @@ fn turn_event_serializes_expected_shape() {
|
||||
personality: Some("pragmatic".to_string()),
|
||||
workspace_kind: Some("projectless".to_string()),
|
||||
num_input_images: 2,
|
||||
image_preparations: vec![ImagePreparationMetadata {
|
||||
message_role: Some("user".to_string()),
|
||||
item_id: None,
|
||||
effective_detail: ImageDetailSetting::High,
|
||||
source_width: 2_048,
|
||||
source_height: 2_048,
|
||||
prepared_width: 1_600,
|
||||
prepared_height: 1_600,
|
||||
}],
|
||||
is_first_turn: true,
|
||||
status: Some(TurnStatus::Completed),
|
||||
explicit_client_interrupt_requested_at_ms: None,
|
||||
@@ -4151,6 +4215,15 @@ fn turn_event_serializes_expected_shape() {
|
||||
"personality": "pragmatic",
|
||||
"workspace_kind": "projectless",
|
||||
"num_input_images": 2,
|
||||
"image_preparations": [{
|
||||
"message_role": "user",
|
||||
"item_id": null,
|
||||
"effective_detail": "high",
|
||||
"source_width": 2048,
|
||||
"source_height": 2048,
|
||||
"prepared_width": 1600,
|
||||
"prepared_height": 1600
|
||||
}],
|
||||
"is_first_turn": true,
|
||||
"status": "completed",
|
||||
"explicit_client_interrupt_requested_at_ms": null,
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::facts::ExternalAgentConfigImportCompletedInput;
|
||||
use crate::facts::ExternalAgentConfigImportFailureInput;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::ImagePreparationFact;
|
||||
use crate::facts::PluginInstallFailedInput;
|
||||
use crate::facts::PluginInstallRequested;
|
||||
use crate::facts::PluginInstallRequestedInput;
|
||||
@@ -393,6 +394,12 @@ impl AnalyticsEventsClient {
|
||||
))));
|
||||
}
|
||||
|
||||
pub fn track_image_preparation(&self, fact: ImagePreparationFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::ImagePreparation(Box::new(fact)),
|
||||
));
|
||||
}
|
||||
|
||||
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::facts::CompactionStrategy;
|
||||
use crate::facts::CompactionTrigger;
|
||||
use crate::facts::GoalEventKind;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::ImagePreparationMetadata;
|
||||
use crate::facts::InvocationType;
|
||||
use crate::facts::PluginInstallRequested;
|
||||
use crate::facts::PluginState;
|
||||
@@ -895,6 +896,7 @@ pub(crate) struct CodexTurnEventParams {
|
||||
pub(crate) personality: Option<String>,
|
||||
pub(crate) workspace_kind: Option<String>,
|
||||
pub(crate) num_input_images: usize,
|
||||
pub(crate) image_preparations: Vec<ImagePreparationMetadata>,
|
||||
pub(crate) is_first_turn: bool,
|
||||
pub(crate) status: Option<TurnStatus>,
|
||||
/// Client wall-clock time for the first non-startup turn/interrupt request
|
||||
|
||||
@@ -59,6 +59,35 @@ pub fn build_track_events_context(
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImageDetailSetting {
|
||||
High,
|
||||
Original,
|
||||
}
|
||||
|
||||
/// Measurements for one successfully decoded image at the point where Codex prepares it for
|
||||
/// durable conversation history.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct ImagePreparationMetadata {
|
||||
/// Set for images embedded in message content.
|
||||
pub message_role: Option<String>,
|
||||
/// Set to the originating call ID for tool-output images. This joins to the `item_id` on
|
||||
/// existing tool events for tool type and provenance.
|
||||
pub item_id: Option<String>,
|
||||
pub effective_detail: ImageDetailSetting,
|
||||
pub source_width: u32,
|
||||
pub source_height: u32,
|
||||
pub prepared_width: u32,
|
||||
pub prepared_height: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ImagePreparationFact {
|
||||
pub turn_id: String,
|
||||
pub metadata: ImagePreparationMetadata,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnSubmissionType {
|
||||
@@ -424,6 +453,7 @@ pub(crate) enum CustomAnalyticsFact {
|
||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||
TurnProfile(Box<TurnProfileFact>),
|
||||
TurnCodexError(Box<TurnCodexErrorFact>),
|
||||
ImagePreparation(Box<ImagePreparationFact>),
|
||||
SkillInvoked(SkillInvokedInput),
|
||||
AppMentioned(AppMentionedInput),
|
||||
AppUsed(AppUsedInput),
|
||||
|
||||
@@ -40,6 +40,9 @@ pub use facts::ExternalAgentConfigImportCompletedInput;
|
||||
pub use facts::ExternalAgentConfigImportFailureInput;
|
||||
pub use facts::GoalEventKind;
|
||||
pub use facts::HookRunFact;
|
||||
pub use facts::ImageDetailSetting;
|
||||
pub use facts::ImagePreparationFact;
|
||||
pub use facts::ImagePreparationMetadata;
|
||||
pub use facts::InputError;
|
||||
pub use facts::InvocationType;
|
||||
pub use facts::PluginInstallRequestSource;
|
||||
|
||||
@@ -77,6 +77,8 @@ use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::ExternalAgentConfigImportCompletedInput;
|
||||
use crate::facts::ExternalAgentConfigImportFailureInput;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::ImagePreparationFact;
|
||||
use crate::facts::ImagePreparationMetadata;
|
||||
use crate::facts::PluginInstallFailedInput;
|
||||
use crate::facts::PluginInstallRequestedInput;
|
||||
use crate::facts::PluginState;
|
||||
@@ -369,6 +371,7 @@ struct TurnState {
|
||||
connection_id: Option<u64>,
|
||||
thread_id: Option<String>,
|
||||
num_input_images: Option<usize>,
|
||||
image_preparations: Vec<ImagePreparationMetadata>,
|
||||
resolved_config: Option<TurnResolvedConfigFact>,
|
||||
started_at: Option<u64>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
@@ -543,6 +546,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::TurnCodexError(input) => {
|
||||
self.ingest_turn_codex_error(*input);
|
||||
}
|
||||
CustomAnalyticsFact::ImagePreparation(input) => {
|
||||
self.ingest_image_preparation(*input);
|
||||
}
|
||||
CustomAnalyticsFact::SkillInvoked(input) => {
|
||||
self.ingest_skill_invoked(input, out).await;
|
||||
}
|
||||
@@ -736,6 +742,11 @@ impl AnalyticsReducer {
|
||||
turn_state.codex_error = Some(error);
|
||||
}
|
||||
|
||||
fn ingest_image_preparation(&mut self, input: ImagePreparationFact) {
|
||||
let turn_state = self.turns.entry(input.turn_id).or_default();
|
||||
turn_state.image_preparations.push(input.metadata);
|
||||
}
|
||||
|
||||
async fn ingest_skill_invoked(
|
||||
&mut self,
|
||||
input: SkillInvokedInput,
|
||||
@@ -2729,6 +2740,7 @@ fn codex_turn_event_params(
|
||||
personality: personality_mode(personality),
|
||||
workspace_kind,
|
||||
num_input_images,
|
||||
image_preparations: turn_state.image_preparations.clone(),
|
||||
is_first_turn,
|
||||
status: completed.status,
|
||||
explicit_client_interrupt_requested_at_ms: turn_state
|
||||
|
||||
@@ -911,6 +911,18 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> {
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert_eq!(event["event_params"]["num_input_images"], 1);
|
||||
assert_eq!(
|
||||
event["event_params"]["image_preparations"],
|
||||
json!([{
|
||||
"message_role": "user",
|
||||
"item_id": null,
|
||||
"effective_detail": "high",
|
||||
"source_width": 1,
|
||||
"source_height": 1,
|
||||
"prepared_width": 1,
|
||||
"prepared_height": 1,
|
||||
}])
|
||||
);
|
||||
assert_eq!(event["event_params"]["status"], "completed");
|
||||
assert!(event["event_params"]["started_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["completed_at"].as_u64().is_some());
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use codex_analytics::ImageDetailSetting;
|
||||
use codex_analytics::ImagePreparationMetadata;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
@@ -24,6 +26,13 @@ const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits
|
||||
max_dimension: 6000,
|
||||
max_patches: 10_000,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ImageOrigin<'a> {
|
||||
message_role: Option<&'a str>,
|
||||
item_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum ImagePreparationError {
|
||||
#[error("remote image URLs are not supported")]
|
||||
@@ -47,14 +56,35 @@ impl ImagePreparationError {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
|
||||
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) -> Vec<ImagePreparationMetadata> {
|
||||
let mut metadata = Vec::new();
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => prepare_message_content(content),
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
ResponseItem::Message { role, content, .. } => {
|
||||
prepare_message_content(
|
||||
content,
|
||||
ImageOrigin {
|
||||
message_role: Some(role),
|
||||
item_id: None,
|
||||
},
|
||||
&mut metadata,
|
||||
);
|
||||
}
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id, output, ..
|
||||
}
|
||||
| ResponseItem::CustomToolCallOutput {
|
||||
call_id, output, ..
|
||||
} => {
|
||||
if let Some(content) = output.content_items_mut() {
|
||||
prepare_tool_output_content(content);
|
||||
prepare_tool_output_content(
|
||||
content,
|
||||
ImageOrigin {
|
||||
message_role: None,
|
||||
item_id: Some(call_id),
|
||||
},
|
||||
&mut metadata,
|
||||
);
|
||||
}
|
||||
}
|
||||
ResponseItem::AdditionalTools { .. }
|
||||
@@ -73,12 +103,17 @@ pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
|
||||
| ResponseItem::Other => {}
|
||||
}
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
fn prepare_message_content(items: &mut [ContentItem]) {
|
||||
fn prepare_message_content(
|
||||
items: &mut [ContentItem],
|
||||
origin: ImageOrigin<'_>,
|
||||
metadata: &mut Vec<ImagePreparationMetadata>,
|
||||
) {
|
||||
for item in items {
|
||||
if let ContentItem::InputImage { image_url, detail } = item
|
||||
&& let Err(error) = prepare_image(image_url, *detail)
|
||||
&& let Err(error) = prepare_image(image_url, *detail, origin, metadata)
|
||||
{
|
||||
warn!(%error, "failed to prepare message image");
|
||||
*item = ContentItem::InputText {
|
||||
@@ -88,10 +123,14 @@ fn prepare_message_content(items: &mut [ContentItem]) {
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
|
||||
fn prepare_tool_output_content(
|
||||
items: &mut [FunctionCallOutputContentItem],
|
||||
origin: ImageOrigin<'_>,
|
||||
metadata: &mut Vec<ImagePreparationMetadata>,
|
||||
) {
|
||||
for item in items {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item
|
||||
&& let Err(error) = prepare_image(image_url, *detail)
|
||||
&& let Err(error) = prepare_image(image_url, *detail, origin, metadata)
|
||||
{
|
||||
warn!(%error, "failed to prepare tool output image");
|
||||
*item = FunctionCallOutputContentItem::InputText {
|
||||
@@ -116,6 +155,8 @@ fn is_data_url(image_url: &str) -> bool {
|
||||
fn prepare_image(
|
||||
image_url: &mut String,
|
||||
detail: Option<ImageDetail>,
|
||||
origin: ImageOrigin<'_>,
|
||||
metadata: &mut Vec<ImagePreparationMetadata>,
|
||||
) -> Result<(), ImagePreparationError> {
|
||||
if is_remote_image_url(image_url) {
|
||||
return Err(ImagePreparationError::RemoteUrlUnsupported);
|
||||
@@ -124,12 +165,23 @@ fn prepare_image(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let limits = match detail {
|
||||
None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS,
|
||||
Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS,
|
||||
let (effective_detail, limits) = match detail {
|
||||
None | Some(ImageDetail::Auto | ImageDetail::High) => {
|
||||
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
|
||||
}
|
||||
Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS),
|
||||
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
|
||||
};
|
||||
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
|
||||
metadata.push(ImagePreparationMetadata {
|
||||
message_role: origin.message_role.map(str::to_string),
|
||||
item_id: origin.item_id.map(str::to_string),
|
||||
effective_detail,
|
||||
source_width: image.source_width,
|
||||
source_height: image.source_height,
|
||||
prepared_width: image.width,
|
||||
prepared_height: image.height,
|
||||
});
|
||||
*image_url = image.into_data_url();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -69,12 +69,32 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
|
||||
|
||||
#[test]
|
||||
fn detail_policies_apply_the_expected_budgets() {
|
||||
for (detail, input_dimensions, expected_dimensions) in [
|
||||
(Some(ImageDetail::High), (2048, 2048), (1600, 1600)),
|
||||
(Some(ImageDetail::Original), (6401, 100), (6000, 94)),
|
||||
(Some(ImageDetail::Original), (3201, 3201), (3200, 3200)),
|
||||
(Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)),
|
||||
(None, (2048, 2048), (1600, 1600)),
|
||||
for (detail, effective_detail, input_dimensions, expected_dimensions) in [
|
||||
(
|
||||
Some(ImageDetail::High),
|
||||
ImageDetailSetting::High,
|
||||
(2048, 2048),
|
||||
(1600, 1600),
|
||||
),
|
||||
(
|
||||
Some(ImageDetail::Original),
|
||||
ImageDetailSetting::Original,
|
||||
(6401, 100),
|
||||
(6000, 94),
|
||||
),
|
||||
(
|
||||
Some(ImageDetail::Original),
|
||||
ImageDetailSetting::Original,
|
||||
(3201, 3201),
|
||||
(3200, 3200),
|
||||
),
|
||||
(
|
||||
Some(ImageDetail::Auto),
|
||||
ImageDetailSetting::High,
|
||||
(2048, 2048),
|
||||
(1600, 1600),
|
||||
),
|
||||
(None, ImageDetailSetting::High, (2048, 2048), (1600, 1600)),
|
||||
] {
|
||||
let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1);
|
||||
let mut items = vec![ResponseItem::Message {
|
||||
@@ -85,7 +105,7 @@ fn detail_policies_apply_the_expected_budgets() {
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
}];
|
||||
|
||||
prepare_response_items(&mut items);
|
||||
let metadata = prepare_response_items(&mut items);
|
||||
|
||||
let ResponseItem::Message { content, .. } = &items[0] else {
|
||||
panic!("expected message");
|
||||
@@ -94,9 +114,52 @@ fn detail_policies_apply_the_expected_budgets() {
|
||||
panic!("expected image");
|
||||
};
|
||||
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
|
||||
assert_eq!(
|
||||
metadata,
|
||||
vec![ImagePreparationMetadata {
|
||||
message_role: Some("user".to_string()),
|
||||
item_id: None,
|
||||
effective_detail,
|
||||
source_width: input_dimensions.0,
|
||||
source_height: input_dimensions.1,
|
||||
prepared_width: expected_dimensions.0,
|
||||
prepared_height: expected_dimensions.1,
|
||||
}]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preparation_reports_tool_output_item_id() {
|
||||
let call_id = "call-image";
|
||||
let (image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
|
||||
let mut items = vec![ResponseItem::FunctionCallOutput {
|
||||
id: None,
|
||||
call_id: call_id.to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
}];
|
||||
let metadata = prepare_response_items(&mut items);
|
||||
|
||||
assert_eq!(
|
||||
metadata,
|
||||
vec![ImagePreparationMetadata {
|
||||
message_role: None,
|
||||
item_id: Some(call_id.to_string()),
|
||||
effective_detail: ImageDetailSetting::High,
|
||||
source_width: 64,
|
||||
source_height: 32,
|
||||
prepared_width: 64,
|
||||
prepared_height: 32,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
|
||||
let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
|
||||
|
||||
@@ -47,6 +47,8 @@ use async_channel::Sender;
|
||||
use chrono::Local;
|
||||
use chrono::Utc;
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_analytics::ImagePreparationFact;
|
||||
use codex_analytics::ImagePreparationMetadata;
|
||||
use codex_analytics::SubAgentThreadStartedInput;
|
||||
use codex_analytics::TurnCodexErrorFact;
|
||||
use codex_async_utils::OrCancelExt;
|
||||
@@ -2907,15 +2909,18 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
items: &'a [ResponseItem],
|
||||
) -> Cow<'a, [ResponseItem]> {
|
||||
) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {
|
||||
let mut items = Cow::Borrowed(items);
|
||||
prepare_image_response_items(items.to_mut());
|
||||
let image_preparations = prepare_image_response_items(items.to_mut());
|
||||
prepare_audio_response_items(items.to_mut());
|
||||
// Most response items get their passthrough turn ID at the durable history boundary.
|
||||
for item in items.to_mut() {
|
||||
item.set_turn_id_if_missing(&turn_context.sub_id);
|
||||
}
|
||||
Self::assign_missing_response_item_ids(items)
|
||||
(
|
||||
Self::assign_missing_response_item_ids(items),
|
||||
image_preparations,
|
||||
)
|
||||
}
|
||||
|
||||
fn assign_missing_response_item_ids(items: Cow<'_, [ResponseItem]>) -> Cow<'_, [ResponseItem]> {
|
||||
@@ -2963,7 +2968,8 @@ impl Session {
|
||||
turn_context: &TurnContext,
|
||||
items: &[ResponseItem],
|
||||
) {
|
||||
let items = self.prepare_conversation_items_for_history(turn_context, items);
|
||||
let (items, image_preparations) =
|
||||
self.prepare_conversation_items_for_history(turn_context, items);
|
||||
let items = items.as_ref();
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
@@ -2973,6 +2979,14 @@ impl Session {
|
||||
turn_context.model_info.truncation_policy.into(),
|
||||
);
|
||||
}
|
||||
for image in image_preparations {
|
||||
self.services
|
||||
.analytics_events_client
|
||||
.track_image_preparation(ImagePreparationFact {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
metadata: image,
|
||||
});
|
||||
}
|
||||
self.persist_rollout_response_items(items).await;
|
||||
self.send_raw_response_items(turn_context, items).await;
|
||||
}
|
||||
@@ -3124,7 +3138,7 @@ impl Session {
|
||||
) {
|
||||
communication.set_turn_id_if_missing(&turn_context.sub_id);
|
||||
let response_item = communication.to_model_input_item();
|
||||
let items = self.prepare_conversation_items_for_history(
|
||||
let (items, _) = self.prepare_conversation_items_for_history(
|
||||
turn_context,
|
||||
std::slice::from_ref(&response_item),
|
||||
);
|
||||
|
||||
@@ -285,7 +285,7 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
|
||||
let (session, turn_context) = make_session_and_context().await;
|
||||
let response_item = user_message("hello");
|
||||
|
||||
let items = session.prepare_conversation_items_for_history(
|
||||
let (items, _) = session.prepare_conversation_items_for_history(
|
||||
&turn_context,
|
||||
std::slice::from_ref(&response_item),
|
||||
);
|
||||
|
||||
@@ -292,7 +292,15 @@ async fn resize_with_limits_respects_dimension_and_patch_budgets() {
|
||||
)
|
||||
.expect("process image with explicit limits");
|
||||
|
||||
assert_eq!((processed.width, processed.height), (1600, 1600));
|
||||
assert_eq!(
|
||||
(
|
||||
processed.source_width,
|
||||
processed.source_height,
|
||||
processed.width,
|
||||
processed.height,
|
||||
),
|
||||
(2048, 2048, 1600, 1600)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
@@ -350,6 +358,8 @@ async fn bounds_cache_by_encoded_byte_size() {
|
||||
let image = |size| EncodedImage {
|
||||
bytes: vec![0; size].into(),
|
||||
mime: "image/png".to_string(),
|
||||
source_width: 1,
|
||||
source_height: 1,
|
||||
width: 1,
|
||||
height: 1,
|
||||
};
|
||||
|
||||
@@ -39,6 +39,8 @@ pub use crate::error::ImageProcessingError;
|
||||
pub struct EncodedImage {
|
||||
pub bytes: Arc<[u8]>,
|
||||
pub mime: String,
|
||||
pub source_width: u32,
|
||||
pub source_height: u32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
@@ -152,7 +154,7 @@ pub fn load_for_prompt_bytes(
|
||||
PromptImageMode::ResizeToFit | PromptImageMode::Original => None,
|
||||
};
|
||||
|
||||
let encoded = if let Some((width, height, resized)) = target_dimensions {
|
||||
let encoded = if let Some((prepared_width, prepared_height, resized)) = target_dimensions {
|
||||
let target_format = format
|
||||
.filter(|format| can_preserve_source_bytes(*format))
|
||||
.unwrap_or(ImageFormat::Png);
|
||||
@@ -161,8 +163,10 @@ pub fn load_for_prompt_bytes(
|
||||
EncodedImage {
|
||||
bytes: bytes.into(),
|
||||
mime,
|
||||
width,
|
||||
height,
|
||||
source_width: width,
|
||||
source_height: height,
|
||||
width: prepared_width,
|
||||
height: prepared_height,
|
||||
}
|
||||
} else {
|
||||
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) {
|
||||
@@ -170,6 +174,8 @@ pub fn load_for_prompt_bytes(
|
||||
EncodedImage {
|
||||
bytes: file_bytes.into(),
|
||||
mime,
|
||||
source_width: width,
|
||||
source_height: height,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
@@ -179,6 +185,8 @@ pub fn load_for_prompt_bytes(
|
||||
EncodedImage {
|
||||
bytes: bytes.into(),
|
||||
mime,
|
||||
source_width: width,
|
||||
source_height: height,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user