diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 75601fb603..a242511b0a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3590,6 +3590,7 @@ dependencies = [ "schemars 0.8.22", "serde", "serde_json", + "serde_with", "ts-rs", ] diff --git a/codex-rs/analytics/src/tests/suite/tools_tests.rs b/codex-rs/analytics/src/tests/suite/tools_tests.rs index cdd20d1176..e85daf1908 100644 --- a/codex-rs/analytics/src/tests/suite/tools_tests.rs +++ b/codex-rs/analytics/src/tests/suite/tools_tests.rs @@ -997,6 +997,7 @@ async fn image_generation_events_preserve_transparent_background_metadata() { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }); reducer diff --git a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs index 5820bce89c..304539246f 100644 --- a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs +++ b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs @@ -574,6 +574,7 @@ async fn turn_event_counts_completed_tool_items() { input_images: Vec::new(), imagegen_request_id: Some("req-imagegen-123".to_string()), generation_id: Some("gen-image-123".to_string()), + image_metadata: Default::default(), }), ]; diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 02d3fc8039..35b83fc1be 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -5748,6 +5748,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 5fbe83cd17..5f416bb6ee 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -24805,6 +24805,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index bc970b5818..0a5f3909cd 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -22501,6 +22501,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 101bfc1039..653fd81ddf 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1841,6 +1841,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index b839fb2e12..44216fbc83 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1841,6 +1841,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index e83b40b4d9..a11d177599 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -2057,6 +2057,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index bedf099526..9fc321da40 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -2671,6 +2671,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index 105c7db09e..c8034e97f5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -1841,6 +1841,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 7430b080f8..e088e2d732 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index 5809d3ee7c..c03d5d428c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index cbbc50f0ce..747387a373 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index e028239607..fe078fb52b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -2723,6 +2723,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index f78ba13763..28dcd5296e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadSearchPartialNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadSearchPartialNotification.json index 11194af2a9..78b0ff782f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadSearchPartialNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadSearchPartialNotification.json @@ -2473,6 +2473,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index ed0c1bcf7a..8ec2a2e0a7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -2671,6 +2671,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 8fd68bf341..726a04d7f6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index 0f6ac7e2d9..5632dda4fc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -2057,6 +2057,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index e9345c0db0..64604f7982 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -2462,6 +2462,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index a358fc9b4f..5699fea692 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -2057,6 +2057,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 3e49122c7f..83121d44ca 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -2057,6 +2057,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 698faaf8c9..0c8a8552a3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -2057,6 +2057,11 @@ "id": { "type": "string" }, + "imageMetadata": { + "additionalProperties": true, + "description": "Per-image metadata selected by the backend for the receiving surface.", + "type": "object" + }, "inputImages": { "description": "Observed local inputs for a completed generation, absent on older items.", "items": { diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 5fad88df6d..40faf74b76 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index b6ea4c65ec..f668b1d0dd 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts index e43fb2995c..4f20f4505a 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts @@ -4,6 +4,7 @@ import type { AbsolutePathBuf } from "./AbsolutePathBuf"; import type { ImageGenerationFailure } from "./ImageGenerationFailure"; import type { ImageGenerationInputImage } from "./ImageGenerationInputImage"; +import type { JsonValue } from "./serde_json/JsonValue"; export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, transparentBackground?: boolean, failure: ImageGenerationFailure | null, savedPath?: AbsolutePathBuf, /** @@ -13,4 +14,8 @@ inputImages?: Array, /** * Image generation identifier for the generated image, when provided by the image backend. */ -generationId: string | null, } & ({ fileId: string, } | { result: string, }); +generationId: string | null, +/** + * Per-image metadata selected by the backend for the receiving surface. + */ +imageMetadata?: { [key in string]?: JsonValue }, } & ({ fileId: string, } | { result: string, }); diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index c7341ba7c5..bd68c591f6 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -1102,6 +1102,7 @@ impl ThreadHistoryBuilder { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }); self.upsert_item_in_current_turn(item); } @@ -1118,6 +1119,7 @@ impl ThreadHistoryBuilder { input_images: payload.input_images.clone(), imagegen_request_id: None, generation_id: payload.generation_id.clone(), + image_metadata: payload.image_metadata.clone(), }); self.upsert_item_in_current_turn(item); } @@ -2751,6 +2753,7 @@ mod tests { input_images: input_images.clone(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }, )), started_at_ms: Some(0), @@ -2793,6 +2796,7 @@ mod tests { input_images, imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), })] ); } @@ -3058,6 +3062,16 @@ mod tests { #[test] fn replays_file_backed_image_generation_end_events_into_turn_history() { + let image_metadata = serde_json::Map::from_iter([ + ( + "entries".to_string(), + serde_json::json!([{"name": "first", "value": 1.5}]), + ), + ( + "futureKey".to_string(), + serde_json::json!({"nestedKey": [0, false, null]}), + ), + ]); let items = vec![ RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { turn_attribution: None, @@ -3096,6 +3110,7 @@ mod tests { sha256: "ab".repeat(32), }], generation_id: None, + image_metadata: image_metadata.clone(), })), RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { root_turn_id: None, @@ -3152,6 +3167,7 @@ mod tests { }], imagegen_request_id: None, generation_id: None, + image_metadata, })), ], } diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index c4eef0021d..3b6509f21c 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -1105,6 +1105,7 @@ impl From for ThreadItemPayload { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }), CoreTurnItem::EnteredReviewMode(review) => Self::EnteredReviewMode { id: review.id, diff --git a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs index d9980749bc..f5f1f9e57d 100644 --- a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs +++ b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs @@ -122,6 +122,7 @@ mod tests { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }), ]); thread.turns[0].items[1].started_at_ms = Some(123); diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index ad0880b565..a472b658f1 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -85,6 +85,14 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( } else { (RESULT, "png") }; + let expected_metadata = if gif_enabled { + json!({}) + } else { + json!({ + "entries": [{"name": "first", "value": 1.5}], + "futureKey": {"nestedKey": [false, 0, null]} + }) + }; let server = responses::start_mock_server().await; Mock::given(method("POST")) .and(path("/api/codex/images/generations")) @@ -95,7 +103,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( "created": 1, "background": "opaque", "data": [ - {"b64_json": result, "generation_id": "gen-image-123"}, + {"b64_json": result, "generation_id": "gen-image-123", "image_metadata": expected_metadata}, {"b64_json": "ignored", "generation_id": "gen-other"}, ], })), @@ -187,6 +195,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( transparent_background, saved_path: Some(saved_path), generation_id, + image_metadata, .. }) = completed.item.payload else { @@ -195,6 +204,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( assert_eq!(status, "completed"); assert_eq!(revised_prompt.as_deref(), Some("paint a blue whale")); assert_eq!(generation_id.as_deref(), Some("gen-image-123")); + assert_eq!(Some(&image_metadata), expected_metadata.as_object()); assert_eq!( image, GeneratedImageRef::Inline { @@ -256,6 +266,10 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( let requests = response_mock.requests(); assert_eq!(requests.len(), 2); + assert!( + !requests[1].body_json().to_string().contains("futureKey"), + "surface metadata must stay out of the model request" + ); let output = requests[1].function_call_output(call_id); if !gif_enabled { assert_eq!( @@ -394,6 +408,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history( image, transparent_background, input_images, + image_metadata, .. }) = completed.item.payload else { @@ -408,6 +423,10 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history( ); assert_eq!(transparent_background, Some(true)); assert_eq!(input_images, expected_inputs); + assert_eq!( + image_metadata.get("futureKey"), + Some(&json!({"nestedKey": [0, false, null]})) + ); drop(mcp); let mut resumed = TestAppServer::builder() @@ -438,6 +457,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history( Some("gen-transparent") ); assert_eq!(persisted_image.input_images, expected_inputs); + assert_eq!(persisted_image.image_metadata, image_metadata); let resume_id = resumed .send_thread_resume_request(ThreadResumeParams { @@ -462,6 +482,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history( Some("gen-transparent") ); assert_eq!(resumed_image.input_images, expected_inputs); + assert_eq!(resumed_image.image_metadata, image_metadata); Ok(()) } @@ -688,6 +709,7 @@ async fn standalone_image_generation_failure_emits_terminal_item( input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }) ); @@ -1217,6 +1239,7 @@ generatedImage(result); input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), } ); assert_eq!( @@ -1472,7 +1495,11 @@ async fn mount_image_response_with_background(server: &MockServer, background: & .set_body_json(json!({ "created": 1, "background": background, - "data": [{"b64_json": RESULT, "generation_id": "gen-transparent"}], + "data": [{ + "b64_json": RESULT, + "generation_id": "gen-transparent", + "image_metadata": {"futureKey": {"nestedKey": [0, false, null]}} + }], })), ) .expect(1) @@ -1486,7 +1513,11 @@ async fn mount_image_edit_response(server: &MockServer) { .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "created": 1, "background": "transparent", - "data": [{"b64_json": RESULT, "generation_id": "gen-transparent"}], + "data": [{ + "b64_json": RESULT, + "generation_id": "gen-transparent", + "image_metadata": {"futureKey": {"nestedKey": [0, false, null]}} + }], }))) .expect(1) .mount(server) diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 16ab0c92ff..97ec7701eb 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -3209,6 +3209,7 @@ fn append_resume_redaction_history( saved_path: Some(test_absolute_path("/tmp/ig-1.png")), input_images: Vec::new(), generation_id: None, + image_metadata: Default::default(), }), ] .into_iter() diff --git a/codex-rs/codex-api/src/endpoint/images.rs b/codex-rs/codex-api/src/endpoint/images.rs index 5366bd75c6..1f42acd60d 100644 --- a/codex-rs/codex-api/src/endpoint/images.rs +++ b/codex-rs/codex-api/src/endpoint/images.rs @@ -229,6 +229,7 @@ mod tests { data: vec![ImageData { b64_json: "REDACT".to_string(), generation_id: None, + image_metadata: Default::default(), }], quality: Some(ImageQuality::Medium), size: Some("1024x1536".to_string()), @@ -296,13 +297,21 @@ mod tests { } #[tokio::test] - async fn preserves_distinct_generation_ids_on_image_data() { + async fn preserves_optional_metadata_on_each_image() { + let first = json!({"entries": [{"name": "first", "value": 1.5}]}); + let second = + json!({"futureKey": {"zero": 0, "disabled": false, "nestedKeys": [null, "x"]}}); let body = serde_json::to_vec(&json!({ "created": 1, "data": [ - {"b64_json": "first", "generation_id": "gen-first"}, - {"b64_json": "second", "generation_id": "gen-second"}, - {"b64_json": "legacy"} + {"b64_json": "first", "generation_id": "gen-first", "image_metadata": first}, + {"b64_json": "second", "generation_id": "gen-second", "image_metadata": second}, + {"b64_json": "legacy"}, + {"b64_json": "empty", "image_metadata": {}}, + {"b64_json": "null", "image_metadata": null}, + {"b64_json": "list", "image_metadata": []}, + {"b64_json": "scalar", "image_metadata": 0}, + {"b64_json": "oversized", "image_metadata": {"value": "x".repeat(1024 * 1024)}} ] })) .expect("serialize response"); @@ -325,14 +334,47 @@ mod tests { .await .expect("image response should parse"); + assert_eq!(response.data.len(), 8); + assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first")); assert_eq!( - response - .data - .iter() - .map(|image| image.generation_id.as_deref()) - .collect::>(), - vec![Some("gen-first"), Some("gen-second"), None] + response.data[1].generation_id.as_deref(), + Some("gen-second") ); + assert_eq!(Some(&response.data[0].image_metadata), first.as_object()); + assert_eq!(Some(&response.data[1].image_metadata), second.as_object()); + for (image, label) in response.data[2..].iter().zip([ + "legacy", + "empty", + "null", + "list", + "scalar", + "oversized", + ]) { + assert_eq!(image.b64_json, label); + assert!(image.image_metadata.is_empty()); + } + } + + #[test] + fn deeply_nested_metadata_does_not_drop_images() { + for (depth, keep_metadata) in [(23, true), (24, false), (120, false), (512, false)] { + let nested = format!("{}null{}", r#"{"child":"#.repeat(depth), "}".repeat(depth)); + let body = format!( + r#"{{"created": 1, "data": [ + {{"image_metadata": {{"nested": {nested}}}, "b64_json": "first", "generation_id": "gen-first"}}, + {{"b64_json": "second", "image_metadata": {{"value": 1}}}} + ]}}"# + ); + let response: ImageResponse = serde_json::from_str(&body).expect("decode images"); + assert_eq!(response.data[0].b64_json, "first"); + assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first")); + assert_eq!( + response.data[0].image_metadata.contains_key("nested"), + keep_metadata + ); + assert_eq!(response.data[1].b64_json, "second"); + assert_eq!(response.data[1].image_metadata["value"], json!(1)); + } } #[tokio::test] diff --git a/codex-rs/codex-api/src/images.rs b/codex-rs/codex-api/src/images.rs index 9cc5c0d6ae..88e6b9e70b 100644 --- a/codex-rs/codex-api/src/images.rs +++ b/codex-rs/codex-api/src/images.rs @@ -2,6 +2,15 @@ use codex_protocol::models::ImageReference; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use serde_json::Map; +use serde_json::Value; +use serde_json::value::RawValue; + +const MAX_IMAGE_METADATA_BYTES: usize = 1024 * 1024; + +// Protobuf Struct objects consume three decode levels per nesting level. +// Leave room for the enclosing CCA and app-server event and history messages. +const MAX_IMAGE_METADATA_DEPTH: usize = 24; #[derive(Debug, Clone, Serialize, PartialEq)] pub struct ImageGenerationRequest { @@ -90,4 +99,40 @@ pub struct ImageData { pub b64_json: String, #[serde(default)] pub generation_id: Option, + #[serde(default, deserialize_with = "deserialize_image_metadata")] + pub image_metadata: Map, +} + +fn deserialize_image_metadata<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + // Read the field independently so a depth error cannot invalidate the surrounding image. + let raw = <&RawValue>::deserialize(deserializer)?; + if raw.get().len() > MAX_IMAGE_METADATA_BYTES { + return Ok(Map::new()); + } + Ok(serde_json::from_str::>(raw.get()) + .ok() + .filter(|metadata| { + metadata + .values() + .all(|value| fits_metadata_depth(value, /*depth*/ 1)) + }) + .unwrap_or_default()) +} + +fn fits_metadata_depth(value: &Value, depth: usize) -> bool { + if depth > MAX_IMAGE_METADATA_DEPTH { + return false; + } + match value { + Value::Array(items) => items + .iter() + .all(|item| fits_metadata_depth(item, depth + 1)), + Value::Object(fields) => fields + .values() + .all(|item| fits_metadata_depth(item, depth + 1)), + _ => true, + } } diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index c7f72f7b32..5f9acc3736 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -622,6 +622,7 @@ mod tests { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }); let expected_completed_item = ExtensionItem::ImageGeneration(ImageGenerationItem { id: "call-image".to_string(), @@ -636,6 +637,7 @@ mod tests { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }); codex_tools::TurnItemEmitter::emit_started( &emitter, @@ -663,6 +665,7 @@ mod tests { saved_path: Some(expected_path.clone()), input_images: Vec::new(), generation_id: None, + image_metadata: Default::default(), })], }, ) diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 96b394df04..f438309297 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -125,6 +125,7 @@ fn legacy_end_event(item: &ImageGenerationItem) -> EventMsg { saved_path: item.saved_path.clone(), input_images: item.input_images.clone(), generation_id: item.generation_id.clone(), + image_metadata: item.image_metadata.clone(), }) } @@ -191,6 +192,7 @@ impl ImageGenerationTool { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }, EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { call_id: call.call_id.clone(), @@ -223,12 +225,7 @@ impl ImageGenerationTool { Some(ImageBackground::Auto) | None => None, }; match response.data.into_iter().next() { - Some(data) => Ok(( - data.b64_json, - transparent_background, - imagegen_request_id, - data.generation_id, - )), + Some(data) => Ok((data, transparent_background, imagegen_request_id)), None => Err(( "image generation returned no image data".to_string(), None, @@ -236,7 +233,7 @@ impl ImageGenerationTool { )), } }); - let (result, transparent_background, imagegen_request_id, generation_id) = match result { + let (data, transparent_background, imagegen_request_id) = match result { Ok(result) => result, Err((message, failure, imagegen_request_id)) => { let item = ImageGenerationItem { @@ -252,6 +249,7 @@ impl ImageGenerationTool { input_images: Vec::new(), imagegen_request_id, generation_id: None, + image_metadata: Default::default(), }; let legacy_event = legacy_end_event(&item); call.turn_item_emitter @@ -260,6 +258,7 @@ impl ImageGenerationTool { return Err(FunctionCallError::RespondToModel(message)); } }; + let result = data.b64_json; // TODO(anp): Migrate image tool path arguments and saved-path events to PathUri so image // operations can use the primary environment even when its paths are foreign to the host. let saved_path = save_image_generation_result( @@ -318,7 +317,8 @@ impl ImageGenerationTool { saved_path: saved_path.clone(), input_images, imagegen_request_id, - generation_id, + generation_id: data.generation_id, + image_metadata: data.image_metadata, }; let legacy_event = legacy_end_event(&item); call.turn_item_emitter diff --git a/codex-rs/ext/items/Cargo.toml b/codex-rs/ext/items/Cargo.toml index 4d58b29bc7..586d2d959c 100644 --- a/codex-rs/ext/items/Cargo.toml +++ b/codex-rs/ext/items/Cargo.toml @@ -17,6 +17,7 @@ codex-utils-absolute-path = { workspace = true } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +serde_with = { workspace = true } ts-rs = { workspace = true } [dev-dependencies] diff --git a/codex-rs/ext/items/src/image_generation.rs b/codex-rs/ext/items/src/image_generation.rs index 9f0ea1f403..88695d9a8d 100644 --- a/codex-rs/ext/items/src/image_generation.rs +++ b/codex-rs/ext/items/src/image_generation.rs @@ -2,6 +2,9 @@ use codex_utils_absolute_path::AbsolutePathBuf; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use serde_json::Map; +use serde_json::Value; +use serde_with::serde_as; use ts_rs::TS; #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] @@ -32,6 +35,7 @@ pub struct ImageGenerationInputImage { // Standalone image-generation item owned by the image extension. This is also // the field-level representation exposed by app-server; core and rollout // persistence only carry it inside an ExtensionItem envelope. +#[serde_as] #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] @@ -64,6 +68,11 @@ pub struct ImageGenerationItem { /// Image generation identifier for the generated image, when provided by the image backend. #[serde(default)] pub generation_id: Option, + /// Per-image metadata selected by the backend for the receiving surface. + #[serde_as(deserialize_as = "serde_with::DefaultOnError")] + #[serde(default, skip_serializing_if = "Map::is_empty")] + #[ts(as = "Option<_>", optional)] + pub image_metadata: Map, } /// Either inline generated-image data or its durable attachment-store identity. diff --git a/codex-rs/ext/items/src/tests.rs b/codex-rs/ext/items/src/tests.rs index c2f18b47f4..3e2c2a855c 100644 --- a/codex-rs/ext/items/src/tests.rs +++ b/codex-rs/ext/items/src/tests.rs @@ -25,6 +25,7 @@ fn completed_image_generation_item() -> ExtensionItem { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }) } @@ -75,6 +76,28 @@ fn image_generation_item_preserves_stable_wire_shape() { ); } +#[test] +fn image_generation_metadata_survives_persistence() { + let base = serde_json::to_value(completed_image_generation_item()).expect("serialize item"); + let mut value = base.clone(); + value["imageMetadata"] = json!({ + "entries": [{"name": "first", "value": 1.5}], + "futureKey": {"nestedKey": [null, false, 0]}, + }); + let item: ExtensionItem = serde_json::from_value(value.clone()).expect("read extended item"); + assert_eq!( + serde_json::to_value(item).expect("persist extended item"), + value + ); + + for malformed in [json!({}), json!(null), json!("bad"), json!([])] { + value["imageMetadata"] = malformed; + let item: ExtensionItem = + serde_json::from_value(value.clone()).expect("read completed image"); + assert_eq!(serde_json::to_value(item).expect("persist image"), base); + } +} + /// Transitional rollouts containing both fields must retain the durable file reference. #[test] fn image_generation_item_reads_transitional_file_wire_shape() { @@ -91,6 +114,7 @@ fn image_generation_item_reads_transitional_file_wire_shape() { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }); assert_eq!( diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index 6f38619550..f432e75f73 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -487,6 +487,7 @@ impl ImageGenerationItem { saved_path: self.saved_path.clone(), input_images: Vec::new(), generation_id: None, + image_metadata: Default::default(), }) } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 35ab5e25ed..098fb6fae1 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -3225,6 +3225,7 @@ pub struct ImageGenerationBeginEvent { pub call_id: String, } +#[serde_as] #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ImageGenerationEndEvent { pub call_id: String, @@ -3249,6 +3250,10 @@ pub struct ImageGenerationEndEvent { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub generation_id: Option, + #[serde_as(deserialize_as = "serde_with::DefaultOnError")] + #[serde(default, skip_serializing_if = "Map::is_empty")] + #[ts(as = "Option<_>", optional)] + pub image_metadata: Map, } // Conversation kept for backward compatibility. diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs index fee292f15e..262eac486a 100644 --- a/codex-rs/rollout/src/policy_tests.rs +++ b/codex-rs/rollout/src/policy_tests.rs @@ -22,6 +22,7 @@ fn image_item(image: GeneratedImageRef) -> ImageGenerationItem { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), } } @@ -71,6 +72,7 @@ fn legacy_image_generation_preserves_file_reference() { saved_path: None, input_images: Vec::new(), generation_id: None, + image_metadata: Default::default(), }); let persisted = persisted_event_msg(&event, ThreadHistoryMode::Legacy) diff --git a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs index af6d199931..e1a0fa9b41 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs @@ -251,6 +251,7 @@ pub(super) fn completed_item( input_images: event.input_images.clone(), imagegen_request_id: None, generation_id: event.generation_id.clone(), + image_metadata: event.image_metadata.clone(), })), None, )), diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs index 214bcae0d9..0b3c95f391 100644 --- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs +++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs @@ -567,6 +567,7 @@ async fn migration_preserves_image_generation_failure_metadata() { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }; let image_completion = RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { @@ -579,6 +580,7 @@ async fn migration_preserves_image_generation_failure_metadata() { saved_path: expected_item.saved_path.clone(), input_images: Vec::new(), generation_id: expected_item.generation_id.clone(), + image_metadata: expected_item.image_metadata.clone(), })); let path = write_rollout( home.path(), @@ -631,6 +633,10 @@ async fn migration_preserves_image_generation_input_observations() { }], imagegen_request_id: None, generation_id: Some("gen-image-123".to_string()), + image_metadata: serde_json::Map::from_iter([( + "futureKey".to_string(), + json!({"nestedKey": [0, false, null]}), + )]), }; let image_completion = RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { @@ -643,6 +649,7 @@ async fn migration_preserves_image_generation_input_observations() { saved_path: expected_item.saved_path.clone(), input_images: expected_item.input_images.clone(), generation_id: expected_item.generation_id.clone(), + image_metadata: expected_item.image_metadata.clone(), })); let path = write_rollout( home.path(), diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index ab2906a0c8..c04a9225fd 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -890,6 +890,7 @@ pub(super) fn handle_image_generation_end( input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }), /*started_at_ms*/ None, /*completed_at_ms*/ 0, diff --git a/codex-rs/tui/src/thread_transcript/other_items_tests.rs b/codex-rs/tui/src/thread_transcript/other_items_tests.rs index 6dfe8ad9ab..66fb8e9b65 100644 --- a/codex-rs/tui/src/thread_transcript/other_items_tests.rs +++ b/codex-rs/tui/src/thread_transcript/other_items_tests.rs @@ -134,6 +134,7 @@ fn tool_and_notice_projection_uses_normal_transcript_presentation() { input_images: Vec::new(), imagegen_request_id: None, generation_id: None, + image_metadata: Default::default(), }; let items = vec![ ThreadItemPayload::EnteredReviewMode { diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index b19aeb5387..646e6f67dc 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -6039,6 +6039,13 @@ class FileIdThreadItem(BaseModel): ), ] = None id: str + image_metadata: Annotated[ + dict[str, Any] | None, + Field( + alias="imageMetadata", + description="Per-image metadata selected by the backend for the receiving surface.", + ), + ] = None input_images: Annotated[ list[ImageGenerationInputImage] | None, Field( @@ -6081,6 +6088,13 @@ class ResultThreadItem(BaseModel): ), ] = None id: str + image_metadata: Annotated[ + dict[str, Any] | None, + Field( + alias="imageMetadata", + description="Per-image metadata selected by the backend for the receiving surface.", + ), + ] = None input_images: Annotated[ list[ImageGenerationInputImage] | None, Field(