mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Preserve per-image metadata in app-server image generation items (#62871)
## Why App-server clients lose metadata supplied by the image backend when Codex turns the response into a completed image generation item. ## What changed Expose the backend's `image_metadata` as optional `imageMetadata` in app-server items and preserve nested values through thread persistence, resume, and legacy history replay and migration. Keep this metadata out of model tool output. Omit empty metadata and discard non-object, oversized, or deeply nested metadata without dropping the generated image. ## Testing Extend coverage for per-image parsing, metadata limits, and round trips through app-server completion, persisted history, and thread resume. GitOrigin-RevId: 7211a9db1149cb05e6176b08b0462ca173ac72d5
This commit is contained in:
committed by
copyberry
parent
e346540867
commit
0bfb8467da
Generated
+1
@@ -3590,6 +3590,7 @@ dependencies = [
|
||||
"schemars 0.8.22",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"ts-rs",
|
||||
]
|
||||
|
||||
|
||||
@@ -997,6 +997,7 @@ async fn image_generation_events_preserve_transparent_background_metadata() {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
|
||||
reducer
|
||||
|
||||
@@ -574,6 +574,7 @@ async fn turn_event_counts_completed_tool_items() {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: Some("req-imagegen-123".to_string()),
|
||||
generation_id: Some("gen-image-123".to_string()),
|
||||
image_metadata: Default::default(),
|
||||
}),
|
||||
];
|
||||
|
||||
|
||||
@@ -5748,6 +5748,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
+5
@@ -24805,6 +24805,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
+5
@@ -22501,6 +22501,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -1841,6 +1841,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -1841,6 +1841,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2057,6 +2057,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2671,6 +2671,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -1841,6 +1841,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
+5
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2723,6 +2723,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
+5
@@ -2473,6 +2473,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2671,6 +2671,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2057,6 +2057,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2462,6 +2462,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2057,6 +2057,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2057,6 +2057,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
@@ -2057,6 +2057,11 @@
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"imageMetadata": {
|
||||
"additionalProperties": true,
|
||||
"description": "Per-image metadata selected by the backend for the receiving surface.",
|
||||
"type": "object"
|
||||
},
|
||||
"inputImages": {
|
||||
"description": "Observed local inputs for a completed generation, absent on older items.",
|
||||
"items": {
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -4,6 +4,7 @@
|
||||
import type { AbsolutePathBuf } from "./AbsolutePathBuf";
|
||||
import type { ImageGenerationFailure } from "./ImageGenerationFailure";
|
||||
import type { ImageGenerationInputImage } from "./ImageGenerationInputImage";
|
||||
import type { JsonValue } from "./serde_json/JsonValue";
|
||||
|
||||
export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, transparentBackground?: boolean, failure: ImageGenerationFailure | null, savedPath?: AbsolutePathBuf,
|
||||
/**
|
||||
@@ -13,4 +14,8 @@ inputImages?: Array<ImageGenerationInputImage>,
|
||||
/**
|
||||
* Image generation identifier for the generated image, when provided by the image backend.
|
||||
*/
|
||||
generationId: string | null, } & ({ fileId: string, } | { result: string, });
|
||||
generationId: string | null,
|
||||
/**
|
||||
* Per-image metadata selected by the backend for the receiving surface.
|
||||
*/
|
||||
imageMetadata?: { [key in string]?: JsonValue }, } & ({ fileId: string, } | { result: string, });
|
||||
|
||||
@@ -1102,6 +1102,7 @@ impl ThreadHistoryBuilder {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
}
|
||||
@@ -1118,6 +1119,7 @@ impl ThreadHistoryBuilder {
|
||||
input_images: payload.input_images.clone(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: payload.generation_id.clone(),
|
||||
image_metadata: payload.image_metadata.clone(),
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
}
|
||||
@@ -2751,6 +2753,7 @@ mod tests {
|
||||
input_images: input_images.clone(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
},
|
||||
)),
|
||||
started_at_ms: Some(0),
|
||||
@@ -2793,6 +2796,7 @@ mod tests {
|
||||
input_images,
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
})]
|
||||
);
|
||||
}
|
||||
@@ -3058,6 +3062,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replays_file_backed_image_generation_end_events_into_turn_history() {
|
||||
let image_metadata = serde_json::Map::from_iter([
|
||||
(
|
||||
"entries".to_string(),
|
||||
serde_json::json!([{"name": "first", "value": 1.5}]),
|
||||
),
|
||||
(
|
||||
"futureKey".to_string(),
|
||||
serde_json::json!({"nestedKey": [0, false, null]}),
|
||||
),
|
||||
]);
|
||||
let items = vec![
|
||||
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_attribution: None,
|
||||
@@ -3096,6 +3110,7 @@ mod tests {
|
||||
sha256: "ab".repeat(32),
|
||||
}],
|
||||
generation_id: None,
|
||||
image_metadata: image_metadata.clone(),
|
||||
})),
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
root_turn_id: None,
|
||||
@@ -3152,6 +3167,7 @@ mod tests {
|
||||
}],
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata,
|
||||
})),
|
||||
],
|
||||
}
|
||||
|
||||
@@ -1105,6 +1105,7 @@ impl From<CoreTurnItem> for ThreadItemPayload {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}),
|
||||
CoreTurnItem::EnteredReviewMode(review) => Self::EnteredReviewMode {
|
||||
id: review.id,
|
||||
|
||||
@@ -122,6 +122,7 @@ mod tests {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}),
|
||||
]);
|
||||
thread.turns[0].items[1].started_at_ms = Some(123);
|
||||
|
||||
@@ -85,6 +85,14 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model(
|
||||
} else {
|
||||
(RESULT, "png")
|
||||
};
|
||||
let expected_metadata = if gif_enabled {
|
||||
json!({})
|
||||
} else {
|
||||
json!({
|
||||
"entries": [{"name": "first", "value": 1.5}],
|
||||
"futureKey": {"nestedKey": [false, 0, null]}
|
||||
})
|
||||
};
|
||||
let server = responses::start_mock_server().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/api/codex/images/generations"))
|
||||
@@ -95,7 +103,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model(
|
||||
"created": 1,
|
||||
"background": "opaque",
|
||||
"data": [
|
||||
{"b64_json": result, "generation_id": "gen-image-123"},
|
||||
{"b64_json": result, "generation_id": "gen-image-123", "image_metadata": expected_metadata},
|
||||
{"b64_json": "ignored", "generation_id": "gen-other"},
|
||||
],
|
||||
})),
|
||||
@@ -187,6 +195,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model(
|
||||
transparent_background,
|
||||
saved_path: Some(saved_path),
|
||||
generation_id,
|
||||
image_metadata,
|
||||
..
|
||||
}) = completed.item.payload
|
||||
else {
|
||||
@@ -195,6 +204,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model(
|
||||
assert_eq!(status, "completed");
|
||||
assert_eq!(revised_prompt.as_deref(), Some("paint a blue whale"));
|
||||
assert_eq!(generation_id.as_deref(), Some("gen-image-123"));
|
||||
assert_eq!(Some(&image_metadata), expected_metadata.as_object());
|
||||
assert_eq!(
|
||||
image,
|
||||
GeneratedImageRef::Inline {
|
||||
@@ -256,6 +266,10 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model(
|
||||
|
||||
let requests = response_mock.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(
|
||||
!requests[1].body_json().to_string().contains("futureKey"),
|
||||
"surface metadata must stay out of the model request"
|
||||
);
|
||||
let output = requests[1].function_call_output(call_id);
|
||||
if !gif_enabled {
|
||||
assert_eq!(
|
||||
@@ -394,6 +408,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history(
|
||||
image,
|
||||
transparent_background,
|
||||
input_images,
|
||||
image_metadata,
|
||||
..
|
||||
}) = completed.item.payload
|
||||
else {
|
||||
@@ -408,6 +423,10 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history(
|
||||
);
|
||||
assert_eq!(transparent_background, Some(true));
|
||||
assert_eq!(input_images, expected_inputs);
|
||||
assert_eq!(
|
||||
image_metadata.get("futureKey"),
|
||||
Some(&json!({"nestedKey": [0, false, null]}))
|
||||
);
|
||||
|
||||
drop(mcp);
|
||||
let mut resumed = TestAppServer::builder()
|
||||
@@ -438,6 +457,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history(
|
||||
Some("gen-transparent")
|
||||
);
|
||||
assert_eq!(persisted_image.input_images, expected_inputs);
|
||||
assert_eq!(persisted_image.image_metadata, image_metadata);
|
||||
|
||||
let resume_id = resumed
|
||||
.send_thread_resume_request(ThreadResumeParams {
|
||||
@@ -462,6 +482,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history(
|
||||
Some("gen-transparent")
|
||||
);
|
||||
assert_eq!(resumed_image.input_images, expected_inputs);
|
||||
assert_eq!(resumed_image.image_metadata, image_metadata);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -688,6 +709,7 @@ async fn standalone_image_generation_failure_emits_terminal_item(
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
})
|
||||
);
|
||||
|
||||
@@ -1217,6 +1239,7 @@ generatedImage(result);
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1472,7 +1495,11 @@ async fn mount_image_response_with_background(server: &MockServer, background: &
|
||||
.set_body_json(json!({
|
||||
"created": 1,
|
||||
"background": background,
|
||||
"data": [{"b64_json": RESULT, "generation_id": "gen-transparent"}],
|
||||
"data": [{
|
||||
"b64_json": RESULT,
|
||||
"generation_id": "gen-transparent",
|
||||
"image_metadata": {"futureKey": {"nestedKey": [0, false, null]}}
|
||||
}],
|
||||
})),
|
||||
)
|
||||
.expect(1)
|
||||
@@ -1486,7 +1513,11 @@ async fn mount_image_edit_response(server: &MockServer) {
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"created": 1,
|
||||
"background": "transparent",
|
||||
"data": [{"b64_json": RESULT, "generation_id": "gen-transparent"}],
|
||||
"data": [{
|
||||
"b64_json": RESULT,
|
||||
"generation_id": "gen-transparent",
|
||||
"image_metadata": {"futureKey": {"nestedKey": [0, false, null]}}
|
||||
}],
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(server)
|
||||
|
||||
@@ -3209,6 +3209,7 @@ fn append_resume_redaction_history(
|
||||
saved_path: Some(test_absolute_path("/tmp/ig-1.png")),
|
||||
input_images: Vec::new(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}),
|
||||
]
|
||||
.into_iter()
|
||||
|
||||
@@ -229,6 +229,7 @@ mod tests {
|
||||
data: vec![ImageData {
|
||||
b64_json: "REDACT".to_string(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}],
|
||||
quality: Some(ImageQuality::Medium),
|
||||
size: Some("1024x1536".to_string()),
|
||||
@@ -296,13 +297,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preserves_distinct_generation_ids_on_image_data() {
|
||||
async fn preserves_optional_metadata_on_each_image() {
|
||||
let first = json!({"entries": [{"name": "first", "value": 1.5}]});
|
||||
let second =
|
||||
json!({"futureKey": {"zero": 0, "disabled": false, "nestedKeys": [null, "x"]}});
|
||||
let body = serde_json::to_vec(&json!({
|
||||
"created": 1,
|
||||
"data": [
|
||||
{"b64_json": "first", "generation_id": "gen-first"},
|
||||
{"b64_json": "second", "generation_id": "gen-second"},
|
||||
{"b64_json": "legacy"}
|
||||
{"b64_json": "first", "generation_id": "gen-first", "image_metadata": first},
|
||||
{"b64_json": "second", "generation_id": "gen-second", "image_metadata": second},
|
||||
{"b64_json": "legacy"},
|
||||
{"b64_json": "empty", "image_metadata": {}},
|
||||
{"b64_json": "null", "image_metadata": null},
|
||||
{"b64_json": "list", "image_metadata": []},
|
||||
{"b64_json": "scalar", "image_metadata": 0},
|
||||
{"b64_json": "oversized", "image_metadata": {"value": "x".repeat(1024 * 1024)}}
|
||||
]
|
||||
}))
|
||||
.expect("serialize response");
|
||||
@@ -325,14 +334,47 @@ mod tests {
|
||||
.await
|
||||
.expect("image response should parse");
|
||||
|
||||
assert_eq!(response.data.len(), 8);
|
||||
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
|
||||
assert_eq!(
|
||||
response
|
||||
.data
|
||||
.iter()
|
||||
.map(|image| image.generation_id.as_deref())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![Some("gen-first"), Some("gen-second"), None]
|
||||
response.data[1].generation_id.as_deref(),
|
||||
Some("gen-second")
|
||||
);
|
||||
assert_eq!(Some(&response.data[0].image_metadata), first.as_object());
|
||||
assert_eq!(Some(&response.data[1].image_metadata), second.as_object());
|
||||
for (image, label) in response.data[2..].iter().zip([
|
||||
"legacy",
|
||||
"empty",
|
||||
"null",
|
||||
"list",
|
||||
"scalar",
|
||||
"oversized",
|
||||
]) {
|
||||
assert_eq!(image.b64_json, label);
|
||||
assert!(image.image_metadata.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deeply_nested_metadata_does_not_drop_images() {
|
||||
for (depth, keep_metadata) in [(23, true), (24, false), (120, false), (512, false)] {
|
||||
let nested = format!("{}null{}", r#"{"child":"#.repeat(depth), "}".repeat(depth));
|
||||
let body = format!(
|
||||
r#"{{"created": 1, "data": [
|
||||
{{"image_metadata": {{"nested": {nested}}}, "b64_json": "first", "generation_id": "gen-first"}},
|
||||
{{"b64_json": "second", "image_metadata": {{"value": 1}}}}
|
||||
]}}"#
|
||||
);
|
||||
let response: ImageResponse = serde_json::from_str(&body).expect("decode images");
|
||||
assert_eq!(response.data[0].b64_json, "first");
|
||||
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
|
||||
assert_eq!(
|
||||
response.data[0].image_metadata.contains_key("nested"),
|
||||
keep_metadata
|
||||
);
|
||||
assert_eq!(response.data[1].b64_json, "second");
|
||||
assert_eq!(response.data[1].image_metadata["value"], json!(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2,6 +2,15 @@ use codex_protocol::models::ImageReference;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value;
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
const MAX_IMAGE_METADATA_BYTES: usize = 1024 * 1024;
|
||||
|
||||
// Protobuf Struct objects consume three decode levels per nesting level.
|
||||
// Leave room for the enclosing CCA and app-server event and history messages.
|
||||
const MAX_IMAGE_METADATA_DEPTH: usize = 24;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct ImageGenerationRequest {
|
||||
@@ -90,4 +99,40 @@ pub struct ImageData {
|
||||
pub b64_json: String,
|
||||
#[serde(default)]
|
||||
pub generation_id: Option<String>,
|
||||
#[serde(default, deserialize_with = "deserialize_image_metadata")]
|
||||
pub image_metadata: Map<String, Value>,
|
||||
}
|
||||
|
||||
fn deserialize_image_metadata<'de, D>(deserializer: D) -> Result<Map<String, Value>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
// Read the field independently so a depth error cannot invalidate the surrounding image.
|
||||
let raw = <&RawValue>::deserialize(deserializer)?;
|
||||
if raw.get().len() > MAX_IMAGE_METADATA_BYTES {
|
||||
return Ok(Map::new());
|
||||
}
|
||||
Ok(serde_json::from_str::<Map<String, Value>>(raw.get())
|
||||
.ok()
|
||||
.filter(|metadata| {
|
||||
metadata
|
||||
.values()
|
||||
.all(|value| fits_metadata_depth(value, /*depth*/ 1))
|
||||
})
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn fits_metadata_depth(value: &Value, depth: usize) -> bool {
|
||||
if depth > MAX_IMAGE_METADATA_DEPTH {
|
||||
return false;
|
||||
}
|
||||
match value {
|
||||
Value::Array(items) => items
|
||||
.iter()
|
||||
.all(|item| fits_metadata_depth(item, depth + 1)),
|
||||
Value::Object(fields) => fields
|
||||
.values()
|
||||
.all(|item| fits_metadata_depth(item, depth + 1)),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -622,6 +622,7 @@ mod tests {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
let expected_completed_item = ExtensionItem::ImageGeneration(ImageGenerationItem {
|
||||
id: "call-image".to_string(),
|
||||
@@ -636,6 +637,7 @@ mod tests {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
codex_tools::TurnItemEmitter::emit_started(
|
||||
&emitter,
|
||||
@@ -663,6 +665,7 @@ mod tests {
|
||||
saved_path: Some(expected_path.clone()),
|
||||
input_images: Vec::new(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
})],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -125,6 +125,7 @@ fn legacy_end_event(item: &ImageGenerationItem) -> EventMsg {
|
||||
saved_path: item.saved_path.clone(),
|
||||
input_images: item.input_images.clone(),
|
||||
generation_id: item.generation_id.clone(),
|
||||
image_metadata: item.image_metadata.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -191,6 +192,7 @@ impl ImageGenerationTool {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
},
|
||||
EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent {
|
||||
call_id: call.call_id.clone(),
|
||||
@@ -223,12 +225,7 @@ impl ImageGenerationTool {
|
||||
Some(ImageBackground::Auto) | None => None,
|
||||
};
|
||||
match response.data.into_iter().next() {
|
||||
Some(data) => Ok((
|
||||
data.b64_json,
|
||||
transparent_background,
|
||||
imagegen_request_id,
|
||||
data.generation_id,
|
||||
)),
|
||||
Some(data) => Ok((data, transparent_background, imagegen_request_id)),
|
||||
None => Err((
|
||||
"image generation returned no image data".to_string(),
|
||||
None,
|
||||
@@ -236,7 +233,7 @@ impl ImageGenerationTool {
|
||||
)),
|
||||
}
|
||||
});
|
||||
let (result, transparent_background, imagegen_request_id, generation_id) = match result {
|
||||
let (data, transparent_background, imagegen_request_id) = match result {
|
||||
Ok(result) => result,
|
||||
Err((message, failure, imagegen_request_id)) => {
|
||||
let item = ImageGenerationItem {
|
||||
@@ -252,6 +249,7 @@ impl ImageGenerationTool {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
};
|
||||
let legacy_event = legacy_end_event(&item);
|
||||
call.turn_item_emitter
|
||||
@@ -260,6 +258,7 @@ impl ImageGenerationTool {
|
||||
return Err(FunctionCallError::RespondToModel(message));
|
||||
}
|
||||
};
|
||||
let result = data.b64_json;
|
||||
// TODO(anp): Migrate image tool path arguments and saved-path events to PathUri so image
|
||||
// operations can use the primary environment even when its paths are foreign to the host.
|
||||
let saved_path = save_image_generation_result(
|
||||
@@ -318,7 +317,8 @@ impl ImageGenerationTool {
|
||||
saved_path: saved_path.clone(),
|
||||
input_images,
|
||||
imagegen_request_id,
|
||||
generation_id,
|
||||
generation_id: data.generation_id,
|
||||
image_metadata: data.image_metadata,
|
||||
};
|
||||
let legacy_event = legacy_end_event(&item);
|
||||
call.turn_item_emitter
|
||||
|
||||
@@ -17,6 +17,7 @@ codex-utils-absolute-path = { workspace = true }
|
||||
schemars = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_with = { workspace = true }
|
||||
ts-rs = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -2,6 +2,9 @@ use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value;
|
||||
use serde_with::serde_as;
|
||||
use ts_rs::TS;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)]
|
||||
@@ -32,6 +35,7 @@ pub struct ImageGenerationInputImage {
|
||||
// Standalone image-generation item owned by the image extension. This is also
|
||||
// the field-level representation exposed by app-server; core and rollout
|
||||
// persistence only carry it inside an ExtensionItem envelope.
|
||||
#[serde_as]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
@@ -64,6 +68,11 @@ pub struct ImageGenerationItem {
|
||||
/// Image generation identifier for the generated image, when provided by the image backend.
|
||||
#[serde(default)]
|
||||
pub generation_id: Option<String>,
|
||||
/// Per-image metadata selected by the backend for the receiving surface.
|
||||
#[serde_as(deserialize_as = "serde_with::DefaultOnError")]
|
||||
#[serde(default, skip_serializing_if = "Map::is_empty")]
|
||||
#[ts(as = "Option<_>", optional)]
|
||||
pub image_metadata: Map<String, Value>,
|
||||
}
|
||||
|
||||
/// Either inline generated-image data or its durable attachment-store identity.
|
||||
|
||||
@@ -25,6 +25,7 @@ fn completed_image_generation_item() -> ExtensionItem {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -75,6 +76,28 @@ fn image_generation_item_preserves_stable_wire_shape() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_metadata_survives_persistence() {
|
||||
let base = serde_json::to_value(completed_image_generation_item()).expect("serialize item");
|
||||
let mut value = base.clone();
|
||||
value["imageMetadata"] = json!({
|
||||
"entries": [{"name": "first", "value": 1.5}],
|
||||
"futureKey": {"nestedKey": [null, false, 0]},
|
||||
});
|
||||
let item: ExtensionItem = serde_json::from_value(value.clone()).expect("read extended item");
|
||||
assert_eq!(
|
||||
serde_json::to_value(item).expect("persist extended item"),
|
||||
value
|
||||
);
|
||||
|
||||
for malformed in [json!({}), json!(null), json!("bad"), json!([])] {
|
||||
value["imageMetadata"] = malformed;
|
||||
let item: ExtensionItem =
|
||||
serde_json::from_value(value.clone()).expect("read completed image");
|
||||
assert_eq!(serde_json::to_value(item).expect("persist image"), base);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transitional rollouts containing both fields must retain the durable file reference.
|
||||
#[test]
|
||||
fn image_generation_item_reads_transitional_file_wire_shape() {
|
||||
@@ -91,6 +114,7 @@ fn image_generation_item_reads_transitional_file_wire_shape() {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
|
||||
@@ -487,6 +487,7 @@ impl ImageGenerationItem {
|
||||
saved_path: self.saved_path.clone(),
|
||||
input_images: Vec::new(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3225,6 +3225,7 @@ pub struct ImageGenerationBeginEvent {
|
||||
pub call_id: String,
|
||||
}
|
||||
|
||||
#[serde_as]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
|
||||
pub struct ImageGenerationEndEvent {
|
||||
pub call_id: String,
|
||||
@@ -3249,6 +3250,10 @@ pub struct ImageGenerationEndEvent {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
pub generation_id: Option<String>,
|
||||
#[serde_as(deserialize_as = "serde_with::DefaultOnError")]
|
||||
#[serde(default, skip_serializing_if = "Map::is_empty")]
|
||||
#[ts(as = "Option<_>", optional)]
|
||||
pub image_metadata: Map<String, Value>,
|
||||
}
|
||||
|
||||
// Conversation kept for backward compatibility.
|
||||
|
||||
@@ -22,6 +22,7 @@ fn image_item(image: GeneratedImageRef) -> ImageGenerationItem {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +72,7 @@ fn legacy_image_generation_preserves_file_reference() {
|
||||
saved_path: None,
|
||||
input_images: Vec::new(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
});
|
||||
|
||||
let persisted = persisted_event_msg(&event, ThreadHistoryMode::Legacy)
|
||||
|
||||
@@ -251,6 +251,7 @@ pub(super) fn completed_item(
|
||||
input_images: event.input_images.clone(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: event.generation_id.clone(),
|
||||
image_metadata: event.image_metadata.clone(),
|
||||
})),
|
||||
None,
|
||||
)),
|
||||
|
||||
@@ -567,6 +567,7 @@ async fn migration_preserves_image_generation_failure_metadata() {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
};
|
||||
let image_completion =
|
||||
RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent {
|
||||
@@ -579,6 +580,7 @@ async fn migration_preserves_image_generation_failure_metadata() {
|
||||
saved_path: expected_item.saved_path.clone(),
|
||||
input_images: Vec::new(),
|
||||
generation_id: expected_item.generation_id.clone(),
|
||||
image_metadata: expected_item.image_metadata.clone(),
|
||||
}));
|
||||
let path = write_rollout(
|
||||
home.path(),
|
||||
@@ -631,6 +633,10 @@ async fn migration_preserves_image_generation_input_observations() {
|
||||
}],
|
||||
imagegen_request_id: None,
|
||||
generation_id: Some("gen-image-123".to_string()),
|
||||
image_metadata: serde_json::Map::from_iter([(
|
||||
"futureKey".to_string(),
|
||||
json!({"nestedKey": [0, false, null]}),
|
||||
)]),
|
||||
};
|
||||
let image_completion =
|
||||
RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent {
|
||||
@@ -643,6 +649,7 @@ async fn migration_preserves_image_generation_input_observations() {
|
||||
saved_path: expected_item.saved_path.clone(),
|
||||
input_images: expected_item.input_images.clone(),
|
||||
generation_id: expected_item.generation_id.clone(),
|
||||
image_metadata: expected_item.image_metadata.clone(),
|
||||
}));
|
||||
let path = write_rollout(
|
||||
home.path(),
|
||||
|
||||
@@ -890,6 +890,7 @@ pub(super) fn handle_image_generation_end(
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}),
|
||||
/*started_at_ms*/ None,
|
||||
/*completed_at_ms*/ 0,
|
||||
|
||||
@@ -134,6 +134,7 @@ fn tool_and_notice_projection_uses_normal_transcript_presentation() {
|
||||
input_images: Vec::new(),
|
||||
imagegen_request_id: None,
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
};
|
||||
let items = vec![
|
||||
ThreadItemPayload::EnteredReviewMode {
|
||||
|
||||
@@ -6039,6 +6039,13 @@ class FileIdThreadItem(BaseModel):
|
||||
),
|
||||
] = None
|
||||
id: str
|
||||
image_metadata: Annotated[
|
||||
dict[str, Any] | None,
|
||||
Field(
|
||||
alias="imageMetadata",
|
||||
description="Per-image metadata selected by the backend for the receiving surface.",
|
||||
),
|
||||
] = None
|
||||
input_images: Annotated[
|
||||
list[ImageGenerationInputImage] | None,
|
||||
Field(
|
||||
@@ -6081,6 +6088,13 @@ class ResultThreadItem(BaseModel):
|
||||
),
|
||||
] = None
|
||||
id: str
|
||||
image_metadata: Annotated[
|
||||
dict[str, Any] | None,
|
||||
Field(
|
||||
alias="imageMetadata",
|
||||
description="Per-image metadata selected by the backend for the receiving surface.",
|
||||
),
|
||||
] = None
|
||||
input_images: Annotated[
|
||||
list[ImageGenerationInputImage] | None,
|
||||
Field(
|
||||
|
||||
Reference in New Issue
Block a user