Preserve per-image metadata in app-server image generation items (#62871)

## Why

App-server clients lose metadata supplied by the image backend when
Codex turns the response into a completed image generation item.

## What changed

Expose the backend's `image_metadata` as optional `imageMetadata` in
app-server items and preserve nested values through thread persistence,
resume, and legacy history replay and migration. Keep this metadata out
of model tool output.

Omit empty metadata and discard non-object, oversized, or deeply nested
metadata without dropping the generated image.

## Testing

Extend coverage for per-image parsing, metadata limits, and round trips
through app-server completion, persisted history, and thread resume.

GitOrigin-RevId: 7211a9db1149cb05e6176b08b0462ca173ac72d5
This commit is contained in:
Sidharth Lakshmanan
2026-11-21 00:23:59 +00:00
committed by copyberry
parent e346540867
commit 0bfb8467da
47 changed files with 341 additions and 22 deletions
+52 -10
View File
@@ -229,6 +229,7 @@ mod tests {
data: vec![ImageData {
b64_json: "REDACT".to_string(),
generation_id: None,
image_metadata: Default::default(),
}],
quality: Some(ImageQuality::Medium),
size: Some("1024x1536".to_string()),
@@ -296,13 +297,21 @@ mod tests {
}
#[tokio::test]
async fn preserves_distinct_generation_ids_on_image_data() {
async fn preserves_optional_metadata_on_each_image() {
let first = json!({"entries": [{"name": "first", "value": 1.5}]});
let second =
json!({"futureKey": {"zero": 0, "disabled": false, "nestedKeys": [null, "x"]}});
let body = serde_json::to_vec(&json!({
"created": 1,
"data": [
{"b64_json": "first", "generation_id": "gen-first"},
{"b64_json": "second", "generation_id": "gen-second"},
{"b64_json": "legacy"}
{"b64_json": "first", "generation_id": "gen-first", "image_metadata": first},
{"b64_json": "second", "generation_id": "gen-second", "image_metadata": second},
{"b64_json": "legacy"},
{"b64_json": "empty", "image_metadata": {}},
{"b64_json": "null", "image_metadata": null},
{"b64_json": "list", "image_metadata": []},
{"b64_json": "scalar", "image_metadata": 0},
{"b64_json": "oversized", "image_metadata": {"value": "x".repeat(1024 * 1024)}}
]
}))
.expect("serialize response");
@@ -325,14 +334,47 @@ mod tests {
.await
.expect("image response should parse");
assert_eq!(response.data.len(), 8);
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
assert_eq!(
response
.data
.iter()
.map(|image| image.generation_id.as_deref())
.collect::<Vec<_>>(),
vec![Some("gen-first"), Some("gen-second"), None]
response.data[1].generation_id.as_deref(),
Some("gen-second")
);
assert_eq!(Some(&response.data[0].image_metadata), first.as_object());
assert_eq!(Some(&response.data[1].image_metadata), second.as_object());
for (image, label) in response.data[2..].iter().zip([
"legacy",
"empty",
"null",
"list",
"scalar",
"oversized",
]) {
assert_eq!(image.b64_json, label);
assert!(image.image_metadata.is_empty());
}
}
#[test]
fn deeply_nested_metadata_does_not_drop_images() {
for (depth, keep_metadata) in [(23, true), (24, false), (120, false), (512, false)] {
let nested = format!("{}null{}", r#"{"child":"#.repeat(depth), "}".repeat(depth));
let body = format!(
r#"{{"created": 1, "data": [
{{"image_metadata": {{"nested": {nested}}}, "b64_json": "first", "generation_id": "gen-first"}},
{{"b64_json": "second", "image_metadata": {{"value": 1}}}}
]}}"#
);
let response: ImageResponse = serde_json::from_str(&body).expect("decode images");
assert_eq!(response.data[0].b64_json, "first");
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
assert_eq!(
response.data[0].image_metadata.contains_key("nested"),
keep_metadata
);
assert_eq!(response.data[1].b64_json, "second");
assert_eq!(response.data[1].image_metadata["value"], json!(1));
}
}
#[tokio::test]
+45
View File
@@ -2,6 +2,15 @@ use codex_protocol::models::ImageReference;
use schemars::JsonSchema;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Map;
use serde_json::Value;
use serde_json::value::RawValue;
const MAX_IMAGE_METADATA_BYTES: usize = 1024 * 1024;
// Protobuf Struct objects consume three decode levels per nesting level.
// Leave room for the enclosing CCA and app-server event and history messages.
const MAX_IMAGE_METADATA_DEPTH: usize = 24;
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct ImageGenerationRequest {
@@ -90,4 +99,40 @@ pub struct ImageData {
pub b64_json: String,
#[serde(default)]
pub generation_id: Option<String>,
#[serde(default, deserialize_with = "deserialize_image_metadata")]
pub image_metadata: Map<String, Value>,
}
fn deserialize_image_metadata<'de, D>(deserializer: D) -> Result<Map<String, Value>, D::Error>
where
D: serde::Deserializer<'de>,
{
// Read the field independently so a depth error cannot invalidate the surrounding image.
let raw = <&RawValue>::deserialize(deserializer)?;
if raw.get().len() > MAX_IMAGE_METADATA_BYTES {
return Ok(Map::new());
}
Ok(serde_json::from_str::<Map<String, Value>>(raw.get())
.ok()
.filter(|metadata| {
metadata
.values()
.all(|value| fits_metadata_depth(value, /*depth*/ 1))
})
.unwrap_or_default())
}
fn fits_metadata_depth(value: &Value, depth: usize) -> bool {
if depth > MAX_IMAGE_METADATA_DEPTH {
return false;
}
match value {
Value::Array(items) => items
.iter()
.all(|item| fits_metadata_depth(item, depth + 1)),
Value::Object(fields) => fields
.values()
.all(|item| fits_metadata_depth(item, depth + 1)),
_ => true,
}
}