mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Preserve per-image metadata in app-server image generation items (#62871)
## Why App-server clients lose metadata supplied by the image backend when Codex turns the response into a completed image generation item. ## What changed Expose the backend's `image_metadata` as optional `imageMetadata` in app-server items and preserve nested values through thread persistence, resume, and legacy history replay and migration. Keep this metadata out of model tool output. Omit empty metadata and discard non-object, oversized, or deeply nested metadata without dropping the generated image. ## Testing Extend coverage for per-image parsing, metadata limits, and round trips through app-server completion, persisted history, and thread resume. GitOrigin-RevId: 7211a9db1149cb05e6176b08b0462ca173ac72d5
This commit is contained in:
committed by
copyberry
parent
e346540867
commit
0bfb8467da
@@ -229,6 +229,7 @@ mod tests {
|
||||
data: vec![ImageData {
|
||||
b64_json: "REDACT".to_string(),
|
||||
generation_id: None,
|
||||
image_metadata: Default::default(),
|
||||
}],
|
||||
quality: Some(ImageQuality::Medium),
|
||||
size: Some("1024x1536".to_string()),
|
||||
@@ -296,13 +297,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preserves_distinct_generation_ids_on_image_data() {
|
||||
async fn preserves_optional_metadata_on_each_image() {
|
||||
let first = json!({"entries": [{"name": "first", "value": 1.5}]});
|
||||
let second =
|
||||
json!({"futureKey": {"zero": 0, "disabled": false, "nestedKeys": [null, "x"]}});
|
||||
let body = serde_json::to_vec(&json!({
|
||||
"created": 1,
|
||||
"data": [
|
||||
{"b64_json": "first", "generation_id": "gen-first"},
|
||||
{"b64_json": "second", "generation_id": "gen-second"},
|
||||
{"b64_json": "legacy"}
|
||||
{"b64_json": "first", "generation_id": "gen-first", "image_metadata": first},
|
||||
{"b64_json": "second", "generation_id": "gen-second", "image_metadata": second},
|
||||
{"b64_json": "legacy"},
|
||||
{"b64_json": "empty", "image_metadata": {}},
|
||||
{"b64_json": "null", "image_metadata": null},
|
||||
{"b64_json": "list", "image_metadata": []},
|
||||
{"b64_json": "scalar", "image_metadata": 0},
|
||||
{"b64_json": "oversized", "image_metadata": {"value": "x".repeat(1024 * 1024)}}
|
||||
]
|
||||
}))
|
||||
.expect("serialize response");
|
||||
@@ -325,14 +334,47 @@ mod tests {
|
||||
.await
|
||||
.expect("image response should parse");
|
||||
|
||||
assert_eq!(response.data.len(), 8);
|
||||
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
|
||||
assert_eq!(
|
||||
response
|
||||
.data
|
||||
.iter()
|
||||
.map(|image| image.generation_id.as_deref())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![Some("gen-first"), Some("gen-second"), None]
|
||||
response.data[1].generation_id.as_deref(),
|
||||
Some("gen-second")
|
||||
);
|
||||
assert_eq!(Some(&response.data[0].image_metadata), first.as_object());
|
||||
assert_eq!(Some(&response.data[1].image_metadata), second.as_object());
|
||||
for (image, label) in response.data[2..].iter().zip([
|
||||
"legacy",
|
||||
"empty",
|
||||
"null",
|
||||
"list",
|
||||
"scalar",
|
||||
"oversized",
|
||||
]) {
|
||||
assert_eq!(image.b64_json, label);
|
||||
assert!(image.image_metadata.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deeply_nested_metadata_does_not_drop_images() {
|
||||
for (depth, keep_metadata) in [(23, true), (24, false), (120, false), (512, false)] {
|
||||
let nested = format!("{}null{}", r#"{"child":"#.repeat(depth), "}".repeat(depth));
|
||||
let body = format!(
|
||||
r#"{{"created": 1, "data": [
|
||||
{{"image_metadata": {{"nested": {nested}}}, "b64_json": "first", "generation_id": "gen-first"}},
|
||||
{{"b64_json": "second", "image_metadata": {{"value": 1}}}}
|
||||
]}}"#
|
||||
);
|
||||
let response: ImageResponse = serde_json::from_str(&body).expect("decode images");
|
||||
assert_eq!(response.data[0].b64_json, "first");
|
||||
assert_eq!(response.data[0].generation_id.as_deref(), Some("gen-first"));
|
||||
assert_eq!(
|
||||
response.data[0].image_metadata.contains_key("nested"),
|
||||
keep_metadata
|
||||
);
|
||||
assert_eq!(response.data[1].b64_json, "second");
|
||||
assert_eq!(response.data[1].image_metadata["value"], json!(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2,6 +2,15 @@ use codex_protocol::models::ImageReference;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value;
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
const MAX_IMAGE_METADATA_BYTES: usize = 1024 * 1024;
|
||||
|
||||
// Protobuf Struct objects consume three decode levels per nesting level.
|
||||
// Leave room for the enclosing CCA and app-server event and history messages.
|
||||
const MAX_IMAGE_METADATA_DEPTH: usize = 24;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct ImageGenerationRequest {
|
||||
@@ -90,4 +99,40 @@ pub struct ImageData {
|
||||
pub b64_json: String,
|
||||
#[serde(default)]
|
||||
pub generation_id: Option<String>,
|
||||
#[serde(default, deserialize_with = "deserialize_image_metadata")]
|
||||
pub image_metadata: Map<String, Value>,
|
||||
}
|
||||
|
||||
fn deserialize_image_metadata<'de, D>(deserializer: D) -> Result<Map<String, Value>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
// Read the field independently so a depth error cannot invalidate the surrounding image.
|
||||
let raw = <&RawValue>::deserialize(deserializer)?;
|
||||
if raw.get().len() > MAX_IMAGE_METADATA_BYTES {
|
||||
return Ok(Map::new());
|
||||
}
|
||||
Ok(serde_json::from_str::<Map<String, Value>>(raw.get())
|
||||
.ok()
|
||||
.filter(|metadata| {
|
||||
metadata
|
||||
.values()
|
||||
.all(|value| fits_metadata_depth(value, /*depth*/ 1))
|
||||
})
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn fits_metadata_depth(value: &Value, depth: usize) -> bool {
|
||||
if depth > MAX_IMAGE_METADATA_DEPTH {
|
||||
return false;
|
||||
}
|
||||
match value {
|
||||
Value::Array(items) => items
|
||||
.iter()
|
||||
.all(|item| fits_metadata_depth(item, depth + 1)),
|
||||
Value::Object(fields) => fields
|
||||
.values()
|
||||
.all(|item| fits_metadata_depth(item, depth + 1)),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user