Store generated image items as attachment references (#54398)

## What changed

- Upload generated images to the attachment store and expose `fileId` in image-generation items when available, with inline `result` fallback on upload or decoding failure.
- Preserve file references through legacy events, persisted rollouts, thread history, and task output summaries. Keep model-facing image preparation separate.
- Add attachment request tags and mark generated-image uploads with `usage=image_gen`, preserving the thread's ephemeral setting.
- Update public schemas and SDK types to accept either image representation, retain the Python `ImageGenerationThreadItem` alias, and prefer `fileId` when reading records containing both fields.

## Testing

Add coverage for file-backed generation and rollout persistence, legacy history replay, inline and transitional serialization, task output visibility, and Python SDK alias compatibility.

GitOrigin-RevId: f7a2bb5707e2f277348a547a988bc33c8e96d891
This commit is contained in:
Krish Chainani
2026-10-15 19:51:22 +00:00
committed by copyberry
parent 1ef30bcbe7
commit aabf13307a
64 changed files with 1229 additions and 125 deletions
+20 -1
View File
@@ -29,7 +29,8 @@ pub struct ImageGenerationItem {
pub id: String,
pub status: String,
pub revised_prompt: Option<String>,
pub result: String,
#[serde(flatten)]
pub image: GeneratedImageRef,
#[serde(default)]
#[ts(optional)]
pub transparent_background: Option<bool>,
@@ -52,3 +53,21 @@ pub struct ImageGenerationItem {
#[ts(skip)]
pub generation_id: Option<String>,
}
/// Either inline generated-image data or its durable attachment-store identity.
///
/// `File` must remain first so rollouts written during the migration, which contain both an empty
/// `result` and a `fileId`, retain the file reference when deserialized.
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)]
#[serde(untagged)]
#[ts(untagged)]
pub enum GeneratedImageRef {
File {
#[serde(rename = "fileId")]
#[ts(rename = "fileId")]
file_id: String,
},
Inline {
result: String,
},
}
+77 -2
View File
@@ -3,6 +3,7 @@ use serde_json::json;
use ts_rs::TS;
use super::ExtensionItem;
use super::image_generation::GeneratedImageRef;
use super::image_generation::ImageGenerationItem;
use super::sleep::SleepItem;
use super::web_search::WebSearchAction;
@@ -13,7 +14,9 @@ fn completed_image_generation_item() -> ExtensionItem {
id: "image-1".to_string(),
status: "completed".to_string(),
revised_prompt: Some("A blue square".to_string()),
result: "cG5n".to_string(),
image: GeneratedImageRef::Inline {
result: "cG5n".to_string(),
},
transparent_background: None,
failure: None,
saved_path: None,
@@ -54,6 +57,73 @@ fn image_generation_item_preserves_stable_wire_shape() {
.expect("deserialize legacy image-generation item without transparency metadata"),
item
);
assert_eq!(
serde_json::from_value::<ExtensionItem>(json!({
"kind": "image_gen.generation",
"id": "image-1",
"status": "completed",
"revisedPrompt": "A blue square",
"result": "cG5n",
"fileId": null,
}))
.expect("deserialize transitional inline image-generation item"),
item
);
}
/// Transitional rollouts containing both fields must retain the durable file reference.
#[test]
fn image_generation_item_reads_transitional_file_wire_shape() {
let item = ExtensionItem::ImageGeneration(ImageGenerationItem {
id: "image-1".to_string(),
status: "completed".to_string(),
revised_prompt: Some("A blue square".to_string()),
image: GeneratedImageRef::File {
file_id: "file-1".to_string(),
},
transparent_background: None,
failure: None,
saved_path: None,
imagegen_request_id: None,
generation_id: None,
});
assert_eq!(
serde_json::from_value::<ExtensionItem>(json!({
"kind": "image_gen.generation",
"id": "image-1",
"status": "completed",
"revisedPrompt": "A blue square",
"result": "",
"fileId": "file-1",
}))
.expect("deserialize transitional file-backed image-generation item"),
item
);
assert_eq!(
serde_json::from_value::<ExtensionItem>(json!({
"kind": "image_gen.generation",
"id": "image-1",
"status": "completed",
"revisedPrompt": "A blue square",
"result": "stale-inline-bytes",
"fileId": "file-1",
}))
.expect("prefer file ID when both image representations are populated"),
item
);
assert_eq!(
serde_json::to_value(&item).expect("serialize file-backed extension item"),
json!({
"kind": "image_gen.generation",
"id": "image-1",
"status": "completed",
"revisedPrompt": "A blue square",
"fileId": "file-1",
"transparentBackground": null,
"failure": null,
})
);
}
#[test]
@@ -84,7 +154,12 @@ fn image_generation_item_preserves_authoritative_transparency() {
}
#[test]
fn image_generation_transparency_is_optional_in_typescript() {
fn image_generation_typescript_fields_preserve_wire_contracts() {
assert!(
ImageGenerationItem::inline().contains("{ fileId: string, } | { result: string, }")
|| ImageGenerationItem::inline().contains("{ result: string, } | { fileId: string, }"),
"v2 image-generation items must require exactly one image reference"
);
assert!(
ImageGenerationItem::inline().contains("transparentBackground?: boolean"),
"image-generation transparency must remain optional for existing TypeScript clients"