Record local image inputs on completed image-generation items (#54420)

## What changed

Add optional `inputImages` metadata containing each referenced local image's path and SHA-256 hash of the bytes read before normalization. Compute the hash from the same bytes used to prepare the edit request.

Preserve this metadata through completion events, persisted thread history, resume, and legacy rollout migration. Update app-server schemas and generated TypeScript and Python types; older items continue to deserialize without the field.

## Testing

Add coverage for input ordering, changed contents at the same path, unreadable or invalid references, serialization, persisted history and resume, and legacy migration.

GitOrigin-RevId: 522be8ec1e39c9b2bbe5b9ccb37ea67d17f0e21a
This commit is contained in:
Matt Kaufer
2026-10-15 21:30:56 +00:00
committed by copyberry
parent 9fcb1f5ed4
commit 592964c0cb
47 changed files with 970 additions and 75 deletions
@@ -28,6 +28,8 @@ use crate::protocol::v2::WebSearchAction;
use crate::protocol::v2::WebSearchItem;
use crate::protocol::v2::web_search_action_from_core;
use codex_extension_items::image_generation::GeneratedImageRef;
#[cfg(test)]
use codex_extension_items::image_generation::ImageGenerationInputImage;
use codex_extension_items::image_generation::ImageGenerationItem;
use codex_protocol::items::parse_hook_prompt_message;
use codex_protocol::protocol::AgentMessageEvent;
@@ -906,6 +908,7 @@ impl ThreadHistoryBuilder {
transparent_background: None,
failure: None,
saved_path: None,
input_images: Vec::new(),
imagegen_request_id: None,
generation_id: None,
});
@@ -921,6 +924,7 @@ impl ThreadHistoryBuilder {
transparent_background: payload.transparent_background,
failure: payload.failure.clone(),
saved_path: payload.saved_path.clone(),
input_images: payload.input_images.clone(),
imagegen_request_id: None,
generation_id: None,
});
@@ -2333,6 +2337,10 @@ mod tests {
let turn_id = "turn-1";
let thread_id = ThreadId::new();
let saved_path = test_path_buf("/tmp/image-1.png").abs();
let input_images = vec![ImageGenerationInputImage {
path: test_path_buf("/tmp/reference.png").abs(),
sha256: "ab".repeat(32),
}];
let events = vec![
EventMsg::TurnStarted(TurnStartedEvent {
turn_attribution: None,
@@ -2357,6 +2365,7 @@ mod tests {
transparent_background: Some(true),
failure: None,
saved_path: Some(saved_path.clone()),
input_images: input_images.clone(),
imagegen_request_id: None,
generation_id: None,
},
@@ -2378,6 +2387,10 @@ mod tests {
let items = events
.into_iter()
.map(RolloutItem::EventMsg)
.map(|item| {
serde_json::from_value(serde_json::to_value(item).expect("serialize rollout"))
.expect("deserialize rollout")
})
.collect::<Vec<_>>();
let turns = build_turns_from_rollout_items(&items);
@@ -2394,6 +2407,7 @@ mod tests {
transparent_background: Some(true),
failure: None,
saved_path: Some(saved_path),
input_images,
imagegen_request_id: None,
generation_id: None,
})]
@@ -2689,6 +2703,10 @@ mod tests {
},
),
saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()),
input_images: vec![ImageGenerationInputImage {
path: test_path_buf("/tmp/reference.png").abs(),
sha256: "ab".repeat(32),
}],
})),
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
root_turn_id: None,
@@ -2739,6 +2757,10 @@ mod tests {
},
),
saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()),
input_images: vec![ImageGenerationInputImage {
path: test_path_buf("/tmp/reference.png").abs(),
sha256: "ab".repeat(32),
}],
imagegen_request_id: None,
generation_id: None,
}),
@@ -17,6 +17,7 @@ use codex_experimental_api_macros::ExperimentalApi;
use codex_extension_items::ExtensionItem;
pub use codex_extension_items::image_generation::GeneratedImageRef;
pub use codex_extension_items::image_generation::ImageGenerationFailure;
pub use codex_extension_items::image_generation::ImageGenerationInputImage;
pub use codex_extension_items::image_generation::ImageGenerationItem;
pub use codex_extension_items::sleep::SleepItem;
pub use codex_extension_items::web_search::WebSearchAction;
@@ -1016,6 +1017,7 @@ impl From<CoreTurnItem> for ThreadItem {
transparent_background: None,
failure: None,
saved_path: image.saved_path,
input_images: Vec::new(),
imagegen_request_id: None,
generation_id: None,
})