Support image references by file ID in inputs and tool outputs (#45794)

## What changed

- Accept `fileId` alongside the existing `url` form for app-server image inputs, and forward file references to the Responses API as `file_id`. Update generated schemas and client types.
- Preserve file references, image detail hints, and mixed inline/file image ordering through user-message events, thread history, and rollout migration. Retain file images when truncating tool output.
- Pass file references through image preparation without resolving them, while keeping resize-notice numbering correct. Omit them from unsupported TUI display and Guardian image context.
- Reject image-edit requests whose recent-image window includes a file reference, preventing selection of an older inline image instead.

## Testing

Add coverage for serialization, request and rollout preservation, mixed-image history ordering, incomplete ordering metadata, tool-output truncation, and rejection of unsupported image-edit selections.

GitOrigin-RevId: 6ca20a8577155cc934b720803c3b7b3bffdf972a
This commit is contained in:
Krish Chainani
2026-09-15 21:19:08 +00:00
committed by copyberry
parent c51cb968e4
commit 7b8b17b97a
95 changed files with 2434 additions and 324 deletions
+7 -2
View File
@@ -29,6 +29,7 @@ use codex_extension_api::TurnStartAdmission;
use codex_protocol::ThreadId;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
@@ -825,7 +826,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_queued_input = user_input_with_media(
"queued attachments",
UserInput::Image {
image_url: TINY_PNG_DATA_URL.to_string(),
image: ImageReference::Inline {
image_url: TINY_PNG_DATA_URL.to_string(),
},
detail: Some(ImageDetail::Original),
},
UserInput::Audio {
@@ -865,7 +868,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_edited_input = user_input_with_media(
"edited attachments",
UserInput::Image {
image_url: TINY_PNG_DATA_URL.to_string(),
image: ImageReference::Inline {
image_url: TINY_PNG_DATA_URL.to_string(),
},
detail: Some(ImageDetail::High),
},
UserInput::Audio {