Support image references by file ID in inputs and tool outputs (#45794)

## What changed

- Accept `fileId` alongside the existing `url` form for app-server image inputs, and forward file references to the Responses API as `file_id`. Update generated schemas and client types.
- Preserve file references, image detail hints, and mixed inline/file image ordering through user-message events, thread history, and rollout migration. Retain file images when truncating tool output.
- Pass file references through image preparation without resolving them, while keeping resize-notice numbering correct. Omit them from unsupported TUI display and Guardian image context.
- Reject image-edit requests whose recent-image window includes a file reference, preventing selection of an older inline image instead.

## Testing

Add coverage for serialization, request and rollout preservation, mixed-image history ordering, incomplete ordering metadata, tool-output truncation, and rejection of unsupported image-edit selections.

GitOrigin-RevId: 6ca20a8577155cc934b720803c3b7b3bffdf972a
This commit is contained in:
Krish Chainani
2026-09-15 21:07:56 +00:00
committed by copyberry
parent c51cb968e4
commit 7b8b17b97a
95 changed files with 2434 additions and 324 deletions

View File

@@ -6240,7 +6240,7 @@ class TextUserInput(BaseModel):
type: Annotated[Literal["text"], Field(title="TextUserInputType")]
class ImageUserInput(BaseModel):
class UrlUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
@@ -6249,6 +6249,15 @@ class ImageUserInput(BaseModel):
url: str
class FileIdUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[Literal["image"], Field(title="ImageUserInputType")]
file_id: Annotated[str, Field(alias="fileId")]
class LocalImageUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6295,7 +6304,8 @@ class MentionUserInput(BaseModel):
class UserInput(
RootModel[
TextUserInput
| ImageUserInput
| UrlUserInput
| FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -6308,7 +6318,8 @@ class UserInput(
)
root: (
TextUserInput
| ImageUserInput
| UrlUserInput
| FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -7794,15 +7805,34 @@ class InputImageContentItem(BaseModel):
image_url: str
class FileIdContentItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")]
file_id: str
class ContentItem(
RootModel[
InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
InputTextContentItem
| InputImageContentItem
| FileIdContentItem
| InputAudioContentItem
| OutputTextContentItem
]
):
model_config = ConfigDict(
populate_by_name=True,
)
root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
root: (
InputTextContentItem
| InputImageContentItem
| FileIdContentItem
| InputAudioContentItem
| OutputTextContentItem
)
class ExperimentalFeature(BaseModel):
@@ -7990,10 +8020,22 @@ class InputImageFunctionCallOutputContentItem(BaseModel):
image_url: str
class FileIdFunctionCallOutputContentItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[
Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType")
]
file_id: str
class FunctionCallOutputContentItem(
RootModel[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem
]
@@ -8004,6 +8046,7 @@ class FunctionCallOutputContentItem(
root: Annotated[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem,
Field(