openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-02 19:43:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support image references by file ID in inputs and tool outputs (#45794)

Browse Source 
## What changed

- Accept `fileId` alongside the existing `url` form for app-server image inputs, and forward file references to the Responses API as `file_id`. Update generated schemas and client types.
- Preserve file references, image detail hints, and mixed inline/file image ordering through user-message events, thread history, and rollout migration. Retain file images when truncating tool output.
- Pass file references through image preparation without resolving them, while keeping resize-notice numbering correct. Omit them from unsupported TUI display and Guardian image context.
- Reject image-edit requests whose recent-image window includes a file reference, preventing selection of an older inline image instead.

## Testing

Add coverage for serialization, request and rollout preservation, mixed-image history ordering, incomplete ordering metadata, tool-output truncation, and rejection of unsupported image-edit selections.

GitOrigin-RevId: 6ca20a8577155cc934b720803c3b7b3bffdf972a
This commit is contained in:
Krish Chainani
2026-09-15 21:19:08 +00:00
committed by copyberry
parent c51cb968e4
commit 7b8b17b97a
95 changed files with 2434 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/analytics/src/analytics_client_tests.rs
+4 -1
View File
@@ -126,6 +126,7 @@ use codex_app_server_protocol::GuardianApprovalReviewAction;
use codex_app_server_protocol::GuardianApprovalReviewStatus;
use codex_app_server_protocol::GuardianCommandSource as AppServerGuardianCommandSource;
use codex_app_server_protocol::ImageGenerationItem;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::ItemCompletedNotification;
@@ -396,7 +397,9 @@ fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest
text_elements: vec![],
},
UserInput::Image {
url: "https://example.com/a.png".to_string(),
image: ImageReference::Inline {
url: "https://example.com/a.png".to_string(),
},
detail: None,
},
],
codex-rs/app-server-protocol/schema/json/ClientRequest.json
Generated
+51 -5
View File
@@ -672,6 +672,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdContentItem",
"type": "object"
}
],
"properties": {
@@ -1442,6 +1454,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -5663,6 +5687,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -5681,14 +5731,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/ServerNotification.json
Generated
+39 -5
View File
@@ -1771,6 +1771,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -6469,6 +6481,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -6487,14 +6525,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
Generated
+51 -5
View File
@@ -10484,6 +10484,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdContentItem",
"type": "object"
}
],
"properties": {
@@ -12355,6 +12367,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -25427,6 +25451,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -25445,14 +25495,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
Generated
+51 -5
View File
@@ -6364,6 +6364,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdContentItem",
"type": "object"
}
],
"properties": {
@@ -8235,6 +8247,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -23141,6 +23165,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -23159,14 +23209,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
Generated
+39 -5
View File
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1614,6 +1626,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1632,14 +1670,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
Generated
+39 -5
View File
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1614,6 +1626,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1632,14 +1670,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
Generated
+24
View File
@@ -92,6 +92,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdContentItem",
"type": "object"
}
],
"properties": {
@@ -210,6 +222,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
Generated
+39 -5
View File
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
Generated
+39 -5
View File
@@ -592,6 +592,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2684,6 +2696,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2702,14 +2740,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
Generated
+39 -5
View File
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1630,6 +1642,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1648,14 +1686,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
Generated
+24
View File
@@ -154,6 +154,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdContentItem",
"type": "object"
}
],
"properties": {
@@ -272,6 +284,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
Generated
+39 -5
View File
@@ -608,6 +608,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2762,6 +2774,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2780,14 +2818,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
Generated
+39 -5
View File
@@ -592,6 +592,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2684,6 +2696,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2702,14 +2740,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
Generated
+39 -5
View File
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
Generated
+39 -5
View File
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
Generated
+39 -5
View File
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
Generated
+39 -5
View File
@@ -178,6 +178,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -559,6 +571,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -577,14 +615,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
Generated
+39 -5
View File
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
Generated
+39 -5
View File
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json
Generated
+27 -5
View File
@@ -105,6 +105,32 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -123,14 +149,10 @@
],
"title": "ImageUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
"type"
],
"title": "ImageUserInput",
"type": "object"
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
Generated
+1 -1
View File
@@ -3,4 +3,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageDetail } from "./ImageDetail";
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
Generated
+1 -1
View File
@@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail";
* Responses API compatible content items that can be returned by a tool call.
* This is a subset of ContentItem with the types we support as function call outputs.
*/
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts
Generated
+1 -1
View File
@@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string,
/**
* UI-defined spans within `text` used to render or persist special elements.
*/
text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, url: string, } | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
codex-rs/app-server-protocol/src/protocol/thread_history.rs
+104 -4
View File
@@ -11,6 +11,7 @@ use crate::protocol::v2::CollabAgentToolCallStatus;
use crate::protocol::v2::CommandExecutionStatus;
use crate::protocol::v2::DynamicToolCallOutputContentItem;
use crate::protocol::v2::DynamicToolCallStatus;
use crate::protocol::v2::ImageReference;
use crate::protocol::v2::McpToolCallAppContext;
use crate::protocol::v2::McpToolCallError;
use crate::protocol::v2::McpToolCallResult;
@@ -54,6 +55,7 @@ use codex_protocol::protocol::TurnAbortedEvent;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::protocol::UserMessageImageKind;
use codex_protocol::protocol::ViewImageToolCallEvent;
use codex_protocol::protocol::WebSearchBeginEvent;
use codex_protocol::protocol::WebSearchEndEvent;
@@ -1511,14 +1513,62 @@ impl ThreadHistoryBuilder {
.collect(),
});
}
if let Some(images) = &payload.images {
let has_complete_image_order = payload.has_complete_image_order();
if has_complete_image_order {
let mut inline_index = 0;
let mut file_index = 0;
for image_kind in &payload.image_order {
let (image, detail) = match image_kind {
UserMessageImageKind::Inline => {
let Some(image) = payload
.images
.as_deref()
.and_then(|images| images.get(inline_index))
else {
continue;
};
let detail = payload.image_details.get(inline_index).copied().flatten();
inline_index += 1;
(ImageReference::Inline { url: image.clone() }, detail)
}
UserMessageImageKind::File => {
let Some(file_id) = payload
.file_ids
.as_deref()
.and_then(|file_ids| file_ids.get(file_index))
else {
continue;
};
let detail = payload.file_id_details.get(file_index).copied().flatten();
file_index += 1;
(
ImageReference::File {
file_id: file_id.clone(),
},
detail,
)
}
};
content.push(UserInput::Image { image, detail });
}
} else if let Some(images) = &payload.images {
for (idx, image) in images.iter().enumerate() {
content.push(UserInput::Image {
url: image.clone(),
image: ImageReference::Inline { url: image.clone() },
detail: payload.image_details.get(idx).copied().flatten(),
});
}
}
if !has_complete_image_order && let Some(file_ids) = &payload.file_ids {
for (idx, file_id) in file_ids.iter().enumerate() {
content.push(UserInput::Image {
image: ImageReference::File {
file_id: file_id.clone(),
},
detail: payload.file_id_details.get(idx).copied().flatten(),
});
}
}
for (idx, path) in payload.local_images.iter().enumerate() {
content.push(UserInput::LocalImage {
path: path.clone(),
@@ -1824,7 +1874,9 @@ mod tests {
text_elements: Vec::new(),
},
UserInput::Image {
url: "https://example.com/one.png".into(),
image: ImageReference::Inline {
url: "https://example.com/one.png".into(),
},
detail: None,
}
],
@@ -1999,6 +2051,9 @@ mod tests {
message: "inspect these".into(),
images: Some(vec!["https://example.com/image.png".into()]),
image_details: vec![Some(ImageDetail::Original)],
file_ids: Some(vec!["file_123".into()]),
file_id_details: vec![Some(ImageDetail::High)],
image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline],
local_images: vec![local_image_path.clone()],
local_image_details: vec![Some(ImageDetail::Original)],
audio: Some(vec!["https://example.com/audio.mp3".into()]),
@@ -2021,7 +2076,15 @@ mod tests {
text_elements: Vec::new(),
},
UserInput::Image {
url: "https://example.com/image.png".into(),
image: ImageReference::File {
file_id: "file_123".into(),
},
detail: Some(ImageDetail::High),
},
UserInput::Image {
image: ImageReference::Inline {
url: "https://example.com/image.png".into(),
},
detail: Some(ImageDetail::Original),
},
UserInput::LocalImage {
@@ -2039,6 +2102,43 @@ mod tests {
);
}
/// Incomplete ordering metadata must not drop references from the legacy split arrays.
#[test]
fn incomplete_image_order_falls_back_to_legacy_order() {
let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage(
UserMessageEvent {
images: Some(vec!["https://example.com/image.png".into()]),
file_ids: Some(vec!["file_123".into()]),
image_order: vec![UserMessageImageKind::File],
..Default::default()
},
))];
let turns = build_turns_from_rollout_items(&events);
assert_eq!(
turns[0].items[0],
ThreadItem::UserMessage {
id: "item-1".into(),
client_id: None,
content: vec![
UserInput::Image {
image: ImageReference::Inline {
url: "https://example.com/image.png".into(),
},
detail: None,
},
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".into(),
},
detail: None,
},
],
}
);
}
#[test]
fn ignores_user_message_item_lifecycle_events() {
let turn_id = "turn-1";
codex-rs/app-server-protocol/src/protocol/v2/tests.rs
+77 -4
View File
@@ -29,6 +29,7 @@ use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermiss
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions;
use codex_protocol::models::WebSearchAction as CoreWebSearchAction;
@@ -3037,7 +3038,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
text_elements: Vec::new(),
},
CoreUserInput::Image {
image_url: "https://example.com/image.png".to_string(),
image: CoreImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Original),
},
CoreUserInput::LocalImage {
@@ -3072,7 +3075,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
text_elements: Vec::new(),
},
UserInput::Image {
url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Original),
},
UserInput::LocalImage {
@@ -3609,16 +3614,84 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() {
);
}
/// Keeps the existing URL form stable while exposing file-backed images as `fileId`.
#[test]
fn user_input_image_references_round_trip_with_stable_wire_shapes() {
let cases = [
(
UserInput::Image {
image: ImageReference::Inline {
url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(ImageDetail::High),
},
json!({
"type": "image",
"url": "data:image/png;base64,AAA",
"detail": "high",
}),
),
(
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::Original),
},
json!({
"type": "image",
"fileId": "file_123",
"detail": "original",
}),
),
];
for (input, wire_value) in cases {
assert_eq!(
serde_json::to_value(&input).expect("user input should serialize"),
wire_value,
);
assert_eq!(
serde_json::from_value::<UserInput>(wire_value).expect("user input should deserialize"),
input,
);
}
}
/// Preserves durable file identity in both directions at the Core boundary.
#[test]
fn file_image_user_input_converts_both_directions() {
let app_server_input = UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::High),
};
let core_input = CoreUserInput::Image {
image: CoreImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::High),
};
assert_eq!(app_server_input.clone().into_core(), core_input);
assert_eq!(UserInput::from(core_input), app_server_input);
}
#[test]
fn user_input_into_core_preserves_media_fields() {
assert_eq!(
UserInput::Image {
url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Original),
}
.into_core(),
CoreUserInput::Image {
image_url: "https://example.com/image.png".to_string(),
image: CoreImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Original),
}
);
codex-rs/app-server-protocol/src/protocol/v2/turn.rs
+30 -5
View File
@@ -11,6 +11,7 @@ use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg;
use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus;
@@ -401,6 +402,21 @@ impl From<TextElement> for CoreTextElement {
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(untagged)]
#[ts(untagged)]
#[ts(export_to = "v2/")]
pub enum ImageReference {
Inline {
url: String,
},
File {
#[serde(rename = "fileId")]
#[ts(rename = "fileId")]
file_id: String,
},
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "camelCase")]
#[ts(tag = "type")]
@@ -413,10 +429,11 @@ pub enum UserInput {
text_elements: Vec<TextElement>,
},
Image {
#[serde(flatten)]
image: ImageReference,
#[serde(default)]
#[ts(optional)]
detail: Option<ImageDetail>,
url: String,
},
LocalImage {
#[serde(default)]
@@ -450,8 +467,11 @@ impl UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
UserInput::Image { url, detail } => CoreUserInput::Image {
image_url: url,
UserInput::Image { image, detail } => CoreUserInput::Image {
image: match image {
ImageReference::Inline { url } => CoreImageReference::Inline { image_url: url },
ImageReference::File { file_id } => CoreImageReference::File { file_id },
},
detail,
},
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
@@ -473,8 +493,13 @@ impl From<CoreUserInput> for UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
CoreUserInput::Image { image_url, detail } => UserInput::Image {
url: image_url,
CoreUserInput::Image { image, detail } => UserInput::Image {
image: match image {
CoreImageReference::Inline { image_url } => {
ImageReference::Inline { url: image_url }
}
CoreImageReference::File { file_id } => ImageReference::File { file_id },
},
detail,
},
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
codex-rs/app-server/src/request_processors/turn_processor.rs
+8 -4
View File
@@ -3,12 +3,13 @@ use super::*;
use codex_agent_extension::AgentInvocation;
use codex_agent_extension::AgentRun;
use codex_agent_extension::AgentRunner;
use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind;
use codex_protocol::protocol::TurnSettingsUpdate;
@@ -24,7 +25,10 @@ pub(super) fn validate_user_input_image_urls(
if input.iter().any(|item| {
matches!(
item,
V2UserInput::Image { url, .. } if is_remote_image_url(url)
V2UserInput::Image {
image: V2ImageReference::Inline { url },
..
} if is_remote_image_url(url)
)
}) {
return Err(invalid_request(REMOTE_IMAGE_URL_ERROR));
@@ -37,7 +41,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
ResponseItem::Message { content, .. } => content.iter().any(|item| {
matches!(
item,
ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
ContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
)
}),
ResponseItem::FunctionCallOutput { output, .. }
@@ -46,7 +50,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
content.iter().any(|item| {
matches!(
item,
FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
FunctionCallOutputContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. }
if is_remote_image_url(image_url)
)
})
codex-rs/app-server/tests/suite/v2/guardian_v2.rs
+4 -1
View File
@@ -25,6 +25,7 @@ use axum::routing::post;
use codex_app_server_protocol::ApprovalsReviewer;
use codex_app_server_protocol::AskForApproval;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemGuardianApprovalReviewStartedNotification;
use codex_app_server_protocol::McpServerElicitationRequest;
@@ -1031,7 +1032,9 @@ async fn guardian_v2_routes_scoped_tool_approvals(
}
if mixed_evidence {
turn_input.push(UserInput::Image {
url: EVIDENCE_IMAGE.to_owned(),
image: ImageReference::Inline {
url: EVIDENCE_IMAGE.to_owned(),
},
detail: None,
});
}
codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
+4 -1
View File
@@ -9,6 +9,7 @@ use app_test_support::TestAppServer;
use app_test_support::write_chatgpt_auth;
use codex_app_server_protocol::ImageGenerationFailure;
use codex_app_server_protocol::ImageGenerationItem;
use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
@@ -657,7 +658,9 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -
text_elements: Vec::new(),
},
V2UserInput::Image {
url: image_url.to_string(),
image: V2ImageReference::Inline {
url: image_url.to_string(),
},
detail: None,
},
],
codex-rs/app-server/tests/suite/v2/thread_read.rs
+75
View File
@@ -12,6 +12,7 @@ use codex_app_server::in_process::InProcessStartArgs;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::DeprecationNoticeNotification;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCError;
@@ -234,6 +235,80 @@ async fn thread_read_can_include_turns() -> Result<()> {
Ok(())
}
/// Preserves a file-backed image across a completed turn, including a cold history read.
#[tokio::test]
async fn thread_read_preserves_file_id_from_completed_turn() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
.build_initialized()
.await?;
let start_id = mcp
.send_thread_start_request_with_auto_env(ThreadStartParams {
model: Some("mock-model".to_string()),
..Default::default()
})
.await?;
let ThreadStartResponse { thread, .. } =
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(start_id)).await??;
let image = UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: None,
};
timeout(
DEFAULT_READ_TIMEOUT,
mcp.start_turn_and_wait_for_completion(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![image.clone()],
..Default::default()
}),
)
.await??;
let read_id = mcp
.send_thread_read_request(ThreadReadParams {
thread_id: thread.id.clone(),
include_turns: true,
})
.await?;
let ThreadReadResponse {
thread: loaded_thread,
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??;
let ThreadItem::UserMessage { content, .. } = &loaded_thread.turns[0].items[0] else {
panic!("expected completed turn to start with a user message");
};
assert_eq!(content, &vec![image.clone()]);
timeout(DEFAULT_READ_TIMEOUT, mcp.shutdown_gracefully()).await??;
drop(mcp);
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
.build_initialized()
.await?;
let read_id = mcp
.send_thread_read_request(ThreadReadParams {
thread_id: thread.id,
include_turns: true,
})
.await?;
let ThreadReadResponse {
thread: restarted_thread,
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??;
let ThreadItem::UserMessage { content, .. } = &restarted_thread.turns[0].items[0] else {
panic!("expected persisted turn to start with a user message");
};
assert_eq!(content, &vec![image]);
Ok(())
}
#[tokio::test]
async fn paginated_stored_thread_routes_projected_turns() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
codex-rs/app-server/tests/suite/v2/turn_start.rs
+26 -4
View File
@@ -34,6 +34,7 @@ use codex_app_server_protocol::CyberAccessProgram;
use codex_app_server_protocol::FileChangeApprovalDecision;
use codex_app_server_protocol::FileChangePatchUpdatedNotification;
use codex_app_server_protocol::FileChangeRequestApprovalResponse;
use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::JSONRPCError;
@@ -267,7 +268,15 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
text_elements: Vec::new(),
},
V2UserInput::Image {
url: TINY_PNG_DATA_URL.to_string(),
image: V2ImageReference::Inline {
url: TINY_PNG_DATA_URL.to_string(),
},
detail: None,
},
V2UserInput::Image {
image: V2ImageReference::File {
file_id: "file_123".to_string(),
},
detail: None,
},
],
@@ -315,6 +324,8 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
content[0]["type"].as_str(),
Some("text" | "input_text")
));
assert!(content[0].get("fileId").is_none());
assert!(content[0].get("file_id").is_none());
user_message_notifications.push(notification.method);
}
}
@@ -332,8 +343,17 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
);
let model_input_images = received_response_input_images(&server).await?;
assert_eq!(model_input_images.len(), 1);
assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL);
assert_eq!(model_input_images.len(), 2);
assert!(
model_input_images
.iter()
.any(|image| image["image_url"] == TINY_PNG_DATA_URL)
);
assert!(
model_input_images
.iter()
.any(|image| image["file_id"] == "file_123")
);
Ok(())
}
@@ -1393,7 +1413,9 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> {
thread_id: thread.id.clone(),
client_user_message_id: None,
input: vec![V2UserInput::Image {
url: TINY_PNG_DATA_URL.to_string(),
image: V2ImageReference::Inline {
url: TINY_PNG_DATA_URL.to_string(),
},
detail: None,
}],
turn_trigger: Some("user".to_string()),
codex-rs/core/src/compact_remote_v2_images.rs
+4
View File
@@ -25,6 +25,10 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
image_url, *detail,
)))
.unwrap_or(usize::MAX),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
ContentItem::InputAudio { .. } => 0,
}
}
codex-rs/core/src/context/node_repl_review_evidence.rs
+17 -9
View File
@@ -63,7 +63,14 @@ impl NodeReplReviewResponse {
.saturating_add(self.items.iter().fold(0_usize, |bytes, item| {
bytes.saturating_add(match item {
Text { text, .. } => text.len(),
Image { image_url, .. } => image_url.len(),
Image {
image: ImageReference::Inline { image_url },
..
} => image_url.len(),
Image {
image: ImageReference::File { file_id },
..
} => file_id.len(),
_ => 0,
})
}))
@@ -116,14 +123,15 @@ impl NodeReplReviewEvidence {
.iter()
.flat_map(|response| &response.items)
.filter_map(|item| match item {
Image { image_url, detail } if seen_images.insert(image_url) => {
Some(ContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: *detail,
})
}
Image {
image: ImageReference::Inline { image_url },
detail,
} if seen_images.insert(image_url) => Some(ContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: *detail,
}),
_ => None,
})
.collect()
codex-rs/core/src/context/node_repl_review_evidence_tests.rs
+43 -1
View File
@@ -1,3 +1,4 @@
use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_utils_output_truncation::approx_bytes_for_tokens;
use pretty_assertions::assert_eq;
@@ -15,7 +16,18 @@ fn text_input(text: &str) -> UserInput {
fn image_input(image_url: &str) -> UserInput {
UserInput::Image {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: None,
}
}
fn file_image_input(file_id: &str) -> UserInput {
UserInput::Image {
image: ImageReference::File {
file_id: file_id.to_string(),
},
detail: None,
}
}
@@ -121,6 +133,36 @@ fn evidence_preserves_tail_text_after_oversized_image_response() {
assert!(rendered_text(&inputs).contains("FINAL IMPORTANT"));
}
#[test]
fn file_backed_images_are_retained_but_omitted_from_guardian_views() {
let evidence = NodeReplReviewEvidence::default();
let file_id = format!("file_{}", "x".repeat(4_096));
let items = vec![text_input("visible"), file_image_input(&file_id)];
evidence.record("browser", "cell", "file", items.clone());
let snapshot = evidence.snapshot_since(/*reviewed_sequence*/ 0).unwrap();
assert_eq!(snapshot.responses.len(), 1);
assert_eq!(snapshot.responses[0].items, items);
assert!(snapshot.responses[0].has_images());
assert!(snapshot.responses[0].retained_bytes() >= file_id.len());
assert_eq!(evidence.images(), Vec::new());
let rendered = snapshot
.context(NodeReplReviewEvidenceMode::Multimodal)
.render_inputs();
assert!(rendered_text(&rendered).contains("visible"));
assert!(!rendered.iter().any(|item| {
matches!(
item,
UserInput::Image {
image: ImageReference::File { .. },
..
}
)
}));
}
#[test]
fn evidence_evicts_complete_oldest_responses_and_rejects_oversized_items() {
let evidence = NodeReplReviewEvidence::default();
codex-rs/core/src/context_manager/history.rs
+9
View File
@@ -930,6 +930,7 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option
});
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
// TODO(kc) Account for file-backed image size after its token-cost contract is defined.
match item {
ResponseItem::Message { content, .. } => content
.iter()
@@ -941,6 +942,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
@@ -1147,6 +1152,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
codex-rs/core/src/event_mapping.rs
+2 -6
View File
@@ -5,7 +5,6 @@ use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -127,12 +126,9 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
text_elements: Vec::new(),
});
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => {
ContentItem::InputImage { image, detail } => {
content.push(UserInput::Image {
image_url: image_url.clone(),
image: image.clone(),
detail: *detail,
});
}
codex-rs/core/src/event_mapping_tests.rs
+47 -4
View File
@@ -110,11 +110,11 @@ fn parses_user_message_with_text_and_two_images() {
text_elements: Vec::new(),
},
UserInput::Image {
image_url: img1,
image: ImageReference::Inline { image_url: img1 },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Image {
image_url: img2,
image: ImageReference::Inline { image_url: img2 },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
@@ -124,6 +124,49 @@ fn parses_user_message_with_text_and_two_images() {
}
}
/// Canonical user-message events must retain opaque file IDs for durable thread history.
#[test]
fn parses_user_message_with_file_image() {
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
text: "describe it".to_string(),
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let turn_item = parse_turn_item(&item).expect("expected user message turn item");
let TurnItem::UserMessage(user) = turn_item else {
panic!("expected TurnItem::UserMessage");
};
assert_eq!(
user.content,
vec![
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Text {
text: "describe it".to_string(),
text_elements: Vec::new(),
},
]
);
}
#[test]
fn skips_local_image_label_text() {
let image_url = "data:image/png;base64,abc".to_string();
@@ -158,7 +201,7 @@ fn skips_local_image_label_text() {
TurnItem::UserMessage(user) => {
let expected_content = vec![
UserInput::Image {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Text {
@@ -287,7 +330,7 @@ fn skips_unnamed_image_label_text() {
TurnItem::UserMessage(user) => {
let expected_content = vec![
UserInput::Image {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Text {
codex-rs/core/src/guardian/review_session.rs
+3 -1
View File
@@ -534,7 +534,9 @@ async fn run_review_on_session(
};
let mut prepared = vec![
ResponseInputItem::from(vec![UserInput::Image {
image_url: image_url.to_owned(),
image: ImageReference::Inline {
image_url: image_url.to_owned(),
},
detail: *detail,
}])
.into(),
codex-rs/core/src/image_preparation.rs
+8 -2
View File
@@ -179,6 +179,8 @@ fn prepare_message_content(
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
) -> Vec<ResizedImage> {
// TODO(kc) Prepare file-backed images once Core has an explicit resolution contract. Until
// then they pass through while retaining their positions in resize notices.
let image_count = items
.iter()
.filter(|item| matches!(item.content(), ContentItem::InputImage { .. }))
@@ -186,12 +188,14 @@ fn prepare_message_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
if matches!(item.content(), ContentItem::InputImage { .. }) {
image_number += 1;
}
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item.content_mut()
{
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
@@ -231,12 +235,14 @@ fn prepare_tool_output_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) {
image_number += 1;
}
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
codex-rs/core/src/image_preparation_tests.rs
+30 -5
View File
@@ -224,8 +224,9 @@ fn preparation_reports_tool_output_item_id() {
);
}
/// File-backed images retain their sequence positions even though preparation skips them.
#[test]
fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
fn resize_notices_count_file_backed_images_and_skip_failed_images() {
let (large_image_url, _) = png_data_url(/*width*/ 2048, /*height*/ 2048);
let (small_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![
@@ -233,6 +234,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_message".to_string(),
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image: ImageReference::Inline {
image_url: small_image_url,
@@ -259,6 +266,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
]),
..Default::default()
},
@@ -270,6 +278,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
name: None,
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_tool".to_string(),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
@@ -294,7 +308,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
);
let expected_user_notice = concat!(
"<image_resize_notice>\n",
"Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
"Image 4 of 4 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
"</image_resize_notice>"
);
@@ -310,6 +324,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
internal_chat_message_metadata_passthrough,
&Some(InternalChatMessageMetadataPassthrough {
content_item_kinds: Some(vec![
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
ContentItemKind("images.preparation_error".to_string()),
ContentItemKind("user.image".to_string()),
@@ -318,6 +333,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
}),
);
let [
ContentItem::InputImage {
image: ImageReference::File { file_id },
..
},
ContentItem::InputImage {
image:
ImageReference::Inline {
@@ -337,8 +356,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
},
] = content.as_slice()
else {
panic!("expected unchanged image, failed image placeholder, and resized image");
panic!("expected file image, unchanged image, failed image placeholder, and resized image");
};
assert_eq!(file_id, "file_message");
assert_eq!(
decoded_image(small_message_image_url).1.dimensions(),
(64, 32)
@@ -373,6 +393,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
panic!("expected function call output");
};
let [
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
},
FunctionCallOutputContentItem::InputText {
text: failed_tool_image,
},
@@ -385,8 +409,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
},
] = output.content_items().expect("tool output content items")
else {
panic!("expected failed image placeholder and resized image in the tool output");
panic!("expected file image, failed image placeholder, and resized image in tool output");
};
assert_eq!(file_id, "file_tool");
assert_eq!(failed_tool_image, IMAGE_PROCESSING_ERROR_PLACEHOLDER);
assert_eq!(
decoded_image(resized_tool_image_url).1.dimensions(),
@@ -400,7 +425,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
content: vec![ContentItem::InputText {
text: concat!(
"<image_resize_notice>\n",
"Image 2 of 2 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n",
"Image 3 of 3 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n",
"</image_resize_notice>"
)
.to_string(),
codex-rs/core/src/session/tests.rs
+3 -1
View File
@@ -11306,7 +11306,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
text_elements: vec![text_element.clone()],
},
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: Some(ImageDetail::High),
},
];
codex-rs/core/src/session/tests/guardian_tests.rs
+4 -1
View File
@@ -40,6 +40,7 @@ use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::error::SandboxErr;
use codex_protocol::models::AdditionalPermissionProfile as PermissionProfile;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ImageReference;
use codex_protocol::models::NetworkPermissions;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::protocol::AskForApproval;
@@ -235,7 +236,9 @@ async fn request_permissions_routes_to_guardian_when_reviewer_is_enabled() {
.thread_extension_data
.get_or_init(crate::context::NodeReplReviewEvidence::default);
let image = UserInput::Image {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: None,
};
evidence.record("js", "cell", "image", vec![image]);
codex-rs/core/src/tools/handlers/mcp.rs
+4 -1
View File
@@ -367,7 +367,10 @@ impl CoreToolRuntime for McpHandler {
load_data_url_for_prompt_uncached(&image_url, PromptImageMode::Original)
.ok()?;
captured_image_bytes = next_image_bytes;
Some(UserInput::Image { image_url, detail })
Some(UserInput::Image {
image: codex_protocol::models::ImageReference::Inline { image_url },
detail,
})
}
_ => None,
}
codex-rs/core/tests/suite/compact.rs
+4 -1
View File
@@ -17,6 +17,7 @@ use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelServiceTier;
@@ -4833,7 +4834,9 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess
codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
},
UserInput::Text {
codex-rs/core/tests/suite/compact_remote.rs
+12 -5
View File
@@ -17,6 +17,7 @@ use codex_model_provider_info::AMAZON_BEDROCK_PROVIDER_ID;
use codex_model_provider_info::ModelProviderInfo;
use codex_protocol::AgentPath;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::EventMsg;
@@ -637,10 +638,12 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
let mut bytes = std::io::Cursor::new(Vec::new());
image.write_to(&mut bytes, image::ImageFormat::Png)?;
Ok(UserInput::Image {
image_url: format!(
"data:image/png;base64,{}",
BASE64_STANDARD.encode(bytes.get_ref())
),
image: ImageReference::Inline {
image_url: format!(
"data:image/png;base64,{}",
BASE64_STANDARD.encode(bytes.get_ref())
),
},
detail: Some(codex_protocol::models::ImageDetail::Original),
})
})
@@ -714,7 +717,11 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
};
let mut expected_images = prepared_images[dropped..].to_vec();
if cycle == 2 {
let UserInput::Image { image_url, .. } = &image_inputs[7] else {
let UserInput::Image {
image: ImageReference::Inline { image_url },
..
} = &image_inputs[7]
else {
unreachable!()
};
expected_images.push(image_url.clone());
codex-rs/core/tests/suite/context_annotations.rs
+4 -1
View File
@@ -4,6 +4,7 @@ use codex_core::config::CurrentTimeReminderConfig;
use codex_core::config::RolloutBudgetConfig;
use codex_core::config::TokenBudgetConfig;
use codex_features::Feature;
use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::AdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind;
@@ -112,7 +113,9 @@ async fn first_request_item_types_roles_and_content_annotations() -> Result<()>
text_elements: Vec::new(),
},
UserInput::Image {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
},
detail: None,
},
UserInput::Audio {
codex-rs/core/tests/suite/guardian_authorization.rs
+4 -1
View File
@@ -11,6 +11,7 @@ use codex_core::context::InternalModelContextFragment;
use codex_features::Feature;
use codex_history::RolloutItem;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::GuardianAssessmentStatus;
@@ -328,7 +329,9 @@ async fn guardian_authorization_revision_survives_compaction_not_user_input() ->
.await;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image {
image_url: "data:image/png;base64,not-an-image".to_owned(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,not-an-image".to_owned(),
},
detail: None,
}]))
.await?;
codex-rs/core/tests/suite/guardian_history.rs
+4 -1
View File
@@ -13,6 +13,7 @@ use codex_history::ResumedHistory;
use codex_history::RolloutItem;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::mcp::ClientMcpExtensions;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
@@ -441,7 +442,9 @@ async fn guardian_history_survives_compaction_and_eviction_but_not_legacy_rollba
text_elements: Vec::new(),
},
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
},
]))
codex-rs/core/tests/suite/image_rollout.rs
+83 -2
View File
@@ -232,7 +232,9 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
.start_or_steer_turn(
TurnInputRequest::user_input(vec![
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
},
UserInput::Text {
@@ -289,6 +291,83 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
Ok(())
}
/// Core must forward an opaque file ID and persist that same reference in canonical history.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn file_image_passes_through_request_and_rollout() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let test = test_codex().build_with_auto_env(&server).await?;
let response_mock = responses::mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-file"),
ev_assistant_message("msg-file", "done"),
ev_completed("resp-file"),
]),
)
.await;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: None,
},
UserInput::Text {
text: "file-backed image".to_string(),
text_elements: Vec::new(),
},
]))
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let request = response_mock.single_request();
assert!(request.input().iter().any(|item| {
item.get("content")
.and_then(Value::as_array)
.is_some_and(|content| {
content.iter().any(|item| {
item.get("type").and_then(Value::as_str) == Some("input_image")
&& item.get("file_id").and_then(Value::as_str) == Some("file_123")
})
})
}));
test.codex.shutdown_and_wait().await?;
let rollout_path = test.codex.rollout_path().expect("rollout path");
let rollout_text = read_rollout_text(&rollout_path).await?;
let actual = find_user_message_with_image(&rollout_text)
.expect("expected user message with file image in rollout");
assert_eq!(
strip_response_item_id(strip_metadata(actual)),
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
text: "file-backed image".to_string(),
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
}
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
@@ -379,7 +458,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
resumed
.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image {
image_url: original_image_url.clone(),
image: ImageReference::Inline {
image_url: original_image_url.clone(),
},
detail: Some(ImageDetail::High),
}]))
.await?;
codex-rs/core/tests/suite/items.rs
+23 -7
View File
@@ -9,6 +9,8 @@ use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Settings;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::WebSearchAction;
use codex_protocol::protocol::AskForApproval;
@@ -81,13 +83,21 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
ByteRange { start: 0, end: 6 },
Some("<file>".into()),
)];
let expected_input = UserInput::Text {
text: "please inspect sample.txt".into(),
text_elements: text_elements.clone(),
};
let expected_input = vec![
UserInput::Text {
text: "please inspect sample.txt".into(),
text_elements: text_elements.clone(),
},
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".into(),
},
detail: Some(ImageDetail::High),
},
];
codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![expected_input.clone()]))
.start_or_steer_turn(TurnInputRequest::user_input(expected_input.clone()))
.await?;
let started_item = wait_for_event_match(&codex, |ev| match ev {
@@ -108,8 +118,8 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
.await;
assert_eq!(started_item.id, completed_item.id);
assert_eq!(started_item.content, vec![expected_input.clone()]);
assert_eq!(completed_item.content, vec![expected_input]);
assert_eq!(started_item.content, expected_input);
assert_eq!(completed_item.content, started_item.content);
let legacy_message = wait_for_event_match(&codex, |ev| match ev {
EventMsg::UserMessage(event) => Some(event.clone()),
@@ -118,6 +128,12 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
.await;
assert_eq!(legacy_message.message, "please inspect sample.txt");
assert_eq!(legacy_message.text_elements, text_elements);
assert_eq!(legacy_message.file_ids, Some(vec!["file_123".into()]));
assert_eq!(
legacy_message.file_id_details,
vec![Some(ImageDetail::High)]
);
Ok(())
}
codex-rs/core/tests/suite/model_switching.rs
+4 -1
View File
@@ -18,6 +18,7 @@ use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::mcp::ClientMcpExtensions;
use codex_protocol::models::BaseInstructionsProvenance;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ConfigShellToolType;
use codex_protocol::openai_models::InputModality;
@@ -906,7 +907,9 @@ async fn model_change_projects_media_without_changing_live_or_replayed_history(
&test,
vec![
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
},
UserInput::Audio {
codex-rs/core/tests/suite/responses_lite.rs
+7 -2
View File
@@ -14,6 +14,7 @@ use codex_login::auth::BedrockApiKeyAuth;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_protocol::config_types::WebSearchMode;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use codex_protocol::openai_models::ToolMode;
use codex_protocol::protocol::EventMsg;
@@ -278,11 +279,15 @@ async fn responses_lite_prepares_images() -> Result<()> {
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![
UserInput::Image {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: Some(ImageDetail::Original),
},
UserInput::Image {
image_url: remote_image_url.to_string(),
image: ImageReference::Inline {
image_url: remote_image_url.to_string(),
},
detail: Some(ImageDetail::High),
},
]))
codex-rs/core/tests/suite/rmcp_client.rs
+3 -1
View File
@@ -1488,7 +1488,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
unreachable!("read_only_user_turn creates user input");
};
content.push(UserInput::Image {
image_url: OPENAI_PNG.to_string(),
image: ImageReference::Inline {
image_url: OPENAI_PNG.to_string(),
},
detail: Some(ImageDetail::High),
});
fixture.codex.start_or_steer_turn(input).await?;
codex-rs/core/tests/suite/scenarios.rs
+4 -1
View File
@@ -23,6 +23,7 @@ use codex_login::CodexAuth;
use codex_models_manager::bundled_models_response;
use codex_protocol::items::AgentMessageDelivery;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::user_input::UserInput;
@@ -382,7 +383,9 @@ async fn astra_kickoff_with_skills_plugins_and_remote_compaction() -> Result<()>
.start_or_steer_turn(TurnInputRequest::user_input(vec![
text("Check the final kickoff brief and attached sketch with $final-check and $calendar:agenda."),
UserInput::Image {
image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"),
},
detail: None,
},
selected_skill("final-check", &skills.final_check),
codex-rs/ext/image-generation/src/tests.rs
+81
View File
@@ -166,6 +166,87 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {
);
}
/// A file-backed image in the requested window must not cause an older inline image to be edited.
#[tokio::test]
async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
referenced_image_paths: None,
num_last_images_to_include: Some(1),
},
&[ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
input_image("older-inline-image"),
ContentItem::InputImage {
image: ImageReference::File {
file_id: "newer-file-backed-image".to_string(),
},
detail: None,
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
}],
&[],
)
.await
.expect_err("a selected file-backed image should fail");
assert_eq!(
error.to_string(),
"requested the last 1 conversation images, but that window includes a file-backed image \
that cannot be used for editing"
);
}
/// Tool-output file references must count toward the window instead of exposing an older image.
#[tokio::test]
async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_window() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
referenced_image_paths: None,
num_last_images_to_include: Some(1),
},
&[
ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("older-inline-output".to_string()),
name: None,
namespace: None,
output: image_output("older-inline-image"),
internal_chat_message_metadata_passthrough: None,
},
ResponseItem::CustomToolCallOutput {
id: None,
call_id: "newer-file-backed-output".to_string(),
name: Some("view_image".to_string()),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "newer-file-backed-image".to_string(),
},
detail: None,
},
]),
internal_chat_message_metadata_passthrough: None,
},
],
&[],
)
.await
.expect_err("a selected file-backed tool output should fail");
assert_eq!(
error.to_string(),
"requested the last 1 conversation images, but that window includes a file-backed image \
that cannot be used for editing"
);
}
#[tokio::test]
async fn conflicting_image_selectors_return_tool_error() {
let error = request_for_call_args(
codex-rs/ext/image-generation/src/tool.rs
+42 -24
View File
@@ -459,14 +459,7 @@ async fn request_for_call_args(
}
// Pathless images have no stable reference, so this bounded window may include newer
// unrelated images. This remains best-effort until the harness provides stable refs.
let images = recent_images(history, count);
if images.len() != count {
return Err(FunctionCallError::RespondToModel(format!(
"requested the last {count} conversation images, but only {} were available",
images.len()
)));
}
images
recent_images(history, count)?
}
(false, Some(_)) => {
return Err(FunctionCallError::RespondToModel(
@@ -488,17 +481,19 @@ async fn request_for_call_args(
}))
}
fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
fn recent_images(
history: &[ResponseItem],
count: usize,
) -> Result<Vec<ImageUrl>, FunctionCallError> {
let mut images = Vec::with_capacity(count);
'history: for item in history.iter().rev() {
let mut image_urls = Vec::new();
match item {
ResponseItem::Message { content, .. } => {
image_urls.extend(content.iter().rev().filter_map(|item| match item {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
ContentItem::InputImage { image, .. } => {
Some(inline_image_url(image).map(str::to_owned))
}
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => None,
@@ -506,10 +501,12 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
image_urls.extend(output_image_urls(output));
image_urls.extend(
output_images(output).map(|image| inline_image_url(image).map(str::to_owned)),
);
}
ResponseItem::ImageGenerationCall { result, .. } if !result.is_empty() => {
image_urls.push(format!("data:image/png;base64,{result}"));
image_urls.push(Some(format!("data:image/png;base64,{result}")));
}
ResponseItem::AdditionalTools { .. }
| ResponseItem::Reasoning { .. }
@@ -528,34 +525,55 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
| ResponseItem::Other => {}
}
for image_url in image_urls {
images.push(ImageUrl { image_url });
images.push(image_url.map(|image_url| ImageUrl { image_url }));
if images.len() == count {
break 'history;
}
}
}
images.reverse();
images
if images.len() != count {
return Err(FunctionCallError::RespondToModel(format!(
"requested the last {count} conversation images, but only {} were available",
images.len()
)));
}
let mut inline_images = Vec::with_capacity(count);
for image in images {
let Some(image) = image else {
return Err(FunctionCallError::RespondToModel(format!(
"requested the last {count} conversation images, but that window includes a \
file-backed image that cannot be used for editing"
)));
};
inline_images.push(image);
}
inline_images.reverse();
Ok(inline_images)
}
/// Extracts image URLs from a tool output in newest-first order.
fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator<Item = String> + '_ {
/// Extracts image references from a tool output in newest-first order.
fn output_images(output: &FunctionCallOutputPayload) -> impl Iterator<Item = &ImageReference> + '_ {
output
.content_items()
.into_iter()
.flatten()
.rev()
.filter_map(|item| match item {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
FunctionCallOutputContentItem::InputImage { image, .. } => Some(image),
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => None,
})
}
fn inline_image_url(image: &ImageReference) -> Option<&str> {
match image {
ImageReference::Inline { image_url } => Some(image_url),
// TODO(kc) Image generation and the Images API only accept inline image URLs.
ImageReference::File { .. } => None,
}
}
async fn image_url(
path: &AbsolutePathBuf,
environment: &ToolEnvironment<'_>,
codex-rs/ext/queue/tests/queue_service.rs
+7 -2
View File
@@ -29,6 +29,7 @@ use codex_extension_api::TurnStartAdmission;
use codex_protocol::ThreadId;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
@@ -825,7 +826,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_queued_input = user_input_with_media(
"queued attachments",
UserInput::Image {
image_url: TINY_PNG_DATA_URL.to_string(),
image: ImageReference::Inline {
image_url: TINY_PNG_DATA_URL.to_string(),
},
detail: Some(ImageDetail::Original),
},
UserInput::Audio {
@@ -865,7 +868,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_edited_input = user_input_with_media(
"edited attachments",
UserInput::Image {
image_url: TINY_PNG_DATA_URL.to_string(),
image: ImageReference::Inline {
image_url: TINY_PNG_DATA_URL.to_string(),
},
detail: Some(ImageDetail::High),
},
UserInput::Audio {
codex-rs/guardian-context/src/budget.rs
+4
View File
@@ -105,6 +105,10 @@ impl SectionCost {
self.image_bytes = self.image_bytes.saturating_add(image_url.len());
self.image_count = self.image_count.saturating_add(1);
}
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => {}
ContentItem::InputAudio { audio_url } => {
// Guardian currently has no audio contributor. Count a future opaque
// payload conservatively until its consumer supplies modality costs.
codex-rs/guardian-context/src/composition.rs
+25 -7
View File
@@ -210,20 +210,27 @@ impl CollectedContext {
let items = evidence
.items
.into_iter()
.map(|item| match item {
.filter_map(|item| match item {
UserInput::Text { text, .. } => {
Ok(Budgeted::required(ContentItem::InputText { text }))
Some(Ok(Budgeted::required(ContentItem::InputText { text })))
}
UserInput::Image { image_url, detail } => Ok(Budgeted::optional(
UserInput::Image {
image: ImageReference::Inline { image_url },
detail,
} => Some(Ok(Budgeted::optional(
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
},
BudgetPriority::Image,
)),
_ => Err(SectionError::UnsupportedDelivery {
))),
UserInput::Image {
image: ImageReference::File { .. },
..
} => None,
_ => Some(Err(SectionError::UnsupportedDelivery {
section: "node_repl_evidence",
}),
})),
})
.collect::<Result<Vec<_>, _>>()?;
(
@@ -288,7 +295,14 @@ impl ComposedContext {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => UserInput::Image { image_url, detail },
} => UserInput::Image {
image: ImageReference::Inline { image_url },
detail,
},
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => continue,
ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {
return Err(SectionError::UnsupportedDelivery {
section: section.id,
@@ -316,6 +330,10 @@ impl ComposedContext {
}
}));
}
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => {}
content => user_content.push(content),
}
}
codex-rs/guardian-context/src/enforcement.rs
+8
View File
@@ -95,6 +95,10 @@ impl ComposedContext {
image: ImageReference::Inline { image_url },
detail,
} => admit(image_url, detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => false,
_ => true,
});
}
@@ -281,6 +285,10 @@ fn retain_content(
image: ImageReference::Inline { image_url },
..
} => image_url.len(),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
ContentItem::InputAudio { audio_url } => audio_url.len(),
};
truncations.push(TruncationObservation {
codex-rs/guardian-context/src/images.rs
+38 -18
View File
@@ -85,18 +85,26 @@ impl TranscriptImages {
});
};
// TODO(kc) Include file-backed images once Guardian can admit them without resolving or
// applying byte-based checks in Core.
for item in items {
match item {
ResponseItem::Message { role, content, .. }
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
include_image(image_url, *detail);
match item {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => include_image(image_url, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
}
| ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => {}
}
}
}
@@ -106,12 +114,18 @@ impl TranscriptImages {
{
if let Some(content) = output.content_items() {
for item in content {
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
include_image(image_url, *detail);
match item {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => include_image(image_url, *detail),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
}
| FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => {}
}
}
}
@@ -121,12 +135,18 @@ impl TranscriptImages {
}
if input.include_tool_outputs {
for image in input.node_repl_images {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = image
{
include_image(image_url, *detail);
match image {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => include_image(image_url, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
}
| ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => {}
}
}
}
codex-rs/guardian-context/src/images_tests.rs
+66
View File
@@ -98,3 +98,69 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {
}
);
}
/// File-backed images stay out of Guardian without consuming its inline-image capacity.
#[test]
fn file_images_are_omitted_without_displacing_inline_images() {
let file_image = ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::High),
};
let history = [
ResponseItem::Message {
id: None,
role: "user".into(),
content: vec![image("first"), file_image.clone(), image("second")],
phase: None,
internal_chat_message_metadata_passthrough: None,
},
ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("screenshot".into()),
name: None,
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_tool".to_string(),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: "third".into(),
},
detail: Some(ImageDetail::High),
},
]),
internal_chat_message_metadata_passthrough: None,
},
];
let repl = [
file_image,
image("fourth"),
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_repl".to_string(),
},
detail: Some(ImageDetail::High),
},
];
assert_eq!(
TranscriptImages::collect(
&history,
TranscriptImageInput {
enabled: true,
include_tool_outputs: true,
node_repl_images: &repl,
}
),
TranscriptImages {
images: ["first", "second", "third", "fourth"].map(image).to_vec(),
omitted_bytes: 0,
}
);
}
codex-rs/guardian-context/src/node_repl.rs
+14 -7
View File
@@ -9,6 +9,7 @@ use crate::SectionInput;
use crate::SectionScope;
use codex_context_fragments::ContextualUserFragment;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_protocol::user_input::UserInput::Image;
use codex_protocol::user_input::UserInput::Text;
@@ -84,10 +85,15 @@ impl NodeReplContext<'_> {
pub fn render_inputs(&self) -> Vec<UserInput> {
if self.mode != NodeReplReviewEvidenceMode::Multimodal
|| !self.responses.iter().any(|response| {
response
.items
.iter()
.any(|item| matches!(item, Image { .. }))
response.items.iter().any(|item| {
matches!(
item,
Image {
image: ImageReference::Inline { .. },
..
}
)
})
})
{
return vec![text_input(self.render())];
@@ -138,9 +144,10 @@ impl NodeReplContext<'_> {
for item in response.items {
match item {
Text { text, .. } => inputs.push(text_input(format!("{text}\n"))),
Image { image_url, .. } if seen_images.insert(image_url) => {
inputs.push(item.clone())
}
Image {
image: ImageReference::Inline { image_url },
..
} if seen_images.insert(image_url) => inputs.push(item.clone()),
_ => {}
}
}
codex-rs/otel/tests/suite/otel_export_routing_policy.rs
+4 -1
View File
@@ -24,6 +24,7 @@ use codex_protocol::ThreadId;
use codex_protocol::ToolName;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ToolResultLogConfig;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
@@ -143,7 +144,9 @@ fn otel_export_routing_policy_routes_user_prompt_log_and_trace_events() {
text_elements: Vec::new(),
},
UserInput::Image {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: None,
},
UserInput::LocalImage {
codex-rs/protocol/src/items.rs
+10 -3
View File
@@ -7,6 +7,7 @@ use crate::memory_citation::MemoryCitation;
use crate::models::ContentItem;
use crate::models::FunctionCallOutputBody;
use crate::models::ImageDetail;
use crate::models::ImageReference;
use crate::models::MessagePhase;
use crate::models::ResponseItem;
use crate::models::WebSearchAction;
@@ -554,7 +555,10 @@ impl UserMessageItem {
self.content
.iter()
.filter_map(|c| match c {
UserInput::Image { image_url, .. } => Some(image_url.clone()),
UserInput::Image {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
_ => None,
})
.collect()
@@ -565,7 +569,10 @@ impl UserMessageItem {
self.content
.iter()
.filter_map(|c| match c {
UserInput::Image { detail, .. } => Some(*detail),
UserInput::Image {
image: ImageReference::Inline { .. },
detail,
} => Some(*detail),
_ => None,
})
.collect(),
@@ -615,7 +622,7 @@ impl UserMessageItem {
}
}
fn trim_trailing_default_image_details(
pub(crate) fn trim_trailing_default_image_details(
mut details: Vec<Option<ImageDetail>>,
) -> Vec<Option<ImageDetail>> {
while matches!(details.last(), Some(None)) {
codex-rs/protocol/src/legacy_events.rs
+25
View File
@@ -20,6 +20,8 @@ use crate::items::SubAgentActivityItem;
use crate::items::TurnItem;
use crate::items::UserMessageItem;
use crate::items::WebSearchItem;
use crate::items::trim_trailing_default_image_details;
use crate::models::ImageReference;
use crate::protocol::AgentMessageContentDeltaEvent;
use crate::protocol::AgentMessageEvent;
use crate::protocol::AgentReasoningEvent;
@@ -58,9 +60,11 @@ use crate::protocol::ReasoningContentDeltaEvent;
use crate::protocol::ReasoningRawContentDeltaEvent;
use crate::protocol::SubAgentActivityEvent;
use crate::protocol::UserMessageEvent;
use crate::protocol::UserMessageImageKind;
use crate::protocol::ViewImageToolCallEvent;
use crate::protocol::WebSearchBeginEvent;
use crate::protocol::WebSearchEndEvent;
use crate::user_input::UserInput;
/// Converts canonical item lifecycle events back into the legacy raw event stream used by
/// compatibility consumers that have not migrated to `TurnItem`.
@@ -78,11 +82,32 @@ impl UserMessageItem {
pub fn as_legacy_user_message_event(&self) -> UserMessageEvent {
// Legacy user-message events flatten only text inputs into `message` and
// rebase text element ranges onto that concatenated text.
let mut file_ids = Vec::new();
let mut file_id_details = Vec::new();
let mut image_order = Vec::new();
for input in &self.content {
if let UserInput::Image { image, detail } = input {
match image {
ImageReference::Inline { .. } => {
image_order.push(UserMessageImageKind::Inline);
}
ImageReference::File { file_id } => {
image_order.push(UserMessageImageKind::File);
file_ids.push(file_id.clone());
file_id_details.push(*detail);
}
}
}
}
let file_id_details = trim_trailing_default_image_details(file_id_details);
UserMessageEvent {
client_id: self.client_id.clone(),
message: self.message(),
images: Some(self.image_urls()),
image_details: self.image_details(),
file_ids: (!file_ids.is_empty()).then_some(file_ids),
file_id_details,
image_order,
local_images: self.local_image_paths(),
local_image_details: self.local_image_details(),
audio: Some(self.audio_urls()),
codex-rs/protocol/src/local_media.rs
+4 -1
View File
@@ -8,6 +8,7 @@ use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_for_prompt_bytes;
use crate::models::ImageDetail;
use crate::models::ImageReference;
use crate::user_input::UserInput;
/// Maximum accepted decoded byte length for prompt audio inputs.
@@ -29,7 +30,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
*input = UserInput::Image {
image_url: image.into_data_url(),
image: ImageReference::Inline {
image_url: image.into_data_url(),
},
detail: image_detail,
};
}
codex-rs/protocol/src/local_media_tests.rs
+4 -2
View File
@@ -31,8 +31,10 @@ fn snapshots_local_image_user_input_with_requested_detail() -> Result<()> {
assert_eq!(
UserInput::Image {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="
.to_string(),
image: crate::models::ImageReference::Inline {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="
.to_string(),
},
detail,
},
input
codex-rs/protocol/src/models.rs
+49 -7
View File
@@ -898,6 +898,7 @@ pub enum ContentItem {
#[ts(untagged)]
pub enum ImageReference {
Inline { image_url: String },
File { file_id: String },
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
@@ -2008,13 +2009,11 @@ impl ResponseInputItem {
.into_iter()
.flat_map(|c| match c {
UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],
UserInput::Image {
image_url, detail, ..
} => {
UserInput::Image { image, detail, .. } => {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
vec![ContentItem::InputImage {
image: ImageReference::Inline { image_url },
image,
detail: Some(detail),
}]
}
@@ -3885,7 +3884,9 @@ mod tests {
let image_url = "data:image/png;base64,abc".to_string();
let item = ResponseInputItem::from(vec![UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
}]);
@@ -3903,6 +3904,43 @@ mod tests {
Ok(())
}
/// A file-backed user image must reach the Responses API without being resolved by Core.
#[test]
fn file_image_user_input_serializes_file_id() -> Result<()> {
let file_id = "file_123".to_string();
let item = ResponseInputItem::from(vec![UserInput::Image {
image: ImageReference::File {
file_id: file_id.clone(),
},
detail: None,
}]);
let expected = ResponseInputItem::Message {
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image: ImageReference::File { file_id },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
};
assert_eq!(item, expected);
assert_eq!(
serde_json::to_value(item)?,
serde_json::json!({
"type": "message",
"role": "user",
"content": [{
"type": "input_image",
"file_id": "file_123",
"detail": "high"
}]
})
);
Ok(())
}
#[test]
fn serializes_audio_user_input_without_tags() -> Result<()> {
let audio_url = "data:audio/wav;base64,abc".to_string();
@@ -4029,7 +4067,9 @@ mod tests {
let image_url = "data:image/png;base64,abc".to_string();
let item = ResponseInputItem::from(vec![UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: Some(ImageDetail::Original),
}]);
@@ -4227,7 +4267,9 @@ mod tests {
let item = ResponseInputItem::from(vec![
UserInput::Image {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: None,
},
UserInput::LocalImage {
codex-rs/protocol/src/protocol.rs
+88 -2
View File
@@ -2521,6 +2521,14 @@ pub struct AgentMessageEvent {
pub questions: Option<Vec<AsyncUserInputQuestion>>,
}
#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
#[ts(rename_all = "snake_case")]
pub enum UserMessageImageKind {
Inline,
File,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
pub struct UserMessageEvent {
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -2535,6 +2543,19 @@ pub struct UserMessageEvent {
/// default image detail behavior.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub image_details: Vec<Option<ImageDetail>>,
/// File IDs sourced from `UserInput::Image`. These are passed through as
/// opaque references and are not created by image preparation.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub file_ids: Option<Vec<String>>,
/// Detail hints for `file_ids`, indexed in parallel. Missing entries imply
/// default image detail behavior.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub file_id_details: Vec<Option<ImageDetail>>,
/// Inline and file-backed image kinds in their original input order.
/// New producers populate this alongside `images` and `file_ids`; when it
/// is absent, consumers retain the legacy inline-then-file ordering.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub image_order: Vec<UserMessageImageKind>,
/// Local file paths sourced from `UserInput::LocalImage`. These are kept so
/// the UI can reattach images when editing history. Local image prompts may
/// include a display form of the path, but these should not be treated as
@@ -2559,6 +2580,27 @@ pub struct UserMessageEvent {
pub text_elements: Vec<crate::user_input::TextElement>,
}
impl UserMessageEvent {
/// Returns whether `image_order` accounts for every split image reference exactly once.
pub fn has_complete_image_order(&self) -> bool {
if self.image_order.is_empty() {
return false;
}
let mut inline_count = 0;
let mut file_count = 0;
for image_kind in &self.image_order {
match image_kind {
UserMessageImageKind::Inline => inline_count += 1,
UserMessageImageKind::File => file_count += 1,
}
}
inline_count == self.images.as_ref().map_or(0, Vec::len)
&& file_count == self.file_ids.as_ref().map_or(0, Vec::len)
}
}
/// Returns the user-facing preview text for a user message.
pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
let message = strip_user_message_prefix(user.message.as_str());
@@ -2569,6 +2611,10 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
.images
.as_ref()
.is_some_and(|images| !images.is_empty())
|| user
.file_ids
.as_ref()
.is_some_and(|file_ids| !file_ids.is_empty())
|| !user.local_images.is_empty()
{
return Some("[Image]".to_string());
@@ -5944,6 +5990,9 @@ mod tests {
Some(vec!["https://example.com/image.png".to_string()])
);
assert_eq!(event.image_details, Vec::<Option<ImageDetail>>::new());
assert_eq!(event.file_ids, None);
assert_eq!(event.file_id_details, Vec::<Option<ImageDetail>>::new());
assert_eq!(event.image_order, Vec::<UserMessageImageKind>::new());
assert_eq!(event.local_images, vec![PathBuf::from("/tmp/local.png")]);
assert_eq!(event.local_image_details, Vec::<Option<ImageDetail>>::new());
assert_eq!(event.audio, None);
@@ -5959,11 +6008,21 @@ mod tests {
let local_audio_path = PathBuf::from("/tmp/local.wav");
let mut item = UserMessageItem::new(&[
crate::user_input::UserInput::Image {
image_url: "https://example.com/first.png".to_string(),
image: crate::models::ImageReference::Inline {
image_url: "https://example.com/first.png".to_string(),
},
detail: Some(ImageDetail::Original),
},
crate::user_input::UserInput::Image {
image_url: "https://example.com/second.png".to_string(),
image: crate::models::ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::Low),
},
crate::user_input::UserInput::Image {
image: crate::models::ImageReference::Inline {
image_url: "https://example.com/second.png".to_string(),
},
detail: None,
},
crate::user_input::UserInput::LocalImage {
@@ -5982,6 +6041,7 @@ mod tests {
let EventMsg::UserMessage(event) = item.as_legacy_event() else {
panic!("expected user message event");
};
let event_json = serde_json::to_value(&event).expect("serialize user message event");
assert_eq!(
event.images,
@@ -5992,6 +6052,22 @@ mod tests {
);
assert_eq!(event.client_id, Some("client-message-1".to_string()));
assert_eq!(event.image_details, vec![Some(ImageDetail::Original)]);
assert_eq!(event.file_ids, Some(vec!["file_123".to_string()]));
assert_eq!(event.file_id_details, vec![Some(ImageDetail::Low)]);
assert_eq!(
event.image_order,
vec![
UserMessageImageKind::Inline,
UserMessageImageKind::File,
UserMessageImageKind::Inline,
]
);
assert_eq!(event_json["file_ids"], json!(["file_123"]));
assert_eq!(event_json["file_id_details"], json!(["low"]));
assert_eq!(
event_json["image_order"],
json!(["inline", "file", "inline"])
);
assert_eq!(event.local_images, vec![local_path]);
assert_eq!(event.local_image_details, vec![Some(ImageDetail::Original)]);
assert_eq!(
@@ -6011,6 +6087,16 @@ mod tests {
assert_eq!(user_message_preview(&event), Some("[Audio]".to_string()));
}
#[test]
fn file_only_user_message_has_placeholder_preview() {
let event = UserMessageEvent {
file_ids: Some(vec!["file_123".to_string()]),
..Default::default()
};
assert_eq!(user_message_preview(&event), Some("[Image]".to_string()));
}
#[test]
fn turn_aborted_event_deserializes_without_turn_id() -> Result<()> {
let event: EventMsg = serde_json::from_value(json!({
codex-rs/protocol/src/user_input.rs
+4 -2
View File
@@ -4,6 +4,7 @@ use serde::Serialize;
use ts_rs::TS;
use crate::models::ImageDetail;
use crate::models::ImageReference;
/// Conservative cap so one user message cannot monopolize a large context window.
pub const MAX_USER_INPUT_TEXT_CHARS: usize = 1 << 20;
@@ -22,9 +23,10 @@ pub enum UserInput {
#[serde(default)]
text_elements: Vec<TextElement>,
},
/// Pre‑encoded data: URI image.
/// Image reference forwarded to the Responses API.
Image {
image_url: String,
#[serde(flatten)]
image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs
+55 -2
View File
@@ -26,8 +26,10 @@ use codex_protocol::items::SubAgentActivityItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::protocol::UserMessageImageKind;
use codex_protocol::user_input::UserInput;
use crate::ThreadStoreResult;
@@ -37,20 +39,71 @@ pub(super) fn user_message_item(
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
) -> ThreadStoreResult<TurnItem> {
let mut content = Vec::new();
let has_image_order = event.has_complete_image_order();
if !event.message.trim().is_empty() {
content.push(UserInput::Text {
text: event.message,
text_elements: event.text_elements,
});
}
if let Some(images) = event.images {
if has_image_order {
let mut inline_index = 0;
let mut file_index = 0;
for image_kind in event.image_order {
let (image, detail) = match image_kind {
UserMessageImageKind::Inline => {
let Some(image_url) = event
.images
.as_deref()
.and_then(|images| images.get(inline_index))
else {
continue;
};
let detail = event.image_details.get(inline_index).copied().flatten();
inline_index += 1;
(
ImageReference::Inline {
image_url: image_url.clone(),
},
detail,
)
}
UserMessageImageKind::File => {
let Some(file_id) = event
.file_ids
.as_deref()
.and_then(|file_ids| file_ids.get(file_index))
else {
continue;
};
let detail = event.file_id_details.get(file_index).copied().flatten();
file_index += 1;
(
ImageReference::File {
file_id: file_id.clone(),
},
detail,
)
}
};
content.push(UserInput::Image { image, detail });
}
} else if let Some(images) = event.images {
for (index, image_url) in images.into_iter().enumerate() {
content.push(UserInput::Image {
image_url,
image: ImageReference::Inline { image_url },
detail: event.image_details.get(index).copied().flatten(),
});
}
}
if !has_image_order && let Some(file_ids) = event.file_ids {
for (index, file_id) in file_ids.into_iter().enumerate() {
content.push(UserInput::Image {
image: ImageReference::File { file_id },
detail: event.file_id_details.get(index).copied().flatten(),
});
}
}
for (index, path) in event.local_images.into_iter().enumerate() {
content.push(UserInput::LocalImage {
path,
codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
+22 -4
View File
@@ -16,6 +16,7 @@ use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::protocol::UserMessageImageKind;
use codex_rollout::CompactedItem;
use codex_rollout::RetainedContextEntry;
use codex_rollout::RetainedInputSource;
@@ -489,19 +490,28 @@ fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {
fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {
let mut text = String::new();
let mut images = Vec::new();
let mut file_ids = Vec::new();
let mut image_order = Vec::new();
let mut audio = Vec::new();
for item in content {
match item {
ContentItem::InputText { text: item_text } => text.push_str(item_text),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => images.push(image_url.as_str()),
ContentItem::InputImage { image, .. } => match image {
ImageReference::Inline { image_url } => {
image_order.push(UserMessageImageKind::Inline);
images.push(image_url.as_str());
}
ImageReference::File { file_id } => {
image_order.push(UserMessageImageKind::File);
file_ids.push(file_id.as_str());
}
},
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
ContentItem::OutputText { .. } => return false,
}
}
text == event.message
&& (!event.has_complete_image_order() || image_order == event.image_order)
&& images
== event
.images
@@ -510,6 +520,14 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
&& file_ids
== event
.file_ids
.as_deref()
.unwrap_or_default()
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
&& audio
== event
.audio
codex-rs/thread-store/src/local/rollout_migration_tests.rs
+65 -3
View File
@@ -20,6 +20,8 @@ use codex_protocol::mcp::McpResourceOrigin;
use codex_protocol::mcp::McpResourceOriginCheckpoint;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AgentMessageEvent;
@@ -39,6 +41,8 @@ use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::protocol::UserMessageImageKind;
use codex_protocol::user_input::UserInput;
use codex_rollout::CompactedItem;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutItem;
@@ -318,12 +322,21 @@ async fn list_active_summary_turns(store: &LocalThreadStore, thread_id: ThreadId
async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
let user_event = UserMessageEvent {
message: "first question".to_string(),
images: Some(vec!["https://example.com/image.png".to_string()]),
image_details: vec![Some(ImageDetail::Original)],
file_ids: Some(vec!["file_123".to_string()]),
file_id_details: vec![Some(ImageDetail::High)],
image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline],
..Default::default()
};
let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
vec![
user_message("first question"),
RolloutItem::EventMsg(EventMsg::UserMessage(user_event)),
agent_message("first answer"),
],
);
@@ -355,6 +368,32 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
.count(),
2
);
let user_item = lines.iter().find_map(|line| match &line.item {
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
item: TurnItem::UserMessage(item),
..
})) => Some(item),
_ => None,
});
let expected_content = vec![
UserInput::Text {
text: "first question".to_string(),
text_elements: Vec::new(),
},
UserInput::Image {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::High),
},
UserInput::Image {
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Original),
},
];
assert_eq!(user_item.map(|item| &item.content), Some(&expected_content));
let turns = list_active_summary_turns(&store, thread_id).await;
assert_eq!(turns.turns.len(), 1);
@@ -847,13 +886,36 @@ async fn migration_drops_trailing_context_when_rollback_arrives_before_next_turn
async fn migration_coalesces_response_first_user_message_rollback_boundary() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
let file_id = "file_123".to_string();
let response = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputText {
text: "remove question".to_string(),
},
ContentItem::InputImage {
image: ImageReference::File {
file_id: file_id.clone(),
},
detail: None,
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let event = UserMessageEvent {
message: "remove question".to_string(),
file_ids: Some(vec![file_id]),
..Default::default()
};
let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
vec![
rollout_response_item(input_response_message("user", "remove question")),
user_message("remove question"),
rollout_response_item(response),
RolloutItem::EventMsg(EventMsg::UserMessage(event)),
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
num_turns: 1,
})),
codex-rs/tools/src/tool_output.rs
+14 -7
View File
@@ -220,13 +220,12 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue
| codex_protocol::models::ContentItem::OutputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
codex_protocol::models::ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
},
codex_protocol::models::ContentItem::InputImage { image, detail } => {
FunctionCallOutputContentItem::InputImage {
image,
detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
}
}
codex_protocol::models::ContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
@@ -260,6 +259,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
image: ImageReference::Inline { image_url },
..
} if !image_url.trim().is_empty() => Some(image_url.clone()),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
} if !file_id.trim().is_empty() => Some(file_id.clone()),
FunctionCallOutputContentItem::InputAudio { audio_url }
if !audio_url.trim().is_empty() =>
{
@@ -274,3 +277,7 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
.join("\n"),
)
}
#[cfg(test)]
#[path = "tool_output_tests.rs"]
mod tests;
codex-rs/tools/src/tool_output_tests.rs
+25
View File
@@ -0,0 +1,25 @@
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use pretty_assertions::assert_eq;
use super::response_input_to_code_mode_result;
#[test]
fn code_mode_text_preserves_file_backed_image_id() {
let response = ResponseInputItem::Message {
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: None,
}],
phase: None,
};
assert_eq!(
response_input_to_code_mode_result(response),
serde_json::json!("file_123")
);
}
codex-rs/tui/src/chatwidget/image_submission.rs
+6 -1
View File
@@ -4,6 +4,7 @@
use super::*;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::snapshot_local_user_input;
use codex_protocol::user_input::UserInput as CoreUserInput;
use tokio::sync::oneshot;
@@ -149,7 +150,11 @@ fn prepare_images(
return Err(too_large());
}
snapshot_local_user_input(&mut input)?;
if let CoreUserInput::Image { image_url, .. } = &input {
if let CoreUserInput::Image {
image: ImageReference::Inline { image_url },
..
} = &input
{
remaining_image_bytes = remaining_image_bytes
.checked_sub(image_url.len())
.ok_or_else(too_large)?;
codex-rs/tui/src/chatwidget/input_submission.rs
+4 -1
View File
@@ -1,6 +1,7 @@
//! User-message and shell-prompt submission behavior for `ChatWidget`.
use super::*;
use codex_app_server_protocol::ImageReference;
impl ChatWidget {
pub(crate) fn set_task_mentions_enabled(&mut self, enabled: bool) {
@@ -239,7 +240,9 @@ impl ChatWidget {
for image_url in &remote_image_urls {
items.push(UserInput::Image {
url: image_url.clone(),
image: ImageReference::Inline {
url: image_url.clone(),
},
detail: None,
});
}
codex-rs/tui/src/chatwidget/tests/composer_submission.rs
+17 -4
View File
@@ -2,6 +2,7 @@ use super::*;
use crate::app_event::ConnectorsSnapshot;
use crate::history_cell::ThreadRecapLoadingCell;
use base64::Engine;
use codex_app_server_protocol::ImageReference;
use codex_protocol::models::ManagedFileSystemPermissions;
use codex_protocol::permissions::FileSystemAccessMode;
use codex_protocol::permissions::FileSystemPath;
@@ -640,7 +641,9 @@ async fn submission_with_remote_and_local_images_keeps_local_placeholder_numberi
assert_eq!(
items[0],
UserInput::Image {
url: remote_url.clone(),
image: ImageReference::Inline {
url: remote_url.clone(),
},
detail: None,
}
);
@@ -727,7 +730,9 @@ async fn enter_with_only_remote_images_submits_user_turn() {
assert_eq!(
items,
vec![UserInput::Image {
url: remote_url.clone(),
image: ImageReference::Inline {
url: remote_url.clone(),
},
detail: None,
}]
);
@@ -2157,7 +2162,9 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() {
text_elements: vec![TextElement::new((0..5).into(), /*placeholder*/ None).into()],
},
UserInput::Image {
url: "https://example.com/remote.png".to_string(),
image: ImageReference::Inline {
url: "https://example.com/remote.png".to_string(),
},
detail: None,
},
UserInput::LocalImage {
@@ -2522,7 +2529,13 @@ async fn image_submission_is_portable_for_new_turns_and_steers() {
_ => {}
}
};
let [UserInput::Image { url, detail }, UserInput::Text { .. }] = items.as_slice()
let [
UserInput::Image {
image: ImageReference::Inline { url },
detail,
},
UserInput::Text { .. },
] = items.as_slice()
else {
panic!("local image must be portable on the wire");
};
codex-rs/tui/src/chatwidget/tests/history_replay.rs
+7 -2
View File
@@ -1,5 +1,6 @@
use super::*;
use crate::app_event::HistoryLookupResponse;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::NetworkAccess;
use codex_app_server_protocol::SandboxPolicy;
use codex_protocol::models::FunctionCallOutputBody;
@@ -523,7 +524,9 @@ async fn replayed_user_message_preserves_remote_image_urls() {
text_elements: Vec::new(),
},
AppServerUserInput::Image {
url: remote_image_urls[0].clone(),
image: ImageReference::Inline {
url: remote_image_urls[0].clone(),
},
detail: None,
},
],
@@ -783,7 +786,9 @@ async fn replayed_user_message_with_only_remote_images_renders_history_cell() {
&mut chat,
"user-1",
vec![AppServerUserInput::Image {
url: remote_image_urls[0].clone(),
image: ImageReference::Inline {
url: remote_image_urls[0].clone(),
},
detail: None,
}],
ReplayKind::ResumeInitialMessages,
codex-rs/tui/src/chatwidget/tests/review_mode.rs
+4 -1
View File
@@ -1,4 +1,5 @@
use super::*;
use codex_app_server_protocol::ImageReference;
use pretty_assertions::assert_eq;
use std::collections::VecDeque;
@@ -646,7 +647,9 @@ async fn item_completed_pops_pending_steer_with_local_image_and_text_elements()
"user-1",
vec![
UserInput::Image {
url: "data:image/png;base64,placeholder".to_string(),
image: ImageReference::Inline {
url: "data:image/png;base64,placeholder".to_string(),
},
detail: None,
},
UserInput::Text {
codex-rs/tui/src/chatwidget/user_messages.rs
+23 -1
View File
@@ -14,6 +14,7 @@ use std::path::PathBuf;
use crate::bottom_pane::LocalImageAttachment;
use crate::bottom_pane::MentionBinding;
use crate::bottom_pane::QueuedInputAction;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::TextElement as AppServerTextElement;
use codex_app_server_protocol::UserInput;
use codex_protocol::config_types::CollaborationMode;
@@ -739,6 +740,20 @@ impl ChatWidget {
{
tracing::warn!("audio user inputs are not supported by the TUI and will be omitted");
}
// TODO(kc) preserve file-backed images when the TUI can resolve or replay them.
if items.iter().any(|item| {
matches!(
item,
UserInput::Image {
image: ImageReference::File { .. },
..
}
)
}) {
tracing::warn!(
"file-backed image inputs are not supported by the TUI and will be omitted"
);
}
let mut message = String::new();
let mut remote_image_urls = Vec::new();
let mut local_images = Vec::new();
@@ -765,7 +780,14 @@ impl ChatWidget {
)
}),
),
UserInput::Image { url, .. } => remote_image_urls.push(url.clone()),
UserInput::Image {
image: ImageReference::Inline { url },
..
} => remote_image_urls.push(url.clone()),
UserInput::Image {
image: ImageReference::File { .. },
..
} => {}
UserInput::LocalImage { path, .. } => local_images.push(path.clone()),
UserInput::Audio { .. } // TODO: Include audio inputs in the user message display.
| UserInput::LocalAudio { .. } // TODO: Include audio inputs in the user message display.
codex-rs/tui/src/dynamic_tools.rs
+9 -1
View File
@@ -12,6 +12,7 @@ use codex_app_server_protocol::DynamicToolFunctionSpec;
use codex_app_server_protocol::DynamicToolNamespaceSpec;
use codex_app_server_protocol::DynamicToolNamespaceTool;
use codex_app_server_protocol::DynamicToolSpec;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxMode;
use codex_app_server_protocol::SandboxPolicy;
@@ -1340,7 +1341,14 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu
}
input
}
UserInput::Image { url, .. } => json!({"type": "image", "url": url}),
UserInput::Image {
image: ImageReference::Inline { url },
..
} => json!({"type": "image", "url": url}),
UserInput::Image {
image: ImageReference::File { file_id },
..
} => json!({"type": "image", "fileId": file_id}),
UserInput::LocalImage { path, .. } => json!({"type": "localImage", "path": path}),
UserInput::Audio { url } => json!({"type": "audio", "url": url}),
UserInput::LocalAudio { path } => json!({"type": "localAudio", "path": path}),
codex-rs/tui/src/resume_picker_transcript_preview_tests.rs
+6 -3
View File
@@ -4,6 +4,7 @@ use app_test_support::create_mock_responses_server_sequence;
use app_test_support::write_mock_responses_config_toml;
use codex_app_server_client::AppServerEvent;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::SortDirection;
@@ -362,9 +363,11 @@ async fn transcript_preview_for_history_mode(
}
} else {
UserInput::Image {
url: String::from(
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
),
image: ImageReference::Inline {
url: String::from(
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
),
},
detail: None,
}
};
codex-rs/utils/output-truncation/src/lib.rs
+8 -17
View File
@@ -3,7 +3,6 @@
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
pub use codex_utils_string::approx_bytes_for_tokens;
pub use codex_utils_string::approx_token_count;
pub use codex_utils_string::approx_tokens_from_byte_count;
@@ -91,15 +90,12 @@ pub fn formatted_truncate_text_content_items_with_policy(
text: formatted_truncate_text(&combined, policy),
}];
out.extend(items.iter().filter_map(|item| match item {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => Some(FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: *detail,
}),
FunctionCallOutputContentItem::InputImage { image, detail } => {
Some(FunctionCallOutputContentItem::InputImage {
image: image.clone(),
detail: *detail,
})
}
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Some(FunctionCallOutputContentItem::InputAudio {
audio_url: audio_url.clone(),
@@ -163,14 +159,9 @@ pub fn truncate_function_output_items_with_policy(
remaining_budget = 0;
}
}
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => {
FunctionCallOutputContentItem::InputImage { image, detail } => {
out.push(FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.clone(),
},
image: image.clone(),
detail: *detail,
});
}
codex-rs/utils/output-truncation/src/truncate_tests.rs
+18
View File
@@ -193,6 +193,12 @@ fn truncate_function_output_items_with_policy_discards_empty_text() {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
@@ -271,6 +277,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
text: "ijkl".to_string(),
},
@@ -298,6 +310,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url: "img:two".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
sdk/python/src/openai_codex/generated/v2_all.py
+48 -5
View File
@@ -6240,7 +6240,7 @@ class TextUserInput(BaseModel):
type: Annotated[Literal["text"], Field(title="TextUserInputType")]
class ImageUserInput(BaseModel):
class UrlUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
@@ -6249,6 +6249,15 @@ class ImageUserInput(BaseModel):
url: str
class FileIdUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[Literal["image"], Field(title="ImageUserInputType")]
file_id: Annotated[str, Field(alias="fileId")]
class LocalImageUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6295,7 +6304,8 @@ class MentionUserInput(BaseModel):
class UserInput(
RootModel[
TextUserInput
| ImageUserInput
| UrlUserInput
| FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -6308,7 +6318,8 @@ class UserInput(
)
root: (
TextUserInput
| ImageUserInput
| UrlUserInput
| FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -7794,15 +7805,34 @@ class InputImageContentItem(BaseModel):
image_url: str
class FileIdContentItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")]
file_id: str
class ContentItem(
RootModel[
InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
InputTextContentItem
| InputImageContentItem
| FileIdContentItem
| InputAudioContentItem
| OutputTextContentItem
]
):
model_config = ConfigDict(
populate_by_name=True,
)
root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
root: (
InputTextContentItem
| InputImageContentItem
| FileIdContentItem
| InputAudioContentItem
| OutputTextContentItem
)
class ExperimentalFeature(BaseModel):
@@ -7990,10 +8020,22 @@ class InputImageFunctionCallOutputContentItem(BaseModel):
image_url: str
class FileIdFunctionCallOutputContentItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
detail: ImageDetail | None = None
type: Annotated[
Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType")
]
file_id: str
class FunctionCallOutputContentItem(
RootModel[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem
]
@@ -8004,6 +8046,7 @@ class FunctionCallOutputContentItem(
root: Annotated[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem,
Field(