diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 0ac74e9b8c..a576251e5a 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -126,6 +126,7 @@ use codex_app_server_protocol::GuardianApprovalReviewAction; use codex_app_server_protocol::GuardianApprovalReviewStatus; use codex_app_server_protocol::GuardianCommandSource as AppServerGuardianCommandSource; use codex_app_server_protocol::ImageGenerationItem; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::ItemCompletedNotification; @@ -396,7 +397,9 @@ fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest text_elements: vec![], }, UserInput::Image { - url: "https://example.com/a.png".to_string(), + image: ImageReference::Inline { + url: "https://example.com/a.png".to_string(), + }, detail: None, }, ], diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index d05de8638d..ac25bc7822 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -672,6 +672,18 @@ ], "title": "ImageUrlContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" } ], "properties": { @@ -1442,6 +1454,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -5663,6 +5687,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -5681,14 +5731,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 6d7bc110d5..b4f02ff44c 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1771,6 +1771,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -6469,6 +6481,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -6487,14 +6525,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 2863a575a9..fa0ba67749 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10484,6 +10484,18 @@ ], "title": "ImageUrlContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" } ], "properties": { @@ -12355,6 +12367,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -25427,6 +25451,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -25445,14 +25495,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index c0787c3bfb..3b8f1e185f 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6364,6 +6364,18 @@ ], "title": "ImageUrlContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" } ], "properties": { @@ -8235,6 +8247,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -23141,6 +23165,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -23159,14 +23209,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index dc2d0dc619..b0757fd2e4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -371,6 +371,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1614,6 +1626,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1632,14 +1670,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index c153e128b3..6018fa212f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -371,6 +371,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1614,6 +1626,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1632,14 +1670,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index c05b788d8a..e9c7bf6e43 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -92,6 +92,18 @@ ], "title": "ImageUrlContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" } ], "properties": { @@ -210,6 +222,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index cddc22f59f..a058b676e6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -511,6 +511,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1944,6 +1956,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1962,14 +2000,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index a10d95cf00..dc0f83ba14 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -592,6 +592,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2684,6 +2696,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2702,14 +2740,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index f467944e67..f8427717dc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -371,6 +371,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1630,6 +1642,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1648,14 +1686,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 537af065c3..9f4be687e0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index c76c1d2491..32cfc6fe96 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 510c26c2ee..a04d78dc33 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 753f062836..2d8a10968f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -154,6 +154,18 @@ ], "title": "ImageUrlContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" } ], "properties": { @@ -272,6 +284,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 3a6fb8bdaa..cb8c0052c3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -608,6 +608,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2762,6 +2774,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2780,14 +2818,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index 8d8a69f51d..2b57af18c2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index da04bf0a27..9e0f4155c2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -592,6 +592,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2684,6 +2696,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2702,14 +2740,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 084cc6522d..6832e8318e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index 3e06a271fa..59ce4d96ba 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -511,6 +511,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1944,6 +1956,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1962,14 +2000,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index d396062af1..f2c1ce42c9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -514,6 +514,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -2475,6 +2487,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -2493,14 +2531,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 9ca8c3b087..2e24d09644 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -511,6 +511,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1944,6 +1956,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1962,14 +2000,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index 68f4b3dff9..5b9105d8e9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -178,6 +178,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -559,6 +571,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -577,14 +615,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index f2d2251e03..c715c11287 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -511,6 +511,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1944,6 +1956,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1962,14 +2000,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index bdf800c52a..0781a4f439 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -511,6 +511,18 @@ ], "title": "ImageUrlFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" } ], "properties": { @@ -1944,6 +1956,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1962,14 +2000,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index 2a3627cbd0..7dbdefa046 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -105,6 +105,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -123,14 +149,10 @@ ], "title": "ImageUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "ImageUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 7b2430db77..96e4c83090 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index e720fc5288..b339e9b4ab 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts index dd5b0a1929..0ad09f18e4 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageDetail } from "./ImageDetail"; -export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, }; +export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts index 8ebefc596e..5fa3d63afe 100644 --- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts @@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail"; * Responses API compatible content items that can be returned by a tool call. * This is a subset of ContentItem with the types we support as function call outputs. */ -export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; +export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts index c268cb4f8d..390d9dd59c 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts @@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string, /** * UI-defined spans within `text` used to render or persist special elements. */ -text_elements: Array, } | { "type": "image", detail?: ImageDetail, url: string, } | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; +text_elements: Array, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 0c46d8c613..4b6b4168f8 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -11,6 +11,7 @@ use crate::protocol::v2::CollabAgentToolCallStatus; use crate::protocol::v2::CommandExecutionStatus; use crate::protocol::v2::DynamicToolCallOutputContentItem; use crate::protocol::v2::DynamicToolCallStatus; +use crate::protocol::v2::ImageReference; use crate::protocol::v2::McpToolCallAppContext; use crate::protocol::v2::McpToolCallError; use crate::protocol::v2::McpToolCallResult; @@ -54,6 +55,7 @@ use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::protocol::UserMessageEvent; +use codex_protocol::protocol::UserMessageImageKind; use codex_protocol::protocol::ViewImageToolCallEvent; use codex_protocol::protocol::WebSearchBeginEvent; use codex_protocol::protocol::WebSearchEndEvent; @@ -1511,14 +1513,62 @@ impl ThreadHistoryBuilder { .collect(), }); } - if let Some(images) = &payload.images { + let has_complete_image_order = payload.has_complete_image_order(); + if has_complete_image_order { + let mut inline_index = 0; + let mut file_index = 0; + for image_kind in &payload.image_order { + let (image, detail) = match image_kind { + UserMessageImageKind::Inline => { + let Some(image) = payload + .images + .as_deref() + .and_then(|images| images.get(inline_index)) + else { + continue; + }; + let detail = payload.image_details.get(inline_index).copied().flatten(); + inline_index += 1; + (ImageReference::Inline { url: image.clone() }, detail) + } + UserMessageImageKind::File => { + let Some(file_id) = payload + .file_ids + .as_deref() + .and_then(|file_ids| file_ids.get(file_index)) + else { + continue; + }; + let detail = payload.file_id_details.get(file_index).copied().flatten(); + file_index += 1; + ( + ImageReference::File { + file_id: file_id.clone(), + }, + detail, + ) + } + }; + content.push(UserInput::Image { image, detail }); + } + } else if let Some(images) = &payload.images { for (idx, image) in images.iter().enumerate() { content.push(UserInput::Image { - url: image.clone(), + image: ImageReference::Inline { url: image.clone() }, detail: payload.image_details.get(idx).copied().flatten(), }); } } + if !has_complete_image_order && let Some(file_ids) = &payload.file_ids { + for (idx, file_id) in file_ids.iter().enumerate() { + content.push(UserInput::Image { + image: ImageReference::File { + file_id: file_id.clone(), + }, + detail: payload.file_id_details.get(idx).copied().flatten(), + }); + } + } for (idx, path) in payload.local_images.iter().enumerate() { content.push(UserInput::LocalImage { path: path.clone(), @@ -1824,7 +1874,9 @@ mod tests { text_elements: Vec::new(), }, UserInput::Image { - url: "https://example.com/one.png".into(), + image: ImageReference::Inline { + url: "https://example.com/one.png".into(), + }, detail: None, } ], @@ -1999,6 +2051,9 @@ mod tests { message: "inspect these".into(), images: Some(vec!["https://example.com/image.png".into()]), image_details: vec![Some(ImageDetail::Original)], + file_ids: Some(vec!["file_123".into()]), + file_id_details: vec![Some(ImageDetail::High)], + image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline], local_images: vec![local_image_path.clone()], local_image_details: vec![Some(ImageDetail::Original)], audio: Some(vec!["https://example.com/audio.mp3".into()]), @@ -2021,7 +2076,15 @@ mod tests { text_elements: Vec::new(), }, UserInput::Image { - url: "https://example.com/image.png".into(), + image: ImageReference::File { + file_id: "file_123".into(), + }, + detail: Some(ImageDetail::High), + }, + UserInput::Image { + image: ImageReference::Inline { + url: "https://example.com/image.png".into(), + }, detail: Some(ImageDetail::Original), }, UserInput::LocalImage { @@ -2039,6 +2102,43 @@ mod tests { ); } + /// Incomplete ordering metadata must not drop references from the legacy split arrays. + #[test] + fn incomplete_image_order_falls_back_to_legacy_order() { + let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage( + UserMessageEvent { + images: Some(vec!["https://example.com/image.png".into()]), + file_ids: Some(vec!["file_123".into()]), + image_order: vec![UserMessageImageKind::File], + ..Default::default() + }, + ))]; + + let turns = build_turns_from_rollout_items(&events); + + assert_eq!( + turns[0].items[0], + ThreadItem::UserMessage { + id: "item-1".into(), + client_id: None, + content: vec![ + UserInput::Image { + image: ImageReference::Inline { + url: "https://example.com/image.png".into(), + }, + detail: None, + }, + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".into(), + }, + detail: None, + }, + ], + } + ); + } + #[test] fn ignores_user_message_item_lifecycle_events() { let turn_id = "turn-1"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 496116f4e5..29cac91c98 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -29,6 +29,7 @@ use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermiss use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference as CoreImageReference; use codex_protocol::models::MessagePhase; use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions; use codex_protocol::models::WebSearchAction as CoreWebSearchAction; @@ -3037,7 +3038,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { text_elements: Vec::new(), }, CoreUserInput::Image { - image_url: "https://example.com/image.png".to_string(), + image: CoreImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::Original), }, CoreUserInput::LocalImage { @@ -3072,7 +3075,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { text_elements: Vec::new(), }, UserInput::Image { - url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::Original), }, UserInput::LocalImage { @@ -3609,16 +3614,84 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() { ); } +/// Keeps the existing URL form stable while exposing file-backed images as `fileId`. +#[test] +fn user_input_image_references_round_trip_with_stable_wire_shapes() { + let cases = [ + ( + UserInput::Image { + image: ImageReference::Inline { + url: "data:image/png;base64,AAA".to_string(), + }, + detail: Some(ImageDetail::High), + }, + json!({ + "type": "image", + "url": "data:image/png;base64,AAA", + "detail": "high", + }), + ), + ( + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::Original), + }, + json!({ + "type": "image", + "fileId": "file_123", + "detail": "original", + }), + ), + ]; + + for (input, wire_value) in cases { + assert_eq!( + serde_json::to_value(&input).expect("user input should serialize"), + wire_value, + ); + assert_eq!( + serde_json::from_value::(wire_value).expect("user input should deserialize"), + input, + ); + } +} + +/// Preserves durable file identity in both directions at the Core boundary. +#[test] +fn file_image_user_input_converts_both_directions() { + let app_server_input = UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::High), + }; + let core_input = CoreUserInput::Image { + image: CoreImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::High), + }; + + assert_eq!(app_server_input.clone().into_core(), core_input); + assert_eq!(UserInput::from(core_input), app_server_input); +} + #[test] fn user_input_into_core_preserves_media_fields() { assert_eq!( UserInput::Image { - url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::Original), } .into_core(), CoreUserInput::Image { - image_url: "https://example.com/image.png".to_string(), + image: CoreImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::Original), } ); diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index a219c5f660..3435ae6402 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -11,6 +11,7 @@ use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference as CoreImageReference; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg; use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus; @@ -401,6 +402,21 @@ impl From for CoreTextElement { } } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(untagged)] +#[ts(untagged)] +#[ts(export_to = "v2/")] +pub enum ImageReference { + Inline { + url: String, + }, + File { + #[serde(rename = "fileId")] + #[ts(rename = "fileId")] + file_id: String, + }, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "camelCase")] #[ts(tag = "type")] @@ -413,10 +429,11 @@ pub enum UserInput { text_elements: Vec, }, Image { + #[serde(flatten)] + image: ImageReference, #[serde(default)] #[ts(optional)] detail: Option, - url: String, }, LocalImage { #[serde(default)] @@ -450,8 +467,11 @@ impl UserInput { text, text_elements: text_elements.into_iter().map(Into::into).collect(), }, - UserInput::Image { url, detail } => CoreUserInput::Image { - image_url: url, + UserInput::Image { image, detail } => CoreUserInput::Image { + image: match image { + ImageReference::Inline { url } => CoreImageReference::Inline { image_url: url }, + ImageReference::File { file_id } => CoreImageReference::File { file_id }, + }, detail, }, UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail }, @@ -473,8 +493,13 @@ impl From for UserInput { text, text_elements: text_elements.into_iter().map(Into::into).collect(), }, - CoreUserInput::Image { image_url, detail } => UserInput::Image { - url: image_url, + CoreUserInput::Image { image, detail } => UserInput::Image { + image: match image { + CoreImageReference::Inline { image_url } => { + ImageReference::Inline { url: image_url } + } + CoreImageReference::File { file_id } => ImageReference::File { file_id }, + }, detail, }, CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail }, diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index a051c867db..1de7f03f1b 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -3,12 +3,13 @@ use super::*; use codex_agent_extension::AgentInvocation; use codex_agent_extension::AgentRun; use codex_agent_extension::AgentRunner; +use codex_app_server_protocol::ImageReference as V2ImageReference; use codex_protocol::error::CodexErrorDetails; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; -use codex_protocol::models::ImageReference; +use codex_protocol::models::ImageReference as CoreImageReference; use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry; use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind; use codex_protocol::protocol::TurnSettingsUpdate; @@ -24,7 +25,10 @@ pub(super) fn validate_user_input_image_urls( if input.iter().any(|item| { matches!( item, - V2UserInput::Image { url, .. } if is_remote_image_url(url) + V2UserInput::Image { + image: V2ImageReference::Inline { url }, + .. + } if is_remote_image_url(url) ) }) { return Err(invalid_request(REMOTE_IMAGE_URL_ERROR)); @@ -37,7 +41,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR ResponseItem::Message { content, .. } => content.iter().any(|item| { matches!( item, - ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url) + ContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url) ) }), ResponseItem::FunctionCallOutput { output, .. } @@ -46,7 +50,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR content.iter().any(|item| { matches!( item, - FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } + FunctionCallOutputContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url) ) }) diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs index bc2d3685ac..90d07c6a50 100644 --- a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs +++ b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs @@ -25,6 +25,7 @@ use axum::routing::post; use codex_app_server_protocol::ApprovalsReviewer; use codex_app_server_protocol::AskForApproval; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemGuardianApprovalReviewStartedNotification; use codex_app_server_protocol::McpServerElicitationRequest; @@ -1031,7 +1032,9 @@ async fn guardian_v2_routes_scoped_tool_approvals( } if mixed_evidence { turn_input.push(UserInput::Image { - url: EVIDENCE_IMAGE.to_owned(), + image: ImageReference::Inline { + url: EVIDENCE_IMAGE.to_owned(), + }, detail: None, }); } diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index b5bae46cd2..e92bb49e8e 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -9,6 +9,7 @@ use app_test_support::TestAppServer; use app_test_support::write_chatgpt_auth; use codex_app_server_protocol::ImageGenerationFailure; use codex_app_server_protocol::ImageGenerationItem; +use codex_app_server_protocol::ImageReference as V2ImageReference; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadReadParams; @@ -657,7 +658,9 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() - text_elements: Vec::new(), }, V2UserInput::Image { - url: image_url.to_string(), + image: V2ImageReference::Inline { + url: image_url.to_string(), + }, detail: None, }, ], diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 85c7a1e86a..860f150869 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -12,6 +12,7 @@ use codex_app_server::in_process::InProcessStartArgs; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::DeprecationNoticeNotification; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::JSONRPCError; @@ -234,6 +235,80 @@ async fn thread_read_can_include_turns() -> Result<()> { Ok(()) } +/// Preserves a file-backed image across a completed turn, including a cold history read. +#[tokio::test] +async fn thread_read_preserves_file_id_from_completed_turn() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + MockResponsesConfig::new(&server.uri()).write(codex_home.path())?; + + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .build_initialized() + .await?; + let start_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let ThreadStartResponse { thread, .. } = + timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(start_id)).await??; + let image = UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: None, + }; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![image.clone()], + ..Default::default() + }), + ) + .await??; + + let read_id = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: thread.id.clone(), + include_turns: true, + }) + .await?; + let ThreadReadResponse { + thread: loaded_thread, + } = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??; + let ThreadItem::UserMessage { content, .. } = &loaded_thread.turns[0].items[0] else { + panic!("expected completed turn to start with a user message"); + }; + assert_eq!(content, &vec![image.clone()]); + + timeout(DEFAULT_READ_TIMEOUT, mcp.shutdown_gracefully()).await??; + drop(mcp); + + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .build_initialized() + .await?; + let read_id = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: thread.id, + include_turns: true, + }) + .await?; + let ThreadReadResponse { + thread: restarted_thread, + } = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??; + let ThreadItem::UserMessage { content, .. } = &restarted_thread.turns[0].items[0] else { + panic!("expected persisted turn to start with a user message"); + }; + assert_eq!(content, &vec![image]); + + Ok(()) +} + #[tokio::test] async fn paginated_stored_thread_routes_projected_turns() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 4bb6eafcfb..311b59cdae 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -34,6 +34,7 @@ use codex_app_server_protocol::CyberAccessProgram; use codex_app_server_protocol::FileChangeApprovalDecision; use codex_app_server_protocol::FileChangePatchUpdatedNotification; use codex_app_server_protocol::FileChangeRequestApprovalResponse; +use codex_app_server_protocol::ImageReference as V2ImageReference; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCError; @@ -267,7 +268,15 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R text_elements: Vec::new(), }, V2UserInput::Image { - url: TINY_PNG_DATA_URL.to_string(), + image: V2ImageReference::Inline { + url: TINY_PNG_DATA_URL.to_string(), + }, + detail: None, + }, + V2UserInput::Image { + image: V2ImageReference::File { + file_id: "file_123".to_string(), + }, detail: None, }, ], @@ -315,6 +324,8 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R content[0]["type"].as_str(), Some("text" | "input_text") )); + assert!(content[0].get("fileId").is_none()); + assert!(content[0].get("file_id").is_none()); user_message_notifications.push(notification.method); } } @@ -332,8 +343,17 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R ); let model_input_images = received_response_input_images(&server).await?; - assert_eq!(model_input_images.len(), 1); - assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL); + assert_eq!(model_input_images.len(), 2); + assert!( + model_input_images + .iter() + .any(|image| image["image_url"] == TINY_PNG_DATA_URL) + ); + assert!( + model_input_images + .iter() + .any(|image| image["file_id"] == "file_123") + ); Ok(()) } @@ -1393,7 +1413,9 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> { thread_id: thread.id.clone(), client_user_message_id: None, input: vec![V2UserInput::Image { - url: TINY_PNG_DATA_URL.to_string(), + image: V2ImageReference::Inline { + url: TINY_PNG_DATA_URL.to_string(), + }, detail: None, }], turn_trigger: Some("user".to_string()), diff --git a/codex-rs/core/src/compact_remote_v2_images.rs b/codex-rs/core/src/compact_remote_v2_images.rs index 0575e07f01..94cbdc239e 100644 --- a/codex-rs/core/src/compact_remote_v2_images.rs +++ b/codex-rs/core/src/compact_remote_v2_images.rs @@ -25,6 +25,10 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize { image_url, *detail, ))) .unwrap_or(usize::MAX), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => 0, ContentItem::InputAudio { .. } => 0, } } diff --git a/codex-rs/core/src/context/node_repl_review_evidence.rs b/codex-rs/core/src/context/node_repl_review_evidence.rs index cd49329a13..d5bad1ea45 100644 --- a/codex-rs/core/src/context/node_repl_review_evidence.rs +++ b/codex-rs/core/src/context/node_repl_review_evidence.rs @@ -63,7 +63,14 @@ impl NodeReplReviewResponse { .saturating_add(self.items.iter().fold(0_usize, |bytes, item| { bytes.saturating_add(match item { Text { text, .. } => text.len(), - Image { image_url, .. } => image_url.len(), + Image { + image: ImageReference::Inline { image_url }, + .. + } => image_url.len(), + Image { + image: ImageReference::File { file_id }, + .. + } => file_id.len(), _ => 0, }) })) @@ -116,14 +123,15 @@ impl NodeReplReviewEvidence { .iter() .flat_map(|response| &response.items) .filter_map(|item| match item { - Image { image_url, detail } if seen_images.insert(image_url) => { - Some(ContentItem::InputImage { - image: ImageReference::Inline { - image_url: image_url.clone(), - }, - detail: *detail, - }) - } + Image { + image: ImageReference::Inline { image_url }, + detail, + } if seen_images.insert(image_url) => Some(ContentItem::InputImage { + image: ImageReference::Inline { + image_url: image_url.clone(), + }, + detail: *detail, + }), _ => None, }) .collect() diff --git a/codex-rs/core/src/context/node_repl_review_evidence_tests.rs b/codex-rs/core/src/context/node_repl_review_evidence_tests.rs index 290aa7d31a..2df586ec34 100644 --- a/codex-rs/core/src/context/node_repl_review_evidence_tests.rs +++ b/codex-rs/core/src/context/node_repl_review_evidence_tests.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::ImageReference; use codex_protocol::user_input::UserInput; use codex_utils_output_truncation::approx_bytes_for_tokens; use pretty_assertions::assert_eq; @@ -15,7 +16,18 @@ fn text_input(text: &str) -> UserInput { fn image_input(image_url: &str) -> UserInput { UserInput::Image { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string(), + }, + detail: None, + } +} + +fn file_image_input(file_id: &str) -> UserInput { + UserInput::Image { + image: ImageReference::File { + file_id: file_id.to_string(), + }, detail: None, } } @@ -121,6 +133,36 @@ fn evidence_preserves_tail_text_after_oversized_image_response() { assert!(rendered_text(&inputs).contains("FINAL IMPORTANT")); } +#[test] +fn file_backed_images_are_retained_but_omitted_from_guardian_views() { + let evidence = NodeReplReviewEvidence::default(); + let file_id = format!("file_{}", "x".repeat(4_096)); + let items = vec![text_input("visible"), file_image_input(&file_id)]; + + evidence.record("browser", "cell", "file", items.clone()); + + let snapshot = evidence.snapshot_since(/*reviewed_sequence*/ 0).unwrap(); + assert_eq!(snapshot.responses.len(), 1); + assert_eq!(snapshot.responses[0].items, items); + assert!(snapshot.responses[0].has_images()); + assert!(snapshot.responses[0].retained_bytes() >= file_id.len()); + assert_eq!(evidence.images(), Vec::new()); + + let rendered = snapshot + .context(NodeReplReviewEvidenceMode::Multimodal) + .render_inputs(); + assert!(rendered_text(&rendered).contains("visible")); + assert!(!rendered.iter().any(|item| { + matches!( + item, + UserInput::Image { + image: ImageReference::File { .. }, + .. + } + ) + })); +} + #[test] fn evidence_evicts_complete_oldest_responses_and_rejects_oversized_items() { let evidence = NodeReplReviewEvidence::default(); diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 69e4e9787c..944efc2929 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -930,6 +930,7 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock i64 { + // TODO(kc) Account for file-backed image size after its token-cost contract is defined. match item { ResponseItem::Message { content, .. } => content .iter() @@ -941,6 +942,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { image: ImageReference::Inline { image_url }, detail, } => estimate_image_bytes(image_url, *detail), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => 0, ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url), }) .fold(0i64, i64::saturating_add), @@ -1147,6 +1152,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { image: ImageReference::Inline { image_url }, detail, } => estimate_image_bytes(image_url, *detail), + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => 0, FunctionCallOutputContentItem::InputAudio { audio_url } => { estimate_audio_bytes(audio_url) } diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index ed983fe644..176e33da17 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -5,7 +5,6 @@ use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::WebSearchItem; use codex_protocol::models::ContentItem; -use codex_protocol::models::ImageReference; use codex_protocol::models::MessagePhase; use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; @@ -127,12 +126,9 @@ fn parse_user_message(message: &[ContentItem]) -> Option { text_elements: Vec::new(), }); } - ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } => { + ContentItem::InputImage { image, detail } => { content.push(UserInput::Image { - image_url: image_url.clone(), + image: image.clone(), detail: *detail, }); } diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index b6ac81c9da..d6d9f59f2c 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -110,11 +110,11 @@ fn parses_user_message_with_text_and_two_images() { text_elements: Vec::new(), }, UserInput::Image { - image_url: img1, + image: ImageReference::Inline { image_url: img1 }, detail: Some(DEFAULT_IMAGE_DETAIL), }, UserInput::Image { - image_url: img2, + image: ImageReference::Inline { image_url: img2 }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]; @@ -124,6 +124,49 @@ fn parses_user_message_with_text_and_two_images() { } } +/// Canonical user-message events must retain opaque file IDs for durable thread history. +#[test] +fn parses_user_message_with_file_image() { + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, + ContentItem::InputText { + text: "describe it".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + let turn_item = parse_turn_item(&item).expect("expected user message turn item"); + + let TurnItem::UserMessage(user) = turn_item else { + panic!("expected TurnItem::UserMessage"); + }; + assert_eq!( + user.content, + vec![ + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, + UserInput::Text { + text: "describe it".to_string(), + text_elements: Vec::new(), + }, + ] + ); +} + #[test] fn skips_local_image_label_text() { let image_url = "data:image/png;base64,abc".to_string(); @@ -158,7 +201,7 @@ fn skips_local_image_label_text() { TurnItem::UserMessage(user) => { let expected_content = vec![ UserInput::Image { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, UserInput::Text { @@ -287,7 +330,7 @@ fn skips_unnamed_image_label_text() { TurnItem::UserMessage(user) => { let expected_content = vec![ UserInput::Image { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, UserInput::Text { diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 458ed9b970..88cc7acf84 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -534,7 +534,9 @@ async fn run_review_on_session( }; let mut prepared = vec![ ResponseInputItem::from(vec![UserInput::Image { - image_url: image_url.to_owned(), + image: ImageReference::Inline { + image_url: image_url.to_owned(), + }, detail: *detail, }]) .into(), diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index adc886f7c1..97098705cb 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -179,6 +179,8 @@ fn prepare_message_content( metadata: &mut Vec, mode: ImagePreparationMode, ) -> Vec { + // TODO(kc) Prepare file-backed images once Core has an explicit resolution contract. Until + // then they pass through while retaining their positions in resize notices. let image_count = items .iter() .filter(|item| matches!(item.content(), ContentItem::InputImage { .. })) @@ -186,12 +188,14 @@ fn prepare_message_content( let mut image_number = 0; let mut resized_images = Vec::new(); for item in items { + if matches!(item.content(), ContentItem::InputImage { .. }) { + image_number += 1; + } if let ContentItem::InputImage { image: ImageReference::Inline { image_url }, detail, } = item.content_mut() { - image_number += 1; match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { resized_images.push(ResizedImage { @@ -231,12 +235,14 @@ fn prepare_tool_output_content( let mut image_number = 0; let mut resized_images = Vec::new(); for item in items { + if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) { + image_number += 1; + } if let FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, detail, } = item { - image_number += 1; match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { resized_images.push(ResizedImage { diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs index a0cfbf5479..ba2324358d 100644 --- a/codex-rs/core/src/image_preparation_tests.rs +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -224,8 +224,9 @@ fn preparation_reports_tool_output_item_id() { ); } +/// File-backed images retain their sequence positions even though preparation skips them. #[test] -fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { +fn resize_notices_count_file_backed_images_and_skip_failed_images() { let (large_image_url, _) = png_data_url(/*width*/ 2048, /*height*/ 2048); let (small_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32); let mut items = vec![ @@ -233,6 +234,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { id: None, role: "user".to_string(), content: vec![ + ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_message".to_string(), + }, + detail: Some(ImageDetail::High), + }, ContentItem::InputImage { image: ImageReference::Inline { image_url: small_image_url, @@ -259,6 +266,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { ContentItemKind("user.image".to_string()), ContentItemKind("user.image".to_string()), ContentItemKind("user.image".to_string()), + ContentItemKind("user.image".to_string()), ]), ..Default::default() }, @@ -270,6 +278,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { name: None, namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "file_tool".to_string(), + }, + detail: Some(ImageDetail::High), + }, FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url: "data:image/png;base64,%%%".to_string(), @@ -294,7 +308,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { ); let expected_user_notice = concat!( "\n", - "Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n", + "Image 4 of 4 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n", "" ); @@ -310,6 +324,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { internal_chat_message_metadata_passthrough, &Some(InternalChatMessageMetadataPassthrough { content_item_kinds: Some(vec![ + ContentItemKind("user.image".to_string()), ContentItemKind("user.image".to_string()), ContentItemKind("images.preparation_error".to_string()), ContentItemKind("user.image".to_string()), @@ -318,6 +333,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { }), ); let [ + ContentItem::InputImage { + image: ImageReference::File { file_id }, + .. + }, ContentItem::InputImage { image: ImageReference::Inline { @@ -337,8 +356,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { }, ] = content.as_slice() else { - panic!("expected unchanged image, failed image placeholder, and resized image"); + panic!("expected file image, unchanged image, failed image placeholder, and resized image"); }; + assert_eq!(file_id, "file_message"); assert_eq!( decoded_image(small_message_image_url).1.dimensions(), (64, 32) @@ -373,6 +393,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { panic!("expected function call output"); }; let [ + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { file_id }, + .. + }, FunctionCallOutputContentItem::InputText { text: failed_tool_image, }, @@ -385,8 +409,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { }, ] = output.content_items().expect("tool output content items") else { - panic!("expected failed image placeholder and resized image in the tool output"); + panic!("expected file image, failed image placeholder, and resized image in tool output"); }; + assert_eq!(file_id, "file_tool"); assert_eq!(failed_tool_image, IMAGE_PROCESSING_ERROR_PLACEHOLDER); assert_eq!( decoded_image(resized_tool_image_url).1.dimensions(), @@ -400,7 +425,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { content: vec![ContentItem::InputText { text: concat!( "\n", - "Image 2 of 2 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n", + "Image 3 of 3 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n", "" ) .to_string(), diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index e4396a92c6..00e7e935ca 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -11306,7 +11306,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() text_elements: vec![text_element.clone()], }, UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: Some(ImageDetail::High), }, ]; diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs index eef4b2cf47..371a1ba0db 100644 --- a/codex-rs/core/src/session/tests/guardian_tests.rs +++ b/codex-rs/core/src/session/tests/guardian_tests.rs @@ -40,6 +40,7 @@ use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::error::SandboxErr; use codex_protocol::models::AdditionalPermissionProfile as PermissionProfile; use codex_protocol::models::ContentItemKind; +use codex_protocol::models::ImageReference; use codex_protocol::models::NetworkPermissions; use codex_protocol::models::ResponseInputItem; use codex_protocol::protocol::AskForApproval; @@ -235,7 +236,9 @@ async fn request_permissions_routes_to_guardian_when_reviewer_is_enabled() { .thread_extension_data .get_or_init(crate::context::NodeReplReviewEvidence::default); let image = UserInput::Image { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string(), + }, detail: None, }; evidence.record("js", "cell", "image", vec![image]); diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index 1d00595a96..509cab24a8 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -367,7 +367,10 @@ impl CoreToolRuntime for McpHandler { load_data_url_for_prompt_uncached(&image_url, PromptImageMode::Original) .ok()?; captured_image_bytes = next_image_bytes; - Some(UserInput::Image { image_url, detail }) + Some(UserInput::Image { + image: codex_protocol::models::ImageReference::Inline { image_url }, + detail, + }) } _ => None, } diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 5016a06375..d8f6bab6ec 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -17,6 +17,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ServiceTier; use codex_protocol::config_types::Settings; use codex_protocol::items::TurnItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelServiceTier; @@ -4833,7 +4834,9 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess codex .start_or_steer_turn(TurnInputRequest::user_input(vec![ UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }, UserInput::Text { diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 53dee6ea97..82ed5f5092 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -17,6 +17,7 @@ use codex_model_provider_info::AMAZON_BEDROCK_PROVIDER_ID; use codex_model_provider_info::ModelProviderInfo; use codex_protocol::AgentPath; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::EventMsg; @@ -637,10 +638,12 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget( let mut bytes = std::io::Cursor::new(Vec::new()); image.write_to(&mut bytes, image::ImageFormat::Png)?; Ok(UserInput::Image { - image_url: format!( - "data:image/png;base64,{}", - BASE64_STANDARD.encode(bytes.get_ref()) - ), + image: ImageReference::Inline { + image_url: format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(bytes.get_ref()) + ), + }, detail: Some(codex_protocol::models::ImageDetail::Original), }) }) @@ -714,7 +717,11 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget( }; let mut expected_images = prepared_images[dropped..].to_vec(); if cycle == 2 { - let UserInput::Image { image_url, .. } = &image_inputs[7] else { + let UserInput::Image { + image: ImageReference::Inline { image_url }, + .. + } = &image_inputs[7] + else { unreachable!() }; expected_images.push(image_url.clone()); diff --git a/codex-rs/core/tests/suite/context_annotations.rs b/codex-rs/core/tests/suite/context_annotations.rs index aeb4a52882..34392c457e 100644 --- a/codex-rs/core/tests/suite/context_annotations.rs +++ b/codex-rs/core/tests/suite/context_annotations.rs @@ -4,6 +4,7 @@ use codex_core::config::CurrentTimeReminderConfig; use codex_core::config::RolloutBudgetConfig; use codex_core::config::TokenBudgetConfig; use codex_features::Feature; +use codex_protocol::models::ImageReference; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::AdditionalContextEntry; use codex_protocol::protocol::AdditionalContextKind; @@ -112,7 +113,9 @@ async fn first_request_item_types_roles_and_content_annotations() -> Result<()> text_elements: Vec::new(), }, UserInput::Image { - image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + }, detail: None, }, UserInput::Audio { diff --git a/codex-rs/core/tests/suite/guardian_authorization.rs b/codex-rs/core/tests/suite/guardian_authorization.rs index 9a8ccfbf55..a252baee34 100644 --- a/codex-rs/core/tests/suite/guardian_authorization.rs +++ b/codex-rs/core/tests/suite/guardian_authorization.rs @@ -11,6 +11,7 @@ use codex_core::context::InternalModelContextFragment; use codex_features::Feature; use codex_history::RolloutItem; use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GuardianAssessmentStatus; @@ -328,7 +329,9 @@ async fn guardian_authorization_revision_survives_compaction_not_user_input() -> .await; test.codex .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image { - image_url: "data:image/png;base64,not-an-image".to_owned(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,not-an-image".to_owned(), + }, detail: None, }])) .await?; diff --git a/codex-rs/core/tests/suite/guardian_history.rs b/codex-rs/core/tests/suite/guardian_history.rs index 5ccc0fa328..94badeabc3 100644 --- a/codex-rs/core/tests/suite/guardian_history.rs +++ b/codex-rs/core/tests/suite/guardian_history.rs @@ -13,6 +13,7 @@ use codex_history::ResumedHistory; use codex_history::RolloutItem; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::mcp::ClientMcpExtensions; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -441,7 +442,9 @@ async fn guardian_history_survives_compaction_and_eviction_but_not_legacy_rollba text_elements: Vec::new(), }, UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }, ])) diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index e12d6a7f57..e9145c1d5d 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -232,7 +232,9 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> .start_or_steer_turn( TurnInputRequest::user_input(vec![ UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }, UserInput::Text { @@ -289,6 +291,83 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> Ok(()) } +/// Core must forward an opaque file ID and persist that same reference in canonical history. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_image_passes_through_request_and_rollout() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let test = test_codex().build_with_auto_env(&server).await?; + let response_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-file"), + ev_assistant_message("msg-file", "done"), + ev_completed("resp-file"), + ]), + ) + .await; + + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![ + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: None, + }, + UserInput::Text { + text: "file-backed image".to_string(), + text_elements: Vec::new(), + }, + ])) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let request = response_mock.single_request(); + assert!(request.input().iter().any(|item| { + item.get("content") + .and_then(Value::as_array) + .is_some_and(|content| { + content.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("input_image") + && item.get("file_id").and_then(Value::as_str) == Some("file_123") + }) + }) + })); + + test.codex.shutdown_and_wait().await?; + let rollout_path = test.codex.rollout_path().expect("rollout path"); + let rollout_text = read_rollout_text(&rollout_path).await?; + let actual = find_user_message_with_image(&rollout_text) + .expect("expected user message with file image in rollout"); + assert_eq!( + strip_response_item_id(strip_metadata(actual)), + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, + ContentItem::InputText { + text: "file-backed image".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + } + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -379,7 +458,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R resumed .codex .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image { - image_url: original_image_url.clone(), + image: ImageReference::Inline { + image_url: original_image_url.clone(), + }, detail: Some(ImageDetail::High), }])) .await?; diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index a3d3d44857..f50baf2ad0 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -9,6 +9,8 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::config_types::Settings; use codex_protocol::items::AgentMessageContent; use codex_protocol::items::TurnItem; +use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::WebSearchAction; use codex_protocol::protocol::AskForApproval; @@ -81,13 +83,21 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> { ByteRange { start: 0, end: 6 }, Some("".into()), )]; - let expected_input = UserInput::Text { - text: "please inspect sample.txt".into(), - text_elements: text_elements.clone(), - }; + let expected_input = vec![ + UserInput::Text { + text: "please inspect sample.txt".into(), + text_elements: text_elements.clone(), + }, + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".into(), + }, + detail: Some(ImageDetail::High), + }, + ]; codex - .start_or_steer_turn(TurnInputRequest::user_input(vec![expected_input.clone()])) + .start_or_steer_turn(TurnInputRequest::user_input(expected_input.clone())) .await?; let started_item = wait_for_event_match(&codex, |ev| match ev { @@ -108,8 +118,8 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> { .await; assert_eq!(started_item.id, completed_item.id); - assert_eq!(started_item.content, vec![expected_input.clone()]); - assert_eq!(completed_item.content, vec![expected_input]); + assert_eq!(started_item.content, expected_input); + assert_eq!(completed_item.content, started_item.content); let legacy_message = wait_for_event_match(&codex, |ev| match ev { EventMsg::UserMessage(event) => Some(event.clone()), @@ -118,6 +128,12 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> { .await; assert_eq!(legacy_message.message, "please inspect sample.txt"); assert_eq!(legacy_message.text_elements, text_elements); + assert_eq!(legacy_message.file_ids, Some(vec!["file_123".into()])); + assert_eq!( + legacy_message.file_id_details, + vec![Some(ImageDetail::High)] + ); + Ok(()) } diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 70e804ee1e..a7a4493e94 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -18,6 +18,7 @@ use codex_protocol::config_types::ServiceTier; use codex_protocol::config_types::Settings; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::models::BaseInstructionsProvenance; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::InputModality; @@ -906,7 +907,9 @@ async fn model_change_projects_media_without_changing_live_or_replayed_history( &test, vec![ UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }, UserInput::Audio { diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index 8ffe25e982..c35be9664b 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -14,6 +14,7 @@ use codex_login::auth::BedrockApiKeyAuth; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_protocol::config_types::WebSearchMode; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ToolMode; use codex_protocol::protocol::EventMsg; @@ -278,11 +279,15 @@ async fn responses_lite_prepares_images() -> Result<()> { test.codex .start_or_steer_turn(TurnInputRequest::user_input(vec![ UserInput::Image { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string(), + }, detail: Some(ImageDetail::Original), }, UserInput::Image { - image_url: remote_image_url.to_string(), + image: ImageReference::Inline { + image_url: remote_image_url.to_string(), + }, detail: Some(ImageDetail::High), }, ])) diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 23405bf9fc..eaa89dd658 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -1488,7 +1488,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history( unreachable!("read_only_user_turn creates user input"); }; content.push(UserInput::Image { - image_url: OPENAI_PNG.to_string(), + image: ImageReference::Inline { + image_url: OPENAI_PNG.to_string(), + }, detail: Some(ImageDetail::High), }); fixture.codex.start_or_steer_turn(input).await?; diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index d339e5b16d..a54147e9e0 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -23,6 +23,7 @@ use codex_login::CodexAuth; use codex_models_manager::bundled_models_response; use codex_protocol::items::AgentMessageDelivery; use codex_protocol::items::TurnItem; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; @@ -382,7 +383,9 @@ async fn astra_kickoff_with_skills_plugins_and_remote_compaction() -> Result<()> .start_or_steer_turn(TurnInputRequest::user_input(vec![ text("Check the final kickoff brief and attached sketch with $final-check and $calendar:agenda."), UserInput::Image { - image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"), + }, detail: None, }, selected_skill("final-check", &skills.final_check), diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index 32b0493775..791244445e 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -166,6 +166,87 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() { ); } +/// A file-backed image in the requested window must not cause an older inline image to be edited. +#[tokio::test] +async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() { + let error = request_for_call_args( + &ImagegenArgs { + prompt: "change the lighting".to_string(), + referenced_image_paths: None, + num_last_images_to_include: Some(1), + }, + &[ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + input_image("older-inline-image"), + ContentItem::InputImage { + image: ImageReference::File { + file_id: "newer-file-backed-image".to_string(), + }, + detail: None, + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }], + &[], + ) + .await + .expect_err("a selected file-backed image should fail"); + + assert_eq!( + error.to_string(), + "requested the last 1 conversation images, but that window includes a file-backed image \ + that cannot be used for editing" + ); +} + +/// Tool-output file references must count toward the window instead of exposing an older image. +#[tokio::test] +async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_window() { + let error = request_for_call_args( + &ImagegenArgs { + prompt: "change the lighting".to_string(), + referenced_image_paths: None, + num_last_images_to_include: Some(1), + }, + &[ + ResponseItem::FunctionCallOutput { + id: None, + call_id: Some("older-inline-output".to_string()), + name: None, + namespace: None, + output: image_output("older-inline-image"), + internal_chat_message_metadata_passthrough: None, + }, + ResponseItem::CustomToolCallOutput { + id: None, + call_id: "newer-file-backed-output".to_string(), + name: Some("view_image".to_string()), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "newer-file-backed-image".to_string(), + }, + detail: None, + }, + ]), + internal_chat_message_metadata_passthrough: None, + }, + ], + &[], + ) + .await + .expect_err("a selected file-backed tool output should fail"); + + assert_eq!( + error.to_string(), + "requested the last 1 conversation images, but that window includes a file-backed image \ + that cannot be used for editing" + ); +} + #[tokio::test] async fn conflicting_image_selectors_return_tool_error() { let error = request_for_call_args( diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 8fc408b442..66864fea6b 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -459,14 +459,7 @@ async fn request_for_call_args( } // Pathless images have no stable reference, so this bounded window may include newer // unrelated images. This remains best-effort until the harness provides stable refs. - let images = recent_images(history, count); - if images.len() != count { - return Err(FunctionCallError::RespondToModel(format!( - "requested the last {count} conversation images, but only {} were available", - images.len() - ))); - } - images + recent_images(history, count)? } (false, Some(_)) => { return Err(FunctionCallError::RespondToModel( @@ -488,17 +481,19 @@ async fn request_for_call_args( })) } -fn recent_images(history: &[ResponseItem], count: usize) -> Vec { +fn recent_images( + history: &[ResponseItem], + count: usize, +) -> Result, FunctionCallError> { let mut images = Vec::with_capacity(count); 'history: for item in history.iter().rev() { let mut image_urls = Vec::new(); match item { ResponseItem::Message { content, .. } => { image_urls.extend(content.iter().rev().filter_map(|item| match item { - ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - .. - } => Some(image_url.clone()), + ContentItem::InputImage { image, .. } => { + Some(inline_image_url(image).map(str::to_owned)) + } ContentItem::InputText { .. } | ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => None, @@ -506,10 +501,12 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec { } ResponseItem::FunctionCallOutput { output, .. } | ResponseItem::CustomToolCallOutput { output, .. } => { - image_urls.extend(output_image_urls(output)); + image_urls.extend( + output_images(output).map(|image| inline_image_url(image).map(str::to_owned)), + ); } ResponseItem::ImageGenerationCall { result, .. } if !result.is_empty() => { - image_urls.push(format!("data:image/png;base64,{result}")); + image_urls.push(Some(format!("data:image/png;base64,{result}"))); } ResponseItem::AdditionalTools { .. } | ResponseItem::Reasoning { .. } @@ -528,34 +525,55 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec { | ResponseItem::Other => {} } for image_url in image_urls { - images.push(ImageUrl { image_url }); + images.push(image_url.map(|image_url| ImageUrl { image_url })); if images.len() == count { break 'history; } } } - images.reverse(); - images + if images.len() != count { + return Err(FunctionCallError::RespondToModel(format!( + "requested the last {count} conversation images, but only {} were available", + images.len() + ))); + } + let mut inline_images = Vec::with_capacity(count); + for image in images { + let Some(image) = image else { + return Err(FunctionCallError::RespondToModel(format!( + "requested the last {count} conversation images, but that window includes a \ + file-backed image that cannot be used for editing" + ))); + }; + inline_images.push(image); + } + inline_images.reverse(); + Ok(inline_images) } -/// Extracts image URLs from a tool output in newest-first order. -fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator + '_ { +/// Extracts image references from a tool output in newest-first order. +fn output_images(output: &FunctionCallOutputPayload) -> impl Iterator + '_ { output .content_items() .into_iter() .flatten() .rev() .filter_map(|item| match item { - FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { image_url }, - .. - } => Some(image_url.clone()), + FunctionCallOutputContentItem::InputImage { image, .. } => Some(image), FunctionCallOutputContentItem::InputText { .. } | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) } +fn inline_image_url(image: &ImageReference) -> Option<&str> { + match image { + ImageReference::Inline { image_url } => Some(image_url), + // TODO(kc) Image generation and the Images API only accept inline image URLs. + ImageReference::File { .. } => None, + } +} + async fn image_url( path: &AbsolutePathBuf, environment: &ToolEnvironment<'_>, diff --git a/codex-rs/ext/queue/tests/queue_service.rs b/codex-rs/ext/queue/tests/queue_service.rs index a597583d39..cadd2fc7fc 100644 --- a/codex-rs/ext/queue/tests/queue_service.rs +++ b/codex-rs/ext/queue/tests/queue_service.rs @@ -29,6 +29,7 @@ use codex_extension_api::TurnStartAdmission; use codex_protocol::ThreadId; use codex_protocol::items::TurnItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -825,7 +826,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow let expected_queued_input = user_input_with_media( "queued attachments", UserInput::Image { - image_url: TINY_PNG_DATA_URL.to_string(), + image: ImageReference::Inline { + image_url: TINY_PNG_DATA_URL.to_string(), + }, detail: Some(ImageDetail::Original), }, UserInput::Audio { @@ -865,7 +868,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow let expected_edited_input = user_input_with_media( "edited attachments", UserInput::Image { - image_url: TINY_PNG_DATA_URL.to_string(), + image: ImageReference::Inline { + image_url: TINY_PNG_DATA_URL.to_string(), + }, detail: Some(ImageDetail::High), }, UserInput::Audio { diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rs index b29ff2c66f..1ebec846e4 100644 --- a/codex-rs/guardian-context/src/budget.rs +++ b/codex-rs/guardian-context/src/budget.rs @@ -105,6 +105,10 @@ impl SectionCost { self.image_bytes = self.image_bytes.saturating_add(image_url.len()); self.image_count = self.image_count.saturating_add(1); } + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => {} ContentItem::InputAudio { audio_url } => { // Guardian currently has no audio contributor. Count a future opaque // payload conservatively until its consumer supplies modality costs. diff --git a/codex-rs/guardian-context/src/composition.rs b/codex-rs/guardian-context/src/composition.rs index 555fbb2163..66892875fd 100644 --- a/codex-rs/guardian-context/src/composition.rs +++ b/codex-rs/guardian-context/src/composition.rs @@ -210,20 +210,27 @@ impl CollectedContext { let items = evidence .items .into_iter() - .map(|item| match item { + .filter_map(|item| match item { UserInput::Text { text, .. } => { - Ok(Budgeted::required(ContentItem::InputText { text })) + Some(Ok(Budgeted::required(ContentItem::InputText { text }))) } - UserInput::Image { image_url, detail } => Ok(Budgeted::optional( + UserInput::Image { + image: ImageReference::Inline { image_url }, + detail, + } => Some(Ok(Budgeted::optional( ContentItem::InputImage { image: ImageReference::Inline { image_url }, detail, }, BudgetPriority::Image, - )), - _ => Err(SectionError::UnsupportedDelivery { + ))), + UserInput::Image { + image: ImageReference::File { .. }, + .. + } => None, + _ => Some(Err(SectionError::UnsupportedDelivery { section: "node_repl_evidence", - }), + })), }) .collect::, _>>()?; ( @@ -288,7 +295,14 @@ impl ComposedContext { ContentItem::InputImage { image: ImageReference::Inline { image_url }, detail, - } => UserInput::Image { image_url, detail }, + } => UserInput::Image { + image: ImageReference::Inline { image_url }, + detail, + }, + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => continue, ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => { return Err(SectionError::UnsupportedDelivery { section: section.id, @@ -316,6 +330,10 @@ impl ComposedContext { } })); } + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => {} content => user_content.push(content), } } diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rs index 54cf00d076..4e59d73463 100644 --- a/codex-rs/guardian-context/src/enforcement.rs +++ b/codex-rs/guardian-context/src/enforcement.rs @@ -95,6 +95,10 @@ impl ComposedContext { image: ImageReference::Inline { image_url }, detail, } => admit(image_url, detail), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => false, _ => true, }); } @@ -281,6 +285,10 @@ fn retain_content( image: ImageReference::Inline { image_url }, .. } => image_url.len(), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } => 0, ContentItem::InputAudio { audio_url } => audio_url.len(), }; truncations.push(TruncationObservation { diff --git a/codex-rs/guardian-context/src/images.rs b/codex-rs/guardian-context/src/images.rs index ccae031095..27314df4db 100644 --- a/codex-rs/guardian-context/src/images.rs +++ b/codex-rs/guardian-context/src/images.rs @@ -85,18 +85,26 @@ impl TranscriptImages { }); }; + // TODO(kc) Include file-backed images once Guardian can admit them without resolving or + // applying byte-based checks in Core. for item in items { match item { ResponseItem::Message { role, content, .. } if matches!(role.as_str(), "user" | "assistant") => { for item in content { - if let ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } = item - { - include_image(image_url, *detail); + match item { + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => include_image(image_url, *detail), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } + | ContentItem::InputText { .. } + | ContentItem::InputAudio { .. } + | ContentItem::OutputText { .. } => {} } } } @@ -106,12 +114,18 @@ impl TranscriptImages { { if let Some(content) = output.content_items() { for item in content { - if let FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } = item - { - include_image(image_url, *detail); + match item { + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => include_image(image_url, *detail), + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } + | FunctionCallOutputContentItem::InputText { .. } + | FunctionCallOutputContentItem::InputAudio { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => {} } } } @@ -121,12 +135,18 @@ impl TranscriptImages { } if input.include_tool_outputs { for image in input.node_repl_images { - if let ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } = image - { - include_image(image_url, *detail); + match image { + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => include_image(image_url, *detail), + ContentItem::InputImage { + image: ImageReference::File { .. }, + .. + } + | ContentItem::InputText { .. } + | ContentItem::InputAudio { .. } + | ContentItem::OutputText { .. } => {} } } } diff --git a/codex-rs/guardian-context/src/images_tests.rs b/codex-rs/guardian-context/src/images_tests.rs index 090b5c0531..5c3fa54d04 100644 --- a/codex-rs/guardian-context/src/images_tests.rs +++ b/codex-rs/guardian-context/src/images_tests.rs @@ -98,3 +98,69 @@ fn image_selection_preserves_source_policy_order_and_both_limits() { } ); } + +/// File-backed images stay out of Guardian without consuming its inline-image capacity. +#[test] +fn file_images_are_omitted_without_displacing_inline_images() { + let file_image = ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::High), + }; + let history = [ + ResponseItem::Message { + id: None, + role: "user".into(), + content: vec![image("first"), file_image.clone(), image("second")], + phase: None, + internal_chat_message_metadata_passthrough: None, + }, + ResponseItem::FunctionCallOutput { + id: None, + call_id: Some("screenshot".into()), + name: None, + namespace: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "file_tool".to_string(), + }, + detail: Some(ImageDetail::High), + }, + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { + image_url: "third".into(), + }, + detail: Some(ImageDetail::High), + }, + ]), + internal_chat_message_metadata_passthrough: None, + }, + ]; + let repl = [ + file_image, + image("fourth"), + ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_repl".to_string(), + }, + detail: Some(ImageDetail::High), + }, + ]; + + assert_eq!( + TranscriptImages::collect( + &history, + TranscriptImageInput { + enabled: true, + include_tool_outputs: true, + node_repl_images: &repl, + } + ), + TranscriptImages { + images: ["first", "second", "third", "fourth"].map(image).to_vec(), + omitted_bytes: 0, + } + ); +} diff --git a/codex-rs/guardian-context/src/node_repl.rs b/codex-rs/guardian-context/src/node_repl.rs index 01b6797d7e..4e7c370379 100644 --- a/codex-rs/guardian-context/src/node_repl.rs +++ b/codex-rs/guardian-context/src/node_repl.rs @@ -9,6 +9,7 @@ use crate::SectionInput; use crate::SectionScope; use codex_context_fragments::ContextualUserFragment; use codex_protocol::models::ContentItemKind; +use codex_protocol::models::ImageReference; use codex_protocol::user_input::UserInput; use codex_protocol::user_input::UserInput::Image; use codex_protocol::user_input::UserInput::Text; @@ -84,10 +85,15 @@ impl NodeReplContext<'_> { pub fn render_inputs(&self) -> Vec { if self.mode != NodeReplReviewEvidenceMode::Multimodal || !self.responses.iter().any(|response| { - response - .items - .iter() - .any(|item| matches!(item, Image { .. })) + response.items.iter().any(|item| { + matches!( + item, + Image { + image: ImageReference::Inline { .. }, + .. + } + ) + }) }) { return vec![text_input(self.render())]; @@ -138,9 +144,10 @@ impl NodeReplContext<'_> { for item in response.items { match item { Text { text, .. } => inputs.push(text_input(format!("{text}\n"))), - Image { image_url, .. } if seen_images.insert(image_url) => { - inputs.push(item.clone()) - } + Image { + image: ImageReference::Inline { image_url }, + .. + } if seen_images.insert(image_url) => inputs.push(item.clone()), _ => {} } } diff --git a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs index 7605fab73e..38c5eb49a7 100644 --- a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs +++ b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs @@ -24,6 +24,7 @@ use codex_protocol::ThreadId; use codex_protocol::ToolName; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ToolResultLogConfig; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; @@ -143,7 +144,9 @@ fn otel_export_routing_policy_routes_user_prompt_log_and_trace_events() { text_elements: Vec::new(), }, UserInput::Image { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: None, }, UserInput::LocalImage { diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index b6c18412f4..2b35643072 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -7,6 +7,7 @@ use crate::memory_citation::MemoryCitation; use crate::models::ContentItem; use crate::models::FunctionCallOutputBody; use crate::models::ImageDetail; +use crate::models::ImageReference; use crate::models::MessagePhase; use crate::models::ResponseItem; use crate::models::WebSearchAction; @@ -554,7 +555,10 @@ impl UserMessageItem { self.content .iter() .filter_map(|c| match c { - UserInput::Image { image_url, .. } => Some(image_url.clone()), + UserInput::Image { + image: ImageReference::Inline { image_url }, + .. + } => Some(image_url.clone()), _ => None, }) .collect() @@ -565,7 +569,10 @@ impl UserMessageItem { self.content .iter() .filter_map(|c| match c { - UserInput::Image { detail, .. } => Some(*detail), + UserInput::Image { + image: ImageReference::Inline { .. }, + detail, + } => Some(*detail), _ => None, }) .collect(), @@ -615,7 +622,7 @@ impl UserMessageItem { } } -fn trim_trailing_default_image_details( +pub(crate) fn trim_trailing_default_image_details( mut details: Vec>, ) -> Vec> { while matches!(details.last(), Some(None)) { diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index bfea90b558..b8f73e0641 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -20,6 +20,8 @@ use crate::items::SubAgentActivityItem; use crate::items::TurnItem; use crate::items::UserMessageItem; use crate::items::WebSearchItem; +use crate::items::trim_trailing_default_image_details; +use crate::models::ImageReference; use crate::protocol::AgentMessageContentDeltaEvent; use crate::protocol::AgentMessageEvent; use crate::protocol::AgentReasoningEvent; @@ -58,9 +60,11 @@ use crate::protocol::ReasoningContentDeltaEvent; use crate::protocol::ReasoningRawContentDeltaEvent; use crate::protocol::SubAgentActivityEvent; use crate::protocol::UserMessageEvent; +use crate::protocol::UserMessageImageKind; use crate::protocol::ViewImageToolCallEvent; use crate::protocol::WebSearchBeginEvent; use crate::protocol::WebSearchEndEvent; +use crate::user_input::UserInput; /// Converts canonical item lifecycle events back into the legacy raw event stream used by /// compatibility consumers that have not migrated to `TurnItem`. @@ -78,11 +82,32 @@ impl UserMessageItem { pub fn as_legacy_user_message_event(&self) -> UserMessageEvent { // Legacy user-message events flatten only text inputs into `message` and // rebase text element ranges onto that concatenated text. + let mut file_ids = Vec::new(); + let mut file_id_details = Vec::new(); + let mut image_order = Vec::new(); + for input in &self.content { + if let UserInput::Image { image, detail } = input { + match image { + ImageReference::Inline { .. } => { + image_order.push(UserMessageImageKind::Inline); + } + ImageReference::File { file_id } => { + image_order.push(UserMessageImageKind::File); + file_ids.push(file_id.clone()); + file_id_details.push(*detail); + } + } + } + } + let file_id_details = trim_trailing_default_image_details(file_id_details); UserMessageEvent { client_id: self.client_id.clone(), message: self.message(), images: Some(self.image_urls()), image_details: self.image_details(), + file_ids: (!file_ids.is_empty()).then_some(file_ids), + file_id_details, + image_order, local_images: self.local_image_paths(), local_image_details: self.local_image_details(), audio: Some(self.audio_urls()), diff --git a/codex-rs/protocol/src/local_media.rs b/codex-rs/protocol/src/local_media.rs index 7e4cbd9dfd..e1b8af25cf 100644 --- a/codex-rs/protocol/src/local_media.rs +++ b/codex-rs/protocol/src/local_media.rs @@ -8,6 +8,7 @@ use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; use crate::models::ImageDetail; +use crate::models::ImageReference; use crate::user_input::UserInput; /// Maximum accepted decoded byte length for prompt audio inputs. @@ -29,7 +30,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> { let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?; let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?; *input = UserInput::Image { - image_url: image.into_data_url(), + image: ImageReference::Inline { + image_url: image.into_data_url(), + }, detail: image_detail, }; } diff --git a/codex-rs/protocol/src/local_media_tests.rs b/codex-rs/protocol/src/local_media_tests.rs index 3d03c32620..47b28c8c25 100644 --- a/codex-rs/protocol/src/local_media_tests.rs +++ b/codex-rs/protocol/src/local_media_tests.rs @@ -31,8 +31,10 @@ fn snapshots_local_image_user_input_with_requested_detail() -> Result<()> { assert_eq!( UserInput::Image { - image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=" - .to_string(), + image: crate::models::ImageReference::Inline { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=" + .to_string(), + }, detail, }, input diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index e2f588eaa3..549d69d553 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -898,6 +898,7 @@ pub enum ContentItem { #[ts(untagged)] pub enum ImageReference { Inline { image_url: String }, + File { file_id: String }, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] @@ -2008,13 +2009,11 @@ impl ResponseInputItem { .into_iter() .flat_map(|c| match c { UserInput::Text { text, .. } => vec![ContentItem::InputText { text }], - UserInput::Image { - image_url, detail, .. - } => { + UserInput::Image { image, detail, .. } => { image_index += 1; let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL); vec![ContentItem::InputImage { - image: ImageReference::Inline { image_url }, + image, detail: Some(detail), }] } @@ -3885,7 +3884,9 @@ mod tests { let image_url = "data:image/png;base64,abc".to_string(); let item = ResponseInputItem::from(vec![UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }]); @@ -3903,6 +3904,43 @@ mod tests { Ok(()) } + /// A file-backed user image must reach the Responses API without being resolved by Core. + #[test] + fn file_image_user_input_serializes_file_id() -> Result<()> { + let file_id = "file_123".to_string(); + + let item = ResponseInputItem::from(vec![UserInput::Image { + image: ImageReference::File { + file_id: file_id.clone(), + }, + detail: None, + }]); + + let expected = ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image: ImageReference::File { file_id }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }], + phase: None, + }; + assert_eq!(item, expected); + assert_eq!( + serde_json::to_value(item)?, + serde_json::json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_image", + "file_id": "file_123", + "detail": "high" + }] + }) + ); + + Ok(()) + } + #[test] fn serializes_audio_user_input_without_tags() -> Result<()> { let audio_url = "data:audio/wav;base64,abc".to_string(); @@ -4029,7 +4067,9 @@ mod tests { let image_url = "data:image/png;base64,abc".to_string(); let item = ResponseInputItem::from(vec![UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: Some(ImageDetail::Original), }]); @@ -4227,7 +4267,9 @@ mod tests { let item = ResponseInputItem::from(vec![ UserInput::Image { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: None, }, UserInput::LocalImage { diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index ceeaa80e31..b08780eb49 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2521,6 +2521,14 @@ pub struct AgentMessageEvent { pub questions: Option>, } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum UserMessageImageKind { + Inline, + File, +} + #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)] pub struct UserMessageEvent { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2535,6 +2543,19 @@ pub struct UserMessageEvent { /// default image detail behavior. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub image_details: Vec>, + /// File IDs sourced from `UserInput::Image`. These are passed through as + /// opaque references and are not created by image preparation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_ids: Option>, + /// Detail hints for `file_ids`, indexed in parallel. Missing entries imply + /// default image detail behavior. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub file_id_details: Vec>, + /// Inline and file-backed image kinds in their original input order. + /// New producers populate this alongside `images` and `file_ids`; when it + /// is absent, consumers retain the legacy inline-then-file ordering. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub image_order: Vec, /// Local file paths sourced from `UserInput::LocalImage`. These are kept so /// the UI can reattach images when editing history. Local image prompts may /// include a display form of the path, but these should not be treated as @@ -2559,6 +2580,27 @@ pub struct UserMessageEvent { pub text_elements: Vec, } +impl UserMessageEvent { + /// Returns whether `image_order` accounts for every split image reference exactly once. + pub fn has_complete_image_order(&self) -> bool { + if self.image_order.is_empty() { + return false; + } + + let mut inline_count = 0; + let mut file_count = 0; + for image_kind in &self.image_order { + match image_kind { + UserMessageImageKind::Inline => inline_count += 1, + UserMessageImageKind::File => file_count += 1, + } + } + + inline_count == self.images.as_ref().map_or(0, Vec::len) + && file_count == self.file_ids.as_ref().map_or(0, Vec::len) + } +} + /// Returns the user-facing preview text for a user message. pub fn user_message_preview(user: &UserMessageEvent) -> Option { let message = strip_user_message_prefix(user.message.as_str()); @@ -2569,6 +2611,10 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option { .images .as_ref() .is_some_and(|images| !images.is_empty()) + || user + .file_ids + .as_ref() + .is_some_and(|file_ids| !file_ids.is_empty()) || !user.local_images.is_empty() { return Some("[Image]".to_string()); @@ -5944,6 +5990,9 @@ mod tests { Some(vec!["https://example.com/image.png".to_string()]) ); assert_eq!(event.image_details, Vec::>::new()); + assert_eq!(event.file_ids, None); + assert_eq!(event.file_id_details, Vec::>::new()); + assert_eq!(event.image_order, Vec::::new()); assert_eq!(event.local_images, vec![PathBuf::from("/tmp/local.png")]); assert_eq!(event.local_image_details, Vec::>::new()); assert_eq!(event.audio, None); @@ -5959,11 +6008,21 @@ mod tests { let local_audio_path = PathBuf::from("/tmp/local.wav"); let mut item = UserMessageItem::new(&[ crate::user_input::UserInput::Image { - image_url: "https://example.com/first.png".to_string(), + image: crate::models::ImageReference::Inline { + image_url: "https://example.com/first.png".to_string(), + }, detail: Some(ImageDetail::Original), }, crate::user_input::UserInput::Image { - image_url: "https://example.com/second.png".to_string(), + image: crate::models::ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::Low), + }, + crate::user_input::UserInput::Image { + image: crate::models::ImageReference::Inline { + image_url: "https://example.com/second.png".to_string(), + }, detail: None, }, crate::user_input::UserInput::LocalImage { @@ -5982,6 +6041,7 @@ mod tests { let EventMsg::UserMessage(event) = item.as_legacy_event() else { panic!("expected user message event"); }; + let event_json = serde_json::to_value(&event).expect("serialize user message event"); assert_eq!( event.images, @@ -5992,6 +6052,22 @@ mod tests { ); assert_eq!(event.client_id, Some("client-message-1".to_string())); assert_eq!(event.image_details, vec![Some(ImageDetail::Original)]); + assert_eq!(event.file_ids, Some(vec!["file_123".to_string()])); + assert_eq!(event.file_id_details, vec![Some(ImageDetail::Low)]); + assert_eq!( + event.image_order, + vec![ + UserMessageImageKind::Inline, + UserMessageImageKind::File, + UserMessageImageKind::Inline, + ] + ); + assert_eq!(event_json["file_ids"], json!(["file_123"])); + assert_eq!(event_json["file_id_details"], json!(["low"])); + assert_eq!( + event_json["image_order"], + json!(["inline", "file", "inline"]) + ); assert_eq!(event.local_images, vec![local_path]); assert_eq!(event.local_image_details, vec![Some(ImageDetail::Original)]); assert_eq!( @@ -6011,6 +6087,16 @@ mod tests { assert_eq!(user_message_preview(&event), Some("[Audio]".to_string())); } + #[test] + fn file_only_user_message_has_placeholder_preview() { + let event = UserMessageEvent { + file_ids: Some(vec!["file_123".to_string()]), + ..Default::default() + }; + + assert_eq!(user_message_preview(&event), Some("[Image]".to_string())); + } + #[test] fn turn_aborted_event_deserializes_without_turn_id() -> Result<()> { let event: EventMsg = serde_json::from_value(json!({ diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index 9cb27e897d..8f5f87c00a 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -4,6 +4,7 @@ use serde::Serialize; use ts_rs::TS; use crate::models::ImageDetail; +use crate::models::ImageReference; /// Conservative cap so one user message cannot monopolize a large context window. pub const MAX_USER_INPUT_TEXT_CHARS: usize = 1 << 20; @@ -22,9 +23,10 @@ pub enum UserInput { #[serde(default)] text_elements: Vec, }, - /// Pre‑encoded data: URI image. + /// Image reference forwarded to the Responses API. Image { - image_url: String, + #[serde(flatten)] + image: ImageReference, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] detail: Option, diff --git a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs index 589509f7ff..734327261a 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs @@ -26,8 +26,10 @@ use codex_protocol::items::SubAgentActivityItem; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::WebSearchItem; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::UserMessageEvent; +use codex_protocol::protocol::UserMessageImageKind; use codex_protocol::user_input::UserInput; use crate::ThreadStoreResult; @@ -37,20 +39,71 @@ pub(super) fn user_message_item( next_item_id: &mut impl FnMut() -> ThreadStoreResult, ) -> ThreadStoreResult { let mut content = Vec::new(); + let has_image_order = event.has_complete_image_order(); if !event.message.trim().is_empty() { content.push(UserInput::Text { text: event.message, text_elements: event.text_elements, }); } - if let Some(images) = event.images { + if has_image_order { + let mut inline_index = 0; + let mut file_index = 0; + for image_kind in event.image_order { + let (image, detail) = match image_kind { + UserMessageImageKind::Inline => { + let Some(image_url) = event + .images + .as_deref() + .and_then(|images| images.get(inline_index)) + else { + continue; + }; + let detail = event.image_details.get(inline_index).copied().flatten(); + inline_index += 1; + ( + ImageReference::Inline { + image_url: image_url.clone(), + }, + detail, + ) + } + UserMessageImageKind::File => { + let Some(file_id) = event + .file_ids + .as_deref() + .and_then(|file_ids| file_ids.get(file_index)) + else { + continue; + }; + let detail = event.file_id_details.get(file_index).copied().flatten(); + file_index += 1; + ( + ImageReference::File { + file_id: file_id.clone(), + }, + detail, + ) + } + }; + content.push(UserInput::Image { image, detail }); + } + } else if let Some(images) = event.images { for (index, image_url) in images.into_iter().enumerate() { content.push(UserInput::Image { - image_url, + image: ImageReference::Inline { image_url }, detail: event.image_details.get(index).copied().flatten(), }); } } + if !has_image_order && let Some(file_ids) = event.file_ids { + for (index, file_id) in file_ids.into_iter().enumerate() { + content.push(UserInput::Image { + image: ImageReference::File { file_id }, + detail: event.file_id_details.get(index).copied().flatten(), + }); + } + } for (index, path) in event.local_images.into_iter().enumerate() { content.push(UserInput::LocalImage { path, diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs index 07da7b7042..11a8c8d6d6 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs @@ -16,6 +16,7 @@ use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::UserMessageEvent; +use codex_protocol::protocol::UserMessageImageKind; use codex_rollout::CompactedItem; use codex_rollout::RetainedContextEntry; use codex_rollout::RetainedInputSource; @@ -489,19 +490,28 @@ fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> { fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool { let mut text = String::new(); let mut images = Vec::new(); + let mut file_ids = Vec::new(); + let mut image_order = Vec::new(); let mut audio = Vec::new(); for item in content { match item { ContentItem::InputText { text: item_text } => text.push_str(item_text), - ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - .. - } => images.push(image_url.as_str()), + ContentItem::InputImage { image, .. } => match image { + ImageReference::Inline { image_url } => { + image_order.push(UserMessageImageKind::Inline); + images.push(image_url.as_str()); + } + ImageReference::File { file_id } => { + image_order.push(UserMessageImageKind::File); + file_ids.push(file_id.as_str()); + } + }, ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()), ContentItem::OutputText { .. } => return false, } } text == event.message + && (!event.has_complete_image_order() || image_order == event.image_order) && images == event .images @@ -510,6 +520,14 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent .iter() .map(String::as_str) .collect::>() + && file_ids + == event + .file_ids + .as_deref() + .unwrap_or_default() + .iter() + .map(String::as_str) + .collect::>() && audio == event .audio diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs index bad11829c6..7c85b6b080 100644 --- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs +++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs @@ -20,6 +20,8 @@ use codex_protocol::mcp::McpResourceOrigin; use codex_protocol::mcp::McpResourceOriginCheckpoint; use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; +use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AgentMessageEvent; @@ -39,6 +41,8 @@ use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::protocol::UserMessageEvent; +use codex_protocol::protocol::UserMessageImageKind; +use codex_protocol::user_input::UserInput; use codex_rollout::CompactedItem; use codex_rollout::RolloutConfig; use codex_rollout::RolloutItem; @@ -318,12 +322,21 @@ async fn list_active_summary_turns(store: &LocalThreadStore, thread_id: ThreadId async fn migration_publishes_canonical_projected_history_and_is_idempotent() { let home = TempDir::new().expect("create Codex home"); let thread_id = ThreadId::new(); + let user_event = UserMessageEvent { + message: "first question".to_string(), + images: Some(vec!["https://example.com/image.png".to_string()]), + image_details: vec![Some(ImageDetail::Original)], + file_ids: Some(vec!["file_123".to_string()]), + file_id_details: vec![Some(ImageDetail::High)], + image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline], + ..Default::default() + }; let path = write_rollout( home.path(), thread_id, SessionSource::Cli, vec![ - user_message("first question"), + RolloutItem::EventMsg(EventMsg::UserMessage(user_event)), agent_message("first answer"), ], ); @@ -355,6 +368,32 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() { .count(), 2 ); + let user_item = lines.iter().find_map(|line| match &line.item { + RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent { + item: TurnItem::UserMessage(item), + .. + })) => Some(item), + _ => None, + }); + let expected_content = vec![ + UserInput::Text { + text: "first question".to_string(), + text_elements: Vec::new(), + }, + UserInput::Image { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(ImageDetail::High), + }, + UserInput::Image { + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, + detail: Some(ImageDetail::Original), + }, + ]; + assert_eq!(user_item.map(|item| &item.content), Some(&expected_content)); let turns = list_active_summary_turns(&store, thread_id).await; assert_eq!(turns.turns.len(), 1); @@ -847,13 +886,36 @@ async fn migration_drops_trailing_context_when_rollback_arrives_before_next_turn async fn migration_coalesces_response_first_user_message_rollback_boundary() { let home = TempDir::new().expect("create Codex home"); let thread_id = ThreadId::new(); + let file_id = "file_123".to_string(); + let response = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "remove question".to_string(), + }, + ContentItem::InputImage { + image: ImageReference::File { + file_id: file_id.clone(), + }, + detail: None, + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + let event = UserMessageEvent { + message: "remove question".to_string(), + file_ids: Some(vec![file_id]), + ..Default::default() + }; let path = write_rollout( home.path(), thread_id, SessionSource::Cli, vec![ - rollout_response_item(input_response_message("user", "remove question")), - user_message("remove question"), + rollout_response_item(response), + RolloutItem::EventMsg(EventMsg::UserMessage(event)), RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent { num_turns: 1, })), diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index a299d172ee..b4cf87be73 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -220,13 +220,12 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue | codex_protocol::models::ContentItem::OutputText { text } => { FunctionCallOutputContentItem::InputText { text } } - codex_protocol::models::ContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } => FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)), - }, + codex_protocol::models::ContentItem::InputImage { image, detail } => { + FunctionCallOutputContentItem::InputImage { + image, + detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)), + } + } codex_protocol::models::ContentItem::InputAudio { audio_url } => { FunctionCallOutputContentItem::InputAudio { audio_url } } @@ -260,6 +259,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> image: ImageReference::Inline { image_url }, .. } if !image_url.trim().is_empty() => Some(image_url.clone()), + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { file_id }, + .. + } if !file_id.trim().is_empty() => Some(file_id.clone()), FunctionCallOutputContentItem::InputAudio { audio_url } if !audio_url.trim().is_empty() => { @@ -274,3 +277,7 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> .join("\n"), ) } + +#[cfg(test)] +#[path = "tool_output_tests.rs"] +mod tests; diff --git a/codex-rs/tools/src/tool_output_tests.rs b/codex-rs/tools/src/tool_output_tests.rs new file mode 100644 index 0000000000..263603e99b --- /dev/null +++ b/codex-rs/tools/src/tool_output_tests.rs @@ -0,0 +1,25 @@ +use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; +use codex_protocol::models::ResponseInputItem; +use pretty_assertions::assert_eq; + +use super::response_input_to_code_mode_result; + +#[test] +fn code_mode_text_preserves_file_backed_image_id() { + let response = ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: None, + }], + phase: None, + }; + + assert_eq!( + response_input_to_code_mode_result(response), + serde_json::json!("file_123") + ); +} diff --git a/codex-rs/tui/src/chatwidget/image_submission.rs b/codex-rs/tui/src/chatwidget/image_submission.rs index 3e5a961158..832104c555 100644 --- a/codex-rs/tui/src/chatwidget/image_submission.rs +++ b/codex-rs/tui/src/chatwidget/image_submission.rs @@ -4,6 +4,7 @@ use super::*; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::snapshot_local_user_input; use codex_protocol::user_input::UserInput as CoreUserInput; use tokio::sync::oneshot; @@ -149,7 +150,11 @@ fn prepare_images( return Err(too_large()); } snapshot_local_user_input(&mut input)?; - if let CoreUserInput::Image { image_url, .. } = &input { + if let CoreUserInput::Image { + image: ImageReference::Inline { image_url }, + .. + } = &input + { remaining_image_bytes = remaining_image_bytes .checked_sub(image_url.len()) .ok_or_else(too_large)?; diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs index 6b43d3793e..d4567b6be9 100644 --- a/codex-rs/tui/src/chatwidget/input_submission.rs +++ b/codex-rs/tui/src/chatwidget/input_submission.rs @@ -1,6 +1,7 @@ //! User-message and shell-prompt submission behavior for `ChatWidget`. use super::*; +use codex_app_server_protocol::ImageReference; impl ChatWidget { pub(crate) fn set_task_mentions_enabled(&mut self, enabled: bool) { @@ -239,7 +240,9 @@ impl ChatWidget { for image_url in &remote_image_urls { items.push(UserInput::Image { - url: image_url.clone(), + image: ImageReference::Inline { + url: image_url.clone(), + }, detail: None, }); } diff --git a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs index e8deed5e9e..1d1db72362 100644 --- a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs +++ b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs @@ -2,6 +2,7 @@ use super::*; use crate::app_event::ConnectorsSnapshot; use crate::history_cell::ThreadRecapLoadingCell; use base64::Engine; +use codex_app_server_protocol::ImageReference; use codex_protocol::models::ManagedFileSystemPermissions; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -640,7 +641,9 @@ async fn submission_with_remote_and_local_images_keeps_local_placeholder_numberi assert_eq!( items[0], UserInput::Image { - url: remote_url.clone(), + image: ImageReference::Inline { + url: remote_url.clone(), + }, detail: None, } ); @@ -727,7 +730,9 @@ async fn enter_with_only_remote_images_submits_user_turn() { assert_eq!( items, vec![UserInput::Image { - url: remote_url.clone(), + image: ImageReference::Inline { + url: remote_url.clone(), + }, detail: None, }] ); @@ -2157,7 +2162,9 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() { text_elements: vec![TextElement::new((0..5).into(), /*placeholder*/ None).into()], }, UserInput::Image { - url: "https://example.com/remote.png".to_string(), + image: ImageReference::Inline { + url: "https://example.com/remote.png".to_string(), + }, detail: None, }, UserInput::LocalImage { @@ -2522,7 +2529,13 @@ async fn image_submission_is_portable_for_new_turns_and_steers() { _ => {} } }; - let [UserInput::Image { url, detail }, UserInput::Text { .. }] = items.as_slice() + let [ + UserInput::Image { + image: ImageReference::Inline { url }, + detail, + }, + UserInput::Text { .. }, + ] = items.as_slice() else { panic!("local image must be portable on the wire"); }; diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index fbc15c4ec4..5979f32662 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -1,5 +1,6 @@ use super::*; use crate::app_event::HistoryLookupResponse; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::NetworkAccess; use codex_app_server_protocol::SandboxPolicy; use codex_protocol::models::FunctionCallOutputBody; @@ -523,7 +524,9 @@ async fn replayed_user_message_preserves_remote_image_urls() { text_elements: Vec::new(), }, AppServerUserInput::Image { - url: remote_image_urls[0].clone(), + image: ImageReference::Inline { + url: remote_image_urls[0].clone(), + }, detail: None, }, ], @@ -783,7 +786,9 @@ async fn replayed_user_message_with_only_remote_images_renders_history_cell() { &mut chat, "user-1", vec![AppServerUserInput::Image { - url: remote_image_urls[0].clone(), + image: ImageReference::Inline { + url: remote_image_urls[0].clone(), + }, detail: None, }], ReplayKind::ResumeInitialMessages, diff --git a/codex-rs/tui/src/chatwidget/tests/review_mode.rs b/codex-rs/tui/src/chatwidget/tests/review_mode.rs index f947b364d2..7107273e6c 100644 --- a/codex-rs/tui/src/chatwidget/tests/review_mode.rs +++ b/codex-rs/tui/src/chatwidget/tests/review_mode.rs @@ -1,4 +1,5 @@ use super::*; +use codex_app_server_protocol::ImageReference; use pretty_assertions::assert_eq; use std::collections::VecDeque; @@ -646,7 +647,9 @@ async fn item_completed_pops_pending_steer_with_local_image_and_text_elements() "user-1", vec![ UserInput::Image { - url: "data:image/png;base64,placeholder".to_string(), + image: ImageReference::Inline { + url: "data:image/png;base64,placeholder".to_string(), + }, detail: None, }, UserInput::Text { diff --git a/codex-rs/tui/src/chatwidget/user_messages.rs b/codex-rs/tui/src/chatwidget/user_messages.rs index 59dec1f6d2..befc76f54a 100644 --- a/codex-rs/tui/src/chatwidget/user_messages.rs +++ b/codex-rs/tui/src/chatwidget/user_messages.rs @@ -14,6 +14,7 @@ use std::path::PathBuf; use crate::bottom_pane::LocalImageAttachment; use crate::bottom_pane::MentionBinding; use crate::bottom_pane::QueuedInputAction; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::TextElement as AppServerTextElement; use codex_app_server_protocol::UserInput; use codex_protocol::config_types::CollaborationMode; @@ -739,6 +740,20 @@ impl ChatWidget { { tracing::warn!("audio user inputs are not supported by the TUI and will be omitted"); } + // TODO(kc) preserve file-backed images when the TUI can resolve or replay them. + if items.iter().any(|item| { + matches!( + item, + UserInput::Image { + image: ImageReference::File { .. }, + .. + } + ) + }) { + tracing::warn!( + "file-backed image inputs are not supported by the TUI and will be omitted" + ); + } let mut message = String::new(); let mut remote_image_urls = Vec::new(); let mut local_images = Vec::new(); @@ -765,7 +780,14 @@ impl ChatWidget { ) }), ), - UserInput::Image { url, .. } => remote_image_urls.push(url.clone()), + UserInput::Image { + image: ImageReference::Inline { url }, + .. + } => remote_image_urls.push(url.clone()), + UserInput::Image { + image: ImageReference::File { .. }, + .. + } => {} UserInput::LocalImage { path, .. } => local_images.push(path.clone()), UserInput::Audio { .. } // TODO: Include audio inputs in the user message display. | UserInput::LocalAudio { .. } // TODO: Include audio inputs in the user message display. diff --git a/codex-rs/tui/src/dynamic_tools.rs b/codex-rs/tui/src/dynamic_tools.rs index 8633fdaf58..e432195d20 100644 --- a/codex-rs/tui/src/dynamic_tools.rs +++ b/codex-rs/tui/src/dynamic_tools.rs @@ -12,6 +12,7 @@ use codex_app_server_protocol::DynamicToolFunctionSpec; use codex_app_server_protocol::DynamicToolNamespaceSpec; use codex_app_server_protocol::DynamicToolNamespaceTool; use codex_app_server_protocol::DynamicToolSpec; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::SandboxPolicy; @@ -1340,7 +1341,14 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu } input } - UserInput::Image { url, .. } => json!({"type": "image", "url": url}), + UserInput::Image { + image: ImageReference::Inline { url }, + .. + } => json!({"type": "image", "url": url}), + UserInput::Image { + image: ImageReference::File { file_id }, + .. + } => json!({"type": "image", "fileId": file_id}), UserInput::LocalImage { path, .. } => json!({"type": "localImage", "path": path}), UserInput::Audio { url } => json!({"type": "audio", "url": url}), UserInput::LocalAudio { path } => json!({"type": "localAudio", "path": path}), diff --git a/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs b/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs index 0c406185e9..401d7ad205 100644 --- a/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs +++ b/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs @@ -4,6 +4,7 @@ use app_test_support::create_mock_responses_server_sequence; use app_test_support::write_mock_responses_config_toml; use codex_app_server_client::AppServerEvent; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ImageReference; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::SortDirection; @@ -362,9 +363,11 @@ async fn transcript_preview_for_history_mode( } } else { UserInput::Image { - url: String::from( - "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", - ), + image: ImageReference::Inline { + url: String::from( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + ), + }, detail: None, } }; diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index da59e7b86d..1cae07741c 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -3,7 +3,6 @@ use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; -use codex_protocol::models::ImageReference; pub use codex_utils_string::approx_bytes_for_tokens; pub use codex_utils_string::approx_token_count; pub use codex_utils_string::approx_tokens_from_byte_count; @@ -91,15 +90,12 @@ pub fn formatted_truncate_text_content_items_with_policy( text: formatted_truncate_text(&combined, policy), }]; out.extend(items.iter().filter_map(|item| match item { - FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } => Some(FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { - image_url: image_url.clone(), - }, - detail: *detail, - }), + FunctionCallOutputContentItem::InputImage { image, detail } => { + Some(FunctionCallOutputContentItem::InputImage { + image: image.clone(), + detail: *detail, + }) + } FunctionCallOutputContentItem::InputAudio { audio_url } => { Some(FunctionCallOutputContentItem::InputAudio { audio_url: audio_url.clone(), @@ -163,14 +159,9 @@ pub fn truncate_function_output_items_with_policy( remaining_budget = 0; } } - FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { image_url }, - detail, - } => { + FunctionCallOutputContentItem::InputImage { image, detail } => { out.push(FunctionCallOutputContentItem::InputImage { - image: ImageReference::Inline { - image_url: image_url.clone(), - }, + image: image.clone(), detail: *detail, }); } diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index bbcf617714..736a283563 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -193,6 +193,12 @@ fn truncate_function_output_items_with_policy_discards_empty_text() { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, FunctionCallOutputContentItem::InputAudio { audio_url: "audio:one".to_string(), }, @@ -271,6 +277,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med FunctionCallOutputContentItem::InputAudio { audio_url: "audio:one".to_string(), }, + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, FunctionCallOutputContentItem::InputText { text: "ijkl".to_string(), }, @@ -298,6 +310,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med FunctionCallOutputContentItem::InputAudio { audio_url: "audio:one".to_string(), }, + FunctionCallOutputContentItem::InputImage { + image: ImageReference::File { + file_id: "file_123".to_string(), + }, + detail: Some(DEFAULT_IMAGE_DETAIL), + }, FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url: "img:two".to_string() }, detail: Some(DEFAULT_IMAGE_DETAIL), diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 5b8e03a633..0a76ae8fc4 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -6240,7 +6240,7 @@ class TextUserInput(BaseModel): type: Annotated[Literal["text"], Field(title="TextUserInputType")] -class ImageUserInput(BaseModel): +class UrlUserInput(BaseModel): model_config = ConfigDict( populate_by_name=True, ) @@ -6249,6 +6249,15 @@ class ImageUserInput(BaseModel): url: str +class FileIdUserInput(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + detail: ImageDetail | None = None + type: Annotated[Literal["image"], Field(title="ImageUserInputType")] + file_id: Annotated[str, Field(alias="fileId")] + + class LocalImageUserInput(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -6295,7 +6304,8 @@ class MentionUserInput(BaseModel): class UserInput( RootModel[ TextUserInput - | ImageUserInput + | UrlUserInput + | FileIdUserInput | LocalImageUserInput | AudioUserInput | LocalAudioUserInput @@ -6308,7 +6318,8 @@ class UserInput( ) root: ( TextUserInput - | ImageUserInput + | UrlUserInput + | FileIdUserInput | LocalImageUserInput | AudioUserInput | LocalAudioUserInput @@ -7794,15 +7805,34 @@ class InputImageContentItem(BaseModel): image_url: str +class FileIdContentItem(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + detail: ImageDetail | None = None + type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")] + file_id: str + + class ContentItem( RootModel[ - InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem + InputTextContentItem + | InputImageContentItem + | FileIdContentItem + | InputAudioContentItem + | OutputTextContentItem ] ): model_config = ConfigDict( populate_by_name=True, ) - root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem + root: ( + InputTextContentItem + | InputImageContentItem + | FileIdContentItem + | InputAudioContentItem + | OutputTextContentItem + ) class ExperimentalFeature(BaseModel): @@ -7990,10 +8020,22 @@ class InputImageFunctionCallOutputContentItem(BaseModel): image_url: str +class FileIdFunctionCallOutputContentItem(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + detail: ImageDetail | None = None + type: Annotated[ + Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType") + ] + file_id: str + + class FunctionCallOutputContentItem( RootModel[ InputTextFunctionCallOutputContentItem | InputImageFunctionCallOutputContentItem + | FileIdFunctionCallOutputContentItem | InputAudioFunctionCallOutputContentItem | EncryptedContentFunctionCallOutputContentItem ] @@ -8004,6 +8046,7 @@ class FunctionCallOutputContentItem( root: Annotated[ InputTextFunctionCallOutputContentItem | InputImageFunctionCallOutputContentItem + | FileIdFunctionCallOutputContentItem | InputAudioFunctionCallOutputContentItem | EncryptedContentFunctionCallOutputContentItem, Field(