diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index d2a9de926a..d05de8638d 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -660,6 +660,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -671,9 +685,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -683,7 +694,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageContentItem", @@ -1420,6 +1430,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1431,9 +1455,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -1443,7 +1464,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 83aa38fdcf..6d7bc110d5 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1759,6 +1759,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -1770,9 +1784,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -1782,7 +1793,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index ac76ffbb2d..813df4d7f3 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10472,6 +10472,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -10483,9 +10497,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -10495,7 +10506,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageContentItem", @@ -12333,6 +12343,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -12344,9 +12368,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -12356,7 +12377,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 7a95a06a2a..0e4c577b0c 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6352,6 +6352,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -6363,9 +6377,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -6375,7 +6386,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageContentItem", @@ -8213,6 +8223,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -8224,9 +8248,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -8236,7 +8257,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index f3caf84e96..dc2d0dc619 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -359,6 +359,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -370,9 +384,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -382,7 +393,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 8432eb5437..c153e128b3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -359,6 +359,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -370,9 +384,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -382,7 +393,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index c6d8c66fe1..c05b788d8a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -80,6 +80,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -91,9 +105,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -103,7 +114,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageContentItem", @@ -188,6 +198,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -199,9 +223,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -211,7 +232,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 7da4ff34d8..cddc22f59f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -499,6 +499,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -510,9 +524,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -522,7 +533,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 8a7fa76e53..a10d95cf00 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -580,6 +580,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -591,9 +605,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -603,7 +614,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index d11f47942a..f467944e67 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -359,6 +359,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -370,9 +384,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -382,7 +393,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 32101efd89..537af065c3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index f411af6d60..c76c1d2491 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index a639e719b8..510c26c2ee 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 5c35287d0e..753f062836 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -142,6 +142,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -153,9 +167,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -165,7 +176,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageContentItem", @@ -250,6 +260,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -261,9 +285,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -273,7 +294,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 2019868698..3a6fb8bdaa 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -596,6 +596,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -607,9 +621,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -619,7 +630,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index 8dd0da9b6a..8d8a69f51d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 3f193b0de8..da04bf0a27 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -580,6 +580,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -591,9 +605,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -603,7 +614,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index e7eb265dcb..084cc6522d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index c8d1dd0354..3e06a271fa 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -499,6 +499,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -510,9 +524,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -522,7 +533,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 7c0f053e20..d396062af1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -502,6 +502,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -513,9 +527,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -525,7 +536,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 7db8c18c82..9ca8c3b087 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -499,6 +499,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -510,9 +524,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -522,7 +533,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index 9f8637ea17..68f4b3dff9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -166,6 +166,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -177,9 +191,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -189,7 +200,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 9552369480..f2d2251e03 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -499,6 +499,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -510,9 +524,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -522,7 +533,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 57052afd9f..bdf800c52a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -499,6 +499,20 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "image_url": { + "type": "string" + } + }, + "required": [ + "image_url" + ], + "title": "ImageUrlFunctionCallOutputContentItem", + "type": "object" + } + ], "properties": { "detail": { "anyOf": [ @@ -510,9 +524,6 @@ } ] }, - "image_url": { - "type": "string" - }, "type": { "enum": [ "input_image" @@ -522,7 +533,6 @@ } }, "required": [ - "image_url", "type" ], "title": "InputImageFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index b4f9045ce0..79d6a3dfb1 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 8222eefe7f..14bdda9860 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts index 9e53b5fc25..dd5b0a1929 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageDetail } from "./ImageDetail"; -export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, }; +export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts index 6c2ab2afa1..8ebefc596e 100644 --- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts @@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail"; * Responses API compatible content items that can be returned by a tool call. * This is a subset of ContentItem with the types we support as function call outputs. */ -export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; +export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index 95e1d62f5c..a051c867db 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -8,6 +8,7 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry; use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind; use codex_protocol::protocol::TurnSettingsUpdate; @@ -36,7 +37,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR ResponseItem::Message { content, .. } => content.iter().any(|item| { matches!( item, - ContentItem::InputImage { image_url, .. } if is_remote_image_url(image_url) + ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url) ) }), ResponseItem::FunctionCallOutput { output, .. } @@ -45,7 +46,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR content.iter().any(|item| { matches!( item, - FunctionCallOutputContentItem::InputImage { image_url, .. } + FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url) ) }) diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 38cad9c430..38f34330ba 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -30,6 +30,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::openai_models::InputModality; use core_test_support::load_default_config_for_test; use core_test_support::responses; @@ -715,7 +716,9 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> { text: "dynamic-ok".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: TINY_PNG_DATA_URL.to_string(), + image: ImageReference::Inline { + image_url: TINY_PNG_DATA_URL.to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { diff --git a/codex-rs/app-server/tests/suite/v2/request_validation.rs b/codex-rs/app-server/tests/suite/v2/request_validation.rs index 1b5abe2f17..042236ed8e 100644 --- a/codex-rs/app-server/tests/suite/v2/request_validation.rs +++ b/codex-rs/app-server/tests/suite/v2/request_validation.rs @@ -14,6 +14,7 @@ use codex_app_server_protocol::ThreadStartResponse; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use pretty_assertions::assert_eq; use serde_json::json; @@ -173,7 +174,9 @@ async fn request_handlers_reject_remote_image_urls() -> Result<()> { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/tool.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/tool.png".to_string(), + }, detail: Some(ImageDetail::High), }, ]), diff --git a/codex-rs/codex-api/src/endpoint/search.rs b/codex-rs/codex-api/src/endpoint/search.rs index 131a335e26..6b2450e649 100644 --- a/codex-rs/codex-api/src/endpoint/search.rs +++ b/codex-rs/codex-api/src/endpoint/search.rs @@ -72,6 +72,7 @@ mod tests { use codex_client::TransportError; use codex_protocol::ResponseItemId; use codex_protocol::models::ContentItem; + use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use http::StatusCode; use pretty_assertions::assert_eq; @@ -164,7 +165,9 @@ mod tests { text: "find this".to_string(), }, ContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: None, }, ], diff --git a/codex-rs/core/src/client_common_tests.rs b/codex-rs/core/src/client_common_tests.rs index e8b030e6a1..1cda4ef13d 100644 --- a/codex-rs/core/src/client_common_tests.rs +++ b/codex-rs/core/src/client_common_tests.rs @@ -6,6 +6,7 @@ use codex_models_manager::model_info::model_info_from_slug; use codex_protocol::config_types::ServiceTier; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use pretty_assertions::assert_eq; use serde_json::value::RawValue; use std::sync::Arc; @@ -27,7 +28,9 @@ fn prompt_with_image_outputs(detail: Option) -> Prompt { text: "Describe this image.".to_string(), }, ContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail, }, ], @@ -41,7 +44,9 @@ fn prompt_with_image_outputs(detail: Option) -> Prompt { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,function".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,function".to_string(), + }, detail, }, ]), @@ -53,7 +58,9 @@ fn prompt_with_image_outputs(detail: Option) -> Prompt { name: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,custom".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,custom".to_string(), + }, detail, }, ]), diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 7a5d3ad416..b367ba4bea 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -45,6 +45,8 @@ use codex_protocol::items::ContextCompactionItem; use codex_protocol::items::TurnItem; use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::ContentItem; +#[cfg(test)] +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::TokenUsage; @@ -981,11 +983,15 @@ mod tests { text: "user".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, }, ContentItem::InputImage { - image_url: "data:image/png;base64,def".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,def".to_string(), + }, detail: None, }, ], @@ -1047,7 +1053,9 @@ mod tests { text: "abcdef".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, }, ContentItem::OutputText { @@ -1057,7 +1065,9 @@ mod tests { text: "discarded after the text budget is exhausted".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,def".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,def".to_string(), + }, detail: None, }, ], @@ -1089,14 +1099,18 @@ mod tests { text: "abcdef".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string() + }, detail: None, }, ContentItem::OutputText { text: "uv…1 tokens truncated…yz".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,def".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,def".to_string() + }, detail: None, }, ], @@ -1123,7 +1137,9 @@ mod tests { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, }], phase: None, @@ -1147,7 +1163,9 @@ mod tests { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, }], phase: None, diff --git a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs index ffc3a6f85a..7c7d94ac8b 100644 --- a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs +++ b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs @@ -1,5 +1,6 @@ use super::*; use codex_protocol::models::ContentItemKind; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_protocol::models::image_close_tag_text; use codex_protocol::models::local_image_open_tag_text_with_path; @@ -17,7 +18,9 @@ fn message(content: Vec) -> ResponseItem { fn image() -> ContentItem { ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, } } diff --git a/codex-rs/core/src/compact_remote_v2_images.rs b/codex-rs/core/src/compact_remote_v2_images.rs index d2fecb03a2..0575e07f01 100644 --- a/codex-rs/core/src/compact_remote_v2_images.rs +++ b/codex-rs/core/src/compact_remote_v2_images.rs @@ -4,6 +4,7 @@ use codex_context_fragments::set_annotated_content; use codex_context_fragments::to_annotated_content; use codex_history::ResponseItemEnvelope; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::is_image_close_tag_text; use codex_protocol::models::is_image_open_tag_text; use codex_protocol::models::is_local_image_open_tag_text; @@ -17,9 +18,12 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize { ContentItem::InputText { text } | ContentItem::OutputText { text } => { approx_token_count(text) } - ContentItem::InputImage { image_url, detail } => usize::try_from( - approx_tokens_from_byte_count_i64(estimate_image_bytes(image_url, *detail)), - ) + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => usize::try_from(approx_tokens_from_byte_count_i64(estimate_image_bytes( + image_url, *detail, + ))) .unwrap_or(usize::MAX), ContentItem::InputAudio { .. } => 0, } diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index b319e75ddb..531d093061 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -10,6 +10,7 @@ use codex_protocol::models::ContentItemKind; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::ExecutedToolCall; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use core_test_support::responses; use core_test_support::skip_if_no_network; @@ -207,7 +208,9 @@ fn content_items_to_text_joins_non_empty_segments() { #[test] fn content_items_to_text_ignores_image_only_content() { let items = vec![ContentItem::InputImage { - image_url: "file://image.png".to_string(), + image: ImageReference::Inline { + image_url: "file://image.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }]; diff --git a/codex-rs/core/src/context/node_repl_review_evidence.rs b/codex-rs/core/src/context/node_repl_review_evidence.rs index fdc2027da2..16cf5f62d1 100644 --- a/codex-rs/core/src/context/node_repl_review_evidence.rs +++ b/codex-rs/core/src/context/node_repl_review_evidence.rs @@ -12,6 +12,7 @@ use codex_guardian_context::NodeReplContext; use codex_guardian_context::NodeReplResponse; pub use codex_guardian_context::NodeReplReviewEvidenceMode; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::user_input::UserInput; use codex_protocol::user_input::UserInput::Image; use codex_protocol::user_input::UserInput::Text; @@ -117,7 +118,9 @@ impl NodeReplReviewEvidence { .filter_map(|item| match item { Image { image_url, detail } if seen_images.insert(image_url) => { Some(ContentItem::InputImage { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: *detail, }) } diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 3695fc7b77..4617e462ab 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -43,6 +43,7 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::InterAgentCommunication; @@ -850,9 +851,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { ContentItem::InputText { text } | ContentItem::OutputText { text } => { text_bytes(text) } - ContentItem::InputImage { image_url, detail } => { - estimate_image_bytes(image_url, *detail) - } + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => estimate_image_bytes(image_url, *detail), ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url), }) .fold(0i64, i64::saturating_add), @@ -1055,9 +1057,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { .iter() .map(|part| match part { FunctionCallOutputContentItem::InputText { text } => text_bytes(text), - FunctionCallOutputContentItem::InputImage { image_url, detail } => { - estimate_image_bytes(image_url, *detail) - } + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => estimate_image_bytes(image_url, *detail), FunctionCallOutputContentItem::InputAudio { audio_url } => { estimate_audio_bytes(audio_url) } diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index c0f35668a9..e0844511bd 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -18,6 +18,7 @@ use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_protocol::models::LocalShellAction; use codex_protocol::models::LocalShellExecAction; @@ -715,7 +716,9 @@ fn for_prompt_annotated_preserves_metadata_while_normalizing_item() { text: "keep".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: None, }, ], @@ -796,7 +799,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "look at this".to_string(), }, ContentItem::InputImage { - image_url: "https://example.com/img.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/img.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputAudio { @@ -838,7 +843,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "image result".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/result.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/result.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { @@ -865,7 +872,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "js repl result".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/js-repl-result.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/js-repl-result.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { @@ -991,7 +1000,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "look".to_string(), }, ContentItem::InputImage { - image_url: "https://example.com/img.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/img.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ], @@ -2414,7 +2425,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() { text: "Here is the screenshot".to_string(), }, ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ], @@ -2455,7 +2466,7 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() { text: "Screenshot captured".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]), @@ -2484,7 +2495,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { text: "Screenshot captured".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]), @@ -2628,7 +2639,9 @@ fn non_base64_image_urls_use_image_estimates() { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: "https://example.com/foo.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/foo.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }], phase: None, @@ -2641,7 +2654,9 @@ fn non_base64_image_urls_use_image_estimates() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "file:///tmp/foo.png".to_string(), + image: ImageReference::Inline { + image_url: "file:///tmp/foo.png".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]), @@ -2746,7 +2761,10 @@ fn data_url_without_base64_marker_uses_image_estimate() { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: "data:image/svg+xml,".to_string(), + image: ImageReference::Inline { + image_url: "data:image/svg+xml," + .to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }], phase: None, @@ -2771,7 +2789,7 @@ fn non_image_base64_data_url_uses_image_estimate() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]), @@ -2791,7 +2809,7 @@ fn mixed_case_data_url_markers_are_adjusted() { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }], phase: None, @@ -2818,11 +2836,15 @@ fn multiple_inline_images_apply_multiple_fixed_costs() { text: "images".to_string(), }, ContentItem::InputImage { - image_url: image_url_one, + image: ImageReference::Inline { + image_url: image_url_one, + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputImage { - image_url: image_url_two, + image: ImageReference::Inline { + image_url: image_url_two, + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ], @@ -2858,7 +2880,7 @@ fn original_detail_images_scale_with_dimensions() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(ImageDetail::Original), }, ]), @@ -2891,7 +2913,7 @@ fn original_detail_images_are_capped_at_max_patch_count() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(ImageDetail::Original), }, ]), @@ -2927,7 +2949,7 @@ fn original_detail_webp_images_scale_with_dimensions() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(ImageDetail::Original), }, ]), diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index d386b08b5d..ed983fe644 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -5,6 +5,7 @@ use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::WebSearchItem; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::MessagePhase; use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; @@ -126,7 +127,10 @@ fn parse_user_message(message: &[ContentItem]) -> Option { text_elements: Vec::new(), }); } - ContentItem::InputImage { image_url, detail } => { + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => { content.push(UserInput::Image { image_url: image_url.clone(), detail: *detail, diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index 972df5e400..b6ac81c9da 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -12,6 +12,7 @@ use codex_protocol::items::WebSearchItem; use codex_protocol::items::build_hook_prompt_message; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; +use codex_protocol::models::ImageReference; use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; @@ -83,11 +84,15 @@ fn parses_user_message_with_text_and_two_images() { text: "Hello world".to_string(), }, ContentItem::InputImage { - image_url: img1.clone(), + image: ImageReference::Inline { + image_url: img1.clone(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputImage { - image_url: img2.clone(), + image: ImageReference::Inline { + image_url: img2.clone(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ], @@ -131,7 +136,9 @@ fn skips_local_image_label_text() { content: vec![ ContentItem::InputText { text: label }, ContentItem::InputImage { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputText { @@ -258,7 +265,9 @@ fn skips_unnamed_image_label_text() { content: vec![ ContentItem::InputText { text: label }, ContentItem::InputImage { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputText { diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 23e756fcf4..397f1eb635 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -35,6 +35,7 @@ use codex_protocol::items::TurnItem; use codex_protocol::mcp::is_node_repl_backed_server; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; @@ -502,7 +503,7 @@ async fn run_review_on_session( _ => &[], }) .filter_map(|item| match item { - ContentItem::InputImage { image_url, .. } => Some(image_url.as_str()), + ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } => Some(image_url.as_str()), _ => None, }) .collect::>(); @@ -550,7 +551,7 @@ async fn run_review_on_session( return false; }; content.iter().any(|item| { - matches!(item, ContentItem::InputImage { image_url, .. } + matches!(item, ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if !reviewer_image_urls.contains(image_url.as_str())) }) }); diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 777b77c54a..cf191851bf 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -45,6 +45,7 @@ use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; use codex_protocol::models::SandboxPermissions; @@ -1130,7 +1131,9 @@ fn collect_guardian_transcript_entries_preserves_named_unpaired_tool_sources() { if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] { *output = codex_protocol::models::FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,image".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,image".to_string(), + }, detail: None, }, ]); diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index 78170db319..adc886f7c1 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -15,6 +15,7 @@ use codex_protocol::models::ContentItemKind; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; use codex_utils_image::ImageProcessingError; @@ -185,7 +186,11 @@ fn prepare_message_content( let mut image_number = 0; let mut resized_images = Vec::new(); for item in items { - if let ContentItem::InputImage { image_url, detail } = item.content_mut() { + if let ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } = item.content_mut() + { image_number += 1; match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { @@ -226,7 +231,11 @@ fn prepare_tool_output_content( let mut image_number = 0; let mut resized_images = Vec::new(); for item in items { - if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item { + if let FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } = item + { image_number += 1; match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs index 67b1f1d4df..a0cfbf5479 100644 --- a/codex-rs/core/src/image_preparation_tests.rs +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -5,6 +5,7 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::ContentItemKind; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_utils_image::data_url_from_bytes; use image::DynamicImage; @@ -41,11 +42,15 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { role: "user".to_string(), content: vec![ ContentItem::InputImage { - image_url: data_url, + image: ImageReference::Inline { + image_url: data_url, + }, detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::Low), }, ], @@ -56,7 +61,9 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { id: None, role: "developer".to_string(), content: vec![ContentItem::InputImage { - image_url: "https://example.com/developer-image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/developer-image.png".to_string(), + }, detail: Some(ImageDetail::High), }], phase: None, @@ -73,7 +80,10 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { panic!("expected message"); }; let [ - ContentItem::InputImage { image_url, .. }, + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + }, ContentItem::InputText { text }, ] = content.as_slice() else { @@ -135,7 +145,10 @@ fn detail_policies_apply_the_expected_budgets() { let mut items = vec![ResponseItem::Message { id: None, role: "user".to_string(), - content: vec![ContentItem::InputImage { image_url, detail }], + content: vec![ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + }], phase: None, internal_chat_message_metadata_passthrough: None, }]; @@ -149,7 +162,13 @@ fn detail_policies_apply_the_expected_budgets() { let ResponseItem::Message { content, .. } = &items[0] else { panic!("expected message"); }; - let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else { + let [ + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + }, + ] = content.as_slice() + else { panic!("expected image"); }; assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions); @@ -179,7 +198,7 @@ fn preparation_reports_tool_output_item_id() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(ImageDetail::High), }, ]), @@ -215,15 +234,21 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { role: "user".to_string(), content: vec![ ContentItem::InputImage { - image_url: small_image_url, + image: ImageReference::Inline { + image_url: small_image_url, + }, detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: "data:image/png;base64,%%%".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,%%%".to_string(), + }, detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: large_image_url.clone(), + image: ImageReference::Inline { + image_url: large_image_url.clone(), + }, detail: Some(ImageDetail::High), }, ], @@ -246,11 +271,15 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,%%%".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,%%%".to_string(), + }, detail: Some(ImageDetail::High), }, FunctionCallOutputContentItem::InputImage { - image_url: large_image_url, + image: ImageReference::Inline { + image_url: large_image_url, + }, detail: Some(ImageDetail::High), }, ]), @@ -290,14 +319,20 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { ); let [ ContentItem::InputImage { - image_url: small_message_image_url, + image: + ImageReference::Inline { + image_url: small_message_image_url, + }, .. }, ContentItem::InputText { text: failed_message_image, }, ContentItem::InputImage { - image_url: resized_message_image_url, + image: + ImageReference::Inline { + image_url: resized_message_image_url, + }, .. }, ] = content.as_slice() @@ -342,7 +377,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { text: failed_tool_image, }, FunctionCallOutputContentItem::InputImage { - image_url: resized_tool_image_url, + image: + ImageReference::Inline { + image_url: resized_tool_image_url, + }, .. }, ] = output.content_items().expect("tool output content items") @@ -394,19 +432,27 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { text: "before".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,%%%".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,%%%".to_string(), + }, detail: Some(ImageDetail::High), }, FunctionCallOutputContentItem::InputImage { - image_url: data_url_from_bytes("image/png", b"not an image"), + image: ImageReference::Inline { + image_url: data_url_from_bytes("image/png", b"not an image"), + }, detail: Some(ImageDetail::High), }, FunctionCallOutputContentItem::InputImage { - image_url: valid_image_url.clone(), + image: ImageReference::Inline { + image_url: valid_image_url.clone(), + }, detail: Some(ImageDetail::Low), }, FunctionCallOutputContentItem::InputImage { - image_url: valid_image_url, + image: ImageReference::Inline { + image_url: valid_image_url, + }, detail: Some(ImageDetail::High), }, ]), @@ -442,7 +488,9 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: expected_valid_image_url, + image: ImageReference::Inline { + image_url: expected_valid_image_url + }, detail: Some(ImageDetail::High), }, ]), diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index f9adf42a67..56ca71705d 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -74,6 +74,7 @@ use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::SandboxEnforcement; use codex_protocol::openai_models::ModelServiceTier; @@ -2575,11 +2576,15 @@ async fn prepares_image_failures_before_history_insertion() { text: "before".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,%%%".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,%%%".to_string(), + }, detail: Some(ImageDetail::High), }, FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::High), }, ]), @@ -2644,11 +2649,15 @@ async fn prepares_resumed_history_before_installing_it() { role: "user".to_string(), content: vec![ ContentItem::InputImage { - image_url: "data:image/png;base64,%%%".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,%%%".to_string(), + }, detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), + image: ImageReference::Inline { + image_url: "https://example.com/image.png".to_string(), + }, detail: Some(ImageDetail::High), }, ContentItem::InputText { @@ -11270,7 +11279,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() text: "late pending input".to_string(), }, ContentItem::InputImage { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: Some(ImageDetail::Original), }, ], diff --git a/codex-rs/core/src/tools/code_mode/response_adapter.rs b/codex-rs/core/src/tools/code_mode/response_adapter.rs index 635eb94900..f28d5da934 100644 --- a/codex-rs/core/src/tools/code_mode/response_adapter.rs +++ b/codex-rs/core/src/tools/code_mode/response_adapter.rs @@ -2,6 +2,7 @@ use codex_code_mode::ImageDetail as CodeModeImageDetail; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; trait IntoProtocol { fn into_protocol(self) -> T; @@ -36,7 +37,7 @@ impl IntoProtocol } codex_code_mode::FunctionCallOutputContentItem::InputImage { image_url, detail } => { FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: detail .map(IntoProtocol::into_protocol) .or(Some(DEFAULT_IMAGE_DETAIL)), diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs index 7a393d35d7..bd0cb0c5b5 100644 --- a/codex-rs/core/src/tools/context_tests.rs +++ b/codex-rs/core/src/tools/context_tests.rs @@ -1,5 +1,6 @@ use super::*; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; +use codex_protocol::models::ImageReference; use codex_protocol::models::SearchToolCallParams; use core_test_support::assert_regex_match; use pretty_assertions::assert_eq; @@ -217,7 +218,9 @@ fn mcp_tool_output_response_item_preserves_content_items() { text: "Wall time: 0.5000 seconds\nOutput:".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string() + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ] @@ -295,7 +298,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() { text: "line 1".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { @@ -315,7 +320,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() { text: "line 1".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index e447208fd4..fd4e3910db 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -7,6 +7,7 @@ use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_protocol::openai_models::InputModality; use codex_utils_image::data_url_from_bytes; @@ -238,7 +239,9 @@ impl ToolOutput for ViewImageOutput { fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem { let body = FunctionCallOutputBody::ContentItems(vec![FunctionCallOutputContentItem::InputImage { - image_url: self.image_url.clone(), + image: ImageReference::Inline { + image_url: self.image_url.clone(), + }, detail: Some(self.image_detail), }]); let output = FunctionCallOutputPayload { diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 033126a16e..79d0eb4f6d 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -40,6 +40,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::LocalShellAction; use codex_protocol::models::LocalShellExecAction; use codex_protocol::models::LocalShellStatus; @@ -1078,7 +1079,9 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() { id: None, role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: legacy_image_url.to_string(), + image: ImageReference::Inline { + image_url: legacy_image_url.to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }], phase: None, @@ -1218,7 +1221,9 @@ async fn resume_replays_image_tool_outputs_with_detail() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string(), + }, detail: Some(ImageDetail::Original), }, ]), @@ -1247,7 +1252,9 @@ async fn resume_replays_image_tool_outputs_with_detail() { name: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: image_url.to_string(), + image: ImageReference::Inline { + image_url: image_url.to_string(), + }, detail: Some(ImageDetail::Original), }, ]), diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 1663fa0a1a..acb1f4b26e 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -42,6 +42,7 @@ use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; @@ -5662,7 +5663,9 @@ async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transc assert_eq!( evidence.images(), vec![ContentItem::InputImage { - image_url: SCREENSHOT.to_owned(), + image: ImageReference::Inline { + image_url: SCREENSHOT.to_owned() + }, detail: Some(ImageDetail::Low), }] ); diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index 3316742feb..e12d6a7f57 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -10,6 +10,7 @@ use codex_protocol::config_types::Settings; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; @@ -69,7 +70,10 @@ fn find_user_message_with_image(text: &str) -> Option { fn extract_image_url(item: &ResponseItem) -> Option { match item { ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span { - ContentItem::InputImage { image_url, .. } => Some(image_url.clone()), + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => Some(image_url.clone()), _ => None, }), _ => None, @@ -178,7 +182,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu ), }, ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputText { @@ -269,7 +273,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> role: "user".to_string(), content: vec![ ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputText { @@ -341,7 +345,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R historical_content.insert( /*index*/ 0, ContentItem::InputImage { - image_url: original_image_url.clone(), + image: ImageReference::Inline { + image_url: original_image_url.clone(), + }, detail: Some(ImageDetail::High), }, ); @@ -480,7 +486,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R ResponseInputItem::Message { role: "user".to_string(), content: vec![ContentItem::InputImage { - image_url: original_image_url, + image: ImageReference::Inline { + image_url: original_image_url, + }, detail: Some(ImageDetail::High), }], phase: None, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 78999fca06..23405bf9fc 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -52,6 +52,7 @@ use codex_protocol::mcp_policy::McpServerRequirement; use codex_protocol::mcp_policy::PluginMcpRequirements; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; use codex_protocol::models::PermissionProfileSnapshot; use codex_protocol::models::ResponseItem; @@ -1549,7 +1550,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history( }; assert!( content.contains(&ContentItem::InputImage { - image_url: OPENAI_PNG.to_string(), + image: ImageReference::Inline { + image_url: OPENAI_PNG.to_string() + }, detail: Some(ImageDetail::Original), }), "interrupted input must use the current model's unified image budget" diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs index 171dc00518..9454713ef8 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::async_scorer::authorization::ScoreAuthorization; +use codex_protocol::models::ImageReference; use pretty_assertions::assert_eq; #[derive(Clone, Copy)] @@ -139,7 +140,7 @@ async fn assert_catalog_budget(evidence: BudgetEvidence) -> Result<()> { unreachable!("user instruction is a message"); }; content.push(ContentItem::InputImage { - image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned(), + image: ImageReference::Inline { image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned() }, detail: None, }); (None, vec!["optional old commentary ".repeat(/*n*/ 500)]) diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs index 31aac3bed6..38440c4f6c 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs @@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_protocol::models::LocalShellAction; use codex_protocol::models::LocalShellExecAction; @@ -1641,7 +1642,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> { text: "Review what is shown on screen.".to_owned(), }, ContentItem::InputImage { - image_url: user_image.to_owned(), + image: ImageReference::Inline { + image_url: user_image.to_owned(), + }, detail: Some(ImageDetail::High), }, ], @@ -1667,7 +1670,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> { text: "Screenshot captured.".to_owned(), }, FunctionCallOutputContentItem::InputImage { - image_url: tool_image.to_owned(), + image: ImageReference::Inline { + image_url: tool_image.to_owned(), + }, detail: Some(ImageDetail::High), }, ]), diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs index ed1bc9ab6a..a3add7fc0a 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs @@ -7,6 +7,7 @@ use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::MessagePhase; use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; @@ -1077,7 +1078,9 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() { if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] { *output = FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,image".to_owned(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,image".to_owned(), + }, detail: None, }, ]); @@ -1222,7 +1225,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() { text: "Review this screenshot.".to_string(), }, ContentItem::InputImage { - image_url: format!("data:image/png;base64,{oversized_image}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{oversized_image}"), + }, detail: None, }, ContentItem::InputAudio { @@ -1242,7 +1247,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() { text: "Screenshot captured.".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,tool-image".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,tool-image".to_string(), + }, detail: None, }, FunctionCallOutputContentItem::InputAudio { diff --git a/codex-rs/ext/history-notes/src/tools.rs b/codex-rs/ext/history-notes/src/tools.rs index ad7422c961..114ee2ece8 100644 --- a/codex-rs/ext/history-notes/src/tools.rs +++ b/codex-rs/ext/history-notes/src/tools.rs @@ -11,6 +11,7 @@ use codex_extension_api::parse_tool_input_schema; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_tools::JsonToolOutput; use codex_tools::ResponsesApiNamespace; @@ -367,7 +368,9 @@ impl HistoryNotesToolOutput { serde_json::from_value(image.get("detail").cloned().unwrap_or(Value::Null)) .map_err(|_| invalid_image())?; content.push(FunctionCallOutputContentItem::InputImage { - image_url: format!("data:{mime_type};base64,{data}"), + image: ImageReference::Inline { + image_url: format!("data:{mime_type};base64,{data}"), + }, detail, }); } diff --git a/codex-rs/ext/history-notes/src/tools_tests.rs b/codex-rs/ext/history-notes/src/tools_tests.rs index 55b99cdc35..9ec63f6621 100644 --- a/codex-rs/ext/history-notes/src/tools_tests.rs +++ b/codex-rs/ext/history-notes/src/tools_tests.rs @@ -1,6 +1,7 @@ use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_tools::ToolOutput; use codex_tools::ToolPayload; @@ -73,19 +74,27 @@ fn preserves_images_as_separate_output_items_without_logging_bytes() { encrypted_content: "enc_payload".to_string() }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,cG5n".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,cG5n".to_string() + }, detail: Some(ImageDetail::Original) }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/jpeg;base64,anBlZw==".to_string(), + image: ImageReference::Inline { + image_url: "data:image/jpeg;base64,anBlZw==".to_string() + }, detail: Some(ImageDetail::Low) }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/gif;base64,Z2lm".to_string(), + image: ImageReference::Inline { + image_url: "data:image/gif;base64,Z2lm".to_string() + }, detail: None }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/webp;base64,d2VicA==".to_string(), + image: ImageReference::Inline { + image_url: "data:image/webp;base64,d2VicA==".to_string() + }, detail: None }, ]) diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index c7bca61e8a..32b0493775 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -12,6 +12,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_tools::ResponsesApiNamespaceTool; @@ -266,7 +267,9 @@ fn generated_output_returns_image_input_and_output_hint() { content_items, vec![ FunctionCallOutputContentItem::InputImage { - image_url: format!("data:image/png;base64,{RESULT}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{RESULT}") + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { text: output_hint }, @@ -311,7 +314,9 @@ fn generated_output_omits_oversized_output_hint() { assert_eq!( content_items, vec![FunctionCallOutputContentItem::InputImage { - image_url: format!("data:image/png;base64,{RESULT}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{RESULT}") + }, detail: Some(DEFAULT_IMAGE_DETAIL), }] ); @@ -319,14 +324,18 @@ fn generated_output_omits_oversized_output_hint() { fn input_image(image: &str) -> ContentItem { ContentItem::InputImage { - image_url: format!("data:image/png;base64,{image}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{image}"), + }, detail: None, } } fn image_output(image: &str) -> FunctionCallOutputPayload { FunctionCallOutputPayload::from_content_items(vec![FunctionCallOutputContentItem::InputImage { - image_url: format!("data:image/png;base64,{image}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{image}"), + }, detail: None, }]) } diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index ac36240cc4..8fc408b442 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -29,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; @@ -494,7 +495,10 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec { match item { ResponseItem::Message { content, .. } => { image_urls.extend(content.iter().rev().filter_map(|item| match item { - ContentItem::InputImage { image_url, .. } => Some(image_url.clone()), + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => Some(image_url.clone()), ContentItem::InputText { .. } | ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => None, @@ -542,7 +546,10 @@ fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator Some(image_url.clone()), + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => Some(image_url.clone()), FunctionCallOutputContentItem::InputText { .. } | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, @@ -651,7 +658,9 @@ impl ToolOutput for GeneratedImageOutput { /// Returns generated bytes and persisted-artifact context for model follow-up. fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem { let mut content = vec![FunctionCallOutputContentItem::InputImage { - image_url: format!("data:image/png;base64,{}", self.result), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{}", self.result), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }]; if let Some(output_hint) = &self.output_hint { diff --git a/codex-rs/ext/web-search/src/history.rs b/codex-rs/ext/web-search/src/history.rs index fb1630c8dc..6d64e22124 100644 --- a/codex-rs/ext/web-search/src/history.rs +++ b/codex-rs/ext/web-search/src/history.rs @@ -84,6 +84,7 @@ mod tests { use codex_api::SearchInput; use codex_protocol::ResponseItemId; use codex_protocol::models::ContentItem; + use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use pretty_assertions::assert_eq; @@ -158,7 +159,9 @@ mod tests { text: "previous user".to_string(), }, ContentItem::InputImage { - image_url: "data:image/png;base64,image".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,image".to_string(), + }, detail: None, }, ], diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rs index 7f172a9c01..b29ff2c66f 100644 --- a/codex-rs/guardian-context/src/budget.rs +++ b/codex-rs/guardian-context/src/budget.rs @@ -5,6 +5,7 @@ use std::io::Write; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TruncationPolicy; @@ -97,7 +98,10 @@ impl SectionCost { ContentItem::InputText { text } | ContentItem::OutputText { text } => { self.text_bytes = self.text_bytes.saturating_add(text.len()); } - ContentItem::InputImage { image_url, .. } => { + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => { self.image_bytes = self.image_bytes.saturating_add(image_url.len()); self.image_count = self.image_count.saturating_add(1); } @@ -206,7 +210,11 @@ pub(super) fn content_framing_tokens(item_count: usize) -> usize { fn adjusted_tokens(mut bytes: usize, content: &[ContentItem]) -> usize { for item in content { - if let ContentItem::InputImage { image_url, .. } = item { + if let ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } = item + { let payload = ByteCount::measure(|counter| serde_json::to_writer(counter, image_url)); if payload == usize::MAX { return usize::MAX; diff --git a/codex-rs/guardian-context/src/budget_tests.rs b/codex-rs/guardian-context/src/budget_tests.rs index 5af8e7460a..33fc174ad7 100644 --- a/codex-rs/guardian-context/src/budget_tests.rs +++ b/codex-rs/guardian-context/src/budget_tests.rs @@ -4,6 +4,7 @@ use super::*; use crate::Budgeted; use crate::composition::SectionOutput; use crate::composition::user_message as message; +use codex_protocol::models::ImageReference; use pretty_assertions::assert_eq; #[test] @@ -17,7 +18,9 @@ fn section_costs_keep_multimodal_payloads_separate() { text: "évidence".to_owned(), }), Budgeted::required(ContentItem::InputImage { - image_url: "data:image/png;base64,AAAA".to_owned(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAAA".to_owned(), + }, detail: None, }), ]), @@ -59,7 +62,9 @@ fn section_costs_keep_multimodal_payloads_separate() { fn request_estimate_reserves_images_independently_of_encoded_size() { let image = |payload: &str| { message(vec![ContentItem::InputImage { - image_url: format!("data:image/png;base64,{payload}"), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{payload}"), + }, detail: None, }]) }; diff --git a/codex-rs/guardian-context/src/composition.rs b/codex-rs/guardian-context/src/composition.rs index 29c7427870..555fbb2163 100644 --- a/codex-rs/guardian-context/src/composition.rs +++ b/codex-rs/guardian-context/src/composition.rs @@ -6,6 +6,7 @@ use codex_context_fragments::ContextualUserFragment; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TruncationPolicy; use codex_protocol::user_input::UserInput; @@ -214,7 +215,10 @@ impl CollectedContext { Ok(Budgeted::required(ContentItem::InputText { text })) } UserInput::Image { image_url, detail } => Ok(Budgeted::optional( - ContentItem::InputImage { image_url, detail }, + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + }, BudgetPriority::Image, )), _ => Err(SectionError::UnsupportedDelivery { @@ -281,9 +285,10 @@ impl ComposedContext { })); continue; } - ContentItem::InputImage { image_url, detail } => { - UserInput::Image { image_url, detail } - } + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => UserInput::Image { image_url, detail }, ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => { return Err(SectionError::UnsupportedDelivery { section: section.id, diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rs index 99c88dff12..54cf00d076 100644 --- a/codex-rs/guardian-context/src/enforcement.rs +++ b/codex-rs/guardian-context/src/enforcement.rs @@ -7,6 +7,7 @@ use std::collections::HashSet; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::protocol::TruncationPolicy; use crate::ComposedContext; @@ -90,7 +91,10 @@ impl ComposedContext { pub fn retain_images(&mut self, mut admit: impl FnMut(&str, &mut Option) -> bool) { for section in &mut self.sections { retain_content(section, &mut self.truncations, |_, item| match item { - ContentItem::InputImage { image_url, detail } => admit(image_url, detail), + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => admit(image_url, detail), _ => true, }); } @@ -273,7 +277,10 @@ fn retain_content( if !keep { let original_bytes = match &item.content { ContentItem::InputText { text } | ContentItem::OutputText { text } => text.len(), - ContentItem::InputImage { image_url, .. } => image_url.len(), + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => image_url.len(), ContentItem::InputAudio { audio_url } => audio_url.len(), }; truncations.push(TruncationObservation { diff --git a/codex-rs/guardian-context/src/enforcement_tests.rs b/codex-rs/guardian-context/src/enforcement_tests.rs index 893e3fea64..88599eb177 100644 --- a/codex-rs/guardian-context/src/enforcement_tests.rs +++ b/codex-rs/guardian-context/src/enforcement_tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::budget::section_tokens; use crate::composition::user_message; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use pretty_assertions::assert_eq; @@ -146,7 +147,9 @@ fn budget_reserves_existing_context_and_preserves_required_messages() { .unwrap() .into_message(); let image = ContentItem::InputImage { - image_url: "data:image/png;base64,AAAA".to_owned(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAAA".to_owned(), + }, detail: None, }; let make_context = || ComposedContext { @@ -241,7 +244,9 @@ fn image_omission_preserves_text_and_later_eviction_policy() { delivery: SectionDelivery::UserContent(vec![ Budgeted::optional( ContentItem::InputImage { - image_url: "rejected-image".to_owned(), + image: ImageReference::Inline { + image_url: "rejected-image".to_owned(), + }, detail: None, }, BudgetPriority::Image, @@ -307,11 +312,15 @@ fn image_omission_preserves_text_and_later_eviction_policy() { ); let older = ContentItem::InputImage { - image_url: "older-image".to_owned(), + image: ImageReference::Inline { + image_url: "older-image".to_owned(), + }, detail: None, }; let newer = ContentItem::InputImage { - image_url: "newer-image".to_owned(), + image: ImageReference::Inline { + image_url: "newer-image".to_owned(), + }, detail: None, }; let image_section = |images: Vec| SectionOutput { diff --git a/codex-rs/guardian-context/src/history_tests.rs b/codex-rs/guardian-context/src/history_tests.rs index 4c03904ba2..f505003037 100644 --- a/codex-rs/guardian-context/src/history_tests.rs +++ b/codex-rs/guardian-context/src/history_tests.rs @@ -1,4 +1,5 @@ use codex_protocol::models::ExecutedToolCall; +use codex_protocol::models::ImageReference; use codex_protocol::models::ReasoningItemContent; use pretty_assertions::assert_eq; use serde_json::json; @@ -147,7 +148,9 @@ fn oversized_user_images_preserve_text_and_metadata_in_order() { content.insert( /*index*/ 1, ContentItem::InputImage { - image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)), + image: ImageReference::Inline { + image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)), + }, detail: Some(codex_protocol::models::ImageDetail::Original), }, ); diff --git a/codex-rs/guardian-context/src/images.rs b/codex-rs/guardian-context/src/images.rs index 1d50b90930..ccae031095 100644 --- a/codex-rs/guardian-context/src/images.rs +++ b/codex-rs/guardian-context/src/images.rs @@ -10,6 +10,7 @@ use crate::SectionScope; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use std::collections::VecDeque; @@ -65,7 +66,11 @@ impl TranscriptImages { while images.len() >= MAX_TRANSCRIPT_IMAGES || image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES { - let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else { + let Some(ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + }) = images.pop_front() + else { break; }; image_bytes -= image_url.len(); @@ -73,7 +78,9 @@ impl TranscriptImages { } image_bytes += image_url.len(); images.push_back(ContentItem::InputImage { - image_url: image_url.to_owned(), + image: ImageReference::Inline { + image_url: image_url.to_owned(), + }, detail, }); }; @@ -84,7 +91,11 @@ impl TranscriptImages { if matches!(role.as_str(), "user" | "assistant") => { for item in content { - if let ContentItem::InputImage { image_url, detail } = item { + if let ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } = item + { include_image(image_url, *detail); } } @@ -95,8 +106,10 @@ impl TranscriptImages { { if let Some(content) = output.content_items() { for item in content { - if let FunctionCallOutputContentItem::InputImage { image_url, detail } = - item + if let FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } = item { include_image(image_url, *detail); } @@ -108,7 +121,11 @@ impl TranscriptImages { } if input.include_tool_outputs { for image in input.node_repl_images { - if let ContentItem::InputImage { image_url, detail } = image { + if let ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } = image + { include_image(image_url, *detail); } } diff --git a/codex-rs/guardian-context/src/images_tests.rs b/codex-rs/guardian-context/src/images_tests.rs index d39b7e8910..090b5c0531 100644 --- a/codex-rs/guardian-context/src/images_tests.rs +++ b/codex-rs/guardian-context/src/images_tests.rs @@ -5,12 +5,15 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use pretty_assertions::assert_eq; fn image(url: &str) -> ContentItem { ContentItem::InputImage { - image_url: url.into(), + image: ImageReference::Inline { + image_url: url.into(), + }, detail: Some(ImageDetail::High), } } @@ -32,7 +35,9 @@ fn image_selection_preserves_source_policy_order_and_both_limits() { namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "tool".into(), + image: ImageReference::Inline { + image_url: "tool".into(), + }, detail: Some(ImageDetail::High), }, ]), diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 7286f9865f..e2f588eaa3 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -879,7 +879,8 @@ pub enum ContentItem { text: String, }, InputImage { - image_url: String, + #[serde(flatten)] + image: ImageReference, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] detail: Option, @@ -892,6 +893,13 @@ pub enum ContentItem { }, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] +#[serde(untagged)] +#[ts(untagged)] +pub enum ImageReference { + Inline { image_url: String }, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] pub enum AgentMessageInputContent { @@ -1843,7 +1851,7 @@ fn local_image_content_items( }); } items.push(ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(detail), }); if label_number.is_some() { @@ -2006,7 +2014,7 @@ impl ResponseInputItem { image_index += 1; let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL); vec![ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(detail), }] } @@ -2080,7 +2088,8 @@ pub enum FunctionCallOutputContentItem { }, // Do not rename, these are serialized and used directly in the responses API. InputImage { - image_url: String, + #[serde(flatten)] + image: ImageReference, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] detail: Option, @@ -2138,7 +2147,7 @@ impl From } crate::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url } => { Self::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), } } @@ -2383,7 +2392,7 @@ fn convert_mcp_content_to_items( format!("data:{mime_type};base64,{data}") }; FunctionCallOutputContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: meta .as_ref() .and_then(serde_json::Value::as_object) @@ -2656,7 +2665,9 @@ mod tests { assert_eq!( content_item, ContentItem::InputImage { - image_url: "data:image/png;base64,abc".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,abc".to_string(), + }, detail: Some(ImageDetail::Auto), } ); @@ -2711,7 +2722,9 @@ mod tests { assert_eq!( items, vec![FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,Zm9v".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,Zm9v".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }] ); @@ -3043,7 +3056,9 @@ mod tests { assert_eq!( items, vec![FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,Zm9v".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,Zm9v".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }] ); @@ -3100,7 +3115,9 @@ mod tests { text: "line 1".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { @@ -3119,7 +3136,9 @@ mod tests { text: " ".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { @@ -3148,7 +3167,9 @@ mod tests { text: "line 1".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]); @@ -3423,7 +3444,9 @@ mod tests { text: "caption".into(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BASE64".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BASE64".into(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ] @@ -3498,7 +3521,9 @@ mod tests { name: None, output: FunctionCallOutputPayload::from_content_items(vec![ FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BASE64".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BASE64".into(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]), @@ -3563,7 +3588,9 @@ mod tests { assert_eq!( items, vec![FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BASE64".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BASE64".into(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }] ); @@ -3595,7 +3622,9 @@ mod tests { assert_eq!( items, vec![FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BASE64".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BASE64".into(), + }, detail: Some(ImageDetail::Original), }] ); @@ -3627,7 +3656,9 @@ mod tests { assert_eq!( items, vec![FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BASE64".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BASE64".into(), + }, detail: Some(ImageDetail::High), }] ); @@ -3650,7 +3681,9 @@ mod tests { text: "note".into(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,XYZ".into(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,XYZ".into(), + }, detail: None, }, ]; @@ -3859,7 +3892,7 @@ mod tests { match item { ResponseInputItem::Message { content, .. } => { let expected = vec![ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }]; assert_eq!(content, expected); @@ -4005,7 +4038,7 @@ mod tests { assert_eq!( content.first(), Some(&ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(ImageDetail::Original), }) ); @@ -4208,7 +4241,7 @@ mod tests { assert_eq!( content.first(), Some(&ContentItem::InputImage { - image_url, + image: ImageReference::Inline { image_url }, detail: Some(DEFAULT_IMAGE_DETAIL), }) ); diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs index 588ac37787..07da7b7042 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs @@ -12,6 +12,7 @@ use std::collections::HashSet; use codex_protocol::ResponseItemId; use codex_protocol::models::ContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::UserMessageEvent; @@ -492,7 +493,10 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent for item in content { match item { ContentItem::InputText { text: item_text } => text.push_str(item_text), - ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()), + ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } => images.push(image_url.as_str()), ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()), ContentItem::OutputText { .. } => return false, } diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs index 0fb26c10c4..c0949576fe 100644 --- a/codex-rs/tools/src/image_detail_tests.rs +++ b/codex-rs/tools/src/image_detail_tests.rs @@ -2,6 +2,7 @@ use super::*; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; +use codex_protocol::models::ImageReference; use codex_protocol::openai_models::ModelInfo; use pretty_assertions::assert_eq; use serde_json::json; @@ -88,11 +89,15 @@ fn sanitize_original_falls_back_to_high_without_support() { text: "header".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string(), + }, detail: Some(ImageDetail::Original), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BBB".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BBB".to_string(), + }, detail: Some(ImageDetail::Low), }, ]; @@ -106,11 +111,15 @@ fn sanitize_original_falls_back_to_high_without_support() { text: "header".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,AAA".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,AAA".to_string() + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputImage { - image_url: "data:image/png;base64,BBB".to_string(), + image: ImageReference::Inline { + image_url: "data:image/png;base64,BBB".to_string() + }, detail: Some(ImageDetail::Low), }, ] diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index e95e662941..a299d172ee 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -2,6 +2,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; use serde_json::Value as JsonValue; @@ -219,12 +220,13 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue | codex_protocol::models::ContentItem::OutputText { text } => { FunctionCallOutputContentItem::InputText { text } } - codex_protocol::models::ContentItem::InputImage { image_url, detail } => { - FunctionCallOutputContentItem::InputImage { - image_url, - detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)), - } - } + codex_protocol::models::ContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)), + }, codex_protocol::models::ContentItem::InputAudio { audio_url } => { FunctionCallOutputContentItem::InputAudio { audio_url } } @@ -254,11 +256,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> FunctionCallOutputContentItem::InputText { text } if !text.trim().is_empty() => { Some(text.clone()) } - FunctionCallOutputContentItem::InputImage { image_url, .. } - if !image_url.trim().is_empty() => - { - Some(image_url.clone()) - } + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + .. + } if !image_url.trim().is_empty() => Some(image_url.clone()), FunctionCallOutputContentItem::InputAudio { audio_url } if !audio_url.trim().is_empty() => { diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index f4f502ab28..da59e7b86d 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -3,6 +3,7 @@ use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageReference; pub use codex_utils_string::approx_bytes_for_tokens; pub use codex_utils_string::approx_token_count; pub use codex_utils_string::approx_tokens_from_byte_count; @@ -90,12 +91,15 @@ pub fn formatted_truncate_text_content_items_with_policy( text: formatted_truncate_text(&combined, policy), }]; out.extend(items.iter().filter_map(|item| match item { - FunctionCallOutputContentItem::InputImage { image_url, detail } => { - Some(FunctionCallOutputContentItem::InputImage { + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => Some(FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url: image_url.clone(), - detail: *detail, - }) - } + }, + detail: *detail, + }), FunctionCallOutputContentItem::InputAudio { audio_url } => { Some(FunctionCallOutputContentItem::InputAudio { audio_url: audio_url.clone(), @@ -159,9 +163,14 @@ pub fn truncate_function_output_items_with_policy( remaining_budget = 0; } } - FunctionCallOutputContentItem::InputImage { image_url, detail } => { + FunctionCallOutputContentItem::InputImage { + image: ImageReference::Inline { image_url }, + detail, + } => { out.push(FunctionCallOutputContentItem::InputImage { - image_url: image_url.clone(), + image: ImageReference::Inline { + image_url: image_url.clone(), + }, detail: *detail, }); } diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index d58dace05d..bbcf617714 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -7,6 +7,7 @@ use crate::truncate_function_output_items_with_policy; use crate::truncate_text; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageReference; use pretty_assertions::assert_eq; #[test] @@ -114,7 +115,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { FunctionCallOutputContentItem::InputText { text: t1.clone() }, FunctionCallOutputContentItem::InputText { text: t2.clone() }, FunctionCallOutputContentItem::InputImage { - image_url: "img:mid".to_string(), + image: ImageReference::Inline { + image_url: "img:mid".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { text: t3 }, @@ -142,7 +145,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { assert_eq!( output[2], FunctionCallOutputContentItem::InputImage { - image_url: "img:mid".to_string(), + image: ImageReference::Inline { + image_url: "img:mid".to_string() + }, detail: Some(DEFAULT_IMAGE_DETAIL), } ); @@ -183,7 +188,9 @@ fn truncate_function_output_items_with_policy_discards_empty_text() { text: "caption".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "img:one".to_string(), + image: ImageReference::Inline { + image_url: "img:one".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { @@ -253,7 +260,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med text: "abcd".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "img:one".to_string(), + image: ImageReference::Inline { + image_url: "img:one".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { @@ -266,7 +275,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med text: "ijkl".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "img:two".to_string(), + image: ImageReference::Inline { + image_url: "img:two".to_string(), + }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ]; @@ -281,14 +292,14 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med text: "Warning: truncated output (original token count: 4)\nTotal output lines: 3\n\nabcd…6 chars truncated…ijkl".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "img:one".to_string(), + image: ImageReference::Inline { image_url: "img:one".to_string() }, detail: Some(DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputAudio { audio_url: "audio:one".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "img:two".to_string(), + image: ImageReference::Inline { image_url: "img:two".to_string() }, detail: Some(DEFAULT_IMAGE_DETAIL), }, ] diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 68234bfc88..61d98e29e4 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -575,6 +575,7 @@ def generate_v2_all(schema_dir: Path) -> None: ], cwd=sdk_root(), ) + _preserve_inline_image_class_names(out_path) _require_nullable_chatgpt_account_email(out_path) _preserve_reasoning_effort_enum(out_path) _preserve_thread_source_enum(out_path) @@ -582,6 +583,23 @@ def generate_v2_all(schema_dir: Path) -> None: _normalize_generated_timestamps(out_path) +def _preserve_inline_image_class_names(out_path: Path) -> None: + """Keep the public class names used before ImageReference was introduced.""" + source = out_path.read_text() + stable_names = { + "ImageUrlContentItem": "InputImageContentItem", + "ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem", + } + for generated_name, stable_name in stable_names.items(): + if source.count(f"class {generated_name}(") != 1: + raise RuntimeError(f"Generated SDK is missing a unique {generated_name} class") + if re.search(rf"\b{re.escape(stable_name)}\b", source): + raise RuntimeError(f"Generated SDK already defines {stable_name}") + source = re.sub(rf"\b{re.escape(generated_name)}\b", stable_name, source) + + out_path.write_text(source) + + def _require_nullable_chatgpt_account_email(out_path: Path) -> None: """Preserve required-but-nullable email semantics in the generated SDK model.""" source = out_path.read_text() diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 621298ce6c..80839540a3 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -7784,8 +7784,8 @@ class InputImageContentItem(BaseModel): populate_by_name=True, ) detail: ImageDetail | None = None - image_url: str type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")] + image_url: str class ContentItem( @@ -7796,9 +7796,7 @@ class ContentItem( model_config = ConfigDict( populate_by_name=True, ) - root: ( - InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem - ) + root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem class ExperimentalFeature(BaseModel): @@ -7980,10 +7978,10 @@ class InputImageFunctionCallOutputContentItem(BaseModel): populate_by_name=True, ) detail: ImageDetail | None = None - image_url: str type: Annotated[ Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType") ] + image_url: str class FunctionCallOutputContentItem( diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index d9c6617ed7..7fae0b7a57 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -536,6 +536,20 @@ def test_generated_chatgpt_account_email_is_required_nullable() -> None: ChatgptAccount.model_validate({"planType": "pro", "type": "chatgpt"}) +def test_generated_inline_image_class_names_remain_stable() -> None: + """Keep the existing Python class names when image references expand.""" + from openai_codex.generated.v2_all import ( + InputImageContentItem, + InputImageFunctionCallOutputContentItem, + ) + + assert InputImageContentItem.__name__ == "InputImageContentItem" + assert ( + InputImageFunctionCallOutputContentItem.__name__ + == "InputImageFunctionCallOutputContentItem" + ) + + def test_runtime_package_template_has_no_checked_in_binaries() -> None: runtime_root = ROOT.parent / "python-runtime" / "src" / "codex_cli_bin" assert sorted(