diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 32ecea8835..d03193a14e 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -719,10 +719,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -732,7 +756,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioContentItem", @@ -6193,6 +6216,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -6200,14 +6249,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index da2f744b3b..7eeaf12a30 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -6943,6 +6943,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -6950,14 +6976,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 1e2d366694..b145a518f9 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10960,10 +10960,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -10973,7 +10997,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioContentItem", @@ -27301,6 +27324,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -27308,14 +27357,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 84707c0e18..766720c551 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6741,10 +6741,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -6754,7 +6778,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioContentItem", @@ -25004,6 +25027,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -25011,14 +25060,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 692617e957..93e26bfb32 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1846,6 +1846,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -1853,14 +1879,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 286ce308fb..6a367ee32f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1846,6 +1846,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -1853,14 +1879,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index cddb265897..a5448c8fdd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -132,10 +132,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -145,7 +169,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index bc7b9ad49b..765a0b422b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -2200,6 +2200,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2207,14 +2233,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index a025f8b78d..3e7b6fa01a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -2940,6 +2940,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2947,14 +2973,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index eb54f49826..49c50b019a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -1878,6 +1878,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -1885,14 +1911,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 8446a6a4da..c08b4cfc83 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -2795,6 +2795,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2802,14 +2828,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index d167771d36..b892ecbf88 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -2731,6 +2731,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2738,14 +2764,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index fffe7e206f..c3f8bd746b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -2795,6 +2795,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2802,14 +2828,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 65211ef5fd..b591294f48 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -194,10 +194,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -207,7 +231,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index baa806edcc..a5f94b1879 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -3018,6 +3018,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -3025,14 +3051,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index b06f6b9efc..62ac26699a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -2731,6 +2731,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2738,14 +2764,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 0e44e3042c..4dd9cb7744 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -2940,6 +2940,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2947,14 +2973,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 326694cf14..a51b8d802b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -2731,6 +2731,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2738,14 +2764,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index 74ef991588..0838003001 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -2200,6 +2200,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2207,14 +2233,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index bcfd2f2571..b377a2d23d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -2731,6 +2731,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2738,14 +2764,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 0c0cf075df..b8b60e3002 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -2200,6 +2200,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2207,14 +2233,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index cdac11cbe7..d299d318de 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -667,6 +667,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -674,14 +700,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index fbe2695c5d..de6e2e5c3d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -2200,6 +2200,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2207,14 +2233,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index d4ba38ae89..28f579b6bb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -2200,6 +2200,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -2207,14 +2233,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index 2fdbb54a1b..6d18a861d3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -196,6 +196,32 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "UrlUserInput", + "type": "object" + }, + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdUserInput", + "type": "object" + } + ], "properties": { "type": { "enum": [ @@ -203,14 +229,10 @@ ], "title": "AudioUserInputType", "type": "string" - }, - "url": { - "type": "string" } }, "required": [ - "type", - "url" + "type" ], "title": "AudioUserInput", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 93ab4e11c0..f8ce6fba36 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 1cadc724fc..62328ddc45 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts index ca3175c1eb..8ca1832b8b 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageDetail } from "./ImageDetail"; -export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, annotations?: unknown[], logprobs?: unknown[], }; +export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", } & ({ audio_url: string, } | { file_id: string, }) | { "type": "output_text", text: string, annotations?: unknown[], logprobs?: unknown[], }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts index 390d9dd59c..e2b034503e 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts @@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string, /** * UI-defined spans within `text` used to render or persist special elements. */ -text_elements: Array, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; +text_elements: Array, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", } & ({ url: string, } | { fileId: string, }) | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index c087791283..369ea4222f 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -59,6 +59,7 @@ use codex_protocol::protocol::ThreadRolledBackEvent; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; +use codex_protocol::protocol::UserMessageAudioKind; use codex_protocol::protocol::UserMessageEvent; use codex_protocol::protocol::UserMessageImageKind; use codex_protocol::protocol::ViewImageToolCallEvent; @@ -1749,8 +1750,48 @@ impl ThreadHistoryBuilder { detail: payload.local_image_details.get(idx).copied().flatten(), }); } - if let Some(audio) = &payload.audio { - content.extend(audio.iter().cloned().map(|url| UserInput::Audio { url })); + if payload.has_complete_audio_order() { + let mut inline_index = 0; + let mut file_index = 0; + for audio_kind in &payload.audio_order { + let audio = match audio_kind { + UserMessageAudioKind::Inline => { + let Some(url) = payload + .audio + .as_deref() + .and_then(|audio| audio.get(inline_index)) + else { + continue; + }; + inline_index += 1; + codex_protocol::models::AudioReference::Inline { + audio_url: url.clone(), + } + } + UserMessageAudioKind::File => { + let Some(audio) = payload + .audio_references + .as_deref() + .and_then(|audio| audio.get(file_index)) + else { + continue; + }; + file_index += 1; + audio.clone() + } + }; + content.push(UserInput::from( + codex_protocol::user_input::UserInput::Audio { audio }, + )); + } + } else if let Some(audio) = &payload.audio_references { + content.extend(audio.iter().cloned().map(|audio| { + UserInput::from(codex_protocol::user_input::UserInput::Audio { audio }) + })); + } else if let Some(audio) = &payload.audio { + content.extend(audio.iter().cloned().map(|url| UserInput::Audio { + audio: crate::AudioReference::Inline { url }, + })); } content.extend( payload @@ -1948,6 +1989,7 @@ mod tests { use codex_protocol::items::UserMessageItem as CoreUserMessageItem; use codex_protocol::items::build_hook_prompt_message; use codex_protocol::mcp::CallToolResult; + use codex_protocol::models::AudioReference as CoreAudioReference; use codex_protocol::models::ImageDetail; use codex_protocol::models::MessagePhase as CoreMessagePhase; use codex_protocol::models::WebSearchAction as CoreWebSearchAction; @@ -1976,6 +2018,7 @@ mod tests { use codex_protocol::protocol::UserMessageEvent; use codex_protocol::protocol::WebSearchBeginEvent; use codex_protocol::protocol::WebSearchEndEvent; + use codex_protocol::user_input::UserInput as CoreUserInput; use codex_rollout::CompactedItem; use codex_utils_absolute_path::test_support::PathBufExt; use codex_utils_absolute_path::test_support::test_path_buf; @@ -2323,6 +2366,8 @@ mod tests { local_images: vec![local_image_path.clone()], local_image_details: vec![Some(ImageDetail::Original)], audio: Some(vec!["https://example.com/audio.mp3".into()]), + audio_references: None, + audio_order: Vec::new(), local_audio: vec![local_audio_path.clone()], text_elements: Vec::new(), }, @@ -2358,7 +2403,9 @@ mod tests { detail: Some(ImageDetail::Original), }, UserInput::Audio { - url: "https://example.com/audio.mp3".into(), + audio: crate::AudioReference::Inline { + url: "https://example.com/audio.mp3".into() + } }, UserInput::LocalAudio { path: local_audio_path, @@ -2405,6 +2452,32 @@ mod tests { ); } + #[test] + fn rebuilds_uploaded_and_inline_audio_in_order_from_legacy_events() { + let item = CoreUserMessageItem::new(&[ + CoreUserInput::Audio { + audio: CoreAudioReference::File { + file_id: "file_audio".into(), + }, + }, + CoreUserInput::Audio { + audio: CoreAudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".into(), + }, + }, + ]); + let turns = + build_turns_from_rollout_items(&[RolloutItem::EventMsg(item.as_legacy_event())]); + + assert_eq!( + serde_json::to_value(&turns[0].items[0]).unwrap()["content"], + serde_json::json!([ + {"type": "audio", "fileId": "file_audio"}, + {"type": "audio", "url": "data:audio/wav;base64,YXVkaW8="}, + ]), + ); + } + #[test] fn ignores_user_message_item_lifecycle_events() { let turn_id = "turn-1"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 4580410ea9..049cef5ed6 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -26,6 +26,7 @@ use codex_protocol::mcp::McpServerInfo; use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation; use codex_protocol::memory_citation::MemoryCitationEntry as CoreMemoryCitationEntry; use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile; +use codex_protocol::models::AudioReference; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions; use codex_protocol::models::ImageDetail; @@ -3142,6 +3143,28 @@ fn network_requirements_serializes_canonical_and_legacy_fields() { ); } +#[test] +fn audio_references_round_trip_through_core_and_json() { + for wire in [ + json!({"type": "audio", "url": "data:audio/wav;base64,AAA"}), + json!({"type": "audio", "fileId": "file_audio"}), + ] { + let input: UserInput = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!( + serde_json::to_value(UserInput::from(input.into_core())).unwrap(), + wire + ); + } + let old_input: UserInput = serde_json::from_value( + json!({"type": "audio", "fileId": "file_audio", "durationMs": 1_000}), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(UserInput::from(old_input.into_core())).unwrap(), + json!({"type": "audio", "fileId": "file_audio"}) + ); +} + #[test] fn core_turn_item_into_thread_item_converts_supported_variants() { let user_item = TurnItem::UserMessage(UserMessageItem { @@ -3163,7 +3186,14 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { detail: Some(ImageDetail::Original), }, CoreUserInput::Audio { - audio_url: "data:audio/wav;base64,AAA".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,AAA".to_string(), + }, + }, + CoreUserInput::Audio { + audio: AudioReference::File { + file_id: "file_audio".into(), + }, }, CoreUserInput::LocalAudio { path: PathBuf::from("local/audio.mp3"), @@ -3200,7 +3230,14 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { detail: Some(ImageDetail::Original), }, UserInput::Audio { - url: "data:audio/wav;base64,AAA".to_string(), + audio: crate::AudioReference::Inline { + url: "data:audio/wav;base64,AAA".to_string() + } + }, + UserInput::Audio { + audio: crate::AudioReference::File { + file_id: "file_audio".into(), + } }, UserInput::LocalAudio { path: PathBuf::from("local/audio.mp3"), @@ -3854,11 +3891,15 @@ fn user_input_into_core_preserves_media_fields() { assert_eq!( UserInput::Audio { - url: "data:audio/wav;base64,AAA".to_string(), + audio: crate::AudioReference::Inline { + url: "data:audio/wav;base64,AAA".to_string() + } } .into_core(), CoreUserInput::Audio { - audio_url: "data:audio/wav;base64,AAA".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,AAA".to_string() + } } ); diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index bf2444553a..c3b3fa7493 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -9,6 +9,7 @@ use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::MultiAgentMode; use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::models::AudioReference as CoreAudioReference; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::ImageDetail; use codex_protocol::models::ImageReference as CoreImageReference; @@ -451,6 +452,21 @@ pub enum ImageReference { }, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(untagged)] +#[ts(untagged)] +#[ts(export_to = "v2/")] +pub enum AudioReference { + Inline { + url: String, + }, + File { + #[serde(rename = "fileId")] + #[ts(rename = "fileId")] + file_id: String, + }, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "camelCase")] #[ts(tag = "type")] @@ -476,7 +492,8 @@ pub enum UserInput { path: PathBuf, }, Audio { - url: String, + #[serde(flatten)] + audio: AudioReference, }, LocalAudio { path: PathBuf, @@ -509,7 +526,12 @@ impl UserInput { detail, }, UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail }, - UserInput::Audio { url } => CoreUserInput::Audio { audio_url: url }, + UserInput::Audio { audio } => CoreUserInput::Audio { + audio: match audio { + AudioReference::Inline { url } => CoreAudioReference::Inline { audio_url: url }, + AudioReference::File { file_id } => CoreAudioReference::File { file_id }, + }, + }, UserInput::LocalAudio { path } => CoreUserInput::LocalAudio { path }, UserInput::Skill { name, path } => CoreUserInput::Skill { name, path }, UserInput::Mention { name, path } => CoreUserInput::Mention { name, path }, @@ -537,7 +559,14 @@ impl From for UserInput { detail, }, CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail }, - CoreUserInput::Audio { audio_url } => UserInput::Audio { url: audio_url }, + CoreUserInput::Audio { audio } => UserInput::Audio { + audio: match audio { + CoreAudioReference::Inline { audio_url } => { + AudioReference::Inline { url: audio_url } + } + CoreAudioReference::File { file_id } => AudioReference::File { file_id }, + }, + }, CoreUserInput::LocalAudio { path } => UserInput::LocalAudio { path }, CoreUserInput::Skill { name, path } => UserInput::Skill { name, path }, CoreUserInput::Mention { name, path } => UserInput::Mention { name, path }, diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 85acd3b319..4e35be4996 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -1083,7 +1083,9 @@ async fn thread_read_loaded_include_turns_reads_store_history_without_rollout_pa text_elements: Vec::new(), }, UserInput::Audio { - url: "https://example.com/recording.mp3".to_string(), + audio: codex_app_server_protocol::AudioReference::Inline { + url: "https://example.com/recording.mp3".to_string() + } }, UserInput::LocalAudio { path: "recording.wav".into(), diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index e694937ba9..a8d5865e7c 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -9,6 +9,7 @@ use crate::session::SessionIo; use crate::session::SessionSettingsUpdate; use crate::session::Submission; use crate::session::emit_subagent_session_started; +use crate::session::inject::validate_injected_response_item; use crate::session::new_submission_id; use crate::session::session::AppServerClientMetadata; use crate::session::session::Session; @@ -673,11 +674,18 @@ impl CodexThread { /// /// This is the thread-level bridge to `Session::inject_if_running` for /// callers that only hold a `CodexThread`. - /// It returns the unchanged items when this thread has no active turn. + /// Returns the unchanged batch when there is no active turn or an item requires typed turn input. pub async fn inject_if_running( &self, items: Vec, ) -> Result<(), Vec> { + if items + .iter() + .try_for_each(validate_injected_response_item) + .is_err() + { + return Err(items); + } self.session.inject_if_running(items).await } @@ -952,6 +960,7 @@ impl CodexThread { "items must not be empty".to_string(), )); } + items.iter().try_for_each(validate_injected_response_item)?; let had_reference_context = self.session.reference_context_item().await.is_some(); let mut turn_context = if had_reference_context { diff --git a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs index a4db94d5e1..cec85187ad 100644 --- a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs +++ b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs @@ -1,4 +1,5 @@ use super::*; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItemKind; use codex_protocol::models::ImageReference; use codex_protocol::models::InternalChatMessageMetadataPassthrough; @@ -88,7 +89,9 @@ fn later_image_parts_preserve_labels_audio_and_annotations() { text(&image_close_tag_text()), text("keep"), ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,abc".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,abc".to_string(), + }, }, ]; let kinds = (0..parts.len()) @@ -137,7 +140,9 @@ fn image_treatment_preserves_text_only_boundary_behavior() { text(&"middle".repeat(20)), text("later text"), ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,abc".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,abc".to_string(), + }, }, ])); for max_tokens in [0, 1, 4, 16, 100] { diff --git a/codex-rs/core/src/context/world_state/world_state_tests.rs b/codex-rs/core/src/context/world_state/world_state_tests.rs index 86a1b00b22..9d31b636c5 100644 --- a/codex-rs/core/src/context/world_state/world_state_tests.rs +++ b/codex-rs/core/src/context/world_state/world_state_tests.rs @@ -1,5 +1,6 @@ use super::*; use codex_context_fragments::AnnotatedContent; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; use codex_protocol::models::input_text; @@ -81,7 +82,9 @@ fn world_state_hash_tracks_typed_content_and_role() { role, ("", ""), ContentItem::InputAudio { - audio_url: audio_url.to_string(), + audio: AudioReference::Inline { + audio_url: audio_url.to_string(), + }, }, ), content_kind: ContentItemKind("generic.test".to_string()), diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index d8be070f41..621d3b7ee1 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -50,6 +50,7 @@ use codex_prompts::render_model_instructions; use codex_protocol::DEFAULT_FUNCTION_NAMESPACE; use codex_protocol::items::TurnItem; use codex_protocol::models::AgentMessageInputContent; +use codex_protocol::models::AudioReference; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputBody; @@ -87,6 +88,11 @@ use codex_features::Feature; static GUARDIAN_REVIEW_CONTEXT_REVISION: AtomicU64 = AtomicU64::new(/*v*/ 1); +// Responses validates uploaded audio against its 20-minute duration limit. +// File durations are unavailable locally, so budget that maximum at the existing +// inline estimate of 10 audio tokens per second. +pub(crate) const FILE_AUDIO_TOKENS_ESTIMATE: usize = 20 * 60 * 10; + fn next_guardian_review_context_revision() -> u64 { GUARDIAN_REVIEW_CONTEXT_REVISION.fetch_add(/*val*/ 1, Ordering::Relaxed) } @@ -1190,7 +1196,7 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { ContentItem::InputImage { image, detail } => { estimate_image_reference_bytes(image, *detail) } - ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url), + ContentItem::InputAudio { audio } => estimate_audio_reference_bytes(audio), }) .fold(0i64, i64::saturating_add), ResponseItem::AgentMessage { @@ -1403,6 +1409,17 @@ fn estimate_audio_bytes(audio_url: &str) -> i64 { .unwrap_or(i64::MAX) } +/// Audio estimate for callers that only have the reference. Uploaded file durations are not +/// available locally, so file references use the maximum duration accepted by Responses. +fn estimate_audio_reference_bytes(audio: &AudioReference) -> i64 { + match audio { + AudioReference::Inline { audio_url } => estimate_audio_bytes(audio_url), + AudioReference::File { .. } => { + i64::try_from(approx_bytes_for_tokens(FILE_AUDIO_TOKENS_ESTIMATE)).unwrap_or(i64::MAX) + } + } +} + fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { match output { FunctionCallOutputBody::Text(text) => text_bytes(text), diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 3aea92f140..42895a8db2 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -15,6 +15,7 @@ use codex_history::CodexHarnessMetadata; use codex_history::ResponseItemEnvelope; use codex_protocol::AgentPath; use codex_protocol::ResponseItemId; +use codex_protocol::models::AudioReference; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ConfigurationReasoning; use codex_protocol::models::ContentItem; @@ -1381,7 +1382,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() { detail: Some(DEFAULT_IMAGE_DETAIL), }, ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, }, ContentItem::InputText { text: "caption".to_string(), @@ -1623,7 +1626,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() { id: None, role: "user".to_string(), content: vec![ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, }], phase: None, internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), @@ -3344,7 +3349,9 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() { encrypted_content: None, id: None, role: "user".to_string(), - content: vec![ContentItem::InputAudio { audio_url }], + content: vec![ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + }], phase: None, internal_chat_message_metadata_passthrough: None, }; @@ -3357,6 +3364,35 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() { assert!(estimated < raw_len); } +#[test] +fn uploaded_audio_estimate_accounts_for_each_recording_instead_of_its_id() { + let item = ResponseItem::Message { + status: None, + encrypted_content: None, + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_a".into(), + }, + }, + ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_a_longer_reference".into(), + }, + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + assert_eq!( + estimate_response_item_model_visible_bytes(&item), + approx_bytes_for_tokens(/*tokens*/ 24_000) as i64, + ); +} + #[test] fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800); @@ -3411,7 +3447,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() { encrypted_content: None, id: None, role: "user".to_string(), - content: vec![ContentItem::InputAudio { audio_url }], + content: vec![ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + }], phase: None, internal_chat_message_metadata_passthrough: None, }; diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 53094a1832..dfc96bebde 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -145,9 +145,9 @@ fn parse_user_message(message: &[ContentItem]) -> Option { detail: *detail, }); } - ContentItem::InputAudio { audio_url } => { + ContentItem::InputAudio { audio } => { content.push(UserInput::Audio { - audio_url: audio_url.clone(), + audio: audio.clone(), }); } ContentItem::OutputText { text, .. } => { diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index 23dd9648ed..fc2ffbe048 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -9,6 +9,7 @@ use codex_protocol::items::HookPromptFragment; use codex_protocol::items::TurnItem; use codex_protocol::items::WebSearchItem; use codex_protocol::items::build_hook_prompt_message; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::ImageReference; @@ -245,7 +246,9 @@ fn skips_local_audio_label_text() { content: vec![ ContentItem::InputText { text: label }, ContentItem::InputAudio { - audio_url: audio_url.clone(), + audio: AudioReference::Inline { + audio_url: audio_url.clone(), + }, }, ContentItem::InputText { text: "".to_string(), @@ -265,7 +268,9 @@ fn skips_local_audio_label_text() { assert_eq!( user.content, vec![ - UserInput::Audio { audio_url }, + UserInput::Audio { + audio: AudioReference::Inline { audio_url } + }, UserInput::Text { text: user_text, text_elements: Vec::new(), diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rs index a7ddaf75d3..e8e61dc643 100644 --- a/codex-rs/core/src/session/inject.rs +++ b/codex-rs/core/src/session/inject.rs @@ -5,9 +5,32 @@ use codex_analytics::ImagePreparationMetadata; use codex_features::Feature; use codex_history::CodexHarnessMetadata; use codex_history::ResponseItemEnvelope; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; +use codex_protocol::models::AudioReference; +use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; +/// Uploaded files need the admission checks on typed turn input before entering history. +pub(crate) fn validate_injected_response_item(item: &ResponseItem) -> CodexResult<()> { + if let ResponseItem::Message { content, .. } = item + && content.iter().any(|part| { + matches!( + part, + ContentItem::InputAudio { + audio: AudioReference::File { .. } + } + ) + }) + { + return Err(CodexErr::InvalidRequest( + "Uploaded audio files must be submitted as turn input.".to_string(), + )); + } + Ok(()) +} + impl Session { /// Returns the input if there is no active turn to inject into. #[expect( diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index c4a0dee278..f365c4e2e8 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -253,7 +253,7 @@ mod extension_interruption; pub(crate) mod extension_metrics; mod guardian_checkpoint; mod handlers; -mod inject; +pub(crate) mod inject; mod reasoning_effort; mod submission; pub(crate) use reasoning_effort::RequestEffortUsage; diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 7227e13784..b25f718d47 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -99,6 +99,7 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::items::PlanItem; use codex_protocol::items::TurnItem; use codex_protocol::items::build_hook_prompt_message; +use codex_protocol::models::AudioReference; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::MessagePhase; @@ -181,6 +182,20 @@ pub(crate) async fn run_turn( prewarmed_client_session: Option, cancellation_token: CancellationToken, ) -> CodexResult> { + if input.iter().any(turn_input_has_uploaded_audio_file) { + let result = run_hooks_and_record_inputs( + &sess, + &turn_context, + &turn_context.capture_current_model_info(), + &input, + PersistContext::TurnStart, + ) + .await; + return match result { + Err(err) => Err(err), + Ok(_) => Err(uploaded_audio_disabled_error()), + }; + } let auto_compaction_enabled = sess .services .thread_extension_data @@ -210,7 +225,7 @@ pub(crate) async fn run_turn( .await { // Compaction runs before the new input is recorded, so preserve it on every failure. - run_hooks_and_record_inputs( + let _ = run_hooks_and_record_inputs( &sess, &turn_context, &turn_context.capture_current_model_info(), @@ -259,7 +274,7 @@ pub(crate) async fn run_turn( { Ok(requirements) => requirements, Err(err) => { - run_hooks_and_record_inputs( + let _ = run_hooks_and_record_inputs( &sess, &turn_context, &turn_context.capture_current_model_info(), @@ -287,7 +302,7 @@ pub(crate) async fn run_turn( { Ok(step_context) => step_context, Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => { - run_hooks_and_record_inputs( + let _ = run_hooks_and_record_inputs( &sess, &turn_context, &turn_context.capture_current_model_info(), @@ -393,7 +408,7 @@ pub(crate) async fn run_turn( &input, PersistContext::TurnStart, ) - .await + .await? { return Ok(None); } @@ -492,7 +507,7 @@ pub(crate) async fn run_turn( &pending_input, PersistContext::SteeredUserInput, ) - .await + .await? { break; } @@ -953,7 +968,8 @@ pub(crate) async fn run_hooks_and_record_inputs( model_info: &ModelInfo, input: &[TurnInput], persist_context: PersistContext, -) -> bool { +) -> CodexResult { + let rejected_uploaded_audio = input.iter().any(turn_input_has_uploaded_audio_file); // Cancellation can reach this path before Guardian's tools and context are // resolved. Only finalized evidence may enter reusable reviewer history. if sess @@ -962,11 +978,31 @@ pub(crate) async fn run_hooks_and_record_inputs( .get::() .is_some() { - return false; + return if rejected_uploaded_audio { + Err(uploaded_audio_disabled_error()) + } else { + Ok(false) + }; } let mut blocked_input = false; let mut accepted_user_input = false; for input_item in input { + let filtered_input; + let input_item = match input_item { + TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file) => { + let mut filtered = input_item.clone(); + let TurnInput::UserInput { content, .. } = &mut filtered else { + unreachable!("cloned user input must remain user input"); + }; + content.retain(|item| !is_uploaded_audio_file(item)); + if content.is_empty() { + continue; + } + filtered_input = filtered; + &filtered_input + } + _ => input_item, + }; let hook_outcome = inspect_pending_input(sess, turn_context, input_item).await; if hook_outcome.should_stop { blocked_input = true; @@ -998,7 +1034,29 @@ pub(crate) async fn run_hooks_and_record_inputs( .await; } } - blocked_input && !accepted_user_input + if rejected_uploaded_audio { + return Err(uploaded_audio_disabled_error()); + } + Ok(blocked_input && !accepted_user_input) +} + +fn turn_input_has_uploaded_audio_file(input: &TurnInput) -> bool { + matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file)) +} + +fn uploaded_audio_disabled_error() -> CodexErr { + CodexErr::UnsupportedOperation( + "Uploaded audio file inputs are not enabled in this build.".to_string(), + ) +} + +fn is_uploaded_audio_file(input: &UserInput) -> bool { + matches!( + input, + UserInput::Audio { + audio: AudioReference::File { .. } + } + ) } fn turn_user_input(input: &[TurnInput]) -> Vec { diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index b82df9b437..74753c50df 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -14,6 +14,7 @@ use super::TurnInput; use super::UserInputMetadata; +use super::inject::validate_injected_response_item; use super::session::Session; use super::session::SessionConfiguration; use super::session::SessionSettingsUpdate; @@ -237,6 +238,12 @@ pub(super) async fn handle( submission_id: String, ) -> CodexResult { let request = request.into(); + if let SubmittedTurnInput::ResponseItem(item) + | SubmittedTurnInput::AnnotatedResponseItem(AnnotatedResponseItem { item, .. }) = + &request.request.input + { + validate_injected_response_item(item)?; + } request .request .additional_context diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 2d2f7e66e3..77ce9b873c 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -715,14 +715,28 @@ impl Session { std::mem::take(&mut ts.token_usage_by_model), ) }; - run_hooks_and_record_inputs( + if let Err(err) = run_hooks_and_record_inputs( self, &turn_context, &turn_context.capture_current_model_info(), &pending_input, PersistContext::Standard, ) - .await; + .await + { + self.emit_turn_error_lifecycle( + turn_context.as_ref(), + err.to_codex_protocol_error(), + err.details(), + ) + .await; + self.track_turn_codex_error(turn_context.as_ref(), &err); + self.send_event( + turn_context.as_ref(), + EventMsg::Error(err.to_error_event(/*message_prefix*/ None)), + ) + .await; + } drop(finish_recording); let turn_telemetry = &turn_context.session_telemetry; // Emit token usage metrics. diff --git a/codex-rs/core/src/tasks/mod_tests.rs b/codex-rs/core/src/tasks/mod_tests.rs index 32e80ee38e..fe6b6dd0da 100644 --- a/codex-rs/core/src/tasks/mod_tests.rs +++ b/codex-rs/core/src/tasks/mod_tests.rs @@ -19,8 +19,14 @@ use codex_otel::TURN_TOOL_CALL_METRIC; use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; +use codex_protocol::models::AudioReference; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TokenUsage; +use codex_protocol::user_input::UserInput; use opentelemetry::KeyValue; use opentelemetry_sdk::metrics::InMemoryMetricExporter; use opentelemetry_sdk::metrics::data::AggregatedMetrics; @@ -54,6 +60,88 @@ impl SessionTask for PendingTask { } } +#[tokio::test] +async fn task_finish_rejects_uploaded_audio_without_losing_other_pending_input() { + let (session, turn_context, receiver) = make_session_and_context_with_rx().await; + session + .spawn_task(Arc::clone(&turn_context), Vec::new(), PendingTask) + .await; + while receiver.try_recv().is_ok() {} + + let turn_state = { + let active_turn = session.active_turn.lock().await; + Arc::clone( + &active_turn + .as_ref() + .expect("active pending task") + .turn_state, + ) + }; + session + .input_queue + .extend_pending_input_for_turn_state( + turn_state.as_ref(), + vec![TurnInput::UserInput { + content: vec![ + UserInput::Text { + text: "preserve pending text".to_string(), + text_elements: Vec::new(), + }, + UserInput::Audio { + audio: AudioReference::File { + file_id: "file_disabled_at_finish".to_string(), + }, + }, + ], + client_id: None, + metadata: Default::default(), + }], + ) + .await; + + session + .on_task_finished(Arc::clone(&turn_context), /*task_result*/ Ok(None)) + .await; + + let history = session.clone_history().await; + assert!(history.raw_items().any(|item| { + matches!(item, ResponseItem::Message { content, .. } if content.iter().any(|part| { + matches!(part, ContentItem::InputText { text } if text == "preserve pending text") + })) + })); + assert!(history.raw_items().all(|item| { + !matches!(item, ResponseItem::Message { content, .. } if content.iter().any(|part| { + matches!(part, ContentItem::InputAudio { + audio: AudioReference::File { file_id, .. }, + } if file_id == "file_disabled_at_finish") + })) + })); + + let (saw_error, completed) = tokio::time::timeout(std::time::Duration::from_secs(2), async { + let mut saw_error = false; + loop { + match receiver.recv().await.expect("event channel open").msg { + EventMsg::Error(error) => { + saw_error = true; + assert_eq!(error.codex_error_info, Some(CodexErrorInfo::BadRequest)); + } + EventMsg::TurnComplete(completed) => break (saw_error, completed), + _ => {} + } + } + }) + .await + .expect("task finish events"); + assert!(saw_error, "the rejected drain should emit an error event"); + assert_eq!( + completed + .error + .as_ref() + .and_then(|error| error.codex_error_info.clone()), + Some(CodexErrorInfo::BadRequest) + ); +} + fn test_session_telemetry() -> SessionTelemetry { let exporter = InMemoryMetricExporter::default(); let metrics = MetricsClient::new( diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 6c6f3d10da..0a05faa8ee 100644 --- a/codex-rs/core/src/tasks/regular.rs +++ b/codex-rs/core/src/tasks/regular.rs @@ -94,7 +94,7 @@ impl SessionTask for RegularTask { .await; let prewarmed_client_session = match prewarmed_client_session { SessionStartupPrewarmResolution::Cancelled => { - run_hooks_and_record_inputs( + let _ = run_hooks_and_record_inputs( &sess, &ctx, &ctx.capture_current_model_info(), diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index d20e1dfb26..630c515384 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -37,6 +37,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::config_types::Verbosity; use codex_protocol::error::CodexErr; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; @@ -528,7 +529,9 @@ async fn sends_audio_urls_to_responses() { codex .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Audio { - audio_url: audio_url.to_string(), + audio: AudioReference::Inline { + audio_url: audio_url.to_string(), + }, }])) .await .unwrap(); diff --git a/codex-rs/core/tests/suite/context_annotations.rs b/codex-rs/core/tests/suite/context_annotations.rs index 30b6f81947..fc982c9b12 100644 --- a/codex-rs/core/tests/suite/context_annotations.rs +++ b/codex-rs/core/tests/suite/context_annotations.rs @@ -4,6 +4,7 @@ use codex_core::config::CurrentTimeReminderConfig; use codex_core::config::RolloutBudgetConfig; use codex_core::config::TokenBudgetConfig; use codex_features::Feature; +use codex_protocol::models::AudioReference; use codex_protocol::models::ImageReference; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::AdditionalContextEntry; @@ -119,9 +120,7 @@ async fn first_request_item_types_roles_and_content_annotations() -> Result<()> }, detail: None, }, - UserInput::Audio { - audio_url: "data:audio/wav;base64,AAAA".to_string(), - }, + UserInput::Audio { audio: AudioReference::Inline { audio_url: "data:audio/wav;base64,AAAA".to_string() } }, ]) .with_additional_context(BTreeMap::from([ ( diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index a45d7d46c9..7f580eedaa 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -18,6 +18,7 @@ use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; use codex_protocol::config_types::ServiceTier; use codex_protocol::config_types::Settings; use codex_protocol::mcp::ClientMcpExtensions; +use codex_protocol::models::AudioReference; use codex_protocol::models::BaseInstructionsProvenance; use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; @@ -908,7 +909,9 @@ async fn model_change_projects_media_without_changing_live_or_replayed_history( detail: None, }, UserInput::Audio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, }, UserInput::Text { text: "first turn".to_string(), diff --git a/codex-rs/core/tests/suite/pending_input.rs b/codex-rs/core/tests/suite/pending_input.rs index 5464800cd2..3217c5bee3 100644 --- a/codex-rs/core/tests/suite/pending_input.rs +++ b/codex-rs/core/tests/suite/pending_input.rs @@ -1530,6 +1530,18 @@ async fn injected_response_item_reopens_turn_after_final_answer() { submit_user_input(&codex, INITIAL_PROMPT).await; wait_for_reasoning_item_started(&codex).await; + let rejected = vec![ + responses::user_message_item("rejected injection prefix"), + serde_json::from_value(json!({ + "type": "message", "role": "user", + "content": [{"type": "input_audio", "file_id": "file_live_injection"}], + })) + .expect("raw audio message"), + ]; + assert_eq!( + codex.inject_if_running(rejected.clone()).await, + Err(rejected) + ); assert!( codex .inject_if_running(vec![responses::user_message_item(INJECTED_CONTEXT)]) @@ -1555,6 +1567,11 @@ async fn injected_response_item_reopens_turn_after_final_answer() { let second: Value = from_slice(&requests[1]).expect("parse second request"); responses::assert_root_turn(&second, Some(first_turn_id)) .expect("external injection should preserve the active turn root"); + assert!( + !message_input_texts(&second, "user") + .iter() + .any(|text| text == "rejected injection prefix") + ); let relevant_user_input = message_input_texts(&second, "user") .into_iter() .filter(|text| { diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index 0bd1df327a..768c5aa70f 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -109,6 +109,9 @@ const ONE_PIXEL_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ #[path = "scenarios_dynamic_tools.rs"] mod dynamic_tools; +#[path = "scenarios_audio_file_tests.rs"] +mod audio_file; + #[path = "scenarios_retained_tool_outputs.rs"] mod retained_tool_outputs; diff --git a/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs new file mode 100644 index 0000000000..c10ebf8f19 --- /dev/null +++ b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs @@ -0,0 +1,179 @@ +//! New uploaded-audio inputs stay inactive until the runtime integration is installed. + +use super::*; +use codex_protocol::models::AudioReference; +use codex_protocol::openai_models::InputModality; +use pretty_assertions::assert_eq; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsupported_steered_audio_is_rejected_before_continuation_and_resume() -> Result<()> { + skip_if_no_network!(Ok(())); + let (release, gate) = tokio::sync::oneshot::channel(); + let (streaming, _completions) = start_streaming_sse_server(vec![ + vec![ + StreamingSseChunk { + gate: None, + body: sse(vec![ + ev_response_created("active"), + ev_assistant_message("working", "I am working on your request."), + ]), + }, + StreamingSseChunk { + gate: Some(gate), + body: sse(vec![ev_completed("active")]), + }, + ], + vec![StreamingSseChunk { + gate: None, + body: sse(vec![ev_completed("follow_up")]), + }], + vec![StreamingSseChunk { + gate: None, + body: sse(vec![ev_completed("resumed")]), + }], + ]) + .await; + let config_server = start_mock_server().await; + let base_url = format!("{}/v1", streaming.uri()); + let builder = || { + let base_url = base_url.clone(); + test_codex() + .with_model_info_override("gpt-5.5", |model| { + model.input_modalities.push(InputModality::Audio); + }) + .with_config(move |config| { + config.model_provider.base_url = Some(base_url); + config.model_provider.supports_websockets = false; + config + .features + .disable(Feature::InstantInterrupt) + .expect("queue steering"); + config + .features + .disable(Feature::EnableRequestCompression) + .expect("record request bodies"); + }) + }; + let test = builder().build_with_auto_env(&config_server).await?; + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![text("Start working.")])) + .await?; + let turn_id = wait_for_event_match(&test.codex, |event| match event { + EventMsg::TurnStarted(event) => Some(event.turn_id.clone()), + _ => None, + }) + .await; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::ItemCompleted(event) + if matches!(&event.item, TurnItem::AgentMessage(message) if message.id == "working")) + }) + .await; + test.codex + .steer_turn( + TurnInputRequest::user_input(vec![ + text("Keep this queued text."), + UserInput::Audio { + audio: AudioReference::File { + file_id: "file_unsupported_steering".into(), + }, + }, + ]), + turn_id, + ) + .await?; + release.send(()).expect("release the active response"); + let completed = wait_for_event_match(&test.codex, |event| match event { + EventMsg::TurnComplete(completed) => Some(completed.clone()), + _ => None, + }) + .await; + assert!(completed.error.is_some()); + assert_eq!(streaming.requests().await.len(), 1); + test.submit_text_turn("Continue without audio.").await?; + let resumed = builder() + .restart_with_auto_env(&config_server, &test) + .await?; + resumed.submit_text_turn("Continue after restart.").await?; + let requests = streaming + .requests() + .await + .iter() + .map(|body| serde_json::from_slice::(body)) + .collect::>>()?; + assert_eq!(requests.len(), 3); + for request in &requests { + assert!(!request["input"].as_array().unwrap().iter().any(|item| { + item["content"] + .as_array() + .is_some_and(|content| content.iter().any(|part| part["type"] == "input_audio")) + })); + } + for request in &requests[1..] { + assert!( + request.to_string().contains("Keep this queued text."), + "rejecting the attachment must preserve unrelated queued input" + ); + } + let entries = requests.iter().map(SnapshotEntry::body).collect::>(); + insta::assert_snapshot!( + "unsupported_steered_audio_follow_up_and_resume", + context_snapshot::format_context_snapshot( + "Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it.", + &entries, + &ContextSnapshotOptions::default().rewrite_known_segments(), + ) + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsupported_uploaded_audio_fails_before_inference_without_poisoning_follow_up() +-> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let mock = mount_sse_sequence(&server, vec![sse(vec![ev_completed("follow_up")])]).await; + let test = test_codex() + .with_model_info_override("gpt-5.5", |model| { + model.input_modalities.push(InputModality::Audio); + }) + .build_with_auto_env(&server) + .await?; + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![ + text("Listen to this recording."), + UserInput::Audio { + audio: AudioReference::File { + file_id: "file_unsupported".into(), + }, + }, + ])) + .await?; + let EventMsg::TurnComplete(completed) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await + else { + unreachable!() + }; + assert!(completed.error.is_some()); + assert!( + mock.requests().is_empty(), + "unsupported audio must not reach inference" + ); + test.submit_text_turn("Continue without the attachment.") + .await?; + let requests = mock.requests(); + assert_eq!(requests.len(), 1); + assert!(!requests[0].input().iter().any(|item| { + item["content"] + .as_array() + .is_some_and(|parts| parts.iter().any(|part| part["type"] == "input_audio")) + })); + assert!( + serde_json::to_string(&requests[0].input()) + .unwrap() + .contains("Listen to this recording."), + "rejecting the attachment must preserve surrounding text" + ); + Ok(()) +} diff --git a/codex-rs/core/tests/suite/scenarios_preparation.rs b/codex-rs/core/tests/suite/scenarios_preparation.rs index d8d5998f24..3c1372a4e1 100644 --- a/codex-rs/core/tests/suite/scenarios_preparation.rs +++ b/codex-rs/core/tests/suite/scenarios_preparation.rs @@ -1,10 +1,70 @@ //! Verifies prepared context reaches both the first model request and a follow-up. use super::*; +use codex_protocol::error::CodexErrorDetails; use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::turn_input::AnnotatedResponseItem; use pretty_assertions::assert_eq; use test_case::test_case; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let test = test_codex().build_with_auto_env(&server).await?; + let before = test.codex.conversation_history_snapshot().await; + let audio: ResponseItem = serde_json::from_value(json!({ + "type": "message", "role": "developer", + "content": [{"type": "input_audio", "file_id": "file_injected"}] + }))?; + let batch = vec![ + core_test_support::responses::user_message_item("must not be partially written"), + audio.clone(), + ]; + + let history_error = test + .codex + .inject_response_items(batch.clone()) + .await + .unwrap_err(); + let preparation_error = test + .codex + .inject_response_items_for_turn(batch) + .await + .unwrap_err(); + for error in [history_error, preparation_error] { + assert!(matches!( + error.details(), + CodexErrorDetails::InvalidRequest(_) + )); + } + for input in [ + TurnInput::ResponseItem(audio.clone()), + TurnInput::AnnotatedResponseItem(AnnotatedResponseItem { + item: audio, + annotations: Default::default(), + }), + ] { + let error = test + .codex + .start_or_steer_turn(TurnInputRequest::new(input)) + .await + .unwrap_err(); + assert!(matches!( + error.details(), + CodexErrorDetails::InvalidRequest(_) + )); + } + let after = test.codex.conversation_history_snapshot().await; + assert_eq!(after.history_version(), before.history_version()); + assert_eq!( + after.items().collect::>(), + before.items().collect::>() + ); + test.codex.shutdown_and_wait().await?; + Ok(()) +} + #[test_case(ThreadHistoryMode::Legacy; "legacy")] #[test_case(ThreadHistoryMode::Paginated; "paginated")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__audio_file__unsupported_steered_audio_follow_up_and_resume.snap b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__audio_file__unsupported_steered_audio_follow_up_and_resume.snap new file mode 100644 index 0000000000..9d29212c2e --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__audio_file__unsupported_steered_audio_follow_up_and_resume.snap @@ -0,0 +1,32 @@ +--- +source: core/tests/suite/scenarios_audio_file_tests.rs +expression: "context_snapshot::format_context_snapshot(\"Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it.\",\n&entries, &ContextSnapshotOptions::default().rewrite_known_segments(),)" +--- +Scenario: Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it. + +## Window 1 +-- request 1 (request) -- +00:message/developer: + +01:message/developer[2]: + [01] + [02] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + +02:message/user: + +03:message/user: + Start working. +-- request 2 (request) -- +04:message/assistant: + I am working on your request. +05:message/user: + Keep this queued text. +06:message/user: + Continue without audio. +-- request 3 (request) -- +07:message/user: + +08:message/user: + Continue after restart. diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__incremental_tools__incremental_tools_suppressed_announcements.snap b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__incremental_tools__incremental_tools_suppressed_announcements.snap index 355ef2eef9..03410da1f7 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__incremental_tools__incremental_tools_suppressed_announcements.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__incremental_tools__incremental_tools_suppressed_announcements.snap @@ -16,7 +16,7 @@ Settings: text: {"verbosity":"low"} tool_choice: "auto" -- request 1 (turn) -- -00:additional_tools/developer (2; hash=49f981f03ae5c5ce): +00:additional_tools/developer (2; hash=b07492b5514a41d0): - namespace/functions; hash=cc6e8cf0e5405f3a - custom/exec - function/wait @@ -25,11 +25,15 @@ Settings: - function/request_user_input - custom/apply_patch - function/view_image - - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9 + - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7 01:message/developer: -02:message/developer: - +02:message/developer[2]: + [01] + [02] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + 03:message/user: 04:message/user: @@ -42,7 +46,7 @@ Settings: Settings: relative to window 1 model: "updated-tools-model" -- request 3 (turn) -- -00:additional_tools/developer (2; hash=49f981f03ae5c5ce): +00:additional_tools/developer (2; hash=b07492b5514a41d0): - namespace/functions; hash=cc6e8cf0e5405f3a - custom/exec - function/wait @@ -51,11 +55,15 @@ Settings: relative to window 1 - function/request_user_input - custom/apply_patch - function/view_image - - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9 + - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7 01:message/developer: -02:message/developer: - +02:message/developer[2]: + [01] + [02] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + 03:message/user: 04:message/user: @@ -84,7 +92,7 @@ Settings: relative to window 1 ## Window 3 (after request 5: input diverged at item 00) Settings: same as window 2 -- request 6 (turn) -- -00:additional_tools/developer (2; hash=1322c3a339378659): +00:additional_tools/developer (2; hash=f9f0840b2e42f7f5): - namespace/functions; hash=9d5d7a0adab05fef - custom/exec - function/wait @@ -93,7 +101,7 @@ Settings: same as window 2 - function/request_user_input - custom/apply_patch - function/view_image - - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9 + - tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7 01:message/developer: 02:message/user: @@ -102,8 +110,12 @@ Settings: same as window 2 Continue with the same tools. 04:message/user: Continue with the updated execution instructions. -05:message/developer: - +05:message/developer[2]: + [01] + [02] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + 06:message/user: 07:message/user: diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__shared_instructions__shared_instructions_update_running_descendants.snap b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__shared_instructions__shared_instructions_update_running_descendants.snap index 59d5e96731..d0bf82c8b9 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__shared_instructions__shared_instructions_update_running_descendants.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__scenarios__shared_instructions__shared_instructions_update_running_descendants.snap @@ -161,7 +161,7 @@ Scenario: A host updates and clears shared instructions for surviving descendant 08:message/user: continue with the current instructions -## Window 3 (after request 8: input diverged at item 03, settings changed (prompt_cache_key, tools)) +## Window 3 (after request 8: input diverged at item 01, settings changed (prompt_cache_key, tools)) -- request 9 (turn) -- 00:message/developer: You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to c...rate with them until their goal is genuinely handled. [hash=2a3efec66e7f8963] @@ -181,24 +181,28 @@ Scenario: A host updates and clears shared instructions for surviving descendant - Before performing file edits of any kind, you provide updates explaining what edits you are making. - Tone of your updates must match your personality. -01:message/developer: - - Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c] - # Escalation Requests +01:message/developer[2]: + [01] + Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c] + # Escalation Requests - Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb] + Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb] - - Pipes: | - - Logical operators: &&, || - - NEVER provide a prefix_rule if your command uses a heredoc or herestring. + - Pipes: | + - Logical operators: &&, || + + NEVER provide a prefix_rule if your command uses a heredoc or herestring. - ### Examples - Good examples of prefixes: - - ["npm", "run", "dev"] - - ["gh", "pr", "check"] - - ["cargo", "test"] - + ### Examples + Good examples of prefixes: + - ["npm", "run", "dev"] + - ["gh", "pr", "check"] + - ["cargo", "test"] + + [02] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + 02:message/user[2]: [01] # AGENTS.md instructions diff --git a/codex-rs/core/tests/suite/user_notification.rs b/codex-rs/core/tests/suite/user_notification.rs index b98da23087..5bfa75707d 100644 --- a/codex-rs/core/tests/suite/user_notification.rs +++ b/codex-rs/core/tests/suite/user_notification.rs @@ -2,6 +2,7 @@ use codex_core::TurnInputRequest; +use codex_protocol::models::AudioReference; use codex_protocol::models::ImageReference; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::EventMsg; @@ -72,7 +73,9 @@ mv "${tmp_path}" "${payload_path}""#, detail: None, }, UserInput::Audio { - audio_url: audio_url.into(), + audio: AudioReference::Inline { + audio_url: audio_url.into(), + }, }, ])) .await?; diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs index a9317981e6..ca96150e17 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs @@ -5,6 +5,7 @@ use codex_guardian_context::ContextTarget; use codex_protocol::AgentPath; use codex_protocol::ResponseItemId; use codex_protocol::models::AgentMessageInputContent; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; @@ -1192,7 +1193,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() { detail: None, }, ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,audio-payload".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,audio-payload".to_string(), + }, }, ], phase: None, diff --git a/codex-rs/ext/queue/tests/queue_service.rs b/codex-rs/ext/queue/tests/queue_service.rs index e46ae8bec9..206146bb03 100644 --- a/codex-rs/ext/queue/tests/queue_service.rs +++ b/codex-rs/ext/queue/tests/queue_service.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::AudioReference; use std::path::Path; use std::sync::Arc; use std::sync::Mutex; @@ -854,7 +855,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow detail: Some(ImageDetail::Original), }, UserInput::Audio { - audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, }, ); @@ -896,7 +899,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow detail: Some(ImageDetail::High), }, UserInput::Audio { - audio_url: "data:audio/mp4;base64,ZWRpdGVkIGF1ZGlv".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/mp4;base64,ZWRpdGVkIGF1ZGlv".to_string(), + }, }, ); @@ -935,7 +940,9 @@ async fn invalid_local_attachments_do_not_mutate_queue() -> anyhow::Result<()> { detail: Some(ImageDetail::Original), }, UserInput::Audio { - audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, }, ); diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rs index eee0c3e8d5..cbb394942e 100644 --- a/codex-rs/guardian-context/src/budget.rs +++ b/codex-rs/guardian-context/src/budget.rs @@ -7,6 +7,7 @@ use codex_protocol::openai_models::CatalogModel; use codex_protocol::openai_models::ModelOrigin; use std::io::Write; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; @@ -295,11 +296,16 @@ impl SectionCost { } => { self.image_count = self.image_count.saturating_add(1); } - ContentItem::InputAudio { audio_url } => { - // Guardian currently has no audio contributor. Count a future opaque - // payload conservatively until its consumer supplies modality costs. + ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } => { + // Guardian has no audio contributor. Keep the existing inline + // transport cost, but do not treat a file handle as evidence. self.text_bytes = self.text_bytes.saturating_add(audio_url.len()); } + ContentItem::InputAudio { + audio: AudioReference::File { .. }, + } => {} } self } diff --git a/codex-rs/guardian-context/src/budget_tests.rs b/codex-rs/guardian-context/src/budget_tests.rs index 9000590b5e..01830bb0d9 100644 --- a/codex-rs/guardian-context/src/budget_tests.rs +++ b/codex-rs/guardian-context/src/budget_tests.rs @@ -4,6 +4,7 @@ use super::*; use crate::Budgeted; use crate::composition::SectionOutput; use crate::composition::user_message as message; +use codex_protocol::models::AudioReference; use codex_protocol::models::ImageReference; use pretty_assertions::assert_eq; @@ -62,6 +63,16 @@ fn section_costs_keep_multimodal_payloads_separate() { }, detail: None, }), + Budgeted::required(ContentItem::InputAudio { + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,AAAA".to_owned(), + }, + }), + Budgeted::required(ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_audio".to_owned(), + }, + }), ]), }, SectionOutput { @@ -82,7 +93,7 @@ fn section_costs_keep_multimodal_payloads_separate() { ( "transcript", SectionCost { - text_bytes: "évidence".len(), + text_bytes: "évidence".len() + "data:audio/wav;base64,AAAA".len(), image_bytes: "data:image/png;base64,AAAA".len(), image_count: 2 } diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rs index 1a81e21ca7..abd394e882 100644 --- a/codex-rs/guardian-context/src/enforcement.rs +++ b/codex-rs/guardian-context/src/enforcement.rs @@ -9,6 +9,7 @@ use std::collections::HashMap; use std::collections::HashSet; use codex_history::ResponseItemEnvelope; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::models::ImageReference; @@ -422,7 +423,12 @@ pub(crate) fn retain_content( image: ImageReference::File { .. }, .. }) => 0, - SectionContent::Other(ContentItem::InputAudio { audio_url }) => audio_url.len(), + SectionContent::Other(ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + }) => audio_url.len(), + SectionContent::Other(ContentItem::InputAudio { + audio: AudioReference::File { .. }, + }) => 0, }; truncations.push(TruncationObservation { component: section.id, diff --git a/codex-rs/history/src/tests.rs b/codex-rs/history/src/tests.rs index 272de15d87..c02aa200cf 100644 --- a/codex-rs/history/src/tests.rs +++ b/codex-rs/history/src/tests.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use codex_protocol::models::AudioReference; use codex_protocol::models::ConfigurationReasoning; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::ThreadSettingsSnapshot; @@ -718,6 +719,79 @@ fn compacted_metadata_remains_compatible_with_legacy_response_item_readers() -> Ok(()) } +#[test] +fn uploaded_audio_round_trips_in_canonical_message_content() -> Result<()> { + let mut message = response_message("user"); + let ResponseItem::Message { content, .. } = &mut message else { + unreachable!() + }; + content.push(ContentItem::InputAudio { + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,aGVsbG8=".to_string(), + }, + }); + content.push(ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, + }); + content.push(ContentItem::InputText { + text: "after audio".to_string(), + }); + let envelope = ResponseItemEnvelope::new(message); + let response = RolloutItem::ResponseItem(envelope.clone()); + let compacted = RolloutItem::Compacted(Box::new(CompactedItem { + message: String::new(), + replacement_history: Some(vec![envelope.clone()]), + guardian_history: Some(GuardianHistoryCheckpoint(vec![envelope])), + retained_context: None, + mcp_resource_origins: None, + window_number: None, + first_window_id: None, + previous_window_id: None, + window_id: None, + compaction_response_id: None, + latest_token_usage_record: None, + resume_metadata: None, + })); + for (item, paths) in [ + (response, vec!["/payload/content"]), + ( + compacted, + vec![ + "/payload/replacement_history/0/content", + "/payload/guardian_history/0/content", + ], + ), + ] { + let mut serialized = serde_json::to_value(&item)?; + for path in paths { + assert_eq!( + serialized.pointer(path), + Some(&json!([ + {"type": "input_text", "text": "hello"}, + {"type": "input_audio", "audio_url": "data:audio/wav;base64,aGVsbG8="}, + {"type": "input_audio", "file_id": "file_audio"}, + {"type": "input_text", "text": "after audio"}, + ])) + ); + // Rollouts from the prototype can contain this extra field. Reading + // them must preserve the same file reference in every history shape. + serialized.pointer_mut(path).unwrap()[2]["duration_ms"] = json!(1_000); + } + match (serde_json::from_value::(serialized)?, item) { + (RolloutItem::ResponseItem(restored), RolloutItem::ResponseItem(original)) => { + assert_eq!(restored, original) + } + (RolloutItem::Compacted(restored), RolloutItem::Compacted(original)) => { + assert_eq!(restored, original) + } + _ => panic!("unexpected rollout variant"), + } + } + Ok(()) +} + #[test] /// Preserves the established tagged payload representation for every rollout variant. fn rollout_item_variants_preserve_existing_payload_shapes() -> Result<()> { diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 220e98a1a3..4905590176 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -4,6 +4,7 @@ use crate::ThreadId; use crate::dynamic_tools::DynamicToolCallOutputContentItem; use crate::mcp::CallToolResult; use crate::memory_citation::MemoryCitation; +use crate::models::AudioReference; use crate::models::ContentItem; use crate::models::ContentItemMetadata; use crate::models::FunctionCallOutputBody; @@ -24,6 +25,7 @@ use crate::protocol::PatchApplyStatus; use crate::protocol::ReviewOutputEvent; use crate::protocol::ReviewTarget; use crate::protocol::SubAgentActivityKind; +use crate::protocol::UserMessageAudioKind; use crate::user_input::ByteRange; use crate::user_input::TextElement; use crate::user_input::UserInput; @@ -635,7 +637,52 @@ impl UserMessageItem { self.content .iter() .filter_map(|c| match c { - UserInput::Audio { audio_url } => Some(audio_url.clone()), + UserInput::Audio { + audio: AudioReference::Inline { audio_url }, + } => Some(audio_url.clone()), + _ => None, + }) + .collect() + } + + /// Returns file-backed audio references when the message includes an upload. + /// Inline payloads remain exclusively in the legacy `audio` field. + pub fn audio_references(&self) -> Option> { + let audio = self + .content + .iter() + .filter_map(|content| match content { + UserInput::Audio { + audio: audio @ AudioReference::File { .. }, + } => Some(audio.clone()), + _ => None, + }) + .collect::>(); + (!audio.is_empty()).then_some(audio) + } + + /// Returns the original inline/file ordering when uploaded audio is present. + pub fn audio_order(&self) -> Vec { + if !self.content.iter().any(|content| { + matches!( + content, + UserInput::Audio { + audio: AudioReference::File { .. } + } + ) + }) { + return Vec::new(); + } + + self.content + .iter() + .filter_map(|content| match content { + UserInput::Audio { + audio: AudioReference::Inline { .. }, + } => Some(UserMessageAudioKind::Inline), + UserInput::Audio { + audio: AudioReference::File { .. }, + } => Some(UserMessageAudioKind::File), _ => None, }) .collect() @@ -812,7 +859,14 @@ mod tests { text_elements: Vec::new(), }, UserInput::Audio { - audio_url: "https://example.com/remote.mp3".to_string(), + audio: AudioReference::Inline { + audio_url: "https://example.com/remote.mp3".to_string(), + }, + }, + UserInput::Audio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, }, UserInput::LocalAudio { path: std::path::PathBuf::from("local.wav"), @@ -826,6 +880,16 @@ mod tests { vec![std::path::PathBuf::from("local.wav")], ) ); + assert_eq!( + item.audio_references(), + Some(vec![AudioReference::File { + file_id: "file_audio".to_string(), + }]) + ); + assert_eq!( + item.audio_order(), + vec![UserMessageAudioKind::Inline, UserMessageAudioKind::File] + ); } #[test] diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index a0372c54f8..6aa205ba1c 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -112,6 +112,8 @@ impl UserMessageItem { local_images: self.local_image_paths(), local_image_details: self.local_image_details(), audio: Some(self.audio_urls()), + audio_references: self.audio_references(), + audio_order: self.audio_order(), local_audio: self.local_audio_paths(), text_elements: self.text_elements(), } diff --git a/codex-rs/protocol/src/local_media.rs b/codex-rs/protocol/src/local_media.rs index e1b8af25cf..286d30351f 100644 --- a/codex-rs/protocol/src/local_media.rs +++ b/codex-rs/protocol/src/local_media.rs @@ -7,6 +7,7 @@ use codex_utils_image::PromptImageMode; use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; +use crate::models::AudioReference; use crate::models::ImageDetail; use crate::models::ImageReference; use crate::user_input::UserInput; @@ -42,7 +43,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> { })?; let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?; *input = UserInput::Audio { - audio_url: data_url_from_bytes(mime, &file_bytes), + audio: AudioReference::Inline { + audio_url: data_url_from_bytes(mime, &file_bytes), + }, }; } UserInput::Text { .. } diff --git a/codex-rs/protocol/src/local_media_tests.rs b/codex-rs/protocol/src/local_media_tests.rs index 47b28c8c25..30a70e8738 100644 --- a/codex-rs/protocol/src/local_media_tests.rs +++ b/codex-rs/protocol/src/local_media_tests.rs @@ -63,7 +63,9 @@ fn snapshots_local_audio_user_input_with_supported_mime_types() -> Result<()> { assert_eq!( UserInput::Audio { - audio_url: format!("data:{mime};base64,YXVkaW8="), + audio: AudioReference::Inline { + audio_url: format!("data:{mime};base64,YXVkaW8=") + }, }, input ); diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 351f45c03c..e1e54f2810 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -893,7 +893,8 @@ pub enum ContentItem { detail: Option, }, InputAudio { - audio_url: String, + #[serde(flatten)] + audio: AudioReference, }, OutputText { text: String, @@ -934,6 +935,15 @@ pub enum ImageReference { File { file_id: String }, } +/// Inline audio data or an uploaded file loaded by the Responses API. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] +#[serde(untagged)] +#[ts(untagged)] +pub enum AudioReference { + Inline { audio_url: String }, + File { file_id: String }, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] pub enum AgentMessageInputContent { @@ -1982,7 +1992,9 @@ fn local_audio_content_items( text: local_audio_open_tag_text_with_path(label_number, path), }, ContentItem::InputAudio { - audio_url: data_url_from_bytes(mime, file_bytes), + audio: AudioReference::Inline { + audio_url: data_url_from_bytes(mime, file_bytes), + }, }, ContentItem::InputText { text: LOCAL_AUDIO_CLOSE_TAG.to_string(), @@ -2199,9 +2211,9 @@ impl ResponseInputItem { )], } } - UserInput::Audio { audio_url } => { + UserInput::Audio { audio } => { audio_index += 1; - vec![ContentItem::InputAudio { audio_url }] + vec![ContentItem::InputAudio { audio }] } UserInput::LocalAudio { path } => { audio_index += 1; @@ -2277,7 +2289,12 @@ impl From for FunctionCallOutputContentItem { Self::InputText { text } } ContentItem::InputImage { image, detail } => Self::InputImage { image, detail }, - ContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url }, + ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } => Self::InputAudio { audio_url }, + ContentItem::InputAudio { + audio: AudioReference::File { file_id, .. }, + } => Self::InputText { text: file_id }, } } } @@ -4147,34 +4164,47 @@ mod tests { #[test] fn serializes_audio_user_input_without_tags() -> Result<()> { - let audio_url = "data:audio/wav;base64,abc".to_string(); - - let item = ResponseInputItem::from(vec![UserInput::Audio { - audio_url: audio_url.clone(), - }]); - - assert_eq!( - item, - ResponseInputItem::Message { - role: "user".to_string(), - content: vec![ContentItem::InputAudio { audio_url }], - phase: None, - } - ); - assert_eq!( - serde_json::to_value(item)?, - serde_json::json!({ - "type": "message", - "role": "user", - "content": [ - { - "type": "input_audio", - "audio_url": "data:audio/wav;base64,abc", - }, - ], - }) - ); - + for (audio, wire) in [ + ( + AudioReference::Inline { + audio_url: "data:audio/wav;base64,abc".into(), + }, + serde_json::json!({"type": "input_audio", "audio_url": "data:audio/wav;base64,abc"}), + ), + ( + AudioReference::File { + file_id: "file_audio".into(), + }, + serde_json::json!({"type": "input_audio", "file_id": "file_audio"}), + ), + ] { + let input = UserInput::Audio { + audio: audio.clone(), + }; + assert_eq!( + serde_json::from_value::(serde_json::to_value(&input)?)?, + input + ); + let item = ResponseInputItem::from(vec![input]); + assert_eq!( + item, + ResponseInputItem::Message { + role: "user".into(), + content: vec![ContentItem::InputAudio { audio }], + phase: None, + } + ); + assert_eq!( + serde_json::to_value(&item)?, + serde_json::json!({ + "type": "message", "role": "user", "content": [wire], + }) + ); + assert_eq!( + serde_json::from_value::(serde_json::to_value(&item)?)?, + item + ); + } Ok(()) } @@ -4207,7 +4237,9 @@ mod tests { ), }, ContentItem::InputAudio { - audio_url: format!("data:{mime};base64,YXVkaW8="), + audio: AudioReference::Inline { + audio_url: format!("data:{mime};base64,YXVkaW8=") + }, }, ContentItem::InputText { text: audio_close_tag_text(), @@ -4583,7 +4615,9 @@ mod tests { let item = ResponseInputItem::from(vec![ UserInput::Audio { - audio_url: audio_url.clone(), + audio: AudioReference::Inline { + audio_url: audio_url.clone(), + }, }, UserInput::LocalAudio { path: local_path.clone(), @@ -4595,7 +4629,9 @@ mod tests { ResponseInputItem::Message { role: "user".to_string(), content: vec![ - ContentItem::InputAudio { audio_url }, + ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url } + }, ContentItem::InputText { text: local_audio_open_tag_text_with_path( /*label_number*/ 2, @@ -4603,7 +4639,9 @@ mod tests { ), }, ContentItem::InputAudio { - audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string() + } }, ContentItem::InputText { text: audio_close_tag_text(), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 1c1282946b..a0af620c34 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -41,6 +41,7 @@ use crate::mcp::RequestId; use crate::memory_citation::MemoryCitation; use crate::models::ActivePermissionProfile; use crate::models::AgentMessageInputContent; +use crate::models::AudioReference; use crate::models::BaseInstructions; use crate::models::ContentItem; use crate::models::ImageDetail; @@ -2777,6 +2778,14 @@ pub enum UserMessageImageKind { File, } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum UserMessageAudioKind { + Inline, + File, +} + #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)] pub struct UserMessageEvent { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2818,6 +2827,16 @@ pub struct UserMessageEvent { /// legacy UI history events and correspond to audio sent to the model. #[serde(skip_serializing_if = "Option::is_none")] pub audio: Option>, + /// File-backed audio references. When `audio_order` is complete, consumers + /// combine this with `audio` without duplicating inline payloads. Older + /// producers may instead store the complete ordered reference list here. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_references: Option>, + /// Inline and file-backed audio kinds in their original input order. + /// New mixed-input producers populate this alongside the split `audio` and + /// `audio_references` fields; when absent, consumers retain the prior shape. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub audio_order: Vec, /// Local file paths sourced from `UserInput::LocalAudio`. These are kept so /// clients can reattach audio when editing history and should not be /// treated as API-ready URLs. @@ -2847,6 +2866,30 @@ impl UserMessageEvent { inline_count == self.images.as_ref().map_or(0, Vec::len) && file_count == self.file_ids.as_ref().map_or(0, Vec::len) } + + /// Returns whether `audio_order` accounts for every split audio reference exactly once. + pub fn has_complete_audio_order(&self) -> bool { + if self.audio_order.is_empty() { + return false; + } + + let mut inline_count = 0; + let mut file_count = 0; + for audio_kind in &self.audio_order { + match audio_kind { + UserMessageAudioKind::Inline => inline_count += 1, + UserMessageAudioKind::File => file_count += 1, + } + } + + inline_count == self.audio.as_ref().map_or(0, Vec::len) + && file_count == self.audio_references.as_ref().map_or(0, Vec::len) + && self.audio_references.as_ref().is_none_or(|references| { + references + .iter() + .all(|reference| matches!(reference, AudioReference::File { .. })) + }) + } } /// Returns the user-facing preview text for a user message. @@ -2867,7 +2910,13 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option { { return Some("[Image]".to_string()); } - if user.audio.as_ref().is_some_and(|audio| !audio.is_empty()) || !user.local_audio.is_empty() { + if user.audio.as_ref().is_some_and(|audio| !audio.is_empty()) + || user + .audio_references + .as_ref() + .is_some_and(|audio| !audio.is_empty()) + || !user.local_audio.is_empty() + { return Some("[Audio]".to_string()); } None @@ -4762,6 +4811,7 @@ mod tests { use crate::items::UserMessageItem; use crate::items::WebSearchItem; use crate::mcp::CallToolResult; + use crate::models::AudioReference; use crate::permissions::FileSystemAccessMode; use crate::permissions::FileSystemPath; use crate::permissions::FileSystemSandboxEntry; @@ -6297,7 +6347,14 @@ mod tests { detail: Some(ImageDetail::Original), }, crate::user_input::UserInput::Audio { - audio_url: "https://example.com/remote.mp3".to_string(), + audio: AudioReference::Inline { + audio_url: "https://example.com/remote.mp3".to_string(), + }, + }, + crate::user_input::UserInput::Audio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, }, crate::user_input::UserInput::LocalAudio { path: local_audio_path.clone(), @@ -6342,16 +6399,41 @@ mod tests { Some(vec!["https://example.com/remote.mp3".to_string()]) ); assert_eq!(event.local_audio, vec![local_audio_path]); + assert_eq!( + event.audio_references, + Some(vec![AudioReference::File { + file_id: "file_audio".to_string(), + }]) + ); + assert_eq!( + event.audio_order, + vec![UserMessageAudioKind::Inline, UserMessageAudioKind::File] + ); + // Earlier readers ignore the new ordered field and retain inline recordings. + #[derive(Deserialize)] + struct EarlierUserMessageEvent { + audio: Option>, + } + let earlier: EarlierUserMessageEvent = serde_json::from_value(event_json).unwrap(); + assert_eq!(earlier.audio, event.audio); } #[test] fn audio_only_user_message_has_placeholder_preview() { - let event = UserMessageEvent { - audio: Some(vec!["https://example.com/remote.mp3".to_string()]), - ..Default::default() - }; - - assert_eq!(user_message_preview(&event), Some("[Audio]".to_string())); + for event in [ + UserMessageEvent { + audio: Some(vec!["https://example.com/remote.mp3".to_string()]), + ..Default::default() + }, + UserMessageEvent { + audio_references: Some(vec![AudioReference::File { + file_id: "file_audio".into(), + }]), + ..Default::default() + }, + ] { + assert_eq!(user_message_preview(&event), Some("[Audio]".to_string())); + } } #[test] diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index 9e28c5ec3d..13b3cacea0 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -3,6 +3,7 @@ use serde::Deserialize; use serde::Serialize; use ts_rs::TS; +use crate::models::AudioReference; use crate::models::ImageDetail; use crate::models::ImageReference; @@ -40,8 +41,11 @@ pub enum UserInput { #[ts(optional)] detail: Option, }, - /// Pre-encoded audio data URI forwarded to the Responses API. - Audio { audio_url: String }, + /// Audio reference forwarded to the Responses API. + Audio { + #[serde(flatten)] + audio: AudioReference, + }, /// Local audio path converted to an `Audio` data URI during request serialization. LocalAudio { path: std::path::PathBuf }, diff --git a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs index 4625d152be..1ef3c25755 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs @@ -26,8 +26,10 @@ use codex_protocol::items::SubAgentActivityItem; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::WebSearchItem; +use codex_protocol::models::AudioReference; use codex_protocol::models::ImageReference; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::UserMessageAudioKind; use codex_protocol::protocol::UserMessageEvent; use codex_protocol::protocol::UserMessageImageKind; use codex_protocol::user_input::UserInput; @@ -40,6 +42,7 @@ pub(super) fn user_message_item( ) -> ThreadStoreResult { let mut content = Vec::new(); let has_image_order = event.has_complete_image_order(); + let has_audio_order = event.has_complete_audio_order(); if !event.message.trim().is_empty() { content.push(UserInput::Text { text: event.message, @@ -110,12 +113,26 @@ pub(super) fn user_message_item( detail: event.local_image_details.get(index).copied().flatten(), }); } - if let Some(audio) = event.audio { - content.extend( - audio - .into_iter() - .map(|audio_url| UserInput::Audio { audio_url }), - ); + if has_audio_order { + let mut inline_audio = event.audio.unwrap_or_default().into_iter(); + let mut file_audio = event.audio_references.unwrap_or_default().into_iter(); + for audio_kind in event.audio_order { + let audio = match audio_kind { + UserMessageAudioKind::Inline => inline_audio + .next() + .map(|audio_url| AudioReference::Inline { audio_url }), + UserMessageAudioKind::File => file_audio.next(), + }; + if let Some(audio) = audio { + content.push(UserInput::Audio { audio }); + } + } + } else if let Some(audio) = event.audio_references { + content.extend(audio.into_iter().map(|audio| UserInput::Audio { audio })); + } else if let Some(audio) = event.audio { + content.extend(audio.into_iter().map(|audio_url| UserInput::Audio { + audio: AudioReference::Inline { audio_url }, + })); } content.extend( event diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs index c7a6b3a0ea..f23d057ae7 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs @@ -11,10 +11,12 @@ use std::collections::HashMap; use std::collections::HashSet; use codex_protocol::ResponseItemId; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::UserMessageAudioKind; use codex_protocol::protocol::UserMessageEvent; use codex_protocol::protocol::UserMessageImageKind; use codex_rollout::CompactedItem; @@ -580,7 +582,7 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent file_ids.push(file_id.as_str()); } }, - ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()), + ContentItem::InputAudio { audio: reference } => audio.push(reference), ContentItem::OutputText { .. } => return false, } } @@ -602,14 +604,48 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent .iter() .map(String::as_str) .collect::>() - && audio - == event - .audio - .as_deref() - .unwrap_or_default() - .iter() - .map(String::as_str) - .collect::>() + && { + if event.has_complete_audio_order() { + let mut actual = audio.into_iter(); + let mut inline = event.audio.as_deref().unwrap_or_default().iter(); + let mut file = event.audio_references.as_deref().unwrap_or_default().iter(); + event.audio_order.iter().all(|kind| { + let Some(actual) = actual.next() else { + return false; + }; + match kind { + UserMessageAudioKind::Inline => matches!( + (actual, inline.next()), + ( + AudioReference::Inline { audio_url }, + Some(expected), + ) if audio_url == expected + ), + UserMessageAudioKind::File => { + file.next().is_some_and(|expected| actual == expected) + } + } + }) && actual.next().is_none() + && inline.next().is_none() + && file.next().is_none() + } else { + match &event.audio_references { + Some(references) => audio.into_iter().eq(references), + None => audio + .into_iter() + .map(|reference| match reference { + AudioReference::Inline { audio_url } => Some(audio_url.as_str()), + AudioReference::File { .. } => None, + }) + .eq(event + .audio + .as_deref() + .unwrap_or_default() + .iter() + .map(|url| Some(url.as_str()))), + } + } + } && event.local_images.is_empty() && event.local_audio.is_empty() } diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs index 7eb881a8d4..dea0da6a59 100644 --- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs +++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs @@ -21,6 +21,7 @@ use codex_protocol::items::ReasoningItem; use codex_protocol::items::TurnItem; use codex_protocol::mcp::McpResourceOrigin; use codex_protocol::mcp::McpResourceOriginCheckpoint; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; use codex_protocol::models::ImageDetail; @@ -43,6 +44,7 @@ use codex_protocol::protocol::ThreadRolledBackEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::TurnStartedEvent; +use codex_protocol::protocol::UserMessageAudioKind; use codex_protocol::protocol::UserMessageEvent; use codex_protocol::protocol::UserMessageImageKind; use codex_protocol::user_input::UserInput; @@ -340,6 +342,11 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() { file_ids: Some(vec!["file_123".to_string()]), file_id_details: vec![Some(ImageDetail::High)], image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline], + audio: Some(vec!["data:audio/wav;base64,YXVkaW8=".into()]), + audio_references: Some(vec![AudioReference::File { + file_id: "file_audio".into(), + }]), + audio_order: vec![UserMessageAudioKind::File, UserMessageAudioKind::Inline], ..Default::default() }; let path = write_rollout( @@ -403,6 +410,16 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() { }, detail: Some(ImageDetail::Original), }, + UserInput::Audio { + audio: AudioReference::File { + file_id: "file_audio".into(), + }, + }, + UserInput::Audio { + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".into(), + }, + }, ]; assert_eq!(user_item.map(|item| &item.content), Some(&expected_content)); @@ -1010,6 +1027,9 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() { let home = TempDir::new().expect("create Codex home"); let thread_id = ThreadId::new(); let file_id = "file_123".to_string(); + let audio = AudioReference::File { + file_id: "file_audio".into(), + }; let response = ResponseItem::Message { status: None, encrypted_content: None, @@ -1025,6 +1045,9 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() { }, detail: None, }, + ContentItem::InputAudio { + audio: audio.clone(), + }, ], phase: None, internal_chat_message_metadata_passthrough: None, @@ -1032,6 +1055,8 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() { let event = UserMessageEvent { message: "remove question".to_string(), file_ids: Some(vec![file_id]), + audio_references: Some(vec![audio]), + audio_order: vec![UserMessageAudioKind::File], ..Default::default() }; let path = write_rollout( diff --git a/codex-rs/tools/src/tool_output_tests.rs b/codex-rs/tools/src/tool_output_tests.rs index bca0e81d97..6549f41a84 100644 --- a/codex-rs/tools/src/tool_output_tests.rs +++ b/codex-rs/tools/src/tool_output_tests.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::ImageReference; use codex_protocol::models::ResponseInputItem; @@ -31,7 +32,14 @@ fn code_mode_text_preserves_message_content() { detail: None, }, ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,def".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,def".to_string(), + }, + }, + ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, }, ], phase: None, @@ -40,7 +48,7 @@ fn code_mode_text_preserves_message_content() { assert_eq!( response_input_to_code_mode_result(response), serde_json::json!( - "Input text\nOutput text\nfile_123\ndata:image/png;base64,abc\ndata:audio/wav;base64,def" + "Input text\nOutput text\nfile_123\ndata:image/png;base64,abc\ndata:audio/wav;base64,def\nfile_audio" ) ); } diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__safety_buffering__safety_recourse_request_history.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__safety_buffering__safety_recourse_request_history.snap index b4ca45aeee..449519422b 100644 --- a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__safety_buffering__safety_recourse_request_history.snap +++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__safety_buffering__safety_recourse_request_history.snap @@ -9,10 +9,14 @@ Scenario: The fork excludes the blocked turn and submits the selected rewrite. -- request 1 (turn) -- 00:message/developer: Help the user. -01:message/developer[3]: +01:message/developer[4]: [01] [02] [03] + [04] + Deferred tool namespaces: + - codex_tui: Manage Codex tasks available through the connected app server. + 02:message/user: 03:message/user: @@ -22,10 +26,14 @@ Scenario: The fork excludes the blocked turn and submits the selected rewrite. -- request 2 (turn) -- 00:message/developer: Help the user. -01:message/developer[3]: +01:message/developer[4]: [01] [02] [03] + [04] + Deferred tool namespaces: + - codex_tui: Manage Codex tasks available through the connected app server. + 02:message/user: 03:message/user: diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index e9cae92a47..2c3f9b000a 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -911,7 +911,9 @@ async fn live_app_server_user_message_omits_unsupported_media() { text_elements: Vec::new(), }, AppServerUserInput::Audio { - url: "https://example.com/one.wav".to_string(), + audio: codex_app_server_protocol::AudioReference::Inline { + url: "https://example.com/one.wav".to_string(), + }, }, AppServerUserInput::LocalAudio { path: test_path_buf("/tmp/two.wav"), diff --git a/codex-rs/tui/src/dynamic_tools.rs b/codex-rs/tui/src/dynamic_tools.rs index 184bc64ec1..5c20f96685 100644 --- a/codex-rs/tui/src/dynamic_tools.rs +++ b/codex-rs/tui/src/dynamic_tools.rs @@ -5,6 +5,7 @@ use crate::app_event_sender::AppEventSender; use codex_app_server_client::AppServerRequestHandle; use codex_app_server_client::TypedRequestError; use codex_app_server_protocol::Account; +use codex_app_server_protocol::AudioReference; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::DynamicToolCallOutputContentItem; use codex_app_server_protocol::DynamicToolCallParams; @@ -1385,7 +1386,10 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu .. } => json!({"type": "image", "fileId": file_id}), UserInput::LocalImage { path, .. } => json!({"type": "localImage", "path": path}), - UserInput::Audio { url } => json!({"type": "audio", "url": url}), + UserInput::Audio { audio: AudioReference::Inline { url } } => json!({"type": "audio", "url": url}), + UserInput::Audio { + audio: AudioReference::File { file_id, .. }, + } => json!({"type": "audio", "fileId": file_id}), UserInput::LocalAudio { path } => json!({"type": "localAudio", "path": path}), UserInput::Skill { name, path } => json!({"type": "skill", "name": name, "path": path}), UserInput::Mention { name, path } => json!({"type": "mention", "name": name, "path": path}), diff --git a/codex-rs/utils/audio/src/audio_preparation_tests.rs b/codex-rs/utils/audio/src/audio_preparation_tests.rs index c1849bcb8d..c277338a26 100644 --- a/codex-rs/utils/audio/src/audio_preparation_tests.rs +++ b/codex-rs/utils/audio/src/audio_preparation_tests.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::AudioReference; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; use pretty_assertions::assert_eq; @@ -13,13 +14,24 @@ fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() { role: "user".to_string(), content: vec![ ContentItem::InputAudio { - audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(), + audio: AudioReference::File { + file_id: "file_audio".into(), + }, }, ContentItem::InputAudio { - audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(), + }, }, ContentItem::InputAudio { - audio_url: "https://example.com/audio.mp3".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + }, + }, + ContentItem::InputAudio { + audio: AudioReference::Inline { + audio_url: "https://example.com/audio.mp3".to_string(), + }, }, ], phase: None, @@ -37,10 +49,19 @@ fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() { role: "user".to_string(), content: vec![ ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + audio: AudioReference::File { + file_id: "file_audio".into(), + } }, ContentItem::InputAudio { - audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string() + } + }, + ContentItem::InputAudio { + audio: AudioReference::Inline { + audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string() + } }, ContentItem::InputText { text: "audio content omitted because it could not be processed".to_string(), @@ -61,10 +82,14 @@ fn preparation_replaces_invalid_message_audio_with_placeholders() { role: "user".to_string(), content: vec![ ContentItem::InputAudio { - audio_url: "data:audio/wav;base64,%%%".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/wav;base64,%%%".to_string(), + }, }, ContentItem::InputAudio { - audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(), + audio: AudioReference::Inline { + audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(), + }, }, ], phase: None, diff --git a/codex-rs/utils/audio/src/lib.rs b/codex-rs/utils/audio/src/lib.rs index e93c6e831d..791fefe610 100644 --- a/codex-rs/utils/audio/src/lib.rs +++ b/codex-rs/utils/audio/src/lib.rs @@ -2,6 +2,7 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES; @@ -91,7 +92,9 @@ pub fn prepare_response_items(items: &mut [ResponseItem]) { fn prepare_message_content(items: &mut [ContentItem]) { for item in items { - if let ContentItem::InputAudio { audio_url } = item + if let ContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } = item && let Err(error) = prepare_audio(audio_url) { warn!(%error, "failed to prepare message audio"); diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 1adac41edc..827724f9fe 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -605,7 +605,7 @@ def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None: + (["--formatters", "ruff-format"] if format_output else []), cwd=sdk_root() if format_output else Path(td), ) - _preserve_inline_image_class_names(out_path) + _preserve_inline_media_class_names(out_path) _add_legacy_image_generation_thread_item_alias(out_path) _preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True) _preserve_reasoning_effort_enum(out_path) @@ -614,13 +614,15 @@ def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None: _normalize_generated_timestamps(out_path) -def _preserve_inline_image_class_names(out_path: Path) -> None: - """Keep the public class names used before ImageReference was introduced.""" +def _preserve_inline_media_class_names(out_path: Path) -> None: + """Keep the public inline class names used before file references were introduced.""" source = out_path.read_text() stable_names = { "UrlUserInput": "ImageUserInput", "ImageUrlContentItem": "InputImageContentItem", "ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem", + "UrlUserInput1": "AudioUserInput", + "AudioUrlContentItem": "InputAudioContentItem", } for generated_name, stable_name in stable_names.items(): if source.count(f"class {generated_name}(") != 1: diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 531fed7eb7..e67b77c99a 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -1101,8 +1101,16 @@ class InputAudioContentItem(BaseModel): model_config = ConfigDict( populate_by_name=True, ) - audio_url: str type: Annotated[Literal["input_audio"], Field(title="InputAudioContentItemType")] + audio_url: str + + +class FileIdContentItem1(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Annotated[Literal["input_audio"], Field(title="InputAudioContentItemType")] + file_id: str class OutputTextContentItem(BaseModel): @@ -6762,6 +6770,14 @@ class AudioUserInput(BaseModel): url: str +class FileIdUserInput1(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Annotated[Literal["audio"], Field(title="AudioUserInputType")] + file_id: Annotated[str, Field(alias="fileId")] + + class LocalAudioUserInput(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -6795,6 +6811,7 @@ class UserInput( | FileIdUserInput | LocalImageUserInput | AudioUserInput + | FileIdUserInput1 | LocalAudioUserInput | SkillUserInput | MentionUserInput @@ -6806,6 +6823,7 @@ class UserInput( | FileIdUserInput | LocalImageUserInput | AudioUserInput + | FileIdUserInput1 | LocalAudioUserInput | SkillUserInput | MentionUserInput @@ -8375,6 +8393,7 @@ class ContentItem( | InputImageContentItem | FileIdContentItem | InputAudioContentItem + | FileIdContentItem1 | OutputTextContentItem ] ): @@ -8383,6 +8402,7 @@ class ContentItem( | InputImageContentItem | FileIdContentItem | InputAudioContentItem + | FileIdContentItem1 | OutputTextContentItem ) diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index 4880b2e24a..740262bbc7 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -555,6 +555,28 @@ def test_generated_inline_image_class_names_remain_stable() -> None: ) +def test_generated_audio_inputs_preserve_inline_and_file_shapes() -> None: + """Existing audio imports still work, and file references round-trip through the unions.""" + from openai_codex.generated.v2_all import ( + AudioUserInput, + ContentItem, + FileIdUserInput1, + InputAudioContentItem, + UserInput, + ) + + audio_url = "data:audio/wav;base64,YXVkaW8=" + for model, wire in [ + (AudioUserInput, {"type": "audio", "url": audio_url}), + (InputAudioContentItem, {"type": "input_audio", "audio_url": audio_url}), + (UserInput, {"type": "audio", "fileId": "file_audio"}), + (FileIdUserInput1, {"type": "audio", "fileId": "file_audio"}), + (ContentItem, {"type": "input_audio", "file_id": "file_audio"}), + ]: + audio = model.model_validate(wire) + assert audio.model_dump(by_alias=True, exclude_unset=True, mode="json") == wire + + def test_generated_image_generation_thread_item_alias_remains_available() -> None: """Existing SDK callers can still construct and validate inline image-generation items.""" from openai_codex.generated.v2_all import (