diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 99333c4ba5..e8502e4d5d 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1613,10 +1613,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -1626,7 +1650,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b4220658bb..52198ba6b5 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1880,10 +1880,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -1893,7 +1917,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 0aebc0f619..0e44ea9dea 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -13178,10 +13178,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -13191,7 +13215,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 12f2594644..89b9537f0b 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -8959,10 +8959,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -8972,7 +8996,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 18590c6053..7d82adcdc2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -463,10 +463,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -476,7 +500,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 03fd67148d..ad9ddd5b05 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -463,10 +463,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -476,7 +500,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index a5448c8fdd..8e495936e1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -299,10 +299,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -312,7 +336,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 7859f862eb..6a027403d1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -620,10 +620,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -633,7 +657,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index a2c3896287..d0cb7ef6ad 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -701,10 +701,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -714,7 +738,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index ccc37db027..5ab4f7ed23 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -463,10 +463,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -476,7 +500,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 1902e4c963..3f7b745ce1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index 8478f0c2fe..b4cef43629 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 9005c3dc67..483f0ad846 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index b591294f48..30600284bd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -361,10 +361,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -374,7 +398,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index eceddd4711..4818843ecd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -717,10 +717,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -730,7 +754,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index e8dc720fe4..b39cff7aac 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index eb28767305..d0b7314bac 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -701,10 +701,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -714,7 +738,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 6c0623606c..c3e1ed1ba6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index 37b21c0367..eb02b3a2fd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -620,10 +620,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -633,7 +657,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 897c7fa8a5..5b8cdb419b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -623,10 +623,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -636,7 +660,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 7ddc02f7ba..5d900b0140 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -620,10 +620,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -633,7 +657,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index d299d318de..488c18cf2d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -225,10 +225,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -238,7 +262,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 2d10343bc1..c6561a7f6d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -620,10 +620,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -633,7 +657,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 5f891ee76f..1f9a38a30a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -620,10 +620,34 @@ "type": "object" }, { - "properties": { - "audio_url": { - "type": "string" + "anyOf": [ + { + "properties": { + "audio_url": { + "type": "string" + } + }, + "required": [ + "audio_url" + ], + "title": "AudioUrlFunctionCallOutputContentItem", + "type": "object" }, + { + "properties": { + "file_id": { + "type": "string" + } + }, + "required": [ + "file_id" + ], + "title": "FileIdFunctionCallOutputContentItem", + "type": "object" + } + ], + "description": "Inline audio data or an uploaded file loaded by the Responses API.", + "properties": { "type": { "enum": [ "input_audio" @@ -633,7 +657,6 @@ } }, "required": [ - "audio_url", "type" ], "title": "InputAudioFunctionCallOutputContentItem", diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index d4060b44db..8052517fed 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index a75004d9b7..8e995c180a 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts index 5fa3d63afe..80d237bed7 100644 --- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts @@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail"; * Responses API compatible content items that can be returned by a tool call. * This is a subset of ContentItem with the types we support as function call outputs. */ -export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; +export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", } & ({ audio_url: string, } | { file_id: string, }) | { "type": "encrypted_content", encrypted_content: string, }; diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 0875bedc17..2e95267df3 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -721,9 +721,7 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: INLINE_AUDIO_DATA_URL.to_string(), - }, + FunctionCallOutputContentItem::inline_audio(INLINE_AUDIO_DATA_URL.to_string()), ]; let response = DynamicToolCallResponse { content_items: response_content_items, diff --git a/codex-rs/code-mode-runtime/src/output/output_render.rs b/codex-rs/code-mode-runtime/src/output/output_render.rs index 0c506f522d..9a15ea30c0 100644 --- a/codex-rs/code-mode-runtime/src/output/output_render.rs +++ b/codex-rs/code-mode-runtime/src/output/output_render.rs @@ -3,10 +3,11 @@ use codex_code_mode_protocol::DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL; use codex_code_mode_protocol::FunctionCallOutputContentItem as OutputItem; use codex_code_mode_protocol::ImageDetail; +use codex_protocol::models::AudioReference; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail as ProtocolImageDetail; use codex_protocol::models::ImageReference; -use codex_utils_audio::estimate_audio_token_count_without_cache; +use codex_utils_audio::estimate_tool_audio_tokens_without_cache; use codex_utils_output_truncation::OutputBoundary; use codex_utils_output_truncation::formatted_output_at_token_boundaries; use codex_utils_output_truncation::mixed_output_at_token_boundaries; @@ -54,7 +55,7 @@ pub(crate) fn render_output( }); } OutputItem::InputAudio { audio_url } => { - content.push(FunctionCallOutputContentItem::InputAudio { audio_url }); + content.push(FunctionCallOutputContentItem::inline_audio(audio_url)); } } } @@ -68,14 +69,14 @@ pub(crate) fn render_output( mixed_output_at_token_boundaries( content, max_tokens, - estimate_audio_token_count_without_cache, + estimate_tool_audio_tokens_without_cache, &mut boundaries, ) } else { formatted_output_at_token_boundaries( content, max_tokens, - estimate_audio_token_count_without_cache, + estimate_tool_audio_tokens_without_cache, &mut boundaries, ) }; @@ -180,10 +181,13 @@ fn into_code_mode_output(item: FunctionCallOutputContentItem) -> OutputItem { ProtocolImageDetail::Original => ImageDetail::Original, }), }, - FunctionCallOutputContentItem::InputAudio { audio_url } => { - OutputItem::InputAudio { audio_url } - } + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } => OutputItem::InputAudio { audio_url }, FunctionCallOutputContentItem::EncryptedContent { .. } + | FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::File { .. }, + } | FunctionCallOutputContentItem::InputImage { image: ImageReference::File { .. }, .. diff --git a/codex-rs/code-mode-runtime/src/runtime/value.rs b/codex-rs/code-mode-runtime/src/runtime/value.rs index 6c93ad9cef..3ad82eb7be 100644 --- a/codex-rs/code-mode-runtime/src/runtime/value.rs +++ b/codex-rs/code-mode-runtime/src/runtime/value.rs @@ -3,6 +3,7 @@ use serde_json::Value as JsonValue; use crate::session_runtime::OutputItem; use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL; use codex_code_mode_protocol::ImageDetail; +use codex_protocol::models::AudioReference; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageReference; @@ -58,7 +59,12 @@ fn mcp_output_item(block: &JsonValue) -> Result { image: ImageReference::Inline { image_url }, detail, } => Ok(OutputItem::InputImage { image_url, detail }), - FunctionCallOutputContentItem::InputAudio { audio_url } => Ok(output_audio_item(audio_url)), + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } => Ok(output_audio_item(audio_url)), + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::File { .. }, + } => Err("content expects inline MCP audio data".to_string()), FunctionCallOutputContentItem::EncryptedContent { .. } => { Err("content cannot emit encrypted MCP text".to_string()) } diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 2ec53d71e9..e6e8a0e27a 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -70,7 +70,7 @@ use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::WorldStateItem; -use codex_utils_audio::estimate_audio_token_count; +use codex_utils_audio::estimate_tool_audio_tokens; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; use codex_utils_output_truncation::TruncationPolicy; @@ -677,7 +677,7 @@ impl ContextManager { .and_then(|metadata| metadata.history_truncation_token_limit) .map(TruncationPolicy::Tokens) .unwrap_or_else(|| with_serialization_allowance(policy)); - truncate_function_output_payload(output, policy, estimate_audio_token_count); + truncate_function_output_payload(output, policy, estimate_tool_audio_tokens); } if let Some(review_history) = &mut self.review_history && !is_guardian_context_message(item) @@ -1411,7 +1411,7 @@ fn estimate_response_item_model_visible_bytes( namespace, output, .. - } => estimate_function_output_bytes(&output.body) + } => estimate_function_output_bytes(&output.body, audio_durations) .saturating_add(text_bytes(call_id.as_deref().unwrap_or_default())) .saturating_add(text_bytes(name.as_deref().unwrap_or_default())) .saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())), @@ -1420,7 +1420,7 @@ fn estimate_response_item_model_visible_bytes( name, output, .. - } => estimate_function_output_bytes(&output.body) + } => estimate_function_output_bytes(&output.body, audio_durations) .saturating_add(text_bytes(call_id)) .saturating_add(text_bytes(name.as_deref().unwrap_or_default())), // These payloads are themselves JSON arguments, rather than transport envelopes @@ -1562,27 +1562,21 @@ pub(crate) fn estimate_image_reference_bytes( } } -fn estimate_audio_bytes(audio_url: &str) -> i64 { - i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count( - audio_url, - ))) - .unwrap_or(i64::MAX) -} - fn estimate_audio_reference_bytes( audio: &AudioReference, audio_durations: &FileAudioDurations, ) -> i64 { - match audio { - AudioReference::Inline { audio_url } => estimate_audio_bytes(audio_url), - AudioReference::File { file_id } => { - i64::try_from(approx_bytes_for_tokens(audio_durations.tokens(file_id))) - .unwrap_or(i64::MAX) - } - } + let tokens = match audio { + AudioReference::Inline { .. } => estimate_tool_audio_tokens(audio), + AudioReference::File { file_id } => audio_durations.tokens(file_id), + }; + i64::try_from(approx_bytes_for_tokens(tokens)).unwrap_or(i64::MAX) } -fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { +fn estimate_function_output_bytes( + output: &FunctionCallOutputBody, + audio_durations: &FileAudioDurations, +) -> i64 { match output { FunctionCallOutputBody::Text(text) => text_bytes(text), FunctionCallOutputBody::ContentItems(items) => items @@ -1592,8 +1586,8 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { FunctionCallOutputContentItem::InputImage { image, detail } => { estimate_image_reference_bytes(image, *detail) } - FunctionCallOutputContentItem::InputAudio { audio_url } => { - estimate_audio_bytes(audio_url) + FunctionCallOutputContentItem::InputAudio { audio } => { + estimate_audio_reference_bytes(audio, audio_durations) } FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { i64::try_from(estimate_encrypted_function_output_length( diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 334a764d00..5aff78bb7f 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1563,9 +1563,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/mpeg;base64,YXVkaW8=".to_string(), + ), ]), internal_chat_message_metadata_passthrough: None, }, @@ -1593,9 +1593,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/ogg;base64,YXVkaW8=".to_string(), + ), ]), internal_chat_message_metadata_passthrough: None, }, @@ -3536,7 +3536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { name: None, namespace: None, output: FunctionCallOutputPayload::from_content_items(vec![ - FunctionCallOutputContentItem::InputAudio { audio_url }, + FunctionCallOutputContentItem::inline_audio(audio_url), ]), internal_chat_message_metadata_passthrough: None, }; @@ -3557,7 +3557,7 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { call_id: "call-custom-audio".to_string(), name: None, output: FunctionCallOutputPayload::from_content_items(vec![ - FunctionCallOutputContentItem::InputAudio { audio_url }, + FunctionCallOutputContentItem::inline_audio(audio_url), ]), internal_chat_message_metadata_passthrough: None, }; @@ -3603,7 +3603,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() { namespace: None, output: FunctionCallOutputPayload { body: FunctionCallOutputBody::ContentItems(vec![ - FunctionCallOutputContentItem::InputAudio { audio_url }, + FunctionCallOutputContentItem::inline_audio(audio_url), ]), success: Some(true), }, diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rs index dab028d7c3..0d2fed11db 100644 --- a/codex-rs/core/src/session/inject.rs +++ b/codex-rs/core/src/session/inject.rs @@ -6,23 +6,12 @@ use codex_features::Feature; use codex_history::ResponseItemEnvelope; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; -use codex_protocol::models::AudioReference; -use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; /// Uploaded files need the admission checks on typed turn input before entering history. pub(crate) fn validate_injected_response_item(item: &ResponseItem) -> CodexResult<()> { - if let ResponseItem::Message { content, .. } = item - && content.iter().any(|part| { - matches!( - part, - ContentItem::InputAudio { - audio: AudioReference::File { .. } - } - ) - }) - { + if item.file_audio_ids().next().is_some() { return Err(CodexErr::InvalidRequest( "Uploaded audio files must be submitted as turn input.".to_string(), )); diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index 0dc3655e2b..fe59c484e6 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -45,7 +45,7 @@ use crate::tools::router::ToolCallSource; use crate::unified_exec::resolve_max_tokens; use codex_protocol::openai_models::ToolMode; use codex_tools::ToolName; -use codex_utils_audio::estimate_audio_token_count; +use codex_utils_audio::estimate_tool_audio_tokens_without_cache; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::formatted_truncate_function_output_items_with_policy; @@ -331,7 +331,11 @@ fn truncate_code_mode_result( ) -> Vec { let max_output_tokens = resolve_max_tokens(max_output_tokens); let policy = TruncationPolicy::Tokens(max_output_tokens); - formatted_truncate_function_output_items_with_policy(items, policy, estimate_audio_token_count) + formatted_truncate_function_output_items_with_policy( + items, + policy, + estimate_tool_audio_tokens_without_cache, + ) } // Submit synchronously so the recorder sees the call before the cell's dispatch gate closes. @@ -568,9 +572,10 @@ mod tests { #[test] fn over_budget_audio_output_is_omitted() { - let items = vec![FunctionCallOutputContentItem::InputAudio { - audio_url: format!("data:audio/wav;base64,{}", "A".repeat(100)), - }]; + let items = vec![FunctionCallOutputContentItem::inline_audio(format!( + "data:audio/wav;base64,{}", + "A".repeat(100) + ))]; assert_eq!( truncate_code_mode_result(items, Some(5)), diff --git a/codex-rs/core/src/tools/code_mode/output_tests.rs b/codex-rs/core/src/tools/code_mode/output_tests.rs index 16c5916c42..b0d7524fb7 100644 --- a/codex-rs/core/src/tools/code_mode/output_tests.rs +++ b/codex-rs/core/src/tools/code_mode/output_tests.rs @@ -16,9 +16,7 @@ fn completed_timing_preserves_content_and_distinguishes_zero_from_unavailable() }, detail: None, }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,audio".to_string(), - }, + FunctionCallOutputContentItem::inline_audio("data:audio/wav;base64,audio".to_string()), ]; for (host_duration, expected_timing) in [ ( diff --git a/codex-rs/core/src/tools/code_mode/response_adapter.rs b/codex-rs/core/src/tools/code_mode/response_adapter.rs index e23ed354f6..37603f2c73 100644 --- a/codex-rs/core/src/tools/code_mode/response_adapter.rs +++ b/codex-rs/core/src/tools/code_mode/response_adapter.rs @@ -28,7 +28,7 @@ impl IntoProtocol } } codex_code_mode::FunctionCallOutputContentItem::InputAudio { audio_url } => { - FunctionCallOutputContentItem::InputAudio { audio_url } + FunctionCallOutputContentItem::inline_audio(audio_url) } } } diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index 252ca66b74..4460563e66 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -16,7 +16,7 @@ use codex_protocol::models::ResponseItem; use codex_protocol::models::function_call_output_content_items_to_text; use codex_tools::LoadableToolSpec; use codex_tools::ToolName; -use codex_utils_audio::estimate_audio_token_count; +use codex_utils_audio::estimate_tool_audio_tokens; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_function_output_payload; use codex_utils_output_truncation::with_serialization_allowance; @@ -252,7 +252,7 @@ impl McpToolOutput { truncate_function_output_payload( &mut payload, with_serialization_allowance(self.truncation_policy), - estimate_audio_token_count, + estimate_tool_audio_tokens, ); payload } diff --git a/codex-rs/core/tests/suite/scenarios_preparation.rs b/codex-rs/core/tests/suite/scenarios_preparation.rs index 3c1372a4e1..cf7f7294a3 100644 --- a/codex-rs/core/tests/suite/scenarios_preparation.rs +++ b/codex-rs/core/tests/suite/scenarios_preparation.rs @@ -1,22 +1,28 @@ //! Verifies prepared context reaches both the first model request and a follow-up. use super::*; +use anyhow::Context as _; use codex_protocol::error::CodexErrorDetails; use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::turn_input::AnnotatedResponseItem; use pretty_assertions::assert_eq; use test_case::test_case; +#[test_case("message")] +#[test_case("function_call_output")] +#[test_case("custom_tool_call_output")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Result<()> { +async fn raw_injection_rejects_uploaded_audio_before_modifying_history(kind: &str) -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; let test = test_codex().build_with_auto_env(&server).await?; let before = test.codex.conversation_history_snapshot().await; - let audio: ResponseItem = serde_json::from_value(json!({ - "type": "message", "role": "developer", - "content": [{"type": "input_audio", "file_id": "file_injected"}] - }))?; + let parts = json!([{"type": "input_audio", "file_id": "file_injected"}]); + let audio: ResponseItem = serde_json::from_value(if kind == "message" { + json!({"type": kind, "role": "developer", "content": parts}) + } else { + json!({"type": kind, "call_id": "untrusted", "output": parts}) + })?; let batch = vec![ core_test_support::responses::user_message_item("must not be partially written"), audio.clone(), @@ -26,12 +32,14 @@ async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Resu .codex .inject_response_items(batch.clone()) .await - .unwrap_err(); + .err() + .context("raw history accepted a caller-supplied audio file")?; let preparation_error = test .codex .inject_response_items_for_turn(batch) .await - .unwrap_err(); + .err() + .context("prepared history accepted a caller-supplied audio file")?; for error in [history_error, preparation_error] { assert!(matches!( error.details(), @@ -49,7 +57,8 @@ async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Resu .codex .start_or_steer_turn(TurnInputRequest::new(input)) .await - .unwrap_err(); + .err() + .context("turn input accepted an untyped audio file")?; assert!(matches!( error.details(), CodexErrorDetails::InvalidRequest(_) diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs index ca96150e17..d91d6832c5 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs @@ -1216,9 +1216,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() { }, detail: None, }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,tool-audio".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav;base64,tool-audio".to_string(), + ), ]), internal_chat_message_metadata_passthrough: None, }, diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 0fed5862a7..13f1eb8a1e 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1368,6 +1368,31 @@ pub enum ResponseItem { } impl ResponseItem { + /// File-backed audio in messages and tool results, in request order. + pub fn file_audio_ids(&self) -> impl Iterator + Clone { + let (message, tool): (&[ContentItem], &[FunctionCallOutputContentItem]) = match self { + Self::Message { content, .. } => (content, &[]), + Self::FunctionCallOutput { output, .. } | Self::CustomToolCallOutput { output, .. } => { + (&[], output.content_items().unwrap_or_default()) + } + _ => (&[], &[]), + }; + message + .iter() + .filter_map(|part| match part { + ContentItem::InputAudio { + audio: AudioReference::File { file_id }, + } => Some(file_id), + _ => None, + }) + .chain(tool.iter().filter_map(|part| match part { + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::File { file_id }, + } => Some(file_id), + _ => None, + })) + } + /// Clears the top-level `encrypted_content` fields. pub fn clear_encrypted_content(&mut self) { match self { @@ -2329,13 +2354,23 @@ pub enum FunctionCallOutputContentItem { }, // Do not rename, these are serialized and used directly in the responses API. InputAudio { - audio_url: String, + #[serde(flatten)] + audio: AudioReference, }, EncryptedContent { encrypted_content: String, }, } +impl FunctionCallOutputContentItem { + /// Audio supplied by an MCP server or Code Mode, before host attachment preparation. + pub fn inline_audio(audio_url: String) -> Self { + Self::InputAudio { + audio: AudioReference::Inline { audio_url }, + } + } +} + /// Converts message content to tool output, preserving media and image detail. /// Output text becomes input text without its sampling annotations or logprobs. impl From for FunctionCallOutputContentItem { @@ -2345,12 +2380,7 @@ impl From for FunctionCallOutputContentItem { Self::InputText { text } } ContentItem::InputImage { image, detail } => Self::InputImage { image, detail }, - ContentItem::InputAudio { - audio: AudioReference::Inline { audio_url }, - } => Self::InputAudio { audio_url }, - ContentItem::InputAudio { - audio: AudioReference::File { file_id, .. }, - } => Self::InputText { text: file_id }, + ContentItem::InputAudio { audio } => Self::InputAudio { audio }, } } } @@ -2404,7 +2434,7 @@ impl From } } crate::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { audio_url } => { - Self::InputAudio { audio_url } + Self::inline_audio(audio_url) } } } @@ -3235,6 +3265,17 @@ mod tests { ); } + #[test] + fn tool_audio_references_round_trip_in_the_responses_wire_shape() { + for wire in [ + serde_json::json!({"type":"input_audio","audio_url":"data:audio/wav;base64,QQ=="}), + serde_json::json!({"type":"input_audio","file_id":"file-clip"}), + ] { + let part: FunctionCallOutputContentItem = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(part).unwrap(), wire); + } + } + #[test] fn convert_mcp_audio_content_builds_data_urls_and_preserves_existing_data_urls() { let contents = vec![ @@ -3254,12 +3295,12 @@ mod tests { assert_eq!( convert_mcp_content_to_items(&contents), vec![ - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,Zm9v".to_string(), - }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/ogg;base64,YmFy".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav;base64,Zm9v".to_string() + ), + FunctionCallOutputContentItem::inline_audio( + "data:audio/ogg;base64,YmFy".to_string() + ), ] ); } @@ -3312,9 +3353,7 @@ mod tests { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,AAA".to_string(), - }, + FunctionCallOutputContentItem::inline_audio("data:audio/wav;base64,AAA".to_string()), FunctionCallOutputContentItem::EncryptedContent { encrypted_content: "enc_opaque".to_string(), }, @@ -3659,9 +3698,9 @@ mod tests { FunctionCallOutputContentItem::InputText { text: "caption".into(), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,BASE64".into(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav;base64,BASE64".into() + ), ]), success: Some(true), } diff --git a/codex-rs/protocol/src/models/mcp_content.rs b/codex-rs/protocol/src/models/mcp_content.rs index 5038db4744..8b78812a44 100644 --- a/codex-rs/protocol/src/models/mcp_content.rs +++ b/codex-rs/protocol/src/models/mcp_content.rs @@ -78,9 +78,7 @@ impl FunctionCallOutputContentItem { .unwrap_or(DEFAULT_IMAGE_DETAIL), ), }, - McpContent::Audio(media) => Self::InputAudio { - audio_url: media.data_url(), - }, + McpContent::Audio(media) => Self::inline_audio(media.data_url()), McpContent::ResourceLink { .. } | McpContent::Resource { .. } => Self::InputText { text: serde_json::to_string(block)?, }, diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index 3a4c1e87c8..01481a1eaf 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -1,3 +1,4 @@ +use codex_protocol::models::AudioReference; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; @@ -268,11 +269,12 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> image: ImageReference::File { file_id }, .. } if !file_id.trim().is_empty() => Some(file_id.clone()), - FunctionCallOutputContentItem::InputAudio { audio_url } - if !audio_url.trim().is_empty() => - { - Some(audio_url.clone()) - } + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } if !audio_url.trim().is_empty() => Some(audio_url.clone()), + FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::File { file_id }, + } if !file_id.trim().is_empty() => Some(file_id.clone()), FunctionCallOutputContentItem::InputText { .. } | FunctionCallOutputContentItem::InputImage { .. } | FunctionCallOutputContentItem::InputAudio { .. } diff --git a/codex-rs/utils/audio/src/audio_preparation_tests.rs b/codex-rs/utils/audio/src/audio_preparation_tests.rs index cbc0d5b4c7..07e609ea9f 100644 --- a/codex-rs/utils/audio/src/audio_preparation_tests.rs +++ b/codex-rs/utils/audio/src/audio_preparation_tests.rs @@ -131,12 +131,12 @@ fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() { FunctionCallOutputContentItem::InputText { text: "before".to_string(), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), - }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav,not-base64".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav;base64,YXVkaW8=".to_string(), + ), + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav,not-base64".to_string(), + ), ]), success: Some(true), }, @@ -157,9 +157,9 @@ fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() { FunctionCallOutputContentItem::InputText { text: "before".to_string(), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), - }, + FunctionCallOutputContentItem::inline_audio( + "data:audio/wav;base64,YXVkaW8=".to_string() + ), FunctionCallOutputContentItem::InputText { text: "audio content omitted because it could not be processed".to_string(), }, diff --git a/codex-rs/utils/audio/src/lib.rs b/codex-rs/utils/audio/src/lib.rs index fa58098260..edd626500a 100644 --- a/codex-rs/utils/audio/src/lib.rs +++ b/codex-rs/utils/audio/src/lib.rs @@ -107,7 +107,9 @@ fn prepare_message_content(items: &mut [ContentItem]) { fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { for item in items { - if let FunctionCallOutputContentItem::InputAudio { audio_url } = item + if let FunctionCallOutputContentItem::InputAudio { + audio: AudioReference::Inline { audio_url }, + } = item && let Err(error) = prepare_audio(audio_url) { warn!(%error, "failed to prepare tool output audio"); @@ -160,8 +162,28 @@ pub fn estimate_file_audio_tokens(duration: Option) -> usiz usize::try_from(tokens).unwrap_or(usize::MAX) } +/// Estimate tool audio on Core's multi-threaded runtime. +pub fn estimate_tool_audio_tokens(audio: &AudioReference) -> usize { + estimate_tool_audio_tokens_with(audio, estimate_audio_token_count) +} + +/// Code Mode can render on a single thread, where blocking cache access is unavailable. +pub fn estimate_tool_audio_tokens_without_cache(audio: &AudioReference) -> usize { + estimate_tool_audio_tokens_with(audio, estimate_audio_token_count_without_cache) +} + +fn estimate_tool_audio_tokens_with( + audio: &AudioReference, + inline_cost: impl FnOnce(&str) -> usize, +) -> usize { + match audio { + AudioReference::Inline { audio_url } => inline_cost(audio_url), + AudioReference::File { .. } => estimate_file_audio_tokens(/*duration*/ None), + } +} + /// Estimates audio tokens from decoded duration, falling back to the data URL size. -pub fn estimate_audio_token_count(audio_url: &str) -> usize { +fn estimate_audio_token_count(audio_url: &str) -> usize { let key = sha1_digest(audio_url.as_bytes()); AUDIO_TOKEN_ESTIMATE_CACHE .get_or_init(key, || estimate_audio_token_count_without_cache(audio_url)) @@ -169,7 +191,7 @@ pub fn estimate_audio_token_count(audio_url: &str) -> usize { /// The same duration-based estimate without accessing the Tokio-dependent cache. /// Suitable for bounded output rendered on a current-thread runtime. -pub fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize { +fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize { let Some(duration_seconds) = audio_duration_seconds(audio_url) else { return approx_token_count(audio_url); }; diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index 6f313059c8..b0dcbfdd11 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use codex_protocol::mcp::CallToolResult; use codex_protocol::mcp::plain_output_text; +use codex_protocol::models::AudioReference; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; @@ -137,7 +138,7 @@ pub fn truncate_mcp_tool_result( pub fn truncate_function_output_payload( output: &mut FunctionCallOutputPayload, policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, ) { match &mut output.body { FunctionCallOutputBody::Text(text) => *text = truncate_text(text, policy), @@ -172,7 +173,7 @@ pub struct OutputBoundary { pub fn formatted_truncate_function_output_items_with_policy<'a>( items: impl Into>, policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, ) -> Vec { formatted_output(items.into(), policy, estimate_audio_token_count, &mut []) } @@ -182,7 +183,7 @@ pub fn formatted_truncate_function_output_items_with_policy<'a>( pub fn formatted_output_at_token_boundaries<'a>( items: impl Into>, max_tokens: usize, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, boundaries: &mut [OutputBoundary], ) -> Vec { let items = items.into(); @@ -198,7 +199,7 @@ pub fn formatted_output_at_token_boundaries<'a>( fn formatted_output( items: Cow<'_, [FunctionCallOutputContentItem]>, policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, boundaries: &mut [OutputBoundary], ) -> Vec { let mut combined_len = 0; @@ -242,7 +243,7 @@ fn formatted_output( pub fn truncate_function_output_items_with_policy( items: &[FunctionCallOutputContentItem], policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, ) -> Vec { truncate_output_items( Cow::Borrowed(items), @@ -257,7 +258,7 @@ pub fn truncate_function_output_items_with_policy( pub fn mixed_output_at_token_boundaries<'a>( items: impl Into>, max_tokens: usize, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, boundaries: &mut [OutputBoundary], ) -> Vec { let items = items.into(); @@ -291,7 +292,7 @@ fn normalize_boundaries( fn truncate_output_items( items: Cow<'_, [FunctionCallOutputContentItem]>, policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, boundaries: &mut [OutputBoundary], ) -> Vec { match items { @@ -313,7 +314,7 @@ fn truncate_output_items( fn truncate_output_iter<'a>( items: impl ExactSizeIterator>, policy: TruncationPolicy, - estimate_audio_token_count: impl Fn(&str) -> usize, + estimate_audio_token_count: impl Fn(&AudioReference) -> usize, boundaries: &mut [OutputBoundary], ) -> Vec { let mut out: Vec = Vec::with_capacity(items.len()); @@ -361,8 +362,8 @@ fn truncate_output_iter<'a>( } FunctionCallOutputContentItem::InputImage { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => true, - FunctionCallOutputContentItem::InputAudio { audio_url } => { - let token_cost = estimate_audio_token_count(audio_url); + FunctionCallOutputContentItem::InputAudio { audio } => { + let token_cost = estimate_audio_token_count(audio); let cost = match policy { TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(token_cost), TruncationPolicy::Tokens(_) => token_cost, diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index f3e1244b82..ddb230eb5c 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -10,6 +10,7 @@ use crate::truncate_function_output_items_with_policy; use crate::truncate_mcp_tool_result; use crate::truncate_text; use codex_protocol::mcp::CallToolResult; +use codex_protocol::models::AudioReference; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageReference; @@ -264,9 +265,7 @@ fn truncate_function_output_items_with_policy_discards_empty_text() { }, detail: Some(DEFAULT_IMAGE_DETAIL), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "audio:one".to_string(), - }, + FunctionCallOutputContentItem::inline_audio("audio:one".to_string()), FunctionCallOutputContentItem::EncryptedContent { encrypted_content: "enc_opaque".to_string(), }, @@ -314,9 +313,7 @@ fn formatted_text_output_preserves_empty_leading_text_behavior() { #[test] fn formatted_mixed_output_budgets_text_and_audio_in_order() { - let audio = FunctionCallOutputContentItem::InputAudio { - audio_url: "audio:one".to_owned(), - }; + let audio = FunctionCallOutputContentItem::inline_audio("audio:one".to_owned()); let file_image = FunctionCallOutputContentItem::InputImage { image: ImageReference::File { file_id: "file_123".to_owned(), @@ -378,9 +375,7 @@ fn truncate_function_output_items_with_policy_omits_audio_over_budget() { FunctionCallOutputContentItem::EncryptedContent { encrypted_content: "enc_opaque".to_string(), }, - FunctionCallOutputContentItem::InputAudio { - audio_url: "audio:one".to_string(), - }, + FunctionCallOutputContentItem::inline_audio("audio:one".to_string()), ]; let output = @@ -403,10 +398,30 @@ fn truncate_function_output_items_with_policy_omits_audio_over_budget() { } #[test] -fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() { +fn repeated_file_audio_and_text_share_one_output_budget() { let audio = FunctionCallOutputContentItem::InputAudio { - audio_url: "audio:one".to_string(), + audio: AudioReference::File { + file_id: "clip".into(), + }, }; + let items = [text("abcd"), audio.clone(), audio.clone()]; + // Text takes one token. The first reference takes two, leaving too little + // for a second copy of the same clip under either budget. + for policy in [TruncationPolicy::Tokens(4), TruncationPolicy::Bytes(16)] { + assert_eq!( + truncate_function_output_items_with_policy(&items, policy, |_| 2), + vec![ + text("abcd"), + audio.clone(), + text("[omitted 1 audio items ...]") + ], + ); + } +} + +#[test] +fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() { + let audio = FunctionCallOutputContentItem::inline_audio("audio:one".to_string()); let items = vec![ audio.clone(), FunctionCallOutputContentItem::InputText { @@ -477,7 +492,7 @@ fn inline_image(image_url: String) -> FunctionCallOutputContentItem { #[test] fn boundary_formatters_normalize_input_and_remap_truncated_text() { - let audio_cost: fn(&str) -> usize = |_| 1; + let audio_cost: fn(&AudioReference) -> usize = |_| 1; let items = [text("left-head"), text("right-tail")]; let max_tokens = 2; let mut boundary = boundaries([(0, 0), (0, 2), (1, 0), (1, 8), (2, 0)]); diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 827724f9fe..9c70b8fc11 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -623,6 +623,7 @@ def _preserve_inline_media_class_names(out_path: Path) -> None: "ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem", "UrlUserInput1": "AudioUserInput", "AudioUrlContentItem": "InputAudioContentItem", + "AudioUrlFunctionCallOutputContentItem": "InputAudioFunctionCallOutputContentItem", } for generated_name, stable_name in stable_names.items(): if source.count(f"class {generated_name}(") != 1: diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 8c9130283c..2297db30c2 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -1781,10 +1781,20 @@ class InputAudioFunctionCallOutputContentItem(BaseModel): model_config = ConfigDict( populate_by_name=True, ) - audio_url: str type: Annotated[ Literal["input_audio"], Field(title="InputAudioFunctionCallOutputContentItemType") ] + audio_url: str + + +class FileIdFunctionCallOutputContentItem1(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Annotated[ + Literal["input_audio"], Field(title="InputAudioFunctionCallOutputContentItemType") + ] + file_id: str class EncryptedContentFunctionCallOutputContentItem(BaseModel): @@ -8930,6 +8940,7 @@ class FunctionCallOutputContentItem( | InputImageFunctionCallOutputContentItem | FileIdFunctionCallOutputContentItem | InputAudioFunctionCallOutputContentItem + | FileIdFunctionCallOutputContentItem1 | EncryptedContentFunctionCallOutputContentItem ] ): @@ -8938,6 +8949,7 @@ class FunctionCallOutputContentItem( | InputImageFunctionCallOutputContentItem | FileIdFunctionCallOutputContentItem | InputAudioFunctionCallOutputContentItem + | FileIdFunctionCallOutputContentItem1 | EncryptedContentFunctionCallOutputContentItem, Field( description="Responses API compatible content items that can be returned by a tool call. This is a subset of ContentItem with the types we support as function call outputs." diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index 740262bbc7..c524d9e299 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -562,11 +562,13 @@ def test_generated_audio_inputs_preserve_inline_and_file_shapes() -> None: ContentItem, FileIdUserInput1, InputAudioContentItem, + InputAudioFunctionCallOutputContentItem, UserInput, ) audio_url = "data:audio/wav;base64,YXVkaW8=" for model, wire in [ + (InputAudioFunctionCallOutputContentItem, {"type": "input_audio", "audio_url": audio_url}), (AudioUserInput, {"type": "audio", "url": audio_url}), (InputAudioContentItem, {"type": "input_audio", "audio_url": audio_url}), (UserInput, {"type": "audio", "fileId": "file_audio"}),