openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-13 21:30:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preserve file-backed audio in tool outputs (#60735)

Browse Source 
## Why

Converting message content to tool output previously reduced file-backed audio to a text file ID, losing its audio type.

## What changed

- Allow `input_audio` tool output items to carry either `audio_url` or `file_id`, preserving both forms during content conversion.
- Account for file-backed tool audio in history estimates and output truncation, and prepare only inline audio data.
- Reject uploaded audio in raw function and custom tool output injection, extending the existing requirement to submit uploaded audio through typed turn input.
- Regenerate app-server schemas and TypeScript/Python types while preserving the Python inline audio class name.

## Testing

Add coverage for tool audio wire-format round trips, repeated file audio sharing text and audio output budgets, and rejection of raw uploaded audio injection across messages and tool outputs. Extend Python SDK coverage for the inline tool audio shape.

GitOrigin-RevId: f4704466e6151662d2c8b8dc3f5ca2ab2f5710a2
This commit is contained in:
Hung
2026-11-11 06:41:15 +00:00
committed by copyberry
parent ce6f7d9503
commit 9018f3a3b0
49 changed files with 882 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-protocol/schema/json/ClientRequest.json
Generated
+27 -4
View File
@@ -1613,10 +1613,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -1626,7 +1650,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/ServerNotification.json
Generated
+27 -4
View File
@@ -1880,10 +1880,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -1893,7 +1917,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
Generated
+27 -4
View File
@@ -13178,10 +13178,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -13191,7 +13215,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
Generated
+27 -4
View File
@@ -8959,10 +8959,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -8972,7 +8996,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
Generated
+27 -4
View File
@@ -463,10 +463,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -476,7 +500,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
Generated
+27 -4
View File
@@ -463,10 +463,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -476,7 +500,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
Generated
+27 -4
View File
@@ -299,10 +299,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -312,7 +336,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
Generated
+27 -4
View File
@@ -620,10 +620,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -633,7 +657,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
Generated
+27 -4
View File
@@ -701,10 +701,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -714,7 +738,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
Generated
+27 -4
View File
@@ -463,10 +463,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -476,7 +500,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
Generated
+27 -4
View File
@@ -361,10 +361,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -374,7 +398,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
Generated
+27 -4
View File
@@ -717,10 +717,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -730,7 +754,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
Generated
+27 -4
View File
@@ -701,10 +701,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -714,7 +738,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
Generated
+27 -4
View File
@@ -620,10 +620,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -633,7 +657,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
Generated
+27 -4
View File
@@ -623,10 +623,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -636,7 +660,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
Generated
+27 -4
View File
@@ -620,10 +620,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -633,7 +657,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
Generated
+27 -4
View File
@@ -225,10 +225,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -238,7 +262,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
Generated
+27 -4
View File
@@ -620,10 +620,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -633,7 +657,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
Generated
+27 -4
View File
@@ -620,10 +620,34 @@
"type": "object"
},
{
"properties": {
"audio_url": {
"type": "string"
"anyOf": [
{
"properties": {
"audio_url": {
"type": "string"
}
},
"required": [
"audio_url"
],
"title": "AudioUrlFunctionCallOutputContentItem",
"type": "object"
},
{
"properties": {
"file_id": {
"type": "string"
}
},
"required": [
"file_id"
],
"title": "FileIdFunctionCallOutputContentItem",
"type": "object"
}
],
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
"properties": {
"type": {
"enum": [
"input_audio"
@@ -633,7 +657,6 @@
}
},
"required": [
"audio_url",
"type"
],
"title": "InputAudioFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
Generated
+1 -1
View File
@@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail";
* Responses API compatible content items that can be returned by a tool call.
* This is a subset of ContentItem with the types we support as function call outputs.
*/
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", } & ({ audio_url: string, } | { file_id: string, }) | { "type": "encrypted_content", encrypted_content: string, };
codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
+1 -3
View File
@@ -721,9 +721,7 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
},
FunctionCallOutputContentItem::inline_audio(INLINE_AUDIO_DATA_URL.to_string()),
];
let response = DynamicToolCallResponse {
content_items: response_content_items,
codex-rs/code-mode-runtime/src/output/output_render.rs
+11 -7
View File
@@ -3,10 +3,11 @@
use codex_code_mode_protocol::DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL;
use codex_code_mode_protocol::FunctionCallOutputContentItem as OutputItem;
use codex_code_mode_protocol::ImageDetail;
use codex_protocol::models::AudioReference;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail as ProtocolImageDetail;
use codex_protocol::models::ImageReference;
use codex_utils_audio::estimate_audio_token_count_without_cache;
use codex_utils_audio::estimate_tool_audio_tokens_without_cache;
use codex_utils_output_truncation::OutputBoundary;
use codex_utils_output_truncation::formatted_output_at_token_boundaries;
use codex_utils_output_truncation::mixed_output_at_token_boundaries;
@@ -54,7 +55,7 @@ pub(crate) fn render_output(
});
}
OutputItem::InputAudio { audio_url } => {
content.push(FunctionCallOutputContentItem::InputAudio { audio_url });
content.push(FunctionCallOutputContentItem::inline_audio(audio_url));
}
}
}
@@ -68,14 +69,14 @@ pub(crate) fn render_output(
mixed_output_at_token_boundaries(
content,
max_tokens,
estimate_audio_token_count_without_cache,
estimate_tool_audio_tokens_without_cache,
&mut boundaries,
)
} else {
formatted_output_at_token_boundaries(
content,
max_tokens,
estimate_audio_token_count_without_cache,
estimate_tool_audio_tokens_without_cache,
&mut boundaries,
)
};
@@ -180,10 +181,13 @@ fn into_code_mode_output(item: FunctionCallOutputContentItem) -> OutputItem {
ProtocolImageDetail::Original => ImageDetail::Original,
}),
},
FunctionCallOutputContentItem::InputAudio { audio_url } => {
OutputItem::InputAudio { audio_url }
}
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} => OutputItem::InputAudio { audio_url },
FunctionCallOutputContentItem::EncryptedContent { .. }
| FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
}
| FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
codex-rs/code-mode-runtime/src/runtime/value.rs
+7 -1
View File
@@ -3,6 +3,7 @@ use serde_json::Value as JsonValue;
use crate::session_runtime::OutputItem;
use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL;
use codex_code_mode_protocol::ImageDetail;
use codex_protocol::models::AudioReference;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageReference;
@@ -58,7 +59,12 @@ fn mcp_output_item(block: &JsonValue) -> Result<OutputItem, String> {
image: ImageReference::Inline { image_url },
detail,
} => Ok(OutputItem::InputImage { image_url, detail }),
FunctionCallOutputContentItem::InputAudio { audio_url } => Ok(output_audio_item(audio_url)),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} => Ok(output_audio_item(audio_url)),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
} => Err("content expects inline MCP audio data".to_string()),
FunctionCallOutputContentItem::EncryptedContent { .. } => {
Err("content cannot emit encrypted MCP text".to_string())
}
codex-rs/core/src/context_manager/history.rs
+15 -21
View File
@@ -70,7 +70,7 @@ use codex_protocol::protocol::TokenUsageInfo;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::WorldStateItem;
use codex_utils_audio::estimate_audio_token_count;
use codex_utils_audio::estimate_tool_audio_tokens;
use codex_utils_cache::BlockingLruCache;
use codex_utils_cache::sha1_digest;
use codex_utils_output_truncation::TruncationPolicy;
@@ -677,7 +677,7 @@ impl ContextManager {
.and_then(|metadata| metadata.history_truncation_token_limit)
.map(TruncationPolicy::Tokens)
.unwrap_or_else(|| with_serialization_allowance(policy));
truncate_function_output_payload(output, policy, estimate_audio_token_count);
truncate_function_output_payload(output, policy, estimate_tool_audio_tokens);
}
if let Some(review_history) = &mut self.review_history
&& !is_guardian_context_message(item)
@@ -1411,7 +1411,7 @@ fn estimate_response_item_model_visible_bytes(
namespace,
output,
..
} => estimate_function_output_bytes(&output.body)
} => estimate_function_output_bytes(&output.body, audio_durations)
.saturating_add(text_bytes(call_id.as_deref().unwrap_or_default()))
.saturating_add(text_bytes(name.as_deref().unwrap_or_default()))
.saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())),
@@ -1420,7 +1420,7 @@ fn estimate_response_item_model_visible_bytes(
name,
output,
..
} => estimate_function_output_bytes(&output.body)
} => estimate_function_output_bytes(&output.body, audio_durations)
.saturating_add(text_bytes(call_id))
.saturating_add(text_bytes(name.as_deref().unwrap_or_default())),
// These payloads are themselves JSON arguments, rather than transport envelopes
@@ -1562,27 +1562,21 @@ pub(crate) fn estimate_image_reference_bytes(
}
}
fn estimate_audio_bytes(audio_url: &str) -> i64 {
i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
audio_url,
)))
.unwrap_or(i64::MAX)
}
fn estimate_audio_reference_bytes(
audio: &AudioReference,
audio_durations: &FileAudioDurations,
) -> i64 {
match audio {
AudioReference::Inline { audio_url } => estimate_audio_bytes(audio_url),
AudioReference::File { file_id } => {
i64::try_from(approx_bytes_for_tokens(audio_durations.tokens(file_id)))
.unwrap_or(i64::MAX)
}
}
let tokens = match audio {
AudioReference::Inline { .. } => estimate_tool_audio_tokens(audio),
AudioReference::File { file_id } => audio_durations.tokens(file_id),
};
i64::try_from(approx_bytes_for_tokens(tokens)).unwrap_or(i64::MAX)
}
fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
fn estimate_function_output_bytes(
output: &FunctionCallOutputBody,
audio_durations: &FileAudioDurations,
) -> i64 {
match output {
FunctionCallOutputBody::Text(text) => text_bytes(text),
FunctionCallOutputBody::ContentItems(items) => items
@@ -1592,8 +1586,8 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
FunctionCallOutputContentItem::InputImage { image, detail } => {
estimate_image_reference_bytes(image, *detail)
}
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
FunctionCallOutputContentItem::InputAudio { audio } => {
estimate_audio_reference_bytes(audio, audio_durations)
}
FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {
i64::try_from(estimate_encrypted_function_output_length(
codex-rs/core/src/context_manager/history_tests.rs
+9 -9
View File
@@ -1563,9 +1563,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/mpeg;base64,YXVkaW8=".to_string(),
),
]),
internal_chat_message_metadata_passthrough: None,
},
@@ -1593,9 +1593,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/ogg;base64,YXVkaW8=".to_string(),
),
]),
internal_chat_message_metadata_passthrough: None,
},
@@ -3536,7 +3536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
name: None,
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputAudio { audio_url },
FunctionCallOutputContentItem::inline_audio(audio_url),
]),
internal_chat_message_metadata_passthrough: None,
};
@@ -3557,7 +3557,7 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
call_id: "call-custom-audio".to_string(),
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputAudio { audio_url },
FunctionCallOutputContentItem::inline_audio(audio_url),
]),
internal_chat_message_metadata_passthrough: None,
};
@@ -3603,7 +3603,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() {
namespace: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputAudio { audio_url },
FunctionCallOutputContentItem::inline_audio(audio_url),
]),
success: Some(true),
},
codex-rs/core/src/session/inject.rs
+1 -12
View File
@@ -6,23 +6,12 @@ use codex_features::Feature;
use codex_history::ResponseItemEnvelope;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
/// Uploaded files need the admission checks on typed turn input before entering history.
pub(crate) fn validate_injected_response_item(item: &ResponseItem) -> CodexResult<()> {
if let ResponseItem::Message { content, .. } = item
&& content.iter().any(|part| {
matches!(
part,
ContentItem::InputAudio {
audio: AudioReference::File { .. }
}
)
})
{
if item.file_audio_ids().next().is_some() {
return Err(CodexErr::InvalidRequest(
"Uploaded audio files must be submitted as turn input.".to_string(),
));
codex-rs/core/src/tools/code_mode/mod.rs
+10 -5
View File
@@ -45,7 +45,7 @@ use crate::tools::router::ToolCallSource;
use crate::unified_exec::resolve_max_tokens;
use codex_protocol::openai_models::ToolMode;
use codex_tools::ToolName;
use codex_utils_audio::estimate_audio_token_count;
use codex_utils_audio::estimate_tool_audio_tokens_without_cache;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::formatted_truncate_function_output_items_with_policy;
@@ -331,7 +331,11 @@ fn truncate_code_mode_result(
) -> Vec<FunctionCallOutputContentItem> {
let max_output_tokens = resolve_max_tokens(max_output_tokens);
let policy = TruncationPolicy::Tokens(max_output_tokens);
formatted_truncate_function_output_items_with_policy(items, policy, estimate_audio_token_count)
formatted_truncate_function_output_items_with_policy(
items,
policy,
estimate_tool_audio_tokens_without_cache,
)
}
// Submit synchronously so the recorder sees the call before the cell's dispatch gate closes.
@@ -568,9 +572,10 @@ mod tests {
#[test]
fn over_budget_audio_output_is_omitted() {
let items = vec![FunctionCallOutputContentItem::InputAudio {
audio_url: format!("data:audio/wav;base64,{}", "A".repeat(100)),
}];
let items = vec![FunctionCallOutputContentItem::inline_audio(format!(
"data:audio/wav;base64,{}",
"A".repeat(100)
))];
assert_eq!(
truncate_code_mode_result(items, Some(5)),
codex-rs/core/src/tools/code_mode/output_tests.rs
+1 -3
View File
@@ -16,9 +16,7 @@ fn completed_timing_preserves_content_and_distinguishes_zero_from_unavailable()
},
detail: None,
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,audio".to_string(),
},
FunctionCallOutputContentItem::inline_audio("data:audio/wav;base64,audio".to_string()),
];
for (host_duration, expected_timing) in [
(
codex-rs/core/src/tools/code_mode/response_adapter.rs
+1 -1
View File
@@ -28,7 +28,7 @@ impl IntoProtocol<FunctionCallOutputContentItem>
}
}
codex_code_mode::FunctionCallOutputContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
FunctionCallOutputContentItem::inline_audio(audio_url)
}
}
}
codex-rs/core/src/tools/context.rs
+2 -2
View File
@@ -16,7 +16,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::models::function_call_output_content_items_to_text;
use codex_tools::LoadableToolSpec;
use codex_tools::ToolName;
use codex_utils_audio::estimate_audio_token_count;
use codex_utils_audio::estimate_tool_audio_tokens;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_function_output_payload;
use codex_utils_output_truncation::with_serialization_allowance;
@@ -252,7 +252,7 @@ impl McpToolOutput {
truncate_function_output_payload(
&mut payload,
with_serialization_allowance(self.truncation_policy),
estimate_audio_token_count,
estimate_tool_audio_tokens,
);
payload
}
codex-rs/core/tests/suite/scenarios_preparation.rs
+17 -8
View File
@@ -1,22 +1,28 @@
//! Verifies prepared context reaches both the first model request and a follow-up.
use super::*;
use anyhow::Context as _;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::turn_input::AnnotatedResponseItem;
use pretty_assertions::assert_eq;
use test_case::test_case;
#[test_case("message")]
#[test_case("function_call_output")]
#[test_case("custom_tool_call_output")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Result<()> {
async fn raw_injection_rejects_uploaded_audio_before_modifying_history(kind: &str) -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let test = test_codex().build_with_auto_env(&server).await?;
let before = test.codex.conversation_history_snapshot().await;
let audio: ResponseItem = serde_json::from_value(json!({
"type": "message", "role": "developer",
"content": [{"type": "input_audio", "file_id": "file_injected"}]
}))?;
let parts = json!([{"type": "input_audio", "file_id": "file_injected"}]);
let audio: ResponseItem = serde_json::from_value(if kind == "message" {
json!({"type": kind, "role": "developer", "content": parts})
} else {
json!({"type": kind, "call_id": "untrusted", "output": parts})
})?;
let batch = vec![
core_test_support::responses::user_message_item("must not be partially written"),
audio.clone(),
@@ -26,12 +32,14 @@ async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Resu
.codex
.inject_response_items(batch.clone())
.await
.unwrap_err();
.err()
.context("raw history accepted a caller-supplied audio file")?;
let preparation_error = test
.codex
.inject_response_items_for_turn(batch)
.await
.unwrap_err();
.err()
.context("prepared history accepted a caller-supplied audio file")?;
for error in [history_error, preparation_error] {
assert!(matches!(
error.details(),
@@ -49,7 +57,8 @@ async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Resu
.codex
.start_or_steer_turn(TurnInputRequest::new(input))
.await
.unwrap_err();
.err()
.context("turn input accepted an untyped audio file")?;
assert!(matches!(
error.details(),
CodexErrorDetails::InvalidRequest(_)
codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs
+3 -3
View File
@@ -1216,9 +1216,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
},
detail: None,
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,tool-audio".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav;base64,tool-audio".to_string(),
),
]),
internal_chat_message_metadata_passthrough: None,
},
codex-rs/protocol/src/models.rs
+59 -20
View File
@@ -1368,6 +1368,31 @@ pub enum ResponseItem {
}
impl ResponseItem {
/// File-backed audio in messages and tool results, in request order.
pub fn file_audio_ids(&self) -> impl Iterator<Item = &String> + Clone {
let (message, tool): (&[ContentItem], &[FunctionCallOutputContentItem]) = match self {
Self::Message { content, .. } => (content, &[]),
Self::FunctionCallOutput { output, .. } | Self::CustomToolCallOutput { output, .. } => {
(&[], output.content_items().unwrap_or_default())
}
_ => (&[], &[]),
};
message
.iter()
.filter_map(|part| match part {
ContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
})
.chain(tool.iter().filter_map(|part| match part {
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
}))
}
/// Clears the top-level `encrypted_content` fields.
pub fn clear_encrypted_content(&mut self) {
match self {
@@ -2329,13 +2354,23 @@ pub enum FunctionCallOutputContentItem {
},
// Do not rename, these are serialized and used directly in the responses API.
InputAudio {
audio_url: String,
#[serde(flatten)]
audio: AudioReference,
},
EncryptedContent {
encrypted_content: String,
},
}
impl FunctionCallOutputContentItem {
/// Audio supplied by an MCP server or Code Mode, before host attachment preparation.
pub fn inline_audio(audio_url: String) -> Self {
Self::InputAudio {
audio: AudioReference::Inline { audio_url },
}
}
}
/// Converts message content to tool output, preserving media and image detail.
/// Output text becomes input text without its sampling annotations or logprobs.
impl From<ContentItem> for FunctionCallOutputContentItem {
@@ -2345,12 +2380,7 @@ impl From<ContentItem> for FunctionCallOutputContentItem {
Self::InputText { text }
}
ContentItem::InputImage { image, detail } => Self::InputImage { image, detail },
ContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} => Self::InputAudio { audio_url },
ContentItem::InputAudio {
audio: AudioReference::File { file_id, .. },
} => Self::InputText { text: file_id },
ContentItem::InputAudio { audio } => Self::InputAudio { audio },
}
}
}
@@ -2404,7 +2434,7 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
}
}
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
Self::inline_audio(audio_url)
}
}
}
@@ -3235,6 +3265,17 @@ mod tests {
);
}
#[test]
fn tool_audio_references_round_trip_in_the_responses_wire_shape() {
for wire in [
serde_json::json!({"type":"input_audio","audio_url":"data:audio/wav;base64,QQ=="}),
serde_json::json!({"type":"input_audio","file_id":"file-clip"}),
] {
let part: FunctionCallOutputContentItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(part).unwrap(), wire);
}
}
#[test]
fn convert_mcp_audio_content_builds_data_urls_and_preserves_existing_data_urls() {
let contents = vec![
@@ -3254,12 +3295,12 @@ mod tests {
assert_eq!(
convert_mcp_content_to_items(&contents),
vec![
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,Zm9v".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/ogg;base64,YmFy".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav;base64,Zm9v".to_string()
),
FunctionCallOutputContentItem::inline_audio(
"data:audio/ogg;base64,YmFy".to_string()
),
]
);
}
@@ -3312,9 +3353,7 @@ mod tests {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,AAA".to_string(),
},
FunctionCallOutputContentItem::inline_audio("data:audio/wav;base64,AAA".to_string()),
FunctionCallOutputContentItem::EncryptedContent {
encrypted_content: "enc_opaque".to_string(),
},
@@ -3659,9 +3698,9 @@ mod tests {
FunctionCallOutputContentItem::InputText {
text: "caption".into(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,BASE64".into(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav;base64,BASE64".into()
),
]),
success: Some(true),
}
codex-rs/protocol/src/models/mcp_content.rs
+1 -3
View File
@@ -78,9 +78,7 @@ impl FunctionCallOutputContentItem {
.unwrap_or(DEFAULT_IMAGE_DETAIL),
),
},
McpContent::Audio(media) => Self::InputAudio {
audio_url: media.data_url(),
},
McpContent::Audio(media) => Self::inline_audio(media.data_url()),
McpContent::ResourceLink { .. } | McpContent::Resource { .. } => Self::InputText {
text: serde_json::to_string(block)?,
},
codex-rs/tools/src/tool_output.rs
+7 -5
View File
@@ -1,3 +1,4 @@
use codex_protocol::models::AudioReference;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
@@ -268,11 +269,12 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
image: ImageReference::File { file_id },
..
} if !file_id.trim().is_empty() => Some(file_id.clone()),
FunctionCallOutputContentItem::InputAudio { audio_url }
if !audio_url.trim().is_empty() =>
{
Some(audio_url.clone())
}
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} if !audio_url.trim().is_empty() => Some(audio_url.clone()),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} if !file_id.trim().is_empty() => Some(file_id.clone()),
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputImage { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
codex-rs/utils/audio/src/audio_preparation_tests.rs
+9 -9
View File
@@ -131,12 +131,12 @@ fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() {
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav,not-base64".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav;base64,YXVkaW8=".to_string(),
),
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav,not-base64".to_string(),
),
]),
success: Some(true),
},
@@ -157,9 +157,9 @@ fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() {
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::inline_audio(
"data:audio/wav;base64,YXVkaW8=".to_string()
),
FunctionCallOutputContentItem::InputText {
text: "audio content omitted because it could not be processed".to_string(),
},
codex-rs/utils/audio/src/lib.rs
+25 -3
View File
@@ -107,7 +107,9 @@ fn prepare_message_content(items: &mut [ContentItem]) {
fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
for item in items {
if let FunctionCallOutputContentItem::InputAudio { audio_url } = item
if let FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} = item
&& let Err(error) = prepare_audio(audio_url)
{
warn!(%error, "failed to prepare tool output audio");
@@ -160,8 +162,28 @@ pub fn estimate_file_audio_tokens(duration: Option<std::time::Duration>) -> usiz
usize::try_from(tokens).unwrap_or(usize::MAX)
}
/// Estimate tool audio on Core's multi-threaded runtime.
pub fn estimate_tool_audio_tokens(audio: &AudioReference) -> usize {
estimate_tool_audio_tokens_with(audio, estimate_audio_token_count)
}
/// Code Mode can render on a single thread, where blocking cache access is unavailable.
pub fn estimate_tool_audio_tokens_without_cache(audio: &AudioReference) -> usize {
estimate_tool_audio_tokens_with(audio, estimate_audio_token_count_without_cache)
}
fn estimate_tool_audio_tokens_with(
audio: &AudioReference,
inline_cost: impl FnOnce(&str) -> usize,
) -> usize {
match audio {
AudioReference::Inline { audio_url } => inline_cost(audio_url),
AudioReference::File { .. } => estimate_file_audio_tokens(/*duration*/ None),
}
}
/// Estimates audio tokens from decoded duration, falling back to the data URL size.
pub fn estimate_audio_token_count(audio_url: &str) -> usize {
fn estimate_audio_token_count(audio_url: &str) -> usize {
let key = sha1_digest(audio_url.as_bytes());
AUDIO_TOKEN_ESTIMATE_CACHE
.get_or_init(key, || estimate_audio_token_count_without_cache(audio_url))
@@ -169,7 +191,7 @@ pub fn estimate_audio_token_count(audio_url: &str) -> usize {
/// The same duration-based estimate without accessing the Tokio-dependent cache.
/// Suitable for bounded output rendered on a current-thread runtime.
pub fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize {
fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize {
let Some(duration_seconds) = audio_duration_seconds(audio_url) else {
return approx_token_count(audio_url);
};
codex-rs/utils/output-truncation/src/lib.rs
+11 -10
View File
@@ -4,6 +4,7 @@ use std::borrow::Cow;
use codex_protocol::mcp::CallToolResult;
use codex_protocol::mcp::plain_output_text;
use codex_protocol::models::AudioReference;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
@@ -137,7 +138,7 @@ pub fn truncate_mcp_tool_result(
pub fn truncate_function_output_payload(
output: &mut FunctionCallOutputPayload,
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
) {
match &mut output.body {
FunctionCallOutputBody::Text(text) => *text = truncate_text(text, policy),
@@ -172,7 +173,7 @@ pub struct OutputBoundary {
pub fn formatted_truncate_function_output_items_with_policy<'a>(
items: impl Into<Cow<'a, [FunctionCallOutputContentItem]>>,
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
) -> Vec<FunctionCallOutputContentItem> {
formatted_output(items.into(), policy, estimate_audio_token_count, &mut [])
}
@@ -182,7 +183,7 @@ pub fn formatted_truncate_function_output_items_with_policy<'a>(
pub fn formatted_output_at_token_boundaries<'a>(
items: impl Into<Cow<'a, [FunctionCallOutputContentItem]>>,
max_tokens: usize,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
boundaries: &mut [OutputBoundary],
) -> Vec<FunctionCallOutputContentItem> {
let items = items.into();
@@ -198,7 +199,7 @@ pub fn formatted_output_at_token_boundaries<'a>(
fn formatted_output(
items: Cow<'_, [FunctionCallOutputContentItem]>,
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
boundaries: &mut [OutputBoundary],
) -> Vec<FunctionCallOutputContentItem> {
let mut combined_len = 0;
@@ -242,7 +243,7 @@ fn formatted_output(
pub fn truncate_function_output_items_with_policy(
items: &[FunctionCallOutputContentItem],
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
) -> Vec<FunctionCallOutputContentItem> {
truncate_output_items(
Cow::Borrowed(items),
@@ -257,7 +258,7 @@ pub fn truncate_function_output_items_with_policy(
pub fn mixed_output_at_token_boundaries<'a>(
items: impl Into<Cow<'a, [FunctionCallOutputContentItem]>>,
max_tokens: usize,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
boundaries: &mut [OutputBoundary],
) -> Vec<FunctionCallOutputContentItem> {
let items = items.into();
@@ -291,7 +292,7 @@ fn normalize_boundaries(
fn truncate_output_items(
items: Cow<'_, [FunctionCallOutputContentItem]>,
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
boundaries: &mut [OutputBoundary],
) -> Vec<FunctionCallOutputContentItem> {
match items {
@@ -313,7 +314,7 @@ fn truncate_output_items(
fn truncate_output_iter<'a>(
items: impl ExactSizeIterator<Item = Cow<'a, FunctionCallOutputContentItem>>,
policy: TruncationPolicy,
estimate_audio_token_count: impl Fn(&str) -> usize,
estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
boundaries: &mut [OutputBoundary],
) -> Vec<FunctionCallOutputContentItem> {
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
@@ -361,8 +362,8 @@ fn truncate_output_iter<'a>(
}
FunctionCallOutputContentItem::InputImage { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => true,
FunctionCallOutputContentItem::InputAudio { audio_url } => {
let token_cost = estimate_audio_token_count(audio_url);
FunctionCallOutputContentItem::InputAudio { audio } => {
let token_cost = estimate_audio_token_count(audio);
let cost = match policy {
TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(token_cost),
TruncationPolicy::Tokens(_) => token_cost,
codex-rs/utils/output-truncation/src/truncate_tests.rs
+27 -12
View File
@@ -10,6 +10,7 @@ use crate::truncate_function_output_items_with_policy;
use crate::truncate_mcp_tool_result;
use crate::truncate_text;
use codex_protocol::mcp::CallToolResult;
use codex_protocol::models::AudioReference;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageReference;
@@ -264,9 +265,7 @@ fn truncate_function_output_items_with_policy_discards_empty_text() {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::inline_audio("audio:one".to_string()),
FunctionCallOutputContentItem::EncryptedContent {
encrypted_content: "enc_opaque".to_string(),
},
@@ -314,9 +313,7 @@ fn formatted_text_output_preserves_empty_leading_text_behavior() {
#[test]
fn formatted_mixed_output_budgets_text_and_audio_in_order() {
let audio = FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_owned(),
};
let audio = FunctionCallOutputContentItem::inline_audio("audio:one".to_owned());
let file_image = FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_owned(),
@@ -378,9 +375,7 @@ fn truncate_function_output_items_with_policy_omits_audio_over_budget() {
FunctionCallOutputContentItem::EncryptedContent {
encrypted_content: "enc_opaque".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::inline_audio("audio:one".to_string()),
];
let output =
@@ -403,10 +398,30 @@ fn truncate_function_output_items_with_policy_omits_audio_over_budget() {
}
#[test]
fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() {
fn repeated_file_audio_and_text_share_one_output_budget() {
let audio = FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
audio: AudioReference::File {
file_id: "clip".into(),
},
};
let items = [text("abcd"), audio.clone(), audio.clone()];
// Text takes one token. The first reference takes two, leaving too little
// for a second copy of the same clip under either budget.
for policy in [TruncationPolicy::Tokens(4), TruncationPolicy::Bytes(16)] {
assert_eq!(
truncate_function_output_items_with_policy(&items, policy, |_| 2),
vec![
text("abcd"),
audio.clone(),
text("[omitted 1 audio items ...]")
],
);
}
}
#[test]
fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() {
let audio = FunctionCallOutputContentItem::inline_audio("audio:one".to_string());
let items = vec![
audio.clone(),
FunctionCallOutputContentItem::InputText {
@@ -477,7 +492,7 @@ fn inline_image(image_url: String) -> FunctionCallOutputContentItem {
#[test]
fn boundary_formatters_normalize_input_and_remap_truncated_text() {
let audio_cost: fn(&str) -> usize = |_| 1;
let audio_cost: fn(&AudioReference) -> usize = |_| 1;
let items = [text("left-head"), text("right-tail")];
let max_tokens = 2;
let mut boundary = boundaries([(0, 0), (0, 2), (1, 0), (1, 8), (2, 0)]);
sdk/python/scripts/update_sdk_artifacts.py
+1
View File
@@ -623,6 +623,7 @@ def _preserve_inline_media_class_names(out_path: Path) -> None:
"ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem",
"UrlUserInput1": "AudioUserInput",
"AudioUrlContentItem": "InputAudioContentItem",
"AudioUrlFunctionCallOutputContentItem": "InputAudioFunctionCallOutputContentItem",
}
for generated_name, stable_name in stable_names.items():
if source.count(f"class {generated_name}(") != 1:
sdk/python/src/openai_codex/generated/v2_all.py
+13 -1
View File
@@ -1781,10 +1781,20 @@ class InputAudioFunctionCallOutputContentItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
audio_url: str
type: Annotated[
Literal["input_audio"], Field(title="InputAudioFunctionCallOutputContentItemType")
]
audio_url: str
class FileIdFunctionCallOutputContentItem1(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
type: Annotated[
Literal["input_audio"], Field(title="InputAudioFunctionCallOutputContentItemType")
]
file_id: str
class EncryptedContentFunctionCallOutputContentItem(BaseModel):
@@ -8930,6 +8940,7 @@ class FunctionCallOutputContentItem(
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem1
| EncryptedContentFunctionCallOutputContentItem
]
):
@@ -8938,6 +8949,7 @@ class FunctionCallOutputContentItem(
| InputImageFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| FileIdFunctionCallOutputContentItem1
| EncryptedContentFunctionCallOutputContentItem,
Field(
description="Responses API compatible content items that can be returned by a tool call. This is a subset of ContentItem with the types we support as function call outputs."
sdk/python/tests/test_artifact_workflow_and_binaries.py
+2
View File
@@ -562,11 +562,13 @@ def test_generated_audio_inputs_preserve_inline_and_file_shapes() -> None:
ContentItem,
FileIdUserInput1,
InputAudioContentItem,
InputAudioFunctionCallOutputContentItem,
UserInput,
)
audio_url = "data:audio/wav;base64,YXVkaW8="
for model, wire in [
(InputAudioFunctionCallOutputContentItem, {"type": "input_audio", "audio_url": audio_url}),
(AudioUserInput, {"type": "audio", "url": audio_url}),
(InputAudioContentItem, {"type": "input_audio", "audio_url": audio_url}),
(UserInput, {"type": "audio", "fileId": "file_audio"}),