From 643de86a190a38a5f4afa5e3a15edf48153f9c64 Mon Sep 17 00:00:00 2001 From: nhamidi-oai Date: Sat, 18 Jul 2026 23:16:37 +0000 Subject: [PATCH] Add audio output support to dynamic tools and code mode (#34080) ## What changed - Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas. - Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks. - Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item. - Reject non-data audio URLs and track audio item counts in dynamic tool analytics. ## Testing - Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling. GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6 --- codex-rs/analytics/src/events.rs | 1 + codex-rs/analytics/src/reducer.rs | 33 +++++- .../schema/json/DynamicToolCallResponse.json | 20 ++++ .../schema/json/ServerNotification.json | 20 ++++ .../codex_app_server_protocol.schemas.json | 20 ++++ .../codex_app_server_protocol.v2.schemas.json | 20 ++++ .../json/v2/ItemCompletedNotification.json | 20 ++++ .../json/v2/ItemStartedNotification.json | 20 ++++ .../schema/json/v2/ReviewStartResponse.json | 20 ++++ .../schema/json/v2/ThreadForkResponse.json | 20 ++++ .../schema/json/v2/ThreadListResponse.json | 20 ++++ .../json/v2/ThreadMetadataUpdateResponse.json | 20 ++++ .../schema/json/v2/ThreadReadResponse.json | 20 ++++ .../schema/json/v2/ThreadResumeResponse.json | 20 ++++ .../json/v2/ThreadRollbackResponse.json | 20 ++++ .../schema/json/v2/ThreadStartResponse.json | 20 ++++ .../json/v2/ThreadStartedNotification.json | 20 ++++ .../json/v2/ThreadUnarchiveResponse.json | 20 ++++ .../json/v2/TurnCompletedNotification.json | 20 ++++ .../schema/json/v2/TurnStartResponse.json | 20 ++++ .../json/v2/TurnStartedNotification.json | 20 ++++ .../v2/DynamicToolCallOutputContentItem.ts | 2 +- .../src/protocol/event_mapping.rs | 3 + .../src/protocol/thread_history.rs | 31 ++++- .../src/protocol/v2/item.rs | 8 ++ .../src/protocol/v2/tests.rs | 29 ++++- codex-rs/app-server/README.md | 5 +- codex-rs/app-server/src/dynamic_tools.rs | 19 +++ .../tests/suite/v2/dynamic_tools.rs | 101 +++++++++++++--- .../code-mode-protocol/src/description.rs | 2 + .../code-mode-protocol/src/host/host_tests.rs | 7 ++ .../code-mode-protocol/src/host/payload.rs | 7 ++ codex-rs/code-mode-protocol/src/response.rs | 3 + .../code-mode/src/cell_actor/conversions.rs | 3 + codex-rs/code-mode/src/runtime/callbacks.rs | 21 ++++ codex-rs/code-mode/src/runtime/globals.rs | 3 + codex-rs/code-mode/src/runtime/value.rs | 101 ++++++++++++++++ codex-rs/code-mode/src/service.rs | 3 + codex-rs/code-mode/src/service_tests.rs | 79 ++++++++++++- .../code-mode/src/session_runtime/types.rs | 3 + codex-rs/core/src/mcp_tool_call.rs | 35 +++--- codex-rs/core/src/mcp_tool_call_tests.rs | 65 +++++++---- .../src/tools/code_mode/response_adapter.rs | 3 + codex-rs/core/tests/suite/code_mode.rs | 1 + codex-rs/protocol/src/dynamic_tools.rs | 2 + codex-rs/protocol/src/models.rs | 109 +++++++++++++++++- 46 files changed, 987 insertions(+), 72 deletions(-) diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 92aad2d041..dda3fa660b 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -707,6 +707,7 @@ pub(crate) struct CodexDynamicToolCallEventParams { pub(crate) output_content_item_count: Option, pub(crate) output_text_item_count: Option, pub(crate) output_image_item_count: Option, + pub(crate) output_audio_item_count: Option, } #[derive(Serialize)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 3ddc7c2350..6e4d084446 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -1968,6 +1968,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { output_content_item_count: counts.map(|counts| counts.total), output_text_item_count: counts.map(|counts| counts.text), output_image_item_count: counts.map(|counts| counts.image), + output_audio_item_count: counts.map(|counts| counts.audio), }, }, )) @@ -2505,26 +2506,30 @@ fn file_change_counts(changes: &[codex_app_server_protocol::FileUpdateChange]) - counts } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] struct DynamicContentCounts { total: u64, text: u64, image: u64, + audio: u64, } fn dynamic_content_counts(items: &[DynamicToolCallOutputContentItem]) -> DynamicContentCounts { let mut text = 0; let mut image = 0; + let mut audio = 0; for item in items { match item { DynamicToolCallOutputContentItem::InputText { .. } => text += 1, DynamicToolCallOutputContentItem::InputImage { .. } => image += 1, + DynamicToolCallOutputContentItem::InputAudio { .. } => audio += 1, } } DynamicContentCounts { total: usize_to_u64(items.len()), text, image, + audio, } } @@ -2830,6 +2835,7 @@ mod tests { use codex_protocol::models::SandboxEnforcement; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; + use pretty_assertions::assert_eq; #[test] fn managed_full_disk_with_restricted_network_reports_external_sandbox() { @@ -2853,4 +2859,29 @@ mod tests { Some((ReviewStatus::TimedOut, ReviewResolution::None)) )); } + + #[test] + fn dynamic_content_counts_include_audio() { + let items = vec![ + DynamicToolCallOutputContentItem::InputText { + text: "ok".to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + ]; + + assert_eq!( + dynamic_content_counts(&items), + DynamicContentCounts { + total: 3, + text: 1, + image: 1, + audio: 1, + } + ); + } } diff --git a/codex-rs/app-server-protocol/schema/json/DynamicToolCallResponse.json b/codex-rs/app-server-protocol/schema/json/DynamicToolCallResponse.json index e0e29641d2..47de6cb30e 100644 --- a/codex-rs/app-server-protocol/schema/json/DynamicToolCallResponse.json +++ b/codex-rs/app-server-protocol/schema/json/DynamicToolCallResponse.json @@ -42,6 +42,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] } diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index db75d4a0d4..3fbb2df3e4 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1129,6 +1129,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index e096b1149d..7f5b7cf081 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -9163,6 +9163,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index ce1a37976e..3d87934c5b 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -5356,6 +5356,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index dd8353e729..a83056ed45 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -240,6 +240,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 5e12386041..0c99b79b89 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -240,6 +240,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 1008400bc9..071b06a4a5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -378,6 +378,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index fcdc2c6630..c697779486 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -459,6 +459,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 0809b9c3ee..5b64c33981 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index 17bc5bb5b6..7423f89fd7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index dd9458d9c9..ed061e1616 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 28081e377a..df2fd0218b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -459,6 +459,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 8bd889b655..4264629db4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index e4a280e1ac..5d40babf4d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -459,6 +459,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 35c8e432d6..3b0288ebcc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 144f0d981c..3fa6217a3a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -381,6 +381,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 554a4cff6b..7ac5c78f33 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -378,6 +378,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 484c34b071..1d38983832 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -378,6 +378,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 06fdc6e81e..d91e66f191 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -378,6 +378,26 @@ ], "title": "InputImageDynamicToolCallOutputContentItem", "type": "object" + }, + { + "properties": { + "audioUrl": { + "type": "string" + }, + "type": { + "enum": [ + "inputAudio" + ], + "title": "InputAudioDynamicToolCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audioUrl", + "type" + ], + "title": "InputAudioDynamicToolCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallOutputContentItem.ts index 8f432109d1..9be1a80996 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallOutputContentItem.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, }; +export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, } | { "type": "inputAudio", audioUrl: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/event_mapping.rs b/codex-rs/app-server-protocol/src/protocol/event_mapping.rs index 6d6012e19b..b32b15272b 100644 --- a/codex-rs/app-server-protocol/src/protocol/event_mapping.rs +++ b/codex-rs/app-server-protocol/src/protocol/event_mapping.rs @@ -59,6 +59,9 @@ pub fn item_event_to_server_notification( CoreDynamicToolCallOutputContentItem::InputImage { image_url } => { DynamicToolCallOutputContentItem::InputImage { image_url } } + CoreDynamicToolCallOutputContentItem::InputAudio { audio_url } => { + DynamicToolCallOutputContentItem::InputAudio { audio_url } + } }) .collect(), ), diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 7bb42c1935..0fc4f12379 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -1498,6 +1498,9 @@ fn convert_dynamic_tool_content_items( codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url, } => DynamicToolCallOutputContentItem::InputImage { image_url }, + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { + audio_url, + } => DynamicToolCallOutputContentItem::InputAudio { audio_url }, }) .collect() } @@ -2879,9 +2882,17 @@ mod tests { namespace: Some("codex_app".into()), tool: "lookup_ticket".into(), arguments: serde_json::json!({"id":"ABC-123"}), - content_items: vec![CoreDynamicToolCallOutputContentItem::InputText { - text: "Ticket is open".into(), - }], + content_items: vec![ + CoreDynamicToolCallOutputContentItem::InputText { + text: "Ticket is open".into(), + }, + CoreDynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".into(), + }, + CoreDynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".into(), + }, + ], success: true, error: None, duration: Duration::from_millis(42), @@ -2903,9 +2914,17 @@ mod tests { tool: "lookup_ticket".into(), arguments: serde_json::json!({"id":"ABC-123"}), status: DynamicToolCallStatus::Completed, - content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { - text: "Ticket is open".into(), - }]), + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: "Ticket is open".into(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".into(), + }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".into(), + }, + ]), success: Some(true), duration_ms: Some(42), } diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index ddf8c9b2ee..2f38a723f3 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -1552,6 +1552,8 @@ pub enum DynamicToolCallOutputContentItem { InputText { text: String }, #[serde(rename_all = "camelCase")] InputImage { image_url: String }, + #[serde(rename_all = "camelCase")] + InputAudio { audio_url: String }, } impl From @@ -1565,6 +1567,9 @@ impl From codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url, } => Self::InputImage { image_url }, + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { + audio_url, + } => Self::InputAudio { audio_url }, } } } @@ -1578,6 +1583,9 @@ impl From DynamicToolCallOutputContentItem::InputImage { image_url } => { Self::InputImage { image_url } } + DynamicToolCallOutputContentItem::InputAudio { audio_url } => { + Self::InputAudio { audio_url } + } } } } diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 65f8c72b32..22f96f159b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -2719,6 +2719,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText { text: "ok".to_string(), }, + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, ]), success: Some(true), error: None, @@ -2733,9 +2739,17 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { tool: "lookup".to_string(), arguments: json!({"id": "123"}), status: DynamicToolCallStatus::Completed, - content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { - text: "ok".to_string(), - }]), + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: "ok".to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + ]), success: Some(true), duration_ms: Some(5), } @@ -3954,7 +3968,7 @@ fn dynamic_tool_response_serializes_content_items() { } #[test] -fn dynamic_tool_response_serializes_text_and_image_content_items() { +fn dynamic_tool_response_serializes_text_image_and_audio_content_items() { let value = serde_json::to_value(DynamicToolCallResponse { content_items: vec![ DynamicToolCallOutputContentItem::InputText { @@ -3963,6 +3977,9 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() { DynamicToolCallOutputContentItem::InputImage { image_url: "data:image/png;base64,AAA".to_string(), }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, ], success: true, }) @@ -3979,6 +3996,10 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() { { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" + }, + { + "type": "inputAudio", + "audioUrl": "data:audio/wav;base64,YXVkaW8=" } ], "success": true, diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 465b82a3ff..2e9abbbaf4 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -1671,7 +1671,7 @@ The server also emits item lifecycle notifications around the request: 3. Client response. 4. `item/completed` with `item.type = "dynamicToolCall"`, final `status`, and the returned `contentItems`/`success`. -The client must respond with content items. Use `inputText` for text and `inputImage` for inline data URLs. Remote HTTP(S) image URLs make the dynamic tool response invalid. +The client must respond with content items. Use `inputText` for text, `inputImage` for inline image data URLs, and `inputAudio` for inline audio data URLs. Audio data URLs accept wav, mp3, m4a, webm, and ogg media types. Remote HTTP(S) image URLs and non-data audio URLs make the dynamic tool response invalid. ```json { @@ -1679,7 +1679,8 @@ The client must respond with content items. Use `inputText` for text and `inputI "result": { "contentItems": [ { "type": "inputText", "text": "Ticket ABC-123 is open." }, - { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" } + { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" }, + { "type": "inputAudio", "audioUrl": "data:audio/wav;base64,AAA" } ], "success": true } diff --git a/codex-rs/app-server/src/dynamic_tools.rs b/codex-rs/app-server/src/dynamic_tools.rs index 0da9cad453..0069b6d3e4 100644 --- a/codex-rs/app-server/src/dynamic_tools.rs +++ b/codex-rs/app-server/src/dynamic_tools.rs @@ -13,6 +13,8 @@ use crate::image_url::is_remote_image_url; use crate::outgoing_message::ClientRequestResult; use crate::server_request_error::is_turn_transition_server_request_error; +const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL"; + pub(crate) async fn on_call_response( call_id: String, receiver: oneshot::Receiver, @@ -71,6 +73,23 @@ fn decode_response(value: serde_json::Value) -> (DynamicToolCallResponse, Option ); fallback_response(REMOTE_IMAGE_URL_ERROR) } + Ok(response) + if response.content_items.iter().any(|item| { + matches!( + item, + DynamicToolCallOutputContentItem::InputAudio { audio_url } + if !audio_url + .get(.."data:".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) + ) + }) => + { + error!( + message = INVALID_AUDIO_URL_ERROR, + "dynamic tool response was invalid" + ); + fallback_response(INVALID_AUDIO_URL_ERROR) + } Ok(response) => (response, None), Err(err) => { error!("failed to deserialize DynamicToolCallResponse: {err}"); diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 58602b4d08..c41be5be71 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -4,6 +4,7 @@ use app_test_support::TestAppServer; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; +use app_test_support::write_models_cache_with_models; use codex_app_server_protocol::DynamicToolCallOutputContentItem; use codex_app_server_protocol::DynamicToolCallParams; use codex_app_server_protocol::DynamicToolCallResponse; @@ -28,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use core_test_support::load_default_config_for_test; use core_test_support::responses; use pretty_assertions::assert_eq; use serde_json::Value; @@ -39,6 +41,8 @@ use tokio::time::timeout; use wiremock::MockServer; const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; +const INLINE_AUDIO_DATA_URL: &str = "data:audio/wav;base64,YXVkaW8="; +const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL"; const REMOTE_IMAGE_URL_ERROR: &str = "remote image URLs are not supported; use an inline data URL instead"; @@ -592,6 +596,9 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result Result Result<()> { +async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> { let call_id = "dyn-call-items-1"; let PendingDynamicToolCall { mut mcp, @@ -696,22 +703,21 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<( DynamicToolCallOutputContentItem::InputImage { image_url: TINY_PNG_DATA_URL.to_string(), }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: INLINE_AUDIO_DATA_URL.to_string(), + }, + ]; + // Audio remains available on the app-server item, but the base truncation policy intentionally + // omits it from the model-visible payload until the history/accounting layer. + let model_content_items = vec![ + FunctionCallOutputContentItem::InputText { + text: "dynamic-ok".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: TINY_PNG_DATA_URL.to_string(), + detail: Some(DEFAULT_IMAGE_DETAIL), + }, ]; - let content_items = response_content_items - .clone() - .into_iter() - .map(|item| match item { - DynamicToolCallOutputContentItem::InputText { text } => { - FunctionCallOutputContentItem::InputText { text } - } - DynamicToolCallOutputContentItem::InputImage { image_url } => { - FunctionCallOutputContentItem::InputImage { - image_url, - detail: Some(DEFAULT_IMAGE_DETAIL), - } - } - }) - .collect::>(); let response = DynamicToolCallResponse { content_items: response_content_items, success: true, @@ -741,6 +747,9 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<( DynamicToolCallOutputContentItem::InputImage { image_url: TINY_PNG_DATA_URL.to_string(), }, + DynamicToolCallOutputContentItem::InputAudio { + audio_url: INLINE_AUDIO_DATA_URL.to_string(), + }, ]) ); assert_eq!(success, Some(true)); @@ -777,12 +786,12 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<( .context("expected function_call_output in follow-up request")?; assert_eq!( payload.body, - FunctionCallOutputBody::ContentItems(content_items.clone()) + FunctionCallOutputBody::ContentItems(model_content_items.clone()) ); assert_eq!(payload.success, None); assert_eq!( serde_json::to_string(&payload)?, - serde_json::to_string(&content_items)? + serde_json::to_string(&model_content_items)? ); Ok(()) @@ -844,6 +853,62 @@ async fn dynamic_tool_remote_image_response_becomes_model_visible_error() -> Res Ok(()) } +#[tokio::test] +async fn dynamic_tool_remote_audio_response_becomes_model_visible_error() -> Result<()> { + let call_id = "dyn-call-remote-audio"; + let PendingDynamicToolCall { + mut mcp, + server, + request_id, + params, + } = start_function_dynamic_tool_call(call_id).await?; + + let response = DynamicToolCallResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputAudio { + audio_url: "https://example.com/tool.wav".to_string(), + }], + success: true, + }; + mcp.send_response(request_id, serde_json::to_value(response)?) + .await?; + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + assert_eq!(completed.thread_id, params.thread_id); + assert_eq!(completed.turn_id, params.turn_id); + let ThreadItem::DynamicToolCall { + status, + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(status, DynamicToolCallStatus::Failed); + assert_eq!( + content_items, + Some(vec![DynamicToolCallOutputContentItem::InputText { + text: INVALID_AUDIO_URL_ERROR.to_string(), + }]) + ); + assert_eq!(success, Some(false)); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let output = responses_bodies(&server) + .await? + .iter() + .find_map(|body| function_call_output_raw_output(body, call_id)) + .context("expected function_call_output output in follow-up request")?; + assert_eq!(output, json!(INVALID_AUDIO_URL_ERROR)); + + Ok(()) +} + async fn responses_bodies(server: &MockServer) -> Result> { let requests = server .received_requests() diff --git a/codex-rs/code-mode-protocol/src/description.rs b/codex-rs/code-mode-protocol/src/description.rs index d0504f54a4..e8db333b3d 100644 --- a/codex-rs/code-mode-protocol/src/description.rs +++ b/codex-rs/code-mode-protocol/src/description.rs @@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. - `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. +- `audio(audioUrlOrItem: string | { audio_url: string } | AudioContent)`: Appends an audio item. `audio_url` should be a base64-encoded `data:` URL. To forward an MCP tool audio block, pass an individual `AudioContent` block from `result.content`, for example `audio(result.content[0])`. - `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. HTTP(S) URLs are not supported. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. @@ -880,6 +881,7 @@ bar" fn exec_description_mentions_timeout_helpers() { let description = build_exec_tool_description(&[], &[], &BTreeMap::new(), /*code_mode_only*/ false); + assert!(description.contains("`audio(audioUrlOrItem:")); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); assert!(description.contains("`clearTimeout(timeoutId?: number)`")); } diff --git a/codex-rs/code-mode-protocol/src/host/host_tests.rs b/codex-rs/code-mode-protocol/src/host/host_tests.rs index d56e56e1fa..dde38e87c9 100644 --- a/codex-rs/code-mode-protocol/src/host/host_tests.rs +++ b/codex-rs/code-mode-protocol/src/host/host_tests.rs @@ -130,6 +130,9 @@ fn content_items() -> Vec { image_url: "data:image/png;base64,original".to_string(), detail: Some(WireImageDetail::Original), }, + WireContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, ] } @@ -157,6 +160,10 @@ fn content_items_json() -> Value { "image_url": "data:image/png;base64,original", "detail": "original", }, + { + "type": "input_audio", + "audio_url": "data:audio/wav;base64,YXVkaW8=", + }, ]) } diff --git a/codex-rs/code-mode-protocol/src/host/payload.rs b/codex-rs/code-mode-protocol/src/host/payload.rs index a5ba410711..7cb15db379 100644 --- a/codex-rs/code-mode-protocol/src/host/payload.rs +++ b/codex-rs/code-mode-protocol/src/host/payload.rs @@ -245,6 +245,9 @@ pub enum WireContentItem { #[serde(default, skip_serializing_if = "Option::is_none")] detail: Option, }, + InputAudio { + audio_url: String, + }, } impl From for WireContentItem { @@ -255,6 +258,9 @@ impl From for WireContentItem { image_url, detail: detail.map(Into::into), }, + FunctionCallOutputContentItem::InputAudio { audio_url } => { + Self::InputAudio { audio_url } + } } } } @@ -267,6 +273,7 @@ impl From for FunctionCallOutputContentItem { image_url, detail: detail.map(Into::into), }, + WireContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url }, } } } diff --git a/codex-rs/code-mode-protocol/src/response.rs b/codex-rs/code-mode-protocol/src/response.rs index 0ac3a03770..9b45032d94 100644 --- a/codex-rs/code-mode-protocol/src/response.rs +++ b/codex-rs/code-mode-protocol/src/response.rs @@ -23,4 +23,7 @@ pub enum FunctionCallOutputContentItem { #[serde(default, skip_serializing_if = "Option::is_none")] detail: Option, }, + InputAudio { + audio_url: String, + }, } diff --git a/codex-rs/code-mode/src/cell_actor/conversions.rs b/codex-rs/code-mode/src/cell_actor/conversions.rs index 7fb2a25c63..781f456226 100644 --- a/codex-rs/code-mode/src/cell_actor/conversions.rs +++ b/codex-rs/code-mode/src/cell_actor/conversions.rs @@ -56,5 +56,8 @@ pub(super) fn output_item(item: FunctionCallOutputContentItem) -> CellOutputItem ImageDetail::Original => CellImageDetail::Original, }), }, + FunctionCallOutputContentItem::InputAudio { audio_url } => { + CellOutputItem::Audio { audio_url } + } } } diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index 7b09ba5ffd..fcd61fd63c 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -5,6 +5,7 @@ use super::RuntimeEvent; use super::RuntimeState; use super::timers; use super::value::json_to_v8; +use super::value::normalize_output_audio; use super::value::normalize_output_image; use super::value::serialize_output_text; use super::value::throw_type_error; @@ -96,6 +97,26 @@ pub(super) fn text_callback( retval.set(v8::undefined(scope).into()); } +pub(super) fn audio_callback( + scope: &mut v8::PinScope<'_, '_>, + args: v8::FunctionCallbackArguments, + mut retval: v8::ReturnValue, +) { + let value = if args.length() == 0 { + v8::undefined(scope).into() + } else { + args.get(0) + }; + let audio_item = match normalize_output_audio(scope, value) { + Ok(audio_item) => audio_item, + Err(()) => return, + }; + if let Some(state) = scope.get_slot::() { + let _ = state.event_tx.send(RuntimeEvent::ContentItem(audio_item)); + } + retval.set(v8::undefined(scope).into()); +} + pub(super) fn image_callback( scope: &mut v8::PinScope<'_, '_>, args: v8::FunctionCallbackArguments, diff --git a/codex-rs/code-mode/src/runtime/globals.rs b/codex-rs/code-mode/src/runtime/globals.rs index fe3df4c958..66601deb49 100644 --- a/codex-rs/code-mode/src/runtime/globals.rs +++ b/codex-rs/code-mode/src/runtime/globals.rs @@ -1,4 +1,5 @@ use super::RuntimeState; +use super::callbacks::audio_callback; use super::callbacks::clear_timeout_callback; use super::callbacks::exit_callback; use super::callbacks::generated_image_callback; @@ -24,6 +25,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?; let text = helper_function(scope, "text", text_callback)?; let image = helper_function(scope, "image", image_callback)?; + let audio = helper_function(scope, "audio", audio_callback)?; let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?; let store = helper_function(scope, "store", store_callback)?; let load = helper_function(scope, "load", load_callback)?; @@ -37,6 +39,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St set_global(scope, global, "setTimeout", set_timeout.into())?; set_global(scope, global, "text", text.into())?; set_global(scope, global, "image", image.into())?; + set_global(scope, global, "audio", audio.into())?; set_global(scope, global, "generatedImage", generated_image.into())?; set_global(scope, global, "store", store.into())?; set_global(scope, global, "load", load.into())?; diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index 6611799631..ac59b09ac3 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -5,9 +5,12 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem; use codex_code_mode_protocol::ImageDetail; const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; +const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block"; const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead"; const INVALID_IMAGE_URL_ERROR: &str = "Tool call failed: invalid image output. Pass a base64 data URI instead"; +const INVALID_AUDIO_URL_ERROR: &str = + "Tool call failed: invalid audio output. Pass a base64 data URI instead"; const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; pub(super) fn serialize_output_text( @@ -182,6 +185,104 @@ fn parse_image_detail_value<'s>( } } +pub(super) fn normalize_output_audio( + scope: &mut v8::PinScope<'_, '_>, + value: v8::Local<'_, v8::Value>, +) -> Result { + let result = (|| -> Result { + let audio_url = if value.is_string() { + value.to_rust_string_lossy(scope) + } else if value.is_object() && !value.is_array() { + let object = v8::Local::::try_from(value) + .map_err(|_| AUDIO_HELPER_EXPECTS_MESSAGE.to_string())?; + if let Some(audio_url) = parse_non_mcp_output_audio(scope, object)? { + audio_url + } else { + parse_mcp_output_audio(scope, value)? + } + } else { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + }; + + if audio_url.is_empty() { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + } + let Some((scheme, _)) = audio_url.split_once(':') else { + return Err(INVALID_AUDIO_URL_ERROR.to_string()); + }; + if !scheme.eq_ignore_ascii_case("data") { + return Err(INVALID_AUDIO_URL_ERROR.to_string()); + } + + Ok(FunctionCallOutputContentItem::InputAudio { audio_url }) + })(); + + match result { + Ok(item) => Ok(item), + Err(error_text) => { + throw_type_error(scope, &error_text); + Err(()) + } + } +} + +fn parse_non_mcp_output_audio( + scope: &mut v8::PinScope<'_, '_>, + object: v8::Local<'_, v8::Object>, +) -> Result, String> { + let audio_url_key = v8::String::new(scope, "audio_url") + .ok_or_else(|| "failed to allocate audio helper keys".to_string())?; + let Some(audio_url) = object.get(scope, audio_url_key.into()) else { + return Ok(None); + }; + if audio_url.is_undefined() { + return Ok(None); + } + if !audio_url.is_string() { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + } + Ok(Some(audio_url.to_rust_string_lossy(scope))) +} + +fn parse_mcp_output_audio( + scope: &mut v8::PinScope<'_, '_>, + value: v8::Local<'_, v8::Value>, +) -> Result { + let Some(result) = v8_value_to_json(scope, value)? else { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let JsonValue::Object(result) = result else { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + }; + let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else { + return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); + }; + if item_type != "audio" { + return Err(format!( + "audio only accepts MCP audio blocks, got \"{item_type}\"" + )); + } + let data = result + .get("data") + .and_then(JsonValue::as_str) + .ok_or_else(|| "audio expected MCP audio data".to_string())?; + if data.is_empty() { + return Err("audio expected MCP audio data".to_string()); + } + + if data.to_ascii_lowercase().starts_with("data:") { + Ok(data.to_string()) + } else { + let mime_type = result + .get("mimeType") + .or_else(|| result.get("mime_type")) + .and_then(JsonValue::as_str) + .filter(|mime_type| !mime_type.is_empty()) + .unwrap_or("application/octet-stream"); + Ok(format!("data:{mime_type};base64,{data}")) + } +} + pub(super) fn v8_value_to_json( scope: &mut v8::PinScope<'_, '_>, value: v8::Local<'_, v8::Value>, diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index c2651758fb..952d9b1d84 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -403,6 +403,9 @@ fn output_item(item: runtime::OutputItem) -> FunctionCallOutputContentItem { }), } } + runtime::OutputItem::Audio { audio_url } => { + FunctionCallOutputContentItem::InputAudio { audio_url } + } } } diff --git a/codex-rs/code-mode/src/service_tests.rs b/codex-rs/code-mode/src/service_tests.rs index 83f8073362..6d0fba172a 100644 --- a/codex-rs/code-mode/src/service_tests.rs +++ b/codex-rs/code-mode/src/service_tests.rs @@ -737,6 +737,7 @@ async fn output_helpers_return_undefined() { const returnsUndefined = [ text("first"), image("data:image/png;base64,AAA"), + audio("data:audio/wav;base64,YXVkaW8="), notify("ping"), ].map((value) => value === undefined); text(JSON.stringify(returnsUndefined)); @@ -760,8 +761,11 @@ text(JSON.stringify(returnsUndefined)); image_url: "data:image/png;base64,AAA".to_string(), detail: Some(crate::DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, FunctionCallOutputContentItem::InputText { - text: "[true,true,true]".to_string(), + text: "[true,true,true,true]".to_string(), }, ], error_text: None, @@ -769,6 +773,79 @@ text(JSON.stringify(returnsUndefined)); ); } +#[tokio::test] +async fn audio_helper_accepts_audio_url_object_and_raw_mcp_audio_block() { + let service = InProcessCodeModeSession::new(); + + let response = execute( + &service, + ExecuteRequest { + source: r#" +audio({ + audio_url: "data:audio/mpeg;base64,YXVkaW8=", +}); +audio({ + type: "audio", + data: "YXVkaW8=", + mimeType: "audio/wav", +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }, + ) + .await; + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: vec![ + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + ], + error_text: None, + } + ); +} + +#[tokio::test] +async fn audio_helper_rejects_non_data_urls() { + for source in [ + r#"audio("https://example.com/audio.wav");"#, + r#"audio({ audio_url: "file:///tmp/audio.wav" });"#, + ] { + let service = InProcessCodeModeSession::new(); + + let response = execute( + &service, + ExecuteRequest { + source: source.to_string(), + yield_time_ms: None, + ..execute_request("") + }, + ) + .await; + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: Vec::new(), + error_text: Some( + "Tool call failed: invalid audio output. Pass a base64 data URI instead" + .to_string(), + ), + } + ); + } +} + #[tokio::test] async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() { let service = InProcessCodeModeSession::new(); diff --git a/codex-rs/code-mode/src/session_runtime/types.rs b/codex-rs/code-mode/src/session_runtime/types.rs index a33ffa1f1f..8b54e1f32e 100644 --- a/codex-rs/code-mode/src/session_runtime/types.rs +++ b/codex-rs/code-mode/src/session_runtime/types.rs @@ -61,6 +61,9 @@ pub(crate) enum OutputItem { image_url: String, detail: Option, }, + Audio { + audio_url: String, + }, } /// Requested image fidelity for an output image. diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 5f9d56d6cd..876f114431 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -612,13 +612,8 @@ async fn execute_mcp_tool_call( ) .await .map_err(|e| format!("tool call error: {e:?}"))?; - let result = sanitize_mcp_tool_result_for_model( - turn_context - .model_info - .input_modalities - .contains(&InputModality::Image), - Ok(result), - )?; + let result = + sanitize_mcp_tool_result_for_model(&turn_context.model_info.input_modalities, Ok(result))?; Ok(maybe_request_codex_apps_auth_elicitation( sess, turn_context, @@ -813,10 +808,12 @@ async fn maybe_mark_thread_memory_mode_polluted( } fn sanitize_mcp_tool_result_for_model( - supports_image_input: bool, + input_modalities: &[InputModality], result: Result, ) -> Result { - if supports_image_input { + let supports_image_input = input_modalities.contains(&InputModality::Image); + let supports_audio_input = input_modalities.contains(&InputModality::Audio); + if supports_image_input && supports_audio_input { return result; } @@ -825,13 +822,19 @@ fn sanitize_mcp_tool_result_for_model( .content .iter() .map(|block| { - if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str) - && content_type == "image" - { - return serde_json::json!({ - "type": "text", - "text": "", - }); + if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str) { + if content_type == "image" && !supports_image_input { + return serde_json::json!({ + "type": "text", + "text": "", + }); + } + if content_type == "audio" && !supports_audio_input { + return serde_json::json!({ + "type": "text", + "text": "