openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-19 17:28:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add audio output support to dynamic tools and code mode (#34080)

Browse Source 
## What changed

- Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas.
- Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks.
- Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item.
- Reject non-data audio URLs and track audio item counts in dynamic tool analytics.

## Testing

- Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling.

GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
nhamidi-oai
2026-07-18 23:22:13 +00:00
committed by copyberry
parent 312caf176a
commit 643de86a19
46 changed files with 987 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/analytics/src/events.rs
+1
View File
@@ -707,6 +707,7 @@ pub(crate) struct CodexDynamicToolCallEventParams {
pub(crate) output_content_item_count: Option<u64>,
pub(crate) output_text_item_count: Option<u64>,
pub(crate) output_image_item_count: Option<u64>,
pub(crate) output_audio_item_count: Option<u64>,
}
#[derive(Serialize)]
codex-rs/analytics/src/reducer.rs
+32 -1
View File
@@ -1968,6 +1968,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option<TrackEventRequest> {
output_content_item_count: counts.map(|counts| counts.total),
output_text_item_count: counts.map(|counts| counts.text),
output_image_item_count: counts.map(|counts| counts.image),
output_audio_item_count: counts.map(|counts| counts.audio),
},
},
))
@@ -2505,26 +2506,30 @@ fn file_change_counts(changes: &[codex_app_server_protocol::FileUpdateChange]) -
counts
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct DynamicContentCounts {
total: u64,
text: u64,
image: u64,
audio: u64,
}
fn dynamic_content_counts(items: &[DynamicToolCallOutputContentItem]) -> DynamicContentCounts {
let mut text = 0;
let mut image = 0;
let mut audio = 0;
for item in items {
match item {
DynamicToolCallOutputContentItem::InputText { .. } => text += 1,
DynamicToolCallOutputContentItem::InputImage { .. } => image += 1,
DynamicToolCallOutputContentItem::InputAudio { .. } => audio += 1,
}
}
DynamicContentCounts {
total: usize_to_u64(items.len()),
text,
image,
audio,
}
}
@@ -2830,6 +2835,7 @@ mod tests {
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::permissions::FileSystemSandboxPolicy;
use codex_protocol::permissions::NetworkSandboxPolicy;
use pretty_assertions::assert_eq;
#[test]
fn managed_full_disk_with_restricted_network_reports_external_sandbox() {
@@ -2853,4 +2859,29 @@ mod tests {
Some((ReviewStatus::TimedOut, ReviewResolution::None))
));
}
#[test]
fn dynamic_content_counts_include_audio() {
let items = vec![
DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
];
assert_eq!(
dynamic_content_counts(&items),
DynamicContentCounts {
total: 3,
text: 1,
image: 1,
audio: 1,
}
);
}
}
codex-rs/app-server-protocol/schema/json/DynamicToolCallResponse.json
Generated
+20
View File
@@ -42,6 +42,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
}
codex-rs/app-server-protocol/schema/json/ServerNotification.json
Generated
+20
View File
@@ -1129,6 +1129,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
Generated
+20
View File
@@ -9163,6 +9163,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
Generated
+20
View File
@@ -5356,6 +5356,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
Generated
+20
View File
@@ -240,6 +240,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
Generated
+20
View File
@@ -240,6 +240,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
Generated
+20
View File
@@ -378,6 +378,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
Generated
+20
View File
@@ -459,6 +459,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
Generated
+20
View File
@@ -459,6 +459,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
Generated
+20
View File
@@ -459,6 +459,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
Generated
+20
View File
@@ -381,6 +381,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
Generated
+20
View File
@@ -378,6 +378,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
Generated
+20
View File
@@ -378,6 +378,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
Generated
+20
View File
@@ -378,6 +378,26 @@
],
"title": "InputImageDynamicToolCallOutputContentItem",
"type": "object"
},
{
"properties": {
"audioUrl": {
"type": "string"
},
"type": {
"enum": [
"inputAudio"
],
"title": "InputAudioDynamicToolCallOutputContentItemType",
"type": "string"
}
},
"required": [
"audioUrl",
"type"
],
"title": "InputAudioDynamicToolCallOutputContentItem",
"type": "object"
}
]
},
codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallOutputContentItem.ts
Generated
+1 -1
View File
@@ -2,4 +2,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, };
export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, } | { "type": "inputAudio", audioUrl: string, };
codex-rs/app-server-protocol/src/protocol/event_mapping.rs
+3
View File
@@ -59,6 +59,9 @@ pub fn item_event_to_server_notification(
CoreDynamicToolCallOutputContentItem::InputImage { image_url } => {
DynamicToolCallOutputContentItem::InputImage { image_url }
}
CoreDynamicToolCallOutputContentItem::InputAudio { audio_url } => {
DynamicToolCallOutputContentItem::InputAudio { audio_url }
}
})
.collect(),
),
codex-rs/app-server-protocol/src/protocol/thread_history.rs
+25 -6
View File
@@ -1498,6 +1498,9 @@ fn convert_dynamic_tool_content_items(
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
image_url,
} => DynamicToolCallOutputContentItem::InputImage { image_url },
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
audio_url,
} => DynamicToolCallOutputContentItem::InputAudio { audio_url },
})
.collect()
}
@@ -2879,9 +2882,17 @@ mod tests {
namespace: Some("codex_app".into()),
tool: "lookup_ticket".into(),
arguments: serde_json::json!({"id":"ABC-123"}),
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
}],
content_items: vec![
CoreDynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
},
CoreDynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".into(),
},
CoreDynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
},
],
success: true,
error: None,
duration: Duration::from_millis(42),
@@ -2903,9 +2914,17 @@ mod tests {
tool: "lookup_ticket".into(),
arguments: serde_json::json!({"id":"ABC-123"}),
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
}]),
content_items: Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".into(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
},
]),
success: Some(true),
duration_ms: Some(42),
}
codex-rs/app-server-protocol/src/protocol/v2/item.rs
+8
View File
@@ -1552,6 +1552,8 @@ pub enum DynamicToolCallOutputContentItem {
InputText { text: String },
#[serde(rename_all = "camelCase")]
InputImage { image_url: String },
#[serde(rename_all = "camelCase")]
InputAudio { audio_url: String },
}
impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
@@ -1565,6 +1567,9 @@ impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
image_url,
} => Self::InputImage { image_url },
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
audio_url,
} => Self::InputAudio { audio_url },
}
}
}
@@ -1578,6 +1583,9 @@ impl From<DynamicToolCallOutputContentItem>
DynamicToolCallOutputContentItem::InputImage { image_url } => {
Self::InputImage { image_url }
}
DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
}
}
}
}
codex-rs/app-server-protocol/src/protocol/v2/tests.rs
+25 -4
View File
@@ -2719,6 +2719,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
},
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]),
success: Some(true),
error: None,
@@ -2733,9 +2739,17 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
tool: "lookup".to_string(),
arguments: json!({"id": "123"}),
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
}]),
content_items: Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]),
success: Some(true),
duration_ms: Some(5),
}
@@ -3954,7 +3968,7 @@ fn dynamic_tool_response_serializes_content_items() {
}
#[test]
fn dynamic_tool_response_serializes_text_and_image_content_items() {
fn dynamic_tool_response_serializes_text_image_and_audio_content_items() {
let value = serde_json::to_value(DynamicToolCallResponse {
content_items: vec![
DynamicToolCallOutputContentItem::InputText {
@@ -3963,6 +3977,9 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
],
success: true,
})
@@ -3979,6 +3996,10 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
{
"type": "inputImage",
"imageUrl": "data:image/png;base64,AAA"
},
{
"type": "inputAudio",
"audioUrl": "data:audio/wav;base64,YXVkaW8="
}
],
"success": true,
codex-rs/app-server/README.md
+3 -2
View File
@@ -1671,7 +1671,7 @@ The server also emits item lifecycle notifications around the request:
3. Client response.
4. `item/completed` with `item.type = "dynamicToolCall"`, final `status`, and the returned `contentItems`/`success`.
The client must respond with content items. Use `inputText` for text and `inputImage` for inline data URLs. Remote HTTP(S) image URLs make the dynamic tool response invalid.
The client must respond with content items. Use `inputText` for text, `inputImage` for inline image data URLs, and `inputAudio` for inline audio data URLs. Audio data URLs accept wav, mp3, m4a, webm, and ogg media types. Remote HTTP(S) image URLs and non-data audio URLs make the dynamic tool response invalid.
```json
{
@@ -1679,7 +1679,8 @@ The client must respond with content items. Use `inputText` for text and `inputI
"result": {
"contentItems": [
{ "type": "inputText", "text": "Ticket ABC-123 is open." },
{ "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" }
{ "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" },
{ "type": "inputAudio", "audioUrl": "data:audio/wav;base64,AAA" }
],
"success": true
}
codex-rs/app-server/src/dynamic_tools.rs
+19
View File
@@ -13,6 +13,8 @@ use crate::image_url::is_remote_image_url;
use crate::outgoing_message::ClientRequestResult;
use crate::server_request_error::is_turn_transition_server_request_error;
const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL";
pub(crate) async fn on_call_response(
call_id: String,
receiver: oneshot::Receiver<ClientRequestResult>,
@@ -71,6 +73,23 @@ fn decode_response(value: serde_json::Value) -> (DynamicToolCallResponse, Option
);
fallback_response(REMOTE_IMAGE_URL_ERROR)
}
Ok(response)
if response.content_items.iter().any(|item| {
matches!(
item,
DynamicToolCallOutputContentItem::InputAudio { audio_url }
if !audio_url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
)
}) =>
{
error!(
message = INVALID_AUDIO_URL_ERROR,
"dynamic tool response was invalid"
);
fallback_response(INVALID_AUDIO_URL_ERROR)
}
Ok(response) => (response, None),
Err(err) => {
error!("failed to deserialize DynamicToolCallResponse: {err}");
codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
+83 -18
View File
@@ -4,6 +4,7 @@ use app_test_support::TestAppServer;
use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::to_response;
use app_test_support::write_models_cache_with_models;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallParams;
use codex_app_server_protocol::DynamicToolCallResponse;
@@ -28,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use core_test_support::load_default_config_for_test;
use core_test_support::responses;
use pretty_assertions::assert_eq;
use serde_json::Value;
@@ -39,6 +41,8 @@ use tokio::time::timeout;
use wiremock::MockServer;
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
const INLINE_AUDIO_DATA_URL: &str = "data:audio/wav;base64,YXVkaW8=";
const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL";
const REMOTE_IMAGE_URL_ERROR: &str =
"remote image URLs are not supported; use an inline data URL instead";
@@ -592,6 +596,9 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result<PendingDynami
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let config = load_default_config_for_test(&codex_home).await;
let model_info = codex_core::test_support::construct_model_info_offline("mock-model", &config);
write_models_cache_with_models(codex_home.path(), vec![model_info])?;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
@@ -680,7 +687,7 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result<PendingDynami
/// Ensures dynamic tool call responses can include structured content items.
#[tokio::test]
async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<()> {
async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
let call_id = "dyn-call-items-1";
let PendingDynamicToolCall {
mut mcp,
@@ -696,22 +703,21 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
DynamicToolCallOutputContentItem::InputImage {
image_url: TINY_PNG_DATA_URL.to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
},
];
// Audio remains available on the app-server item, but the base truncation policy intentionally
// omits it from the model-visible payload until the history/accounting layer.
let model_content_items = vec![
FunctionCallOutputContentItem::InputText {
text: "dynamic-ok".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: TINY_PNG_DATA_URL.to_string(),
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
let content_items = response_content_items
.clone()
.into_iter()
.map(|item| match item {
DynamicToolCallOutputContentItem::InputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
DynamicToolCallOutputContentItem::InputImage { image_url } => {
FunctionCallOutputContentItem::InputImage {
image_url,
detail: Some(DEFAULT_IMAGE_DETAIL),
}
}
})
.collect::<Vec<FunctionCallOutputContentItem>>();
let response = DynamicToolCallResponse {
content_items: response_content_items,
success: true,
@@ -741,6 +747,9 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
DynamicToolCallOutputContentItem::InputImage {
image_url: TINY_PNG_DATA_URL.to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
},
])
);
assert_eq!(success, Some(true));
@@ -777,12 +786,12 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
.context("expected function_call_output in follow-up request")?;
assert_eq!(
payload.body,
FunctionCallOutputBody::ContentItems(content_items.clone())
FunctionCallOutputBody::ContentItems(model_content_items.clone())
);
assert_eq!(payload.success, None);
assert_eq!(
serde_json::to_string(&payload)?,
serde_json::to_string(&content_items)?
serde_json::to_string(&model_content_items)?
);
Ok(())
@@ -844,6 +853,62 @@ async fn dynamic_tool_remote_image_response_becomes_model_visible_error() -> Res
Ok(())
}
#[tokio::test]
async fn dynamic_tool_remote_audio_response_becomes_model_visible_error() -> Result<()> {
let call_id = "dyn-call-remote-audio";
let PendingDynamicToolCall {
mut mcp,
server,
request_id,
params,
} = start_function_dynamic_tool_call(call_id).await?;
let response = DynamicToolCallResponse {
content_items: vec![DynamicToolCallOutputContentItem::InputAudio {
audio_url: "https://example.com/tool.wav".to_string(),
}],
success: true,
};
mcp.send_response(request_id, serde_json::to_value(response)?)
.await?;
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
assert_eq!(completed.thread_id, params.thread_id);
assert_eq!(completed.turn_id, params.turn_id);
let ThreadItem::DynamicToolCall {
status,
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(status, DynamicToolCallStatus::Failed);
assert_eq!(
content_items,
Some(vec![DynamicToolCallOutputContentItem::InputText {
text: INVALID_AUDIO_URL_ERROR.to_string(),
}])
);
assert_eq!(success, Some(false));
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let output = responses_bodies(&server)
.await?
.iter()
.find_map(|body| function_call_output_raw_output(body, call_id))
.context("expected function_call_output output in follow-up request")?;
assert_eq!(output, json!(INVALID_AUDIO_URL_ERROR));
Ok(())
}
async fn responses_bodies(server: &MockServer) -> Result<Vec<Value>> {
let requests = server
.received_requests()
codex-rs/code-mode-protocol/src/description.rs
+2
View File
@@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
- `audio(audioUrlOrItem: string | { audio_url: string } | AudioContent)`: Appends an audio item. `audio_url` should be a base64-encoded `data:` URL. To forward an MCP tool audio block, pass an individual `AudioContent` block from `result.content`, for example `audio(result.content[0])`.
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. HTTP(S) URLs are not supported.
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
@@ -880,6 +881,7 @@ bar"
fn exec_description_mentions_timeout_helpers() {
let description =
build_exec_tool_description(&[], &[], &BTreeMap::new(), /*code_mode_only*/ false);
assert!(description.contains("`audio(audioUrlOrItem:"));
assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`"));
assert!(description.contains("`clearTimeout(timeoutId?: number)`"));
}
codex-rs/code-mode-protocol/src/host/host_tests.rs
+7
View File
@@ -130,6 +130,9 @@ fn content_items() -> Vec<WireContentItem> {
image_url: "data:image/png;base64,original".to_string(),
detail: Some(WireImageDetail::Original),
},
WireContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]
}
@@ -157,6 +160,10 @@ fn content_items_json() -> Value {
"image_url": "data:image/png;base64,original",
"detail": "original",
},
{
"type": "input_audio",
"audio_url": "data:audio/wav;base64,YXVkaW8=",
},
])
}
codex-rs/code-mode-protocol/src/host/payload.rs
+7
View File
@@ -245,6 +245,9 @@ pub enum WireContentItem {
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<WireImageDetail>,
},
InputAudio {
audio_url: String,
},
}
impl From<FunctionCallOutputContentItem> for WireContentItem {
@@ -255,6 +258,9 @@ impl From<FunctionCallOutputContentItem> for WireContentItem {
image_url,
detail: detail.map(Into::into),
},
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
}
}
}
}
@@ -267,6 +273,7 @@ impl From<WireContentItem> for FunctionCallOutputContentItem {
image_url,
detail: detail.map(Into::into),
},
WireContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url },
}
}
}
codex-rs/code-mode-protocol/src/response.rs
+3
View File
@@ -23,4 +23,7 @@ pub enum FunctionCallOutputContentItem {
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<ImageDetail>,
},
InputAudio {
audio_url: String,
},
}
codex-rs/code-mode/src/cell_actor/conversions.rs
+3
View File
@@ -56,5 +56,8 @@ pub(super) fn output_item(item: FunctionCallOutputContentItem) -> CellOutputItem
ImageDetail::Original => CellImageDetail::Original,
}),
},
FunctionCallOutputContentItem::InputAudio { audio_url } => {
CellOutputItem::Audio { audio_url }
}
}
}
codex-rs/code-mode/src/runtime/callbacks.rs
+21
View File
@@ -5,6 +5,7 @@ use super::RuntimeEvent;
use super::RuntimeState;
use super::timers;
use super::value::json_to_v8;
use super::value::normalize_output_audio;
use super::value::normalize_output_image;
use super::value::serialize_output_text;
use super::value::throw_type_error;
@@ -96,6 +97,26 @@ pub(super) fn text_callback(
retval.set(v8::undefined(scope).into());
}
pub(super) fn audio_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
mut retval: v8::ReturnValue<v8::Value>,
) {
let value = if args.length() == 0 {
v8::undefined(scope).into()
} else {
args.get(0)
};
let audio_item = match normalize_output_audio(scope, value) {
Ok(audio_item) => audio_item,
Err(()) => return,
};
if let Some(state) = scope.get_slot::<RuntimeState>() {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(audio_item));
}
retval.set(v8::undefined(scope).into());
}
pub(super) fn image_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
codex-rs/code-mode/src/runtime/globals.rs
+3
View File
@@ -1,4 +1,5 @@
use super::RuntimeState;
use super::callbacks::audio_callback;
use super::callbacks::clear_timeout_callback;
use super::callbacks::exit_callback;
use super::callbacks::generated_image_callback;
@@ -24,6 +25,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
let text = helper_function(scope, "text", text_callback)?;
let image = helper_function(scope, "image", image_callback)?;
let audio = helper_function(scope, "audio", audio_callback)?;
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
let store = helper_function(scope, "store", store_callback)?;
let load = helper_function(scope, "load", load_callback)?;
@@ -37,6 +39,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
set_global(scope, global, "setTimeout", set_timeout.into())?;
set_global(scope, global, "text", text.into())?;
set_global(scope, global, "image", image.into())?;
set_global(scope, global, "audio", audio.into())?;
set_global(scope, global, "generatedImage", generated_image.into())?;
set_global(scope, global, "store", store.into())?;
set_global(scope, global, "load", load.into())?;
codex-rs/code-mode/src/runtime/value.rs
+101
View File
@@ -5,9 +5,12 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem;
use codex_code_mode_protocol::ImageDetail;
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
const INVALID_IMAGE_URL_ERROR: &str =
"Tool call failed: invalid image output. Pass a base64 data URI instead";
const INVALID_AUDIO_URL_ERROR: &str =
"Tool call failed: invalid audio output. Pass a base64 data URI instead";
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
pub(super) fn serialize_output_text(
@@ -182,6 +185,104 @@ fn parse_image_detail_value<'s>(
}
}
pub(super) fn normalize_output_audio(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<FunctionCallOutputContentItem, ()> {
let result = (|| -> Result<FunctionCallOutputContentItem, String> {
let audio_url = if value.is_string() {
value.to_rust_string_lossy(scope)
} else if value.is_object() && !value.is_array() {
let object = v8::Local::<v8::Object>::try_from(value)
.map_err(|_| AUDIO_HELPER_EXPECTS_MESSAGE.to_string())?;
if let Some(audio_url) = parse_non_mcp_output_audio(scope, object)? {
audio_url
} else {
parse_mcp_output_audio(scope, value)?
}
} else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
if audio_url.is_empty() {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
}
let Some((scheme, _)) = audio_url.split_once(':') else {
return Err(INVALID_AUDIO_URL_ERROR.to_string());
};
if !scheme.eq_ignore_ascii_case("data") {
return Err(INVALID_AUDIO_URL_ERROR.to_string());
}
Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
})();
match result {
Ok(item) => Ok(item),
Err(error_text) => {
throw_type_error(scope, &error_text);
Err(())
}
}
}
fn parse_non_mcp_output_audio(
scope: &mut v8::PinScope<'_, '_>,
object: v8::Local<'_, v8::Object>,
) -> Result<Option<String>, String> {
let audio_url_key = v8::String::new(scope, "audio_url")
.ok_or_else(|| "failed to allocate audio helper keys".to_string())?;
let Some(audio_url) = object.get(scope, audio_url_key.into()) else {
return Ok(None);
};
if audio_url.is_undefined() {
return Ok(None);
}
if !audio_url.is_string() {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
}
Ok(Some(audio_url.to_rust_string_lossy(scope)))
}
fn parse_mcp_output_audio(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<String, String> {
let Some(result) = v8_value_to_json(scope, value)? else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
let JsonValue::Object(result) = result else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
if item_type != "audio" {
return Err(format!(
"audio only accepts MCP audio blocks, got \"{item_type}\""
));
}
let data = result
.get("data")
.and_then(JsonValue::as_str)
.ok_or_else(|| "audio expected MCP audio data".to_string())?;
if data.is_empty() {
return Err("audio expected MCP audio data".to_string());
}
if data.to_ascii_lowercase().starts_with("data:") {
Ok(data.to_string())
} else {
let mime_type = result
.get("mimeType")
.or_else(|| result.get("mime_type"))
.and_then(JsonValue::as_str)
.filter(|mime_type| !mime_type.is_empty())
.unwrap_or("application/octet-stream");
Ok(format!("data:{mime_type};base64,{data}"))
}
}
pub(super) fn v8_value_to_json(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
codex-rs/code-mode/src/service.rs
+3
View File
@@ -403,6 +403,9 @@ fn output_item(item: runtime::OutputItem) -> FunctionCallOutputContentItem {
}),
}
}
runtime::OutputItem::Audio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
}
}
codex-rs/code-mode/src/service_tests.rs
+78 -1
View File
@@ -737,6 +737,7 @@ async fn output_helpers_return_undefined() {
const returnsUndefined = [
text("first"),
image("data:image/png;base64,AAA"),
audio("data:audio/wav;base64,YXVkaW8="),
notify("ping"),
].map((value) => value === undefined);
text(JSON.stringify(returnsUndefined));
@@ -760,8 +761,11 @@ text(JSON.stringify(returnsUndefined));
image_url: "data:image/png;base64,AAA".to_string(),
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "[true,true,true]".to_string(),
text: "[true,true,true,true]".to_string(),
},
],
error_text: None,
@@ -769,6 +773,79 @@ text(JSON.stringify(returnsUndefined));
);
}
#[tokio::test]
async fn audio_helper_accepts_audio_url_object_and_raw_mcp_audio_block() {
let service = InProcessCodeModeSession::new();
let response = execute(
&service,
ExecuteRequest {
source: r#"
audio({
audio_url: "data:audio/mpeg;base64,YXVkaW8=",
});
audio({
type: "audio",
data: "YXVkaW8=",
mimeType: "audio/wav",
});
"#
.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;
assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: vec![
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
],
error_text: None,
}
);
}
#[tokio::test]
async fn audio_helper_rejects_non_data_urls() {
for source in [
r#"audio("https://example.com/audio.wav");"#,
r#"audio({ audio_url: "file:///tmp/audio.wav" });"#,
] {
let service = InProcessCodeModeSession::new();
let response = execute(
&service,
ExecuteRequest {
source: source.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;
assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: Vec::new(),
error_text: Some(
"Tool call failed: invalid audio output. Pass a base64 data URI instead"
.to_string(),
),
}
);
}
}
#[tokio::test]
async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() {
let service = InProcessCodeModeSession::new();
codex-rs/code-mode/src/session_runtime/types.rs
+3
View File
@@ -61,6 +61,9 @@ pub(crate) enum OutputItem {
image_url: String,
detail: Option<ImageDetail>,
},
Audio {
audio_url: String,
},
}
/// Requested image fidelity for an output image.
codex-rs/core/src/mcp_tool_call.rs
+19 -16
View File
@@ -612,13 +612,8 @@ async fn execute_mcp_tool_call(
)
.await
.map_err(|e| format!("tool call error: {e:?}"))?;
let result = sanitize_mcp_tool_result_for_model(
turn_context
.model_info
.input_modalities
.contains(&InputModality::Image),
Ok(result),
)?;
let result =
sanitize_mcp_tool_result_for_model(&turn_context.model_info.input_modalities, Ok(result))?;
Ok(maybe_request_codex_apps_auth_elicitation(
sess,
turn_context,
@@ -813,10 +808,12 @@ async fn maybe_mark_thread_memory_mode_polluted(
}
fn sanitize_mcp_tool_result_for_model(
supports_image_input: bool,
input_modalities: &[InputModality],
result: Result<CallToolResult, String>,
) -> Result<CallToolResult, String> {
if supports_image_input {
let supports_image_input = input_modalities.contains(&InputModality::Image);
let supports_audio_input = input_modalities.contains(&InputModality::Audio);
if supports_image_input && supports_audio_input {
return result;
}
@@ -825,13 +822,19 @@ fn sanitize_mcp_tool_result_for_model(
.content
.iter()
.map(|block| {
if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str)
&& content_type == "image"
{
return serde_json::json!({
"type": "text",
"text": "<image content omitted because you do not support image input>",
});
if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str) {
if content_type == "image" && !supports_image_input {
return serde_json::json!({
"type": "text",
"text": "<image content omitted because you do not support image input>",
});
}
if content_type == "audio" && !supports_audio_input {
return serde_json::json!({
"type": "text",
"text": "<audio content omitted because you do not support audio input>",
});
}
}
block.clone()
codex-rs/core/src/mcp_tool_call_tests.rs
+45 -20
View File
@@ -962,7 +962,7 @@ fn codex_apps_connectors_support_persistent_approval() {
}
#[test]
fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
fn sanitize_mcp_tool_result_for_model_rewrites_unsupported_media_content() {
let result = Ok(CallToolResult {
content: vec![
serde_json::json!({
@@ -970,6 +970,11 @@ fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
"data": "Zm9v",
"mimeType": "image/png",
}),
serde_json::json!({
"type": "audio",
"data": "YmFy",
"mimeType": "audio/wav",
}),
serde_json::json!({
"type": "text",
"text": "hello",
@@ -980,39 +985,59 @@ fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
meta: None,
});
let got = sanitize_mcp_tool_result_for_model(/*supports_image_input*/ false, result)
let got = sanitize_mcp_tool_result_for_model(&[InputModality::Text], result)
.expect("sanitized result");
assert_eq!(
got.content,
vec![
serde_json::json!({
"type": "text",
"text": "<image content omitted because you do not support image input>",
}),
serde_json::json!({
"type": "text",
"text": "hello",
}),
]
got,
CallToolResult {
content: vec![
serde_json::json!({
"type": "text",
"text": "<image content omitted because you do not support image input>",
}),
serde_json::json!({
"type": "text",
"text": "<audio content omitted because you do not support audio input>",
}),
serde_json::json!({
"type": "text",
"text": "hello",
}),
],
structured_content: None,
is_error: Some(false),
meta: None,
}
);
}
#[test]
fn sanitize_mcp_tool_result_for_model_preserves_image_when_supported() {
fn sanitize_mcp_tool_result_for_model_preserves_supported_media() {
let original = CallToolResult {
content: vec![serde_json::json!({
"type": "image",
"data": "Zm9v",
"mimeType": "image/png",
})],
content: vec![
serde_json::json!({
"type": "image",
"data": "Zm9v",
"mimeType": "image/png",
}),
serde_json::json!({
"type": "audio",
"data": "YmFy",
"mimeType": "audio/wav",
}),
],
structured_content: Some(serde_json::json!({"x": 1})),
is_error: Some(false),
meta: Some(serde_json::json!({"k": "v"})),
};
let got = sanitize_mcp_tool_result_for_model(
/*supports_image_input*/ true,
&[
InputModality::Text,
InputModality::Image,
InputModality::Audio,
],
Ok(original.clone()),
)
.expect("unsanitized result");
codex-rs/core/src/tools/code_mode/response_adapter.rs
+3
View File
@@ -42,6 +42,9 @@ impl IntoProtocol<FunctionCallOutputContentItem>
.or(Some(DEFAULT_IMAGE_DETAIL)),
}
}
codex_code_mode::FunctionCallOutputContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
}
}
}
codex-rs/core/tests/suite/code_mode.rs
+1
View File
@@ -3500,6 +3500,7 @@ text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort()));
"WeakSet",
"__codexContentItems",
"add_content",
"audio",
"decodeURI",
"decodeURIComponent",
"encodeURI",
codex-rs/protocol/src/dynamic_tools.rs
+2
View File
@@ -70,6 +70,8 @@ pub enum DynamicToolCallOutputContentItem {
InputText { text: String },
#[serde(rename_all = "camelCase")]
InputImage { image_url: String },
#[serde(rename_all = "camelCase")]
InputAudio { audio_url: String },
}
/// Former flat `SessionMeta` shape, including the old `exposeToContext` flag.
codex-rs/protocol/src/models.rs
+106 -3
View File
@@ -1856,7 +1856,7 @@ pub enum FunctionCallOutputContentItem {
///
/// This conversion is intentionally lossy:
/// - only `input_text` items are included
/// - image items are ignored
/// - image and audio items are ignored
///
/// We use this helper where callers still need a string representation (for
/// example telemetry previews or legacy string-only output paths) while keeping
@@ -1899,6 +1899,9 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
detail: Some(DEFAULT_IMAGE_DETAIL),
}
}
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
}
}
}
}
@@ -2118,6 +2121,14 @@ fn convert_mcp_content_to_items(
#[serde(rename = "_meta", default)]
meta: Option<serde_json::Value>,
},
#[serde(rename = "audio")]
Audio {
data: String,
#[serde(rename = "mimeType", alias = "mime_type")]
mime_type: Option<String>,
#[serde(rename = "_meta", default)]
_meta: Option<serde_json::Value>,
},
#[serde(other)]
Unknown,
}
@@ -2171,6 +2182,18 @@ fn convert_mcp_content_to_items(
.or(Some(DEFAULT_IMAGE_DETAIL)),
}
}
Ok(McpContent::Audio {
data, mime_type, ..
}) => {
saw_content_item = true;
let audio_url = if data.starts_with("data:") {
data
} else {
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
format!("data:{mime_type};base64,{data}")
};
FunctionCallOutputContentItem::InputAudio { audio_url }
}
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
},
@@ -2726,7 +2749,36 @@ mod tests {
}
#[test]
fn convert_mcp_content_to_items_returns_none_without_images() {
fn convert_mcp_audio_content_builds_data_urls_and_preserves_existing_data_urls() {
let contents = vec![
serde_json::json!({
"type": "audio",
"data": "Zm9v",
"mimeType": "audio/wav",
"_meta": {"source": "microphone"},
}),
serde_json::json!({
"type": "audio",
"data": "data:audio/ogg;base64,YmFy",
"mimeType": "audio/ogg",
}),
];
assert_eq!(
convert_mcp_content_to_items(&contents),
Some(vec![
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,Zm9v".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/ogg;base64,YmFy".to_string(),
},
])
);
}
#[test]
fn convert_mcp_content_to_items_returns_none_without_media() {
let contents = vec![serde_json::json!({
"type": "text",
"text": "hello",
@@ -2755,7 +2807,7 @@ mod tests {
}
#[test]
fn function_call_output_content_items_to_text_ignores_blank_text_and_images() {
fn function_call_output_content_items_to_text_ignores_blank_text_and_media() {
let content_items = vec![
FunctionCallOutputContentItem::InputText {
text: " ".to_string(),
@@ -2764,6 +2816,9 @@ mod tests {
image_url: "data:image/png;base64,AAA".to_string(),
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,AAA".to_string(),
},
FunctionCallOutputContentItem::EncryptedContent {
encrypted_content: "enc_opaque".to_string(),
},
@@ -2953,6 +3008,54 @@ mod tests {
Ok(())
}
#[test]
fn serializes_audio_outputs_as_array() -> Result<()> {
let call_tool_result = CallToolResult {
content: vec![
serde_json::json!({"type":"text","text":"caption"}),
serde_json::json!({"type":"audio","data":"BASE64","mimeType":"audio/wav"}),
],
structured_content: None,
is_error: Some(false),
meta: None,
};
let payload = call_tool_result.into_function_call_output_payload();
assert_eq!(
payload,
FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "caption".into(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,BASE64".into(),
},
]),
success: Some(true),
}
);
let item = ResponseInputItem::FunctionCallOutput {
call_id: "call1".into(),
output: payload,
};
assert_eq!(
serde_json::to_value(item)?,
serde_json::json!({
"type": "function_call_output",
"call_id": "call1",
"output": [
{"type": "input_text", "text": "caption"},
{"type": "input_audio", "audio_url": "data:audio/wav;base64,BASE64"},
],
})
);
Ok(())
}
#[test]
fn serializes_custom_tool_image_outputs_as_array() -> Result<()> {
let item = ResponseInputItem::CustomToolCallOutput {