openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-08-23 13:09:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add audio output support to dynamic tools and code mode (#34080)

Browse Source 
## What changed

- Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas.
- Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks.
- Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item.
- Reject non-data audio URLs and track audio item counts in dynamic tool analytics.

## Testing

- Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling.

GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
nhamidi-oai
2026-07-18 23:16:37 +00:00
committed by copyberry
parent 312caf176a
commit 643de86a19
46 changed files with 987 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
codex-rs/code-mode/src/cell_actor/conversions.rs
View File

@@ -56,5 +56,8 @@ pub(super) fn output_item(item: FunctionCallOutputContentItem) -> CellOutputItem
ImageDetail::Original => CellImageDetail::Original,
}),
},
FunctionCallOutputContentItem::InputAudio { audio_url } => {
CellOutputItem::Audio { audio_url }
}
}
}

21
codex-rs/code-mode/src/runtime/callbacks.rs
View File

@@ -5,6 +5,7 @@ use super::RuntimeEvent;
use super::RuntimeState;
use super::timers;
use super::value::json_to_v8;
use super::value::normalize_output_audio;
use super::value::normalize_output_image;
use super::value::serialize_output_text;
use super::value::throw_type_error;
@@ -96,6 +97,26 @@ pub(super) fn text_callback(
retval.set(v8::undefined(scope).into());
}
pub(super) fn audio_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
mut retval: v8::ReturnValue<v8::Value>,
) {
let value = if args.length() == 0 {
v8::undefined(scope).into()
} else {
args.get(0)
};
let audio_item = match normalize_output_audio(scope, value) {
Ok(audio_item) => audio_item,
Err(()) => return,
};
if let Some(state) = scope.get_slot::<RuntimeState>() {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(audio_item));
}
retval.set(v8::undefined(scope).into());
}
pub(super) fn image_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,

3
codex-rs/code-mode/src/runtime/globals.rs
View File

@@ -1,4 +1,5 @@
use super::RuntimeState;
use super::callbacks::audio_callback;
use super::callbacks::clear_timeout_callback;
use super::callbacks::exit_callback;
use super::callbacks::generated_image_callback;
@@ -24,6 +25,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
let text = helper_function(scope, "text", text_callback)?;
let image = helper_function(scope, "image", image_callback)?;
let audio = helper_function(scope, "audio", audio_callback)?;
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
let store = helper_function(scope, "store", store_callback)?;
let load = helper_function(scope, "load", load_callback)?;
@@ -37,6 +39,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
set_global(scope, global, "setTimeout", set_timeout.into())?;
set_global(scope, global, "text", text.into())?;
set_global(scope, global, "image", image.into())?;
set_global(scope, global, "audio", audio.into())?;
set_global(scope, global, "generatedImage", generated_image.into())?;
set_global(scope, global, "store", store.into())?;
set_global(scope, global, "load", load.into())?;

101
codex-rs/code-mode/src/runtime/value.rs
View File

@@ -5,9 +5,12 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem;
use codex_code_mode_protocol::ImageDetail;
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
const INVALID_IMAGE_URL_ERROR: &str =
"Tool call failed: invalid image output. Pass a base64 data URI instead";
const INVALID_AUDIO_URL_ERROR: &str =
"Tool call failed: invalid audio output. Pass a base64 data URI instead";
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
pub(super) fn serialize_output_text(
@@ -182,6 +185,104 @@ fn parse_image_detail_value<'s>(
}
}
pub(super) fn normalize_output_audio(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<FunctionCallOutputContentItem, ()> {
let result = (|| -> Result<FunctionCallOutputContentItem, String> {
let audio_url = if value.is_string() {
value.to_rust_string_lossy(scope)
} else if value.is_object() && !value.is_array() {
let object = v8::Local::<v8::Object>::try_from(value)
.map_err(|_| AUDIO_HELPER_EXPECTS_MESSAGE.to_string())?;
if let Some(audio_url) = parse_non_mcp_output_audio(scope, object)? {
audio_url
} else {
parse_mcp_output_audio(scope, value)?
}
} else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
if audio_url.is_empty() {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
}
let Some((scheme, _)) = audio_url.split_once(':') else {
return Err(INVALID_AUDIO_URL_ERROR.to_string());
};
if !scheme.eq_ignore_ascii_case("data") {
return Err(INVALID_AUDIO_URL_ERROR.to_string());
}
Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
})();
match result {
Ok(item) => Ok(item),
Err(error_text) => {
throw_type_error(scope, &error_text);
Err(())
}
}
}
fn parse_non_mcp_output_audio(
scope: &mut v8::PinScope<'_, '_>,
object: v8::Local<'_, v8::Object>,
) -> Result<Option<String>, String> {
let audio_url_key = v8::String::new(scope, "audio_url")
.ok_or_else(|| "failed to allocate audio helper keys".to_string())?;
let Some(audio_url) = object.get(scope, audio_url_key.into()) else {
return Ok(None);
};
if audio_url.is_undefined() {
return Ok(None);
}
if !audio_url.is_string() {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
}
Ok(Some(audio_url.to_rust_string_lossy(scope)))
}
fn parse_mcp_output_audio(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<String, String> {
let Some(result) = v8_value_to_json(scope, value)? else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
let JsonValue::Object(result) = result else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else {
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
};
if item_type != "audio" {
return Err(format!(
"audio only accepts MCP audio blocks, got \"{item_type}\""
));
}
let data = result
.get("data")
.and_then(JsonValue::as_str)
.ok_or_else(|| "audio expected MCP audio data".to_string())?;
if data.is_empty() {
return Err("audio expected MCP audio data".to_string());
}
if data.to_ascii_lowercase().starts_with("data:") {
Ok(data.to_string())
} else {
let mime_type = result
.get("mimeType")
.or_else(|| result.get("mime_type"))
.and_then(JsonValue::as_str)
.filter(|mime_type| !mime_type.is_empty())
.unwrap_or("application/octet-stream");
Ok(format!("data:{mime_type};base64,{data}"))
}
}
pub(super) fn v8_value_to_json(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,

3
codex-rs/code-mode/src/service.rs
View File

@@ -403,6 +403,9 @@ fn output_item(item: runtime::OutputItem) -> FunctionCallOutputContentItem {
}),
}
}
runtime::OutputItem::Audio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
}
}

79
codex-rs/code-mode/src/service_tests.rs
View File

@@ -737,6 +737,7 @@ async fn output_helpers_return_undefined() {
const returnsUndefined = [
text("first"),
image("data:image/png;base64,AAA"),
audio("data:audio/wav;base64,YXVkaW8="),
notify("ping"),
].map((value) => value === undefined);
text(JSON.stringify(returnsUndefined));
@@ -760,8 +761,11 @@ text(JSON.stringify(returnsUndefined));
image_url: "data:image/png;base64,AAA".to_string(),
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "[true,true,true]".to_string(),
text: "[true,true,true,true]".to_string(),
},
],
error_text: None,
@@ -769,6 +773,79 @@ text(JSON.stringify(returnsUndefined));
);
}
#[tokio::test]
async fn audio_helper_accepts_audio_url_object_and_raw_mcp_audio_block() {
let service = InProcessCodeModeSession::new();
let response = execute(
&service,
ExecuteRequest {
source: r#"
audio({
audio_url: "data:audio/mpeg;base64,YXVkaW8=",
});
audio({
type: "audio",
data: "YXVkaW8=",
mimeType: "audio/wav",
});
"#
.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;
assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: vec![
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
],
error_text: None,
}
);
}
#[tokio::test]
async fn audio_helper_rejects_non_data_urls() {
for source in [
r#"audio("https://example.com/audio.wav");"#,
r#"audio({ audio_url: "file:///tmp/audio.wav" });"#,
] {
let service = InProcessCodeModeSession::new();
let response = execute(
&service,
ExecuteRequest {
source: source.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;
assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: Vec::new(),
error_text: Some(
"Tool call failed: invalid audio output. Pass a base64 data URI instead"
.to_string(),
),
}
);
}
}
#[tokio::test]
async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() {
let service = InProcessCodeModeSession::new();

3
codex-rs/code-mode/src/session_runtime/types.rs
View File

@@ -61,6 +61,9 @@ pub(crate) enum OutputItem {
image_url: String,
detail: Option<ImageDetail>,
},
Audio {
audio_url: String,
},
}
/// Requested image fidelity for an output image.