From 981d6b652b710991e5e1c09e12ee7a7ee1a6c0da Mon Sep 17 00:00:00 2001 From: Jennifer Zhao Date: Mon, 31 Aug 2026 23:08:18 +0000 Subject: [PATCH] Omit undersized WAV output from Code Mode (#41934) ## Why Audio models cannot reliably encode tool-generated clips shorter than 25 ms. ## What changed - Measure the available audio frames in base64-encoded PCM and IEEE float WAV outputs. - Replace clips under 25 ms with explanatory text while preserving surrounding Code Mode output. - Leave clips at or above the threshold and unrecognized audio formats unchanged. ## Testing - Cover the duration boundary across multiple sample rates and all supported `audio()` input forms. - Verify bounded and truncated WAV chunks, output after `yield_control()`, and end-to-end mixed text and audio output. GitOrigin-RevId: 342b4bdbc65db4238580f9cdc372763c2576eca5 --- codex-rs/Cargo.lock | 1 + codex-rs/code-mode-runtime/Cargo.toml | 1 + .../code-mode-runtime/src/runtime/audio.rs | 63 +++++++ codex-rs/code-mode-runtime/src/runtime/mod.rs | 1 + .../code-mode-runtime/src/runtime/value.rs | 10 ++ .../src/service_audio_tests.rs | 163 ++++++++++++++++++ .../code-mode-runtime/src/service_tests.rs | 3 + codex-rs/core/tests/suite/code_mode.rs | 84 +++++++++ 8 files changed, 326 insertions(+) create mode 100644 codex-rs/code-mode-runtime/src/runtime/audio.rs create mode 100644 codex-rs/code-mode-runtime/src/service_audio_tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 8a34615829..07b46199c9 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2638,6 +2638,7 @@ dependencies = [ name = "codex-code-mode-runtime" version = "0.0.0" dependencies = [ + "base64 0.22.1", "codex-code-mode-protocol", "codex-protocol", "deno_core_icudata", diff --git a/codex-rs/code-mode-runtime/Cargo.toml b/codex-rs/code-mode-runtime/Cargo.toml index e3708b657e..20c1e6cb4c 100644 --- a/codex-rs/code-mode-runtime/Cargo.toml +++ b/codex-rs/code-mode-runtime/Cargo.toml @@ -13,6 +13,7 @@ path = "src/lib.rs" workspace = true [dependencies] +base64 = { workspace = true } codex-code-mode-protocol = { workspace = true } codex-protocol = { workspace = true } deno_core_icudata = { workspace = true } diff --git a/codex-rs/code-mode-runtime/src/runtime/audio.rs b/codex-rs/code-mode-runtime/src/runtime/audio.rs new file mode 100644 index 0000000000..c17b4bc4ba --- /dev/null +++ b/codex-rs/code-mode-runtime/src/runtime/audio.rs @@ -0,0 +1,63 @@ +//! Measures tool-generated PCM WAV clips using the audio bytes actually present. +//! Unknown formats retain the existing audio output behavior. + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES; + +pub(super) fn wav_duration_seconds(audio_url: &str) -> Option { + let (metadata, payload) = audio_url.split_once(',')?; + if !metadata + .split(';') + .skip(1) + .any(|part| part.eq_ignore_ascii_case("base64")) + || payload.len() > MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4 + { + return None; + } + let bytes = BASE64_STANDARD.decode(payload).ok()?; + if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" { + return None; + } + + let mut chunks = bytes.get(12..)?; + let mut format = None; + while chunks.len() >= 8 { + let chunk_id = &chunks[..4]; + let size = u32::from_le_bytes(chunks[4..8].try_into().ok()?) as usize; + let remaining = &chunks[8..]; + // Streaming WAV headers can declare more data than the file contains. + let chunk = &remaining[..size.min(remaining.len())]; + match chunk_id { + b"fmt " => { + let mut encoding = u16::from_le_bytes(chunk.get(..2)?.try_into().ok()?); + if encoding == 0xfffe { + // WAVE_FORMAT_EXTENSIBLE stores the encoding in a subtype GUID. + if chunk.get(26..40)? + != [0, 0, 0, 0, 0x10, 0, 0x80, 0, 0, 0xaa, 0, 0x38, 0x9b, 0x71] + { + return None; + } + encoding = u16::from_le_bytes(chunk.get(24..26)?.try_into().ok()?); + } + if !matches!(encoding, 1 | 3) { + return None; + } + let sample_rate = u32::from_le_bytes(chunk.get(4..8)?.try_into().ok()?); + let block_align = u16::from_le_bytes(chunk.get(12..14)?.try_into().ok()?); + if sample_rate == 0 || block_align == 0 { + return None; + } + format = Some((sample_rate, block_align)); + } + b"data" => { + let (sample_rate, block_align) = format?; + let frames = chunk.len() / usize::from(block_align); + return Some(frames as f64 / f64::from(sample_rate)); + } + _ => {} + } + chunks = remaining.get(size.checked_add(size % 2)?..)?; + } + None +} diff --git a/codex-rs/code-mode-runtime/src/runtime/mod.rs b/codex-rs/code-mode-runtime/src/runtime/mod.rs index 09e9f4aeed..3077b23330 100644 --- a/codex-rs/code-mode-runtime/src/runtime/mod.rs +++ b/codex-rs/code-mode-runtime/src/runtime/mod.rs @@ -1,3 +1,4 @@ +mod audio; mod callbacks; mod globals; mod module_loader; diff --git a/codex-rs/code-mode-runtime/src/runtime/value.rs b/codex-rs/code-mode-runtime/src/runtime/value.rs index ac59b09ac3..f7e63d0429 100644 --- a/codex-rs/code-mode-runtime/src/runtime/value.rs +++ b/codex-rs/code-mode-runtime/src/runtime/value.rs @@ -4,6 +4,8 @@ use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL; use codex_code_mode_protocol::FunctionCallOutputContentItem; use codex_code_mode_protocol::ImageDetail; +use super::audio::wav_duration_seconds; + const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block"; const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead"; @@ -214,6 +216,14 @@ pub(super) fn normalize_output_audio( return Err(INVALID_AUDIO_URL_ERROR.to_string()); } + // Tiny tool-generated clips cannot be encoded reliably by audio models. + if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) { + return Ok(FunctionCallOutputContentItem::InputText { + text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip." + .to_string(), + }); + } + Ok(FunctionCallOutputContentItem::InputAudio { audio_url }) })(); diff --git a/codex-rs/code-mode-runtime/src/service_audio_tests.rs b/codex-rs/code-mode-runtime/src/service_audio_tests.rs new file mode 100644 index 0000000000..45c1bc453a --- /dev/null +++ b/codex-rs/code-mode-runtime/src/service_audio_tests.rs @@ -0,0 +1,163 @@ +//! Covers per-item short-WAV omissions without disrupting other Code Mode outputs. + +use super::cell_id; +use super::execute; +use super::execute_request; +use crate::ExecuteRequest; +use crate::FunctionCallOutputContentItem; +use crate::InProcessCodeModeSession; +use crate::RuntimeResponse; +use crate::WaitOutcome; +use crate::WaitRequest; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use pretty_assertions::assert_eq; + +const SHORT_AUDIO_OMISSION_TEXT: &str = + "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."; + +fn pcm16_wav(sample_rate: u32, frames: usize) -> Vec { + let data_size = u32::try_from(frames * 2).unwrap(); + let mut wav = b"RIFF".to_vec(); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + wav.resize(44 + frames * 2, /*value*/ 0); + wav +} + +#[tokio::test] +async fn audio_helper_omits_short_wav_in_all_input_forms() { + for (sample_rate, frames, omitted) in [ + (24_000, 0, true), + (24_000, 1, true), + (24_000, 599, true), + (24_000, 600, false), + (24_000, 601, false), + (44_100, 223, true), + (44_100, 1_102, true), + (44_100, 1_103, false), + (30_870, 223, true), + (30_870, 771, true), + (30_870, 772, false), + (34_398, 221, true), + (34_398, 859, true), + (34_398, 860, false), + ] { + let payload = BASE64_STANDARD.encode(pcm16_wav(sample_rate, frames)); + let audio_url = format!("data:audio/wav;base64,{payload}"); + let service = InProcessCodeModeSession::new(); + let response = execute( + &service, + ExecuteRequest { + source: format!( + r#"audio({audio_url:?}); audio({{audio_url:{audio_url:?}}}); +audio({{type:"audio",mimeType:"audio/wav",data:{payload:?}}});"# + ), + yield_time_ms: None, + ..execute_request("") + }, + ) + .await; + let expected = if omitted { + FunctionCallOutputContentItem::InputText { + text: SHORT_AUDIO_OMISSION_TEXT.to_string(), + } + } else { + FunctionCallOutputContentItem::InputAudio { audio_url } + }; + assert_eq!( + response, + RuntimeResponse::Result { + code_mode_host_duration: None, + cell_id: cell_id("1"), + content_items: vec![expected; 3], + error_text: None, + }, + "sample rate {sample_rate}, frames {frames}" + ); + } +} + +#[tokio::test] +async fn audio_helper_bounds_wav_data_and_preserves_outputs_after_yield() { + let mut metadata_short = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 1); + let mut valid = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600); + let mut list = b"LIST".to_vec(); + list.extend_from_slice(&(4_u32 + 8 + 2048).to_le_bytes()); + list.extend_from_slice(b"INFOINAM"); + list.extend_from_slice(&2048_u32.to_le_bytes()); + list.resize(8 + 4 + 8 + 2048, /*value*/ 0); + list.extend_from_slice(b"JUNK\x01\0\0\0x\0"); + for wav in [&mut metadata_short, &mut valid] { + drop(wav.splice(36..36, list.iter().copied())); + let riff_size = u32::try_from(wav.len() - 8).unwrap(); + wav[4..8].copy_from_slice(&riff_size.to_le_bytes()); + } + let mut truncated = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 0); + truncated[4..8].copy_from_slice(&48_036_u32.to_le_bytes()); + truncated[40..44].copy_from_slice(&48_000_u32.to_le_bytes()); + let mut bounded = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600); + bounded[40..44].copy_from_slice(&2_u32.to_le_bytes()); + let urls = [metadata_short, truncated, bounded, valid] + .map(|wav| format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav))); + let audio_urls = serde_json::to_string(&urls).unwrap(); + let service = InProcessCodeModeSession::new(); + let response = execute( + &service, + ExecuteRequest { + source: format!( + "text('before'); await yield_control(); \ + for (const audio_url of {audio_urls}) audio({{audio_url}}); text('after');" + ), + yield_time_ms: None, + ..execute_request("") + }, + ) + .await; + assert_eq!( + response, + RuntimeResponse::Yielded { + code_mode_host_duration: None, + cell_id: cell_id("1"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }], + } + ); + let mut expected = vec![ + FunctionCallOutputContentItem::InputText { + text: SHORT_AUDIO_OMISSION_TEXT.to_string(), + }; + 3 + ]; + expected.push(FunctionCallOutputContentItem::InputAudio { + audio_url: urls[3].clone(), + }); + expected.push(FunctionCallOutputContentItem::InputText { + text: "after".to_string(), + }); + assert_eq!( + service + .wait(WaitRequest { + cell_id: cell_id("1"), + yield_time_ms: 60_000, + }) + .await + .unwrap(), + WaitOutcome::LiveCell(RuntimeResponse::Result { + code_mode_host_duration: None, + cell_id: cell_id("1"), + content_items: expected, + error_text: None, + }) + ); +} diff --git a/codex-rs/code-mode-runtime/src/service_tests.rs b/codex-rs/code-mode-runtime/src/service_tests.rs index c3c687b3ab..2ce272efe4 100644 --- a/codex-rs/code-mode-runtime/src/service_tests.rs +++ b/codex-rs/code-mode-runtime/src/service_tests.rs @@ -1189,6 +1189,9 @@ audio({ ); } +#[path = "service_audio_tests.rs"] +mod audio_tests; + #[tokio::test] async fn audio_helper_rejects_non_data_urls() { for source in [ diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 57f98df1d1..112740dfea 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ToolMode; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -3856,6 +3857,89 @@ text("after"); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_omits_short_audio_and_preserves_other_output() -> Result<()> { + skip_if_no_network!(Ok(())); + + let [short_audio_url, valid_audio_url] = [120_u32, 600].map(|frames| { + let sample_rate = 24_000_u32; + let data_size = frames * 2; + let mut wav = b"RIFF".to_vec(); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + wav.resize(44 + data_size as usize, /*value*/ 0); + format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav)) + }); + let code = format!( + r#" +text("before"); +audio({short_audio_url:?}); +text("between"); +audio({valid_audio_url:?}); +text("after"); +"#, + ); + + let server = responses::start_mock_server().await; + let mut builder = test_codex() + .with_model_info_override("gpt-5.4", |model| { + model.input_modalities.push(InputModality::Audio); + }) + .with_config(|config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build_with_auto_env(&server).await?; + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &code), + ev_completed("resp-1"), + ]), + ) + .await; + let second_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("use exec to return short and valid audio with text") + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!(success, Some(false)); + assert_eq!( + &items[1..], + &[ + serde_json::json!({ "type": "input_text", "text": "before" }), + serde_json::json!({ + "type": "input_text", + "text": "Audio output omitted because the clip is shorter than 25 ms; use a longer clip." + }), + serde_json::json!({ "type": "input_text", "text": "between" }), + serde_json::json!({ "type": "input_audio", "audio_url": valid_audio_url }), + serde_json::json!({ "type": "input_text", "text": "after" }), + ] + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_replaces_malformed_image() -> Result<()> { skip_if_no_network!(Ok(()));