Omit undersized WAV output from Code Mode (#41934)

## Why

Audio models cannot reliably encode tool-generated clips shorter than 25 ms.

## What changed

- Measure the available audio frames in base64-encoded PCM and IEEE float WAV outputs.
- Replace clips under 25 ms with explanatory text while preserving surrounding Code Mode output.
- Leave clips at or above the threshold and unrecognized audio formats unchanged.

## Testing

- Cover the duration boundary across multiple sample rates and all supported `audio()` input forms.
- Verify bounded and truncated WAV chunks, output after `yield_control()`, and end-to-end mixed text and audio output.

GitOrigin-RevId: 342b4bdbc65db4238580f9cdc372763c2576eca5
This commit is contained in:
Jennifer Zhao
2026-08-31 23:36:04 +00:00
committed by copyberry
parent a62ff9a5d4
commit 981d6b652b
8 changed files with 326 additions and 0 deletions
@@ -0,0 +1,63 @@
//! Measures tool-generated PCM WAV clips using the audio bytes actually present.
//! Unknown formats retain the existing audio output behavior.
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES;
pub(super) fn wav_duration_seconds(audio_url: &str) -> Option<f64> {
let (metadata, payload) = audio_url.split_once(',')?;
if !metadata
.split(';')
.skip(1)
.any(|part| part.eq_ignore_ascii_case("base64"))
|| payload.len() > MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4
{
return None;
}
let bytes = BASE64_STANDARD.decode(payload).ok()?;
if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {
return None;
}
let mut chunks = bytes.get(12..)?;
let mut format = None;
while chunks.len() >= 8 {
let chunk_id = &chunks[..4];
let size = u32::from_le_bytes(chunks[4..8].try_into().ok()?) as usize;
let remaining = &chunks[8..];
// Streaming WAV headers can declare more data than the file contains.
let chunk = &remaining[..size.min(remaining.len())];
match chunk_id {
b"fmt " => {
let mut encoding = u16::from_le_bytes(chunk.get(..2)?.try_into().ok()?);
if encoding == 0xfffe {
// WAVE_FORMAT_EXTENSIBLE stores the encoding in a subtype GUID.
if chunk.get(26..40)?
!= [0, 0, 0, 0, 0x10, 0, 0x80, 0, 0, 0xaa, 0, 0x38, 0x9b, 0x71]
{
return None;
}
encoding = u16::from_le_bytes(chunk.get(24..26)?.try_into().ok()?);
}
if !matches!(encoding, 1 | 3) {
return None;
}
let sample_rate = u32::from_le_bytes(chunk.get(4..8)?.try_into().ok()?);
let block_align = u16::from_le_bytes(chunk.get(12..14)?.try_into().ok()?);
if sample_rate == 0 || block_align == 0 {
return None;
}
format = Some((sample_rate, block_align));
}
b"data" => {
let (sample_rate, block_align) = format?;
let frames = chunk.len() / usize::from(block_align);
return Some(frames as f64 / f64::from(sample_rate));
}
_ => {}
}
chunks = remaining.get(size.checked_add(size % 2)?..)?;
}
None
}
@@ -1,3 +1,4 @@
mod audio;
mod callbacks;
mod globals;
mod module_loader;
@@ -4,6 +4,8 @@ use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL;
use codex_code_mode_protocol::FunctionCallOutputContentItem;
use codex_code_mode_protocol::ImageDetail;
use super::audio::wav_duration_seconds;
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
@@ -214,6 +216,14 @@ pub(super) fn normalize_output_audio(
return Err(INVALID_AUDIO_URL_ERROR.to_string());
}
// Tiny tool-generated clips cannot be encoded reliably by audio models.
if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) {
return Ok(FunctionCallOutputContentItem::InputText {
text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."
.to_string(),
});
}
Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
})();