Add opt-in file-backed tool audio (#61394)

## What changed

- Add the disabled-by-default `file_backed_tool_audio` feature to prepare fresh function and custom tool audio through the attachment store before persistence. Store returned file IDs in history and preserve them on replay without uploading again.
- Validate and budget mixed text and audio output before uploading retained clips, preserving audio token charges and producer limits. Resolve tool audio durations before continuation compaction checks.
- Keep inline audio on backend upload failures and use text placeholders for invalid or rejected clips. Replace unavailable tool audio files with placeholders in prompts while retaining the original history, so restored files can be used again.
- Treat file-backed tool audio as a competing source when attributing audio or unknown-file errors to user uploads.

## Testing

Add coverage for direct tool and Code Mode audio across follow-up turns, file expiration, restoration, and resume in both legacy and paginated history. Extend tests for truncation before upload, upload failure handling, bounded error placeholders, and error attribution.

GitOrigin-RevId: 66c1e7699c7d92f98c3a2d0f5cecbbd63147eb6c

codex-rs/utils/audio/src/audio_preparation_tests.rs

@@ -210,3 +210,13 @@ fn file_audio_uses_the_measured_length_or_the_full_file_budget() {
         assert_eq!(super::estimate_file_audio_tokens(duration), expected);
     }
 }
+
+#[test]
+fn audio_data_url_encodes_all_padding_lengths_directly_into_the_url() {
+    for (bytes, expected) in [(b"a".as_slice(), "YQ=="), (b"ab", "YWI="), (b"abc", "YWJj")] {
+        assert_eq!(
+            super::audio_data_url("audio/wav", bytes),
+            format!("data:audio/wav;base64,{expected}")
+        );
+    }
+}

codex-rs/utils/audio/src/lib.rs

@@ -41,7 +41,7 @@ static AUDIO_TOKEN_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Arc<OnceL
     });
 
 #[derive(Debug, thiserror::Error)]
-enum AudioPreparationError {
+pub enum AudioPreparationError {
     #[error("invalid audio data URL: {reason}")]
     InvalidDataUrl { reason: &'static str },
     #[error("unsupported audio format")]
@@ -51,7 +51,7 @@ enum AudioPreparationError {
 }
 
 impl AudioPreparationError {
-    fn placeholder(&self) -> &'static str {
+    pub fn placeholder(&self) -> &'static str {
         match self {
             AudioPreparationError::InvalidDataUrl { .. } => AUDIO_PROCESSING_ERROR_PLACEHOLDER,
             AudioPreparationError::UnsupportedFormat => UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER,
@@ -192,7 +192,14 @@ fn estimate_audio_token_count(audio_url: &str) -> usize {
 /// The same duration-based estimate without accessing the Tokio-dependent cache.
 /// Suitable for bounded output rendered on a current-thread runtime.
 fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize {
-    let Some(duration_seconds) = audio_duration_seconds(audio_url) else {
+    let Ok((mime, bytes)) = decode_audio(audio_url) else {
+        return approx_token_count(audio_url);
+    };
+    decoded_audio_token_count(mime, bytes, audio_url)
+}
+
+fn decoded_audio_token_count(mime: &str, bytes: Vec<u8>, audio_url: &str) -> usize {
+    let Some(duration_seconds) = audio_duration_seconds(mime, bytes) else {
         return approx_token_count(audio_url);
     };
     let token_count = (duration_seconds * f64::from(AUDIO_TOKENS_PER_SECOND)).ceil();
@@ -203,22 +210,7 @@ fn estimate_audio_token_count_without_cache(audio_url: &str) -> usize {
     }
 }
 
-fn audio_duration_seconds(audio_url: &str) -> Option<f64> {
-    let (metadata, payload) = audio_url.split_once(',')?;
-    let metadata = metadata.get("data:".len()..)?;
-    let mut metadata_parts = metadata.split(';');
-    let canonical_mime = canonical_audio_mime(metadata_parts.next()?)?;
-    if !metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")) {
-        return None;
-    }
-
-    let bytes = match BASE64_STANDARD.decode(payload) {
-        Ok(bytes) => bytes,
-        Err(error) => {
-            tracing::trace!(%error, "failed to decode audio payload for token estimation");
-            return None;
-        }
-    };
+fn audio_duration_seconds(canonical_mime: &str, bytes: Vec<u8>) -> Option<f64> {
     let media_source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
     let mut hint = Hint::new();
     hint.mime_type(canonical_mime);
@@ -247,7 +239,44 @@ fn audio_duration_seconds(audio_url: &str) -> Option<f64> {
     duration_seconds.is_finite().then_some(duration_seconds)
 }
 
+/// Validate the data URL and measure its decoded duration in the same pass.
+pub fn prepare_and_measure_inline_audio(
+    audio_url: &mut String,
+) -> Result<usize, AudioPreparationError> {
+    let (mime, bytes) = decode_audio(audio_url)?;
+    normalize_validated_header(audio_url, mime);
+    Ok(decoded_audio_token_count(mime, bytes, audio_url))
+}
+
 fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> {
+    let (mime, _) = decode_audio(audio_url)?;
+    normalize_validated_header(audio_url, mime);
+    Ok(())
+}
+
+// The standard base64 decoder has already validated the payload, including padding.
+fn normalize_validated_header(url: &mut String, mime: &str) {
+    let canonical = format!("data:{mime};base64");
+    if let Some((header, _)) = url.split_once(',')
+        && header != canonical
+    {
+        url.replace_range(..header.len(), &canonical);
+    }
+}
+
+/// Canonical inline representation for a validated payload.
+pub fn audio_data_url(mime: &str, data: &[u8]) -> String {
+    let mut url =
+        String::with_capacity("data:;base64,".len() + mime.len() + data.len().div_ceil(3) * 4);
+    url.push_str("data:");
+    url.push_str(mime);
+    url.push_str(";base64,");
+    BASE64_STANDARD.encode_string(data, &mut url);
+    url
+}
+
+/// Parses a size-bounded audio data URL. A file-writing host also verifies its media container.
+pub fn decode_audio(audio_url: &str) -> Result<(&'static str, Vec<u8>), AudioPreparationError> {
     if !is_data_url(audio_url) {
         return Err(AudioPreparationError::InvalidDataUrl {
             reason: "audio input must be a data URL",
@@ -300,9 +329,7 @@ fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> {
         return Err(AudioPreparationError::AudioTooLarge { size: bytes.len() });
     }
 
-    let encoded = BASE64_STANDARD.encode(bytes);
-    *audio_url = format!("data:{canonical_mime};base64,{encoded}");
-    Ok(())
+    Ok((canonical_mime, bytes))
 }
 
 #[cfg(test)]

codex-rs/utils/output-truncation/src/lib.rs

@@ -311,13 +311,47 @@ fn truncate_output_items(
     }
 }
 
+/// Apply the mixed-output policy to owned, premeasured parts. Audio keeps its original
+/// token charge through truncation and storage. The charge for other parts is zero.
+pub fn truncate_measured_tool_output(
+    items: Vec<(FunctionCallOutputContentItem, usize)>,
+    policy: TruncationPolicy,
+) -> Vec<(FunctionCallOutputContentItem, usize)> {
+    truncate_measured_output_iter(
+        items
+            .into_iter()
+            .map(|(item, charge)| (Cow::Owned(item), charge)),
+        policy,
+        &mut [],
+        |item, charge| (item, charge),
+    )
+}
+
 fn truncate_output_iter<'a>(
     items: impl ExactSizeIterator<Item = Cow<'a, FunctionCallOutputContentItem>>,
     policy: TruncationPolicy,
     estimate_audio_token_count: impl Fn(&AudioReference) -> usize,
     boundaries: &mut [OutputBoundary],
 ) -> Vec<FunctionCallOutputContentItem> {
-    let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
+    let measured = items.map(|item| {
+        let charge = match item.as_ref() {
+            FunctionCallOutputContentItem::InputAudio { audio } => {
+                estimate_audio_token_count(audio)
+            }
+            _ => 0,
+        };
+        (item, charge)
+    });
+    truncate_measured_output_iter(measured, policy, boundaries, |item, _| item)
+}
+
+fn truncate_measured_output_iter<'a, T>(
+    items: impl ExactSizeIterator<Item = (Cow<'a, FunctionCallOutputContentItem>, usize)>,
+    policy: TruncationPolicy,
+    boundaries: &mut [OutputBoundary],
+    emit: impl Fn(FunctionCallOutputContentItem, usize) -> T,
+) -> Vec<T> {
+    let mut out = Vec::with_capacity(items.len());
     let mut remaining_budget = match policy {
         TruncationPolicy::Bytes(_) => policy.byte_budget(),
         TruncationPolicy::Tokens(_) => policy.token_budget(),
@@ -326,7 +360,7 @@ fn truncate_output_iter<'a>(
     let mut omitted_audio_items = 0usize;
     let mut next_boundary = 0;
 
-    for (index, item) in items.enumerate() {
+    for (index, (item, audio_tokens)) in items.enumerate() {
         let item_out = out.len();
         let item_policy = match policy {
             TruncationPolicy::Bytes(_) => TruncationPolicy::Bytes(remaining_budget),
@@ -354,7 +388,10 @@ fn truncate_output_iter<'a>(
                     if snippet.is_empty() {
                         omitted_text_items += 1;
                     } else {
-                        out.push(FunctionCallOutputContentItem::InputText { text: snippet });
+                        out.push(emit(
+                            FunctionCallOutputContentItem::InputText { text: snippet },
+                            0,
+                        ));
                     }
                     remaining_budget = 0;
                     false
@@ -362,11 +399,10 @@ fn truncate_output_iter<'a>(
             }
             FunctionCallOutputContentItem::InputImage { .. }
             | FunctionCallOutputContentItem::EncryptedContent { .. } => true,
-            FunctionCallOutputContentItem::InputAudio { audio } => {
-                let token_cost = estimate_audio_token_count(audio);
+            FunctionCallOutputContentItem::InputAudio { .. } => {
                 let cost = match policy {
-                    TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(token_cost),
-                    TruncationPolicy::Tokens(_) => token_cost,
+                    TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(audio_tokens),
+                    TruncationPolicy::Tokens(_) => audio_tokens,
                 };
                 if cost <= remaining_budget {
                     remaining_budget = remaining_budget.saturating_sub(cost);
@@ -390,7 +426,7 @@ fn truncate_output_iter<'a>(
             next_boundary += 1;
         }
         if keep {
-            out.push(item.into_owned());
+            out.push(emit(item.into_owned(), audio_tokens));
         }
     }
     // End boundaries precede generated omission counts: counts are not original
@@ -401,14 +437,20 @@ fn truncate_output_iter<'a>(
     }
 
     if omitted_text_items > 0 {
-        out.push(FunctionCallOutputContentItem::InputText {
-            text: format!("[omitted {omitted_text_items} text items ...]"),
-        });
+        out.push(emit(
+            FunctionCallOutputContentItem::InputText {
+                text: format!("[omitted {omitted_text_items} text items ...]"),
+            },
+            0,
+        ));
     }
     if omitted_audio_items > 0 {
-        out.push(FunctionCallOutputContentItem::InputText {
-            text: format!("[omitted {omitted_audio_items} audio items ...]"),
-        });
+        out.push(emit(
+            FunctionCallOutputContentItem::InputText {
+                text: format!("[omitted {omitted_audio_items} audio items ...]"),
+            },
+            0,
+        ));
     }
     out
 }

codex-rs/utils/output-truncation/src/truncate_tests.rs

@@ -8,6 +8,7 @@ use crate::formatted_truncate_text;
 use crate::mixed_output_at_token_boundaries;
 use crate::truncate_function_output_items_with_policy;
 use crate::truncate_mcp_tool_result;
+use crate::truncate_measured_tool_output;
 use crate::truncate_text;
 use codex_protocol::mcp::CallToolResult;
 use codex_protocol::models::AudioReference;
@@ -577,3 +578,26 @@ fn owned_mixed_output_moves_a_preserved_image_across_text_truncation() {
     };
     assert_eq!(image_url.as_ptr(), url_ptr);
 }
+
+#[test]
+fn measured_tool_output_keeps_charges_paired_with_retained_clips() {
+    let clip = FunctionCallOutputContentItem::inline_audio("clip".into());
+    let text = FunctionCallOutputContentItem::InputText {
+        text: "keep".into(),
+    };
+    let omitted = FunctionCallOutputContentItem::InputText {
+        text: "[omitted 1 audio items ...]".into(),
+    };
+    assert_eq!(
+        truncate_measured_tool_output(
+            vec![
+                (clip.clone(), 9),
+                (text.clone(), 0),
+                (clip.clone(), 2),
+                (clip.clone(), 1)
+            ],
+            TruncationPolicy::Tokens(4)
+        ),
+        vec![(text, 0), (clip.clone(), 2), (clip, 1), (omitted, 0)]
+    );
+}