Add opt-in file-backed tool audio (#61394)

## What changed

- Add the disabled-by-default `file_backed_tool_audio` feature to prepare fresh function and custom tool audio through the attachment store before persistence. Store returned file IDs in history and preserve them on replay without uploading again.
- Validate and budget mixed text and audio output before uploading retained clips, preserving audio token charges and producer limits. Resolve tool audio durations before continuation compaction checks.
- Keep inline audio on backend upload failures and use text placeholders for invalid or rejected clips. Replace unavailable tool audio files with placeholders in prompts while retaining the original history, so restored files can be used again.
- Treat file-backed tool audio as a competing source when attributing audio or unknown-file errors to user uploads.

## Testing

Add coverage for direct tool and Code Mode audio across follow-up turns, file expiration, restoration, and resume in both legacy and paginated history. Extend tests for truncation before upload, upload failure handling, bounded error placeholders, and error attribution.

GitOrigin-RevId: 66c1e7699c7d92f98c3a2d0f5cecbbd63147eb6c
This commit is contained in:
Hung
2026-11-13 23:02:16 +00:00
committed by copyberry
parent f941a5fb92
commit 5158e9780e
32 changed files with 1283 additions and 110 deletions
+5 -4
View File
@@ -9,7 +9,7 @@
//! Live usage boundaries advance only when a response supplies usable usage.
//! Oversized instructions keep an incomplete excerpt for bounded root review, including
//! sources recovered from legacy Guardian checkpoints before their raw history is dropped.
//! Prompt projections replace unavailable file images without changing retained history.
//! Prompt projections replace unavailable file images and tool audio without changing retained history.
#[path = "history_rejected_audio.rs"]
pub(crate) mod rejected_audio;
@@ -25,7 +25,7 @@ use crate::context::world_state::WorldState;
use crate::context::world_state::WorldStateSnapshot;
use crate::context::world_state::WorldStateUpdate;
use crate::context_manager::normalize;
use crate::context_manager::replace_unresolvable_image_files;
use crate::context_manager::replace_unresolvable_media_files;
use crate::event_mapping::contextual_dev_fragments;
use crate::event_mapping::is_contextual_user_message_content;
use crate::event_mapping::parse_turn_item;
@@ -680,6 +680,7 @@ impl ContextManager {
self.sampling.record(&mut processed);
if let ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } = &mut processed.item
&& !metadata.is_some_and(|metadata| metadata.tool_audio_budget_applied)
{
// The override already includes the tool's serialization allowance.
let policy = metadata
@@ -716,7 +717,7 @@ impl ContextManager {
/// Returns the history prepared for sending to the model. This applies a proper
/// normalization and drops un-suited items. Unsupported image and audio content
/// is stripped from messages and tool outputs according to `input_modalities`.
/// Unavailable file images are replaced with text placeholders.
/// Unavailable file images and tool audio are replaced with text placeholders.
pub(crate) async fn for_prompt(
self,
input_modalities: &[InputModality],
@@ -736,7 +737,7 @@ impl ContextManager {
image_store: &dyn AttachmentStore,
) -> Vec<ResponseItemEnvelope> {
self.normalize_history(input_modalities);
replace_unresolvable_image_files(
replace_unresolvable_media_files(
Arc::make_mut(&mut self.items)
.iter_mut()
.map(|envelope| &mut envelope.item),
@@ -58,6 +58,7 @@ pub(crate) fn pending_audio_confirmation(
enum FileErrorSource<'a> {
Audio(&'a str),
InlineAudio,
ToolFileAudio,
FileImage,
Opaque,
}
@@ -85,9 +86,12 @@ fn visit_sources<'a>(item: &'a ResponseItem, mut visit: impl FnMut(FileErrorSour
| ResponseItem::CustomToolCallOutput { output, .. } => {
for part in output.content_items().unwrap_or_default() {
match part {
FunctionCallOutputContentItem::InputAudio { .. } => {
visit(FileErrorSource::InlineAudio)
}
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
} => visit(FileErrorSource::ToolFileAudio),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { .. },
} => visit(FileErrorSource::InlineAudio),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
@@ -141,8 +145,8 @@ fn opaque_content(item: &ResponseItem) -> Option<&str> {
pub(crate) struct SubmittedFileAudio {
file_ids: Vec<String>,
candidate: Option<usize>,
has_inline_audio: bool,
has_file_image: bool,
has_competing_audio: bool,
has_competing_file: bool,
}
impl SubmittedFileAudio {
@@ -152,14 +156,18 @@ impl SubmittedFileAudio {
clear_checkpoints: impl Iterator<Item = &'a str> + Clone,
) -> Option<Self> {
let mut file_ids = Vec::new();
let mut has_inline_audio = false;
let mut has_file_image = false;
let mut has_competing_audio = false;
let mut has_competing_file = false;
let mut unknown_checkpoint = false;
for item in items {
visit_sources(item, |source| match source {
FileErrorSource::Audio(id) => file_ids.push(id),
FileErrorSource::InlineAudio => has_inline_audio = true,
FileErrorSource::FileImage => has_file_image = true,
FileErrorSource::InlineAudio => has_competing_audio = true,
FileErrorSource::ToolFileAudio => {
has_competing_audio = true;
has_competing_file = true;
}
FileErrorSource::FileImage => has_competing_file = true,
FileErrorSource::Opaque => {
unknown_checkpoint |= !opaque_content(item).is_some_and(|ciphertext| {
clear_checkpoints.clone().any(|known| known == ciphertext)
@@ -185,8 +193,8 @@ impl SubmittedFileAudio {
Some(Self {
file_ids: file_ids.into_iter().map(str::to_owned).collect(),
candidate,
has_inline_audio,
has_file_image,
has_competing_audio,
has_competing_file,
})
}
@@ -195,9 +203,9 @@ impl SubmittedFileAudio {
let unique_file = self.file_ids.len() == 1;
let attributable = match error {
CodexErrorDetails::InvalidAudioInput { per_recording, .. } => {
!self.has_inline_audio && (unique_file || *per_recording)
!self.has_competing_audio && (unique_file || *per_recording)
}
_ => unique_file && !self.has_file_image && is_unknown_file_error(error),
_ => unique_file && !self.has_competing_file && is_unknown_file_error(error),
};
attributable.then_some(index)
}
@@ -108,6 +108,38 @@ fn tool_output_audio_makes_an_invalid_audio_error_ambiguous() {
}
}
#[test]
fn only_file_backed_tool_audio_competes_with_a_user_upload_for_an_unknown_file_error() {
use serde_json::json;
for kind in ["function_call_output", "custom_tool_call_output"] {
for (part, expected) in [
(
json!({"type":"input_audio","file_id":"expired_tool_clip"}),
None,
),
(
json!({"type":"input_audio","audio_url":"data:audio/wav;base64,AA=="}),
Some("valid_upload"),
),
] {
let output =
serde_json::from_value(json!({"type":kind, "call_id":"call", "output":[part]}))
.unwrap();
let items = [message(vec![file_audio("valid_upload")]), output];
assert_eq!(
rejected_file_audio_id(&unknown_file_error(), &items).as_deref(),
expected,
"{kind}"
);
assert_eq!(
rejected_file_audio_id(&audio_error(), &items),
None,
"{kind}"
);
}
}
}
#[test]
fn attributes_audio_error_only_to_one_visible_file_audio() {
let error = audio_error();
@@ -145,9 +177,8 @@ fn attributes_audio_error_only_to_one_visible_file_audio() {
);
}
#[test]
fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
let error = CodexErrorDetails::UnexpectedStatus(codex_protocol::error::UnexpectedResponseError {
fn unknown_file_error() -> CodexErrorDetails {
CodexErrorDetails::UnexpectedStatus(codex_protocol::error::UnexpectedResponseError {
status: http::StatusCode::NOT_FOUND,
body: serde_json::json!({"error": {"message": "Invalid input: Unknown `file_id` in input content."}}).to_string(),
url: None,
@@ -156,7 +187,12 @@ fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
request_id: None,
identity_authorization_error: None,
identity_error_code: None,
});
})
}
#[test]
fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
let error = unknown_file_error();
let unavailable_image_projection = message(vec![
ContentItem::InputText {
text: "[Image unavailable]".to_string(),
@@ -1,4 +1,4 @@
//! Projects unavailable file images into placeholders without changing tool attribution.
//! Projects unavailable images and tool audio into placeholders without changing tool attribution.
use std::collections::HashSet;
@@ -6,6 +6,7 @@ use codex_attachment_store::AttachmentStore;
use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ContentItemMetadata;
@@ -15,12 +16,12 @@ use codex_protocol::models::ResponseItem;
use futures::StreamExt;
use futures::stream;
const MAX_CONCURRENT_IMAGE_AVAILABILITY_CHECKS: usize = 8;
const MAX_CONCURRENT_FILE_AVAILABILITY_CHECKS: usize = 8;
/// Checks distinct file IDs concurrently and replaces only unavailable image content.
pub(crate) async fn replace_unresolvable_image_files<'a>(
/// Checks distinct file IDs concurrently and replaces only media the store confirms is unavailable.
pub(crate) async fn replace_unresolvable_media_files<'a>(
items: impl Iterator<Item = &'a mut ResponseItem>,
image_store: &dyn AttachmentStore,
store: &dyn AttachmentStore,
) {
let items: Vec<_> = items.collect();
let mut file_ids = HashSet::new();
@@ -44,6 +45,9 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
}
| FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} = part
{
file_ids.insert(file_id.clone());
@@ -56,10 +60,10 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
}
let mut checks = stream::iter(file_ids)
.map(|file_id| async move {
let resolvable = image_store.is_resolvable(&file_id).await;
let resolvable = store.is_resolvable(&file_id).await;
(file_id, resolvable)
})
.buffer_unordered(MAX_CONCURRENT_IMAGE_AVAILABILITY_CHECKS);
.buffer_unordered(MAX_CONCURRENT_FILE_AVAILABILITY_CHECKS);
let mut unavailable_file_ids = HashSet::new();
while let Some((file_id, resolvable)) = checks.next().await {
if !resolvable {
@@ -103,14 +107,22 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let Some(content) = output.content_items_mut() {
for part in content {
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
} = part
&& unavailable_file_ids.contains(file_id)
{
let (file_id, placeholder) = match part {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
} => (file_id, "[image unavailable]"),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => (
file_id,
"audio content omitted because its file is unavailable",
),
_ => continue,
};
if unavailable_file_ids.contains(file_id) {
*part = FunctionCallOutputContentItem::InputText {
text: "[image unavailable]".to_string(),
text: placeholder.into(),
};
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod history;
mod image_availability;
mod media_availability;
mod normalize;
mod sampling;
pub(crate) mod updates;
@@ -11,5 +11,5 @@ pub(crate) use history::estimate_item_token_count;
pub(crate) use history::estimate_item_token_count_with_audio;
pub(crate) use history::is_user_turn_boundary;
pub(crate) use history::rejected_audio;
pub(crate) use image_availability::replace_unresolvable_image_files;
pub(crate) use media_availability::replace_unresolvable_media_files;
pub(crate) use sampling::sampled_through;