Add opt-in file-backed tool audio (#61394)

## What changed

- Add the disabled-by-default `file_backed_tool_audio` feature to prepare fresh function and custom tool audio through the attachment store before persistence. Store returned file IDs in history and preserve them on replay without uploading again.
- Validate and budget mixed text and audio output before uploading retained clips, preserving audio token charges and producer limits. Resolve tool audio durations before continuation compaction checks.
- Keep inline audio on backend upload failures and use text placeholders for invalid or rejected clips. Replace unavailable tool audio files with placeholders in prompts while retaining the original history, so restored files can be used again.
- Treat file-backed tool audio as a competing source when attributing audio or unknown-file errors to user uploads.

## Testing

Add coverage for direct tool and Code Mode audio across follow-up turns, file expiration, restoration, and resume in both legacy and paginated history. Extend tests for truncation before upload, upload failure handling, bounded error placeholders, and error attribution.

GitOrigin-RevId: 66c1e7699c7d92f98c3a2d0f5cecbbd63147eb6c
This commit is contained in:
Hung
2026-11-13 23:02:16 +00:00
committed by copyberry
parent f941a5fb92
commit 5158e9780e
32 changed files with 1283 additions and 110 deletions
+205 -10
View File
@@ -8,13 +8,19 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use codex_attachment_store::AttachmentStore;
use codex_attachment_store::FormatSpecificMetadata;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ResponseItem;
use codex_utils_audio::estimate_file_audio_tokens;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::approx_token_count;
use codex_utils_output_truncation::truncate_measured_tool_output;
use futures::StreamExt;
use futures::stream;
@@ -37,17 +43,206 @@ pub(crate) async fn snapshot(session: &Session) -> Arc<FileAudioDurations> {
pub(super) fn file_ids(items: &[ResponseItemEnvelope]) -> impl Iterator<Item = &String> {
items
.iter()
.filter_map(|envelope| match &envelope.item {
ResponseItem::Message { content, .. } => Some(content),
_ => None,
.flat_map(|envelope| envelope.item.file_audio_ids())
}
/// Only fresh results pass through the store. A replay keeps the representation in the rollout.
pub(super) async fn prepare_new_items(
items: &mut [ResponseItemEnvelope],
store: &dyn AttachmentStore,
thread_id: &str,
ephemeral: bool,
default_token_limit: usize,
) {
for ResponseItemEnvelope { item, metadata } in items {
let (ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. }) = item
else {
codex_utils_audio::prepare_response_items(std::slice::from_mut(item));
continue;
};
if !output.contains_audio() {
continue;
}
let metadata = metadata.get_or_insert_default();
// The producer's output limit can be larger than the history limit (e.g. Code Mode).
// Apply the latter to the whole result while audio still has its inline cost.
let limit = metadata
.history_truncation_token_limit
.unwrap_or(default_token_limit);
let producer_budgeted = std::mem::take(&mut metadata.producer_bounded_tool_audio);
// Validate and measure on a worker. Only clips retained by history need their
// decoded bytes materialized for upload, one at a time.
let Some(parts) = output.content_items_mut() else {
continue;
};
let parts = std::mem::take(parts);
let preserve_producer_limit =
producer_budgeted && metadata.history_truncation_token_limit.is_none();
let Ok((mut admitted, repair_limit)) = tokio::task::spawn_blocking(move || {
admit_tool_audio(parts, limit, preserve_producer_limit)
})
.flatten()
.filter_map(|part| match part {
ContentItem::InputAudio {
.await
else {
*output = codex_protocol::models::FunctionCallOutputPayload::from_text(
"audio content omitted because it could not be processed".into(),
);
continue;
};
let mut replaced_with_text = false;
for (part, charge) in &mut admitted {
if let FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} = part
{
let url = std::mem::take(audio_url);
*part = store_tool_audio(store, thread_id, ephemeral, url).await;
if matches!(part, FunctionCallOutputContentItem::InputText { .. }) {
*charge = 0;
replaced_with_text = true;
}
}
}
if replaced_with_text {
admitted =
truncate_measured_tool_output(admitted, TruncationPolicy::Tokens(repair_limit));
}
output.body = codex_protocol::models::FunctionCallOutputBody::ContentItems(
admitted.into_iter().map(|(part, _)| part).collect(),
);
metadata.tool_audio_budget_applied = true;
}
}
fn admit_tool_audio(
parts: Vec<FunctionCallOutputContentItem>,
history_limit: usize,
preserve_producer_limit: bool,
) -> (Vec<(FunctionCallOutputContentItem, usize)>, usize) {
let mut measured = Vec::with_capacity(parts.len());
for mut part in parts {
let charge = match &mut part {
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} => match codex_utils_audio::prepare_and_measure_inline_audio(audio_url) {
Ok(tokens) => tokens,
Err(error) => {
part = FunctionCallOutputContentItem::InputText {
text: error.placeholder().into(),
};
0
}
},
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
} => estimate_file_audio_tokens(/*duration*/ None),
_ => 0,
};
measured.push((part, charge));
}
let admitted = truncate_measured_tool_output(measured, TruncationPolicy::Tokens(history_limit));
// Admission matches inline history. Only storage can introduce a new placeholder
// after that point. With no explicit producer limit, keep such repairs within
// the size of the result the model would otherwise have received.
let repair_limit = if preserve_producer_limit {
admitted
.iter()
.fold(0_usize, |sum, (part, charge)| {
let cost = match part {
FunctionCallOutputContentItem::InputText { text } => approx_token_count(text),
_ => *charge,
};
sum.saturating_add(cost)
})
.min(history_limit)
} else {
history_limit
};
(admitted, repair_limit)
}
async fn store_tool_audio(
store: &dyn AttachmentStore,
thread_id: &str,
ephemeral: bool,
inline_url: String,
) -> FunctionCallOutputContentItem {
let (mime, data, inline_url) = match tokio::task::spawn_blocking(move || {
codex_utils_audio::decode_audio(&inline_url).map(|(mime, data)| (mime, data, inline_url))
})
.await
{
Ok(Ok(decoded)) => decoded,
Ok(Err(error)) => {
return FunctionCallOutputContentItem::InputText {
text: error.placeholder().into(),
};
}
Err(_) => {
return FunctionCallOutputContentItem::InputText {
text: "audio content omitted because it could not be processed".into(),
};
}
};
// These tags come from Core. MCP servers and Code Mode never supply attachment IDs or policy tags.
let tags = HashMap::from([
("usage".to_string(), "tool_audio".to_string()),
("mime_type".to_string(), mime.to_string()),
]);
let result = store
.upload(UploadRequest {
thread_id: thread_id.to_owned(),
ephemeral,
file_name: None,
tags,
data,
})
.await;
tool_audio_after_upload(result, mime, inline_url)
}
fn tool_audio_after_upload(
result: Result<UploadResult, codex_attachment_store::AttachmentStoreError>,
mime: &str,
mut inline_url: String,
) -> FunctionCallOutputContentItem {
match result {
Ok(UploadResult::File { file_id }) if !file_id.trim().is_empty() => {
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
})
}
}
Ok(UploadResult::Inline { bytes }) => FunctionCallOutputContentItem::inline_audio(
codex_utils_audio::audio_data_url(mime, &bytes),
),
Err(error) if error.kind() == codex_attachment_store::AttachmentStoreErrorKind::Backend => {
tracing::warn!(kind = ?error.kind(), "tool audio upload failed; using inline audio");
// decode_audio validated the base64 and normalized the MIME alias.
if let Some(separator) = inline_url.find(',') {
inline_url.replace_range(..separator, &format!("data:{mime};base64"));
}
FunctionCallOutputContentItem::inline_audio(inline_url)
}
Err(error)
if error.kind()
== codex_attachment_store::AttachmentStoreErrorKind::InvalidAttachment =>
{
FunctionCallOutputContentItem::InputText {
text: "audio content omitted because the attachment store rejected the clip".into(),
}
}
Ok(UploadResult::File { .. }) | Err(_) => {
tracing::warn!("could not store tool audio attachment");
FunctionCallOutputContentItem::InputText {
text: "audio content omitted because it could not be stored".to_owned(),
}
}
}
}
pub(super) async fn resolve_pending(session: &Session) {
let ids = std::mem::take(&mut session.state.lock().await.pending_audio_duration_ids);
resolve(session, ids).await;
}
pub(super) async fn resolve(session: &Session, mut file_ids: BTreeSet<String>) {
@@ -44,3 +44,123 @@ fn history_sums_measured_and_unknown_file_audio() {
);
assert_eq!(history.estimated_tokens_since_last_usage(), 12_300);
}
#[test]
fn transient_upload_failure_retains_inline_audio_but_invalid_attachments_are_omitted() {
use codex_attachment_store::AttachmentStoreError;
use codex_attachment_store::AttachmentStoreErrorKind;
use serde_json::json;
for (kind, expected) in [
(
AttachmentStoreErrorKind::Backend,
json!({"type":"input_audio", "audio_url":"data:audio/wav;base64,QQ=="}),
),
(
AttachmentStoreErrorKind::InvalidAttachment,
json!({"type":"input_text", "text":"audio content omitted because the attachment store rejected the clip"}),
),
] {
let result = Err(AttachmentStoreError::new(kind, "upload failed"));
let part =
tool_audio_after_upload(result, "audio/wav", "data:audio/x-wav;base64,QQ==".into());
assert_eq!(serde_json::to_value(part).unwrap(), expected);
}
}
async fn prepared_tool_output(
parts: serde_json::Value,
metadata: codex_history::CodexHarnessMetadata,
limit: usize,
) -> serde_json::Value {
let mut item = ResponseItemEnvelope {
item: serde_json::from_value(
serde_json::json!({"type":"function_call_output","call_id":"clip","output":parts}),
)
.unwrap(),
metadata: Some(metadata),
};
prepare_new_items(
std::slice::from_mut(&mut item),
&codex_attachment_store::InlineAttachmentStore,
"thread",
/*ephemeral*/ false,
limit,
)
.await;
let metadata = item.metadata.unwrap();
assert!(!metadata.producer_bounded_tool_audio);
assert!(metadata.tool_audio_budget_applied);
assert_eq!(
serde_json::to_value(&metadata).unwrap()["tool_audio_budget_applied"],
true
);
serde_json::to_value(item.item).unwrap()["output"].take()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn producer_bounded_audio_keeps_the_same_parts_as_inline_history() {
use codex_history::CodexHarnessMetadata;
use serde_json::json;
let clip = json!({"type":"input_audio","audio_url":"data:audio/wav;base64,UklGRiYAAABXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YQIAAAAAAA=="});
let caption = json!({"type":"input_text","text":"caption ".repeat(100)});
let policy = TruncationPolicy::Tokens(20);
let limit = codex_utils_output_truncation::with_serialization_allowance(policy).token_budget();
let malformed = json!({"type":"input_audio","audio_url":format!("data:audio/wav;base64,{}", "!".repeat(8192))});
for (parts, keep_clip) in [
(json!([malformed, clip]), true),
(json!([{"type":"input_audio","audio_url":""}, clip]), true),
(json!([clip, caption]), true),
(json!([caption, clip]), false),
] {
let mut inline = serde_json::from_value(
json!({"type":"function_call_output","call_id":"clip","output":parts}),
)
.unwrap();
codex_utils_audio::prepare_response_items(std::slice::from_mut(&mut inline));
let mut history = ContextManager::new(uuid_v7_item_id_generator_for_test());
history.record_items([&inline], policy);
let expected =
serde_json::to_value(history.raw_items().last().unwrap()).unwrap()["output"].take();
assert_eq!(expected.as_array().unwrap().contains(&clip), keep_clip);
let metadata = CodexHarnessMetadata {
producer_bounded_tool_audio: true,
..Default::default()
};
assert_eq!(prepared_tool_output(parts, metadata, limit).await, expected);
}
}
#[tokio::test]
async fn zero_cost_invalid_audio_cannot_expand_into_unbounded_error_text() {
use codex_history::CodexHarnessMetadata;
use serde_json::json;
for producer_budgeted in [false, true] {
let mut parts = vec![json!({"type":"input_text","text":"keep"})];
parts.extend((0..100).map(|_| json!({"type":"input_audio","audio_url":""})));
let metadata = CodexHarnessMetadata {
producer_bounded_tool_audio: producer_budgeted,
..Default::default()
};
assert_eq!(
prepared_tool_output(parts.into(), metadata, /*limit*/ 1).await,
json!([{"type":"input_text","text":"keep"},{"type":"input_text","text":"[omitted 100 text items ...]"}])
);
}
}
#[tokio::test]
async fn known_producer_limit_keeps_the_replacement_and_following_caption() {
use serde_json::json;
let caption = json!({"type":"input_text","text":"caption"});
let parts = json!([{"type":"input_audio","audio_url":""}, caption]);
let metadata = codex_history::CodexHarnessMetadata {
producer_bounded_tool_audio: true,
history_truncation_token_limit: Some(100),
..Default::default()
};
let error = codex_utils_audio::decode_audio("").unwrap_err();
assert_eq!(
prepared_tool_output(parts, metadata, /*limit*/ 1).await,
json!([{"type":"input_text","text":error.placeholder()},caption])
);
}
+17 -1
View File
@@ -3961,9 +3961,25 @@ impl Session {
self.services.image_store.as_ref(),
))
.await;
let file_backed = turn_context
.config
.features
.enabled(Feature::FileBackedToolAudio);
if file_backed {
Box::pin(file_audio::prepare_new_items(
&mut items,
self.services.image_store.as_ref(),
&self.thread_id.to_string(),
turn_context.config.ephemeral,
with_serialization_allowance(model_info.truncation_policy.into()).token_budget(),
))
.await;
}
for envelope in &mut items {
let item = &mut envelope.item;
prepare_audio_response_items(std::slice::from_mut(item));
if !file_backed {
prepare_audio_response_items(std::slice::from_mut(item));
}
// Most response items get their passthrough turn ID at the durable history boundary.
Self::stamp_response_item_for_history(item, &turn_context.sub_id);
self.assign_missing_response_item_id(item);
+4 -2
View File
@@ -678,6 +678,9 @@ pub(crate) async fn run_turn(
// Process async hooks only after sampling and its tools have finished.
drain_async_hook_results(&sess, &turn_context, /*before_user_prompt*/ false).await;
let (has_pending_input, token_status) = async {
// Tool output can add audio without a queued user input. Resolve its
// duration before checking the continuation's compaction threshold.
super::file_audio::resolve_pending(&sess).await;
let has_pending_input =
sess.input_queue.has_pending_input(&sess.active_turn).await;
let token_status = super::context_window::context_window_token_status(
@@ -1079,8 +1082,7 @@ pub(crate) async fn run_hooks_and_record_inputs(
// The batch has left the input queue. Persist every accepted item before the
// optional lookup can suspend this task long enough for an interrupt to abort it.
if !input.is_empty() {
let ids = std::mem::take(&mut sess.state.lock().await.pending_audio_duration_ids);
super::file_audio::resolve(sess, ids).await;
super::file_audio::resolve_pending(sess).await;
}
Ok(blocked_input && !accepted_user_input)
}