Add opt-in file-backed tool audio (#61394)

## What changed

- Add the disabled-by-default `file_backed_tool_audio` feature to prepare fresh function and custom tool audio through the attachment store before persistence. Store returned file IDs in history and preserve them on replay without uploading again.
- Validate and budget mixed text and audio output before uploading retained clips, preserving audio token charges and producer limits. Resolve tool audio durations before continuation compaction checks.
- Keep inline audio on backend upload failures and use text placeholders for invalid or rejected clips. Replace unavailable tool audio files with placeholders in prompts while retaining the original history, so restored files can be used again.
- Treat file-backed tool audio as a competing source when attributing audio or unknown-file errors to user uploads.

## Testing

Add coverage for direct tool and Code Mode audio across follow-up turns, file expiration, restoration, and resume in both legacy and paginated history. Extend tests for truncation before upload, upload failure handling, bounded error placeholders, and error attribution.

GitOrigin-RevId: 66c1e7699c7d92f98c3a2d0f5cecbbd63147eb6c
This commit is contained in:
Hung
2026-11-13 23:02:16 +00:00
committed by copyberry
parent f941a5fb92
commit 5158e9780e
32 changed files with 1283 additions and 110 deletions
@@ -18,7 +18,7 @@ use crate::compact_remote_history::trim_function_call_history_to_fit_context_win
use crate::context::world_state::WorldState;
use crate::context_manager::ContextManager;
use crate::context_manager::rejected_audio;
use crate::context_manager::replace_unresolvable_image_files;
use crate::context_manager::replace_unresolvable_media_files;
use crate::responses_metadata::CompactionTurnMetadata;
use crate::session::CompactionPrefix;
use crate::session::session::Session;
@@ -340,7 +340,7 @@ pub(crate) async fn run_background_compaction(
) -> CodexResult<BackgroundCompactionCandidate> {
tokio::select! {
_ = shutdown_cancellation.cancelled() => return Err(CodexErr::TurnAborted),
() = replace_unresolvable_image_files(
() = replace_unresolvable_media_files(
prepared_input.input.iter_mut(),
sess.services.image_store.as_ref(),
) => {},
+5 -4
View File
@@ -9,7 +9,7 @@
//! Live usage boundaries advance only when a response supplies usable usage.
//! Oversized instructions keep an incomplete excerpt for bounded root review, including
//! sources recovered from legacy Guardian checkpoints before their raw history is dropped.
//! Prompt projections replace unavailable file images without changing retained history.
//! Prompt projections replace unavailable file images and tool audio without changing retained history.
#[path = "history_rejected_audio.rs"]
pub(crate) mod rejected_audio;
@@ -25,7 +25,7 @@ use crate::context::world_state::WorldState;
use crate::context::world_state::WorldStateSnapshot;
use crate::context::world_state::WorldStateUpdate;
use crate::context_manager::normalize;
use crate::context_manager::replace_unresolvable_image_files;
use crate::context_manager::replace_unresolvable_media_files;
use crate::event_mapping::contextual_dev_fragments;
use crate::event_mapping::is_contextual_user_message_content;
use crate::event_mapping::parse_turn_item;
@@ -680,6 +680,7 @@ impl ContextManager {
self.sampling.record(&mut processed);
if let ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } = &mut processed.item
&& !metadata.is_some_and(|metadata| metadata.tool_audio_budget_applied)
{
// The override already includes the tool's serialization allowance.
let policy = metadata
@@ -716,7 +717,7 @@ impl ContextManager {
/// Returns the history prepared for sending to the model. This applies a proper
/// normalization and drops un-suited items. Unsupported image and audio content
/// is stripped from messages and tool outputs according to `input_modalities`.
/// Unavailable file images are replaced with text placeholders.
/// Unavailable file images and tool audio are replaced with text placeholders.
pub(crate) async fn for_prompt(
self,
input_modalities: &[InputModality],
@@ -736,7 +737,7 @@ impl ContextManager {
image_store: &dyn AttachmentStore,
) -> Vec<ResponseItemEnvelope> {
self.normalize_history(input_modalities);
replace_unresolvable_image_files(
replace_unresolvable_media_files(
Arc::make_mut(&mut self.items)
.iter_mut()
.map(|envelope| &mut envelope.item),
@@ -58,6 +58,7 @@ pub(crate) fn pending_audio_confirmation(
enum FileErrorSource<'a> {
Audio(&'a str),
InlineAudio,
ToolFileAudio,
FileImage,
Opaque,
}
@@ -85,9 +86,12 @@ fn visit_sources<'a>(item: &'a ResponseItem, mut visit: impl FnMut(FileErrorSour
| ResponseItem::CustomToolCallOutput { output, .. } => {
for part in output.content_items().unwrap_or_default() {
match part {
FunctionCallOutputContentItem::InputAudio { .. } => {
visit(FileErrorSource::InlineAudio)
}
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
} => visit(FileErrorSource::ToolFileAudio),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { .. },
} => visit(FileErrorSource::InlineAudio),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
@@ -141,8 +145,8 @@ fn opaque_content(item: &ResponseItem) -> Option<&str> {
pub(crate) struct SubmittedFileAudio {
file_ids: Vec<String>,
candidate: Option<usize>,
has_inline_audio: bool,
has_file_image: bool,
has_competing_audio: bool,
has_competing_file: bool,
}
impl SubmittedFileAudio {
@@ -152,14 +156,18 @@ impl SubmittedFileAudio {
clear_checkpoints: impl Iterator<Item = &'a str> + Clone,
) -> Option<Self> {
let mut file_ids = Vec::new();
let mut has_inline_audio = false;
let mut has_file_image = false;
let mut has_competing_audio = false;
let mut has_competing_file = false;
let mut unknown_checkpoint = false;
for item in items {
visit_sources(item, |source| match source {
FileErrorSource::Audio(id) => file_ids.push(id),
FileErrorSource::InlineAudio => has_inline_audio = true,
FileErrorSource::FileImage => has_file_image = true,
FileErrorSource::InlineAudio => has_competing_audio = true,
FileErrorSource::ToolFileAudio => {
has_competing_audio = true;
has_competing_file = true;
}
FileErrorSource::FileImage => has_competing_file = true,
FileErrorSource::Opaque => {
unknown_checkpoint |= !opaque_content(item).is_some_and(|ciphertext| {
clear_checkpoints.clone().any(|known| known == ciphertext)
@@ -185,8 +193,8 @@ impl SubmittedFileAudio {
Some(Self {
file_ids: file_ids.into_iter().map(str::to_owned).collect(),
candidate,
has_inline_audio,
has_file_image,
has_competing_audio,
has_competing_file,
})
}
@@ -195,9 +203,9 @@ impl SubmittedFileAudio {
let unique_file = self.file_ids.len() == 1;
let attributable = match error {
CodexErrorDetails::InvalidAudioInput { per_recording, .. } => {
!self.has_inline_audio && (unique_file || *per_recording)
!self.has_competing_audio && (unique_file || *per_recording)
}
_ => unique_file && !self.has_file_image && is_unknown_file_error(error),
_ => unique_file && !self.has_competing_file && is_unknown_file_error(error),
};
attributable.then_some(index)
}
@@ -108,6 +108,38 @@ fn tool_output_audio_makes_an_invalid_audio_error_ambiguous() {
}
}
#[test]
fn only_file_backed_tool_audio_competes_with_a_user_upload_for_an_unknown_file_error() {
use serde_json::json;
for kind in ["function_call_output", "custom_tool_call_output"] {
for (part, expected) in [
(
json!({"type":"input_audio","file_id":"expired_tool_clip"}),
None,
),
(
json!({"type":"input_audio","audio_url":"data:audio/wav;base64,AA=="}),
Some("valid_upload"),
),
] {
let output =
serde_json::from_value(json!({"type":kind, "call_id":"call", "output":[part]}))
.unwrap();
let items = [message(vec![file_audio("valid_upload")]), output];
assert_eq!(
rejected_file_audio_id(&unknown_file_error(), &items).as_deref(),
expected,
"{kind}"
);
assert_eq!(
rejected_file_audio_id(&audio_error(), &items),
None,
"{kind}"
);
}
}
}
#[test]
fn attributes_audio_error_only_to_one_visible_file_audio() {
let error = audio_error();
@@ -145,9 +177,8 @@ fn attributes_audio_error_only_to_one_visible_file_audio() {
);
}
#[test]
fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
let error = CodexErrorDetails::UnexpectedStatus(codex_protocol::error::UnexpectedResponseError {
fn unknown_file_error() -> CodexErrorDetails {
CodexErrorDetails::UnexpectedStatus(codex_protocol::error::UnexpectedResponseError {
status: http::StatusCode::NOT_FOUND,
body: serde_json::json!({"error": {"message": "Invalid input: Unknown `file_id` in input content."}}).to_string(),
url: None,
@@ -156,7 +187,12 @@ fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
request_id: None,
identity_authorization_error: None,
identity_error_code: None,
});
})
}
#[test]
fn unknown_file_error_counts_the_submitted_projection_not_raw_history() {
let error = unknown_file_error();
let unavailable_image_projection = message(vec![
ContentItem::InputText {
text: "[Image unavailable]".to_string(),
@@ -1,4 +1,4 @@
//! Projects unavailable file images into placeholders without changing tool attribution.
//! Projects unavailable images and tool audio into placeholders without changing tool attribution.
use std::collections::HashSet;
@@ -6,6 +6,7 @@ use codex_attachment_store::AttachmentStore;
use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ContentItemMetadata;
@@ -15,12 +16,12 @@ use codex_protocol::models::ResponseItem;
use futures::StreamExt;
use futures::stream;
const MAX_CONCURRENT_IMAGE_AVAILABILITY_CHECKS: usize = 8;
const MAX_CONCURRENT_FILE_AVAILABILITY_CHECKS: usize = 8;
/// Checks distinct file IDs concurrently and replaces only unavailable image content.
pub(crate) async fn replace_unresolvable_image_files<'a>(
/// Checks distinct file IDs concurrently and replaces only media the store confirms is unavailable.
pub(crate) async fn replace_unresolvable_media_files<'a>(
items: impl Iterator<Item = &'a mut ResponseItem>,
image_store: &dyn AttachmentStore,
store: &dyn AttachmentStore,
) {
let items: Vec<_> = items.collect();
let mut file_ids = HashSet::new();
@@ -44,6 +45,9 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
}
| FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} = part
{
file_ids.insert(file_id.clone());
@@ -56,10 +60,10 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
}
let mut checks = stream::iter(file_ids)
.map(|file_id| async move {
let resolvable = image_store.is_resolvable(&file_id).await;
let resolvable = store.is_resolvable(&file_id).await;
(file_id, resolvable)
})
.buffer_unordered(MAX_CONCURRENT_IMAGE_AVAILABILITY_CHECKS);
.buffer_unordered(MAX_CONCURRENT_FILE_AVAILABILITY_CHECKS);
let mut unavailable_file_ids = HashSet::new();
while let Some((file_id, resolvable)) = checks.next().await {
if !resolvable {
@@ -103,14 +107,22 @@ pub(crate) async fn replace_unresolvable_image_files<'a>(
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let Some(content) = output.content_items_mut() {
for part in content {
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
} = part
&& unavailable_file_ids.contains(file_id)
{
let (file_id, placeholder) = match part {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { file_id },
..
} => (file_id, "[image unavailable]"),
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => (
file_id,
"audio content omitted because its file is unavailable",
),
_ => continue,
};
if unavailable_file_ids.contains(file_id) {
*part = FunctionCallOutputContentItem::InputText {
text: "[image unavailable]".to_string(),
text: placeholder.into(),
};
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod history;
mod image_availability;
mod media_availability;
mod normalize;
mod sampling;
pub(crate) mod updates;
@@ -11,5 +11,5 @@ pub(crate) use history::estimate_item_token_count;
pub(crate) use history::estimate_item_token_count_with_audio;
pub(crate) use history::is_user_turn_boundary;
pub(crate) use history::rejected_audio;
pub(crate) use image_availability::replace_unresolvable_image_files;
pub(crate) use media_availability::replace_unresolvable_media_files;
pub(crate) use sampling::sampled_through;
+205 -10
View File
@@ -8,13 +8,19 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use codex_attachment_store::AttachmentStore;
use codex_attachment_store::FormatSpecificMetadata;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ResponseItem;
use codex_utils_audio::estimate_file_audio_tokens;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::approx_token_count;
use codex_utils_output_truncation::truncate_measured_tool_output;
use futures::StreamExt;
use futures::stream;
@@ -37,17 +43,206 @@ pub(crate) async fn snapshot(session: &Session) -> Arc<FileAudioDurations> {
pub(super) fn file_ids(items: &[ResponseItemEnvelope]) -> impl Iterator<Item = &String> {
items
.iter()
.filter_map(|envelope| match &envelope.item {
ResponseItem::Message { content, .. } => Some(content),
_ => None,
.flat_map(|envelope| envelope.item.file_audio_ids())
}
/// Only fresh results pass through the store. A replay keeps the representation in the rollout.
pub(super) async fn prepare_new_items(
items: &mut [ResponseItemEnvelope],
store: &dyn AttachmentStore,
thread_id: &str,
ephemeral: bool,
default_token_limit: usize,
) {
for ResponseItemEnvelope { item, metadata } in items {
let (ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. }) = item
else {
codex_utils_audio::prepare_response_items(std::slice::from_mut(item));
continue;
};
if !output.contains_audio() {
continue;
}
let metadata = metadata.get_or_insert_default();
// The producer's output limit can be larger than the history limit (e.g. Code Mode).
// Apply the latter to the whole result while audio still has its inline cost.
let limit = metadata
.history_truncation_token_limit
.unwrap_or(default_token_limit);
let producer_budgeted = std::mem::take(&mut metadata.producer_bounded_tool_audio);
// Validate and measure on a worker. Only clips retained by history need their
// decoded bytes materialized for upload, one at a time.
let Some(parts) = output.content_items_mut() else {
continue;
};
let parts = std::mem::take(parts);
let preserve_producer_limit =
producer_budgeted && metadata.history_truncation_token_limit.is_none();
let Ok((mut admitted, repair_limit)) = tokio::task::spawn_blocking(move || {
admit_tool_audio(parts, limit, preserve_producer_limit)
})
.flatten()
.filter_map(|part| match part {
ContentItem::InputAudio {
.await
else {
*output = codex_protocol::models::FunctionCallOutputPayload::from_text(
"audio content omitted because it could not be processed".into(),
);
continue;
};
let mut replaced_with_text = false;
for (part, charge) in &mut admitted {
if let FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} = part
{
let url = std::mem::take(audio_url);
*part = store_tool_audio(store, thread_id, ephemeral, url).await;
if matches!(part, FunctionCallOutputContentItem::InputText { .. }) {
*charge = 0;
replaced_with_text = true;
}
}
}
if replaced_with_text {
admitted =
truncate_measured_tool_output(admitted, TruncationPolicy::Tokens(repair_limit));
}
output.body = codex_protocol::models::FunctionCallOutputBody::ContentItems(
admitted.into_iter().map(|(part, _)| part).collect(),
);
metadata.tool_audio_budget_applied = true;
}
}
fn admit_tool_audio(
parts: Vec<FunctionCallOutputContentItem>,
history_limit: usize,
preserve_producer_limit: bool,
) -> (Vec<(FunctionCallOutputContentItem, usize)>, usize) {
let mut measured = Vec::with_capacity(parts.len());
for mut part in parts {
let charge = match &mut part {
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::Inline { audio_url },
} => match codex_utils_audio::prepare_and_measure_inline_audio(audio_url) {
Ok(tokens) => tokens,
Err(error) => {
part = FunctionCallOutputContentItem::InputText {
text: error.placeholder().into(),
};
0
}
},
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { .. },
} => estimate_file_audio_tokens(/*duration*/ None),
_ => 0,
};
measured.push((part, charge));
}
let admitted = truncate_measured_tool_output(measured, TruncationPolicy::Tokens(history_limit));
// Admission matches inline history. Only storage can introduce a new placeholder
// after that point. With no explicit producer limit, keep such repairs within
// the size of the result the model would otherwise have received.
let repair_limit = if preserve_producer_limit {
admitted
.iter()
.fold(0_usize, |sum, (part, charge)| {
let cost = match part {
FunctionCallOutputContentItem::InputText { text } => approx_token_count(text),
_ => *charge,
};
sum.saturating_add(cost)
})
.min(history_limit)
} else {
history_limit
};
(admitted, repair_limit)
}
async fn store_tool_audio(
store: &dyn AttachmentStore,
thread_id: &str,
ephemeral: bool,
inline_url: String,
) -> FunctionCallOutputContentItem {
let (mime, data, inline_url) = match tokio::task::spawn_blocking(move || {
codex_utils_audio::decode_audio(&inline_url).map(|(mime, data)| (mime, data, inline_url))
})
.await
{
Ok(Ok(decoded)) => decoded,
Ok(Err(error)) => {
return FunctionCallOutputContentItem::InputText {
text: error.placeholder().into(),
};
}
Err(_) => {
return FunctionCallOutputContentItem::InputText {
text: "audio content omitted because it could not be processed".into(),
};
}
};
// These tags come from Core. MCP servers and Code Mode never supply attachment IDs or policy tags.
let tags = HashMap::from([
("usage".to_string(), "tool_audio".to_string()),
("mime_type".to_string(), mime.to_string()),
]);
let result = store
.upload(UploadRequest {
thread_id: thread_id.to_owned(),
ephemeral,
file_name: None,
tags,
data,
})
.await;
tool_audio_after_upload(result, mime, inline_url)
}
fn tool_audio_after_upload(
result: Result<UploadResult, codex_attachment_store::AttachmentStoreError>,
mime: &str,
mut inline_url: String,
) -> FunctionCallOutputContentItem {
match result {
Ok(UploadResult::File { file_id }) if !file_id.trim().is_empty() => {
FunctionCallOutputContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
})
}
}
Ok(UploadResult::Inline { bytes }) => FunctionCallOutputContentItem::inline_audio(
codex_utils_audio::audio_data_url(mime, &bytes),
),
Err(error) if error.kind() == codex_attachment_store::AttachmentStoreErrorKind::Backend => {
tracing::warn!(kind = ?error.kind(), "tool audio upload failed; using inline audio");
// decode_audio validated the base64 and normalized the MIME alias.
if let Some(separator) = inline_url.find(',') {
inline_url.replace_range(..separator, &format!("data:{mime};base64"));
}
FunctionCallOutputContentItem::inline_audio(inline_url)
}
Err(error)
if error.kind()
== codex_attachment_store::AttachmentStoreErrorKind::InvalidAttachment =>
{
FunctionCallOutputContentItem::InputText {
text: "audio content omitted because the attachment store rejected the clip".into(),
}
}
Ok(UploadResult::File { .. }) | Err(_) => {
tracing::warn!("could not store tool audio attachment");
FunctionCallOutputContentItem::InputText {
text: "audio content omitted because it could not be stored".to_owned(),
}
}
}
}
pub(super) async fn resolve_pending(session: &Session) {
let ids = std::mem::take(&mut session.state.lock().await.pending_audio_duration_ids);
resolve(session, ids).await;
}
pub(super) async fn resolve(session: &Session, mut file_ids: BTreeSet<String>) {
@@ -44,3 +44,123 @@ fn history_sums_measured_and_unknown_file_audio() {
);
assert_eq!(history.estimated_tokens_since_last_usage(), 12_300);
}
#[test]
fn transient_upload_failure_retains_inline_audio_but_invalid_attachments_are_omitted() {
use codex_attachment_store::AttachmentStoreError;
use codex_attachment_store::AttachmentStoreErrorKind;
use serde_json::json;
for (kind, expected) in [
(
AttachmentStoreErrorKind::Backend,
json!({"type":"input_audio", "audio_url":"data:audio/wav;base64,QQ=="}),
),
(
AttachmentStoreErrorKind::InvalidAttachment,
json!({"type":"input_text", "text":"audio content omitted because the attachment store rejected the clip"}),
),
] {
let result = Err(AttachmentStoreError::new(kind, "upload failed"));
let part =
tool_audio_after_upload(result, "audio/wav", "data:audio/x-wav;base64,QQ==".into());
assert_eq!(serde_json::to_value(part).unwrap(), expected);
}
}
async fn prepared_tool_output(
parts: serde_json::Value,
metadata: codex_history::CodexHarnessMetadata,
limit: usize,
) -> serde_json::Value {
let mut item = ResponseItemEnvelope {
item: serde_json::from_value(
serde_json::json!({"type":"function_call_output","call_id":"clip","output":parts}),
)
.unwrap(),
metadata: Some(metadata),
};
prepare_new_items(
std::slice::from_mut(&mut item),
&codex_attachment_store::InlineAttachmentStore,
"thread",
/*ephemeral*/ false,
limit,
)
.await;
let metadata = item.metadata.unwrap();
assert!(!metadata.producer_bounded_tool_audio);
assert!(metadata.tool_audio_budget_applied);
assert_eq!(
serde_json::to_value(&metadata).unwrap()["tool_audio_budget_applied"],
true
);
serde_json::to_value(item.item).unwrap()["output"].take()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn producer_bounded_audio_keeps_the_same_parts_as_inline_history() {
use codex_history::CodexHarnessMetadata;
use serde_json::json;
let clip = json!({"type":"input_audio","audio_url":"data:audio/wav;base64,UklGRiYAAABXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YQIAAAAAAA=="});
let caption = json!({"type":"input_text","text":"caption ".repeat(100)});
let policy = TruncationPolicy::Tokens(20);
let limit = codex_utils_output_truncation::with_serialization_allowance(policy).token_budget();
let malformed = json!({"type":"input_audio","audio_url":format!("data:audio/wav;base64,{}", "!".repeat(8192))});
for (parts, keep_clip) in [
(json!([malformed, clip]), true),
(json!([{"type":"input_audio","audio_url":""}, clip]), true),
(json!([clip, caption]), true),
(json!([caption, clip]), false),
] {
let mut inline = serde_json::from_value(
json!({"type":"function_call_output","call_id":"clip","output":parts}),
)
.unwrap();
codex_utils_audio::prepare_response_items(std::slice::from_mut(&mut inline));
let mut history = ContextManager::new(uuid_v7_item_id_generator_for_test());
history.record_items([&inline], policy);
let expected =
serde_json::to_value(history.raw_items().last().unwrap()).unwrap()["output"].take();
assert_eq!(expected.as_array().unwrap().contains(&clip), keep_clip);
let metadata = CodexHarnessMetadata {
producer_bounded_tool_audio: true,
..Default::default()
};
assert_eq!(prepared_tool_output(parts, metadata, limit).await, expected);
}
}
#[tokio::test]
async fn zero_cost_invalid_audio_cannot_expand_into_unbounded_error_text() {
use codex_history::CodexHarnessMetadata;
use serde_json::json;
for producer_budgeted in [false, true] {
let mut parts = vec![json!({"type":"input_text","text":"keep"})];
parts.extend((0..100).map(|_| json!({"type":"input_audio","audio_url":""})));
let metadata = CodexHarnessMetadata {
producer_bounded_tool_audio: producer_budgeted,
..Default::default()
};
assert_eq!(
prepared_tool_output(parts.into(), metadata, /*limit*/ 1).await,
json!([{"type":"input_text","text":"keep"},{"type":"input_text","text":"[omitted 100 text items ...]"}])
);
}
}
#[tokio::test]
async fn known_producer_limit_keeps_the_replacement_and_following_caption() {
use serde_json::json;
let caption = json!({"type":"input_text","text":"caption"});
let parts = json!([{"type":"input_audio","audio_url":""}, caption]);
let metadata = codex_history::CodexHarnessMetadata {
producer_bounded_tool_audio: true,
history_truncation_token_limit: Some(100),
..Default::default()
};
let error = codex_utils_audio::decode_audio("").unwrap_err();
assert_eq!(
prepared_tool_output(parts, metadata, /*limit*/ 1).await,
json!([{"type":"input_text","text":error.placeholder()},caption])
);
}
+17 -1
View File
@@ -3961,9 +3961,25 @@ impl Session {
self.services.image_store.as_ref(),
))
.await;
let file_backed = turn_context
.config
.features
.enabled(Feature::FileBackedToolAudio);
if file_backed {
Box::pin(file_audio::prepare_new_items(
&mut items,
self.services.image_store.as_ref(),
&self.thread_id.to_string(),
turn_context.config.ephemeral,
with_serialization_allowance(model_info.truncation_policy.into()).token_budget(),
))
.await;
}
for envelope in &mut items {
let item = &mut envelope.item;
prepare_audio_response_items(std::slice::from_mut(item));
if !file_backed {
prepare_audio_response_items(std::slice::from_mut(item));
}
// Most response items get their passthrough turn ID at the durable history boundary.
Self::stamp_response_item_for_history(item, &turn_context.sub_id);
self.assign_missing_response_item_id(item);
+4 -2
View File
@@ -678,6 +678,9 @@ pub(crate) async fn run_turn(
// Process async hooks only after sampling and its tools have finished.
drain_async_hook_results(&sess, &turn_context, /*before_user_prompt*/ false).await;
let (has_pending_input, token_status) = async {
// Tool output can add audio without a queued user input. Resolve its
// duration before checking the continuation's compaction threshold.
super::file_audio::resolve_pending(&sess).await;
let has_pending_input =
sess.input_queue.has_pending_input(&sess.active_turn).await;
let token_status = super::context_window::context_window_token_status(
@@ -1079,8 +1082,7 @@ pub(crate) async fn run_hooks_and_record_inputs(
// The batch has left the input queue. Persist every accepted item before the
// optional lookup can suspend this task long enough for an interrupt to abort it.
if !input.is_empty() {
let ids = std::mem::take(&mut sess.state.lock().await.pending_audio_duration_ids);
super::file_audio::resolve(sess, ids).await;
super::file_audio::resolve_pending(sess).await;
}
Ok(blocked_input && !accepted_user_input)
}
+1 -1
View File
@@ -393,7 +393,7 @@ pub(crate) async fn handle_output_item_done(
// The call is already in history; resume needs its output even if we never dispatch.
let output =
ToolCallRuntime::aborted_response(&call, /*elapsed_secs*/ None);
return Ok(output.into_response());
return Ok(output.into_response(/*file_backed_tool_audio*/ false));
}
runtime.handle_tool_call(call, cancellation_token).await
})
@@ -40,6 +40,10 @@ impl CodeModeToolOutput {
}
impl ToolOutput for CodeModeToolOutput {
fn output_budget_applied(&self) -> bool {
true
}
fn log_output(&self) -> String {
self.output.log_output()
}
+4
View File
@@ -178,6 +178,10 @@ pub struct McpToolOutput {
}
impl ToolOutput for McpToolOutput {
fn output_budget_applied(&self) -> bool {
true
}
fn log_output(&self) -> String {
// Logging has its own budget; do not first apply the model-context budget.
let output = self.result.log_output();
+7 -1
View File
@@ -112,6 +112,12 @@ impl ToolCallRuntime {
) -> impl std::future::Future<Output = Result<ResponseItemEnvelope, CodexErr>> {
let error_call = call.clone();
let source = call.direct_source();
let file_backed_tool_audio = self
.step_context
.turn
.config
.features
.enabled(codex_features::Feature::FileBackedToolAudio);
let recorder = self.session.services.executed_tool_calls.clone();
let item_id_generator = Arc::clone(&self.session.item_id_generator);
let recorded_call = recorder.prepare_direct_call(&call, &source, &self.step_context);
@@ -135,7 +141,7 @@ impl ToolCallRuntime {
{
call.set_tool_result_metadata(ToolResultMetadata::new(metadata));
}
result.into_response()
result.into_response(file_backed_tool_audio)
}
Err(FunctionCallError::Fatal(message)) => return Err(CodexErr::Fatal(message)),
Err(other) => {
+14 -6
View File
@@ -35,6 +35,7 @@ use codex_history::CodexHarnessMetadata;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::EventMsg;
use codex_rollout::state_db;
@@ -327,7 +328,7 @@ pub(crate) struct AnyToolResult {
}
impl AnyToolResult {
pub(crate) fn into_response(self) -> ResponseItemEnvelope {
pub(crate) fn into_response(self, file_backed_tool_audio: bool) -> ResponseItemEnvelope {
let Self {
call_id,
payload,
@@ -335,12 +336,19 @@ impl AnyToolResult {
..
} = self;
let history_truncation_token_limit = result.fallback_token_limit_override();
let item = result.to_response_item(&call_id, &payload).into();
let producer_bounded_tool_audio = file_backed_tool_audio
&& result.output_budget_applied()
&& matches!(&item, ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } if output.contains_audio());
ResponseItemEnvelope {
item: result.to_response_item(&call_id, &payload).into(),
metadata: history_truncation_token_limit.map(|limit| CodexHarnessMetadata {
history_truncation_token_limit: Some(limit),
..Default::default()
}),
item,
metadata: (producer_bounded_tool_audio || history_truncation_token_limit.is_some())
.then_some(CodexHarnessMetadata {
producer_bounded_tool_audio,
history_truncation_token_limit,
..Default::default()
}),
}
}
+41 -2
View File
@@ -640,7 +640,7 @@ fn post_tool_use_feedback_output_preserves_fallback_token_limit_override(
};
assert_eq!(
result.into_response(),
result.into_response(/*file_backed_tool_audio*/ false),
ResponseItemEnvelope {
item: ResponseItem::from(ResponseInputItem::FunctionCallOutput {
call_id: "call-1".to_string(),
@@ -654,6 +654,45 @@ fn post_tool_use_feedback_output_preserves_fallback_token_limit_override(
);
}
#[test_case::test_case(true; "file-backed audio enabled")]
#[test_case::test_case(false; "file-backed audio disabled")]
fn mcp_audio_producer_limit_is_transient_until_history_admission(file_backed_tool_audio: bool) {
let result = AnyToolResult {
call_id: "clip".into(),
payload: ToolPayload::Function {
arguments: "{}".into(),
},
result: Box::new(crate::tools::context::McpToolOutput {
result: codex_protocol::mcp::CallToolResult {
content: vec![serde_json::json!({
"type": "audio",
"mimeType": "audio/wav",
"data": "UklGRiYAAABXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YQIAAAAAAA==",
})],
structured_content: None,
is_error: None,
meta: None,
},
tool_input: serde_json::json!({}),
result_metadata_capture_allowed: false,
wall_time: Duration::ZERO,
original_image_detail_supported: false,
truncation_policy: TruncationPolicy::Tokens(100),
}),
post_tool_use_payload: None,
};
let metadata = result
.into_response(file_backed_tool_audio)
.metadata
.unwrap();
assert_eq!(metadata.producer_bounded_tool_audio, file_backed_tool_audio);
assert!(!metadata.tool_audio_budget_applied);
let serialized = serde_json::to_value(&metadata).unwrap();
assert!(serialized.get("producer_bounded_tool_audio").is_none());
assert!(serialized.get("tool_audio_budget_applied").is_none());
}
#[test]
fn post_tool_use_feedback_output_keeps_code_mode_result_typed() {
let result = AnyToolResult {
@@ -674,7 +713,7 @@ fn post_tool_use_feedback_output_keeps_code_mode_result_typed() {
};
assert_eq!(
result.into_response(),
result.into_response(/*file_backed_tool_audio*/ false),
ResponseItemEnvelope::new(
ResponseInputItem::FunctionCallOutput {
call_id: "call-1".to_string(),
+1 -1
View File
@@ -692,7 +692,7 @@ async fn extension_tool_executors_are_model_visible_and_dispatchable() -> anyhow
)
.await?;
let response = result.into_response().item;
let response = result.into_response(/*file_backed_tool_audio*/ false).item;
match response {
ResponseItem::FunctionCallOutput {
call_id, output, ..