Add opt-in file-backed tool audio (#61394)

## What changed

- Add the disabled-by-default `file_backed_tool_audio` feature to prepare fresh function and custom tool audio through the attachment store before persistence. Store returned file IDs in history and preserve them on replay without uploading again.
- Validate and budget mixed text and audio output before uploading retained clips, preserving audio token charges and producer limits. Resolve tool audio durations before continuation compaction checks.
- Keep inline audio on backend upload failures and use text placeholders for invalid or rejected clips. Replace unavailable tool audio files with placeholders in prompts while retaining the original history, so restored files can be used again.
- Treat file-backed tool audio as a competing source when attributing audio or unknown-file errors to user uploads.

## Testing

Add coverage for direct tool and Code Mode audio across follow-up turns, file expiration, restoration, and resume in both legacy and paginated history. Extend tests for truncation before upload, upload failure handling, bounded error placeholders, and error attribution.

GitOrigin-RevId: 66c1e7699c7d92f98c3a2d0f5cecbbd63147eb6c
This commit is contained in:
Hung
2026-11-13 23:02:16 +00:00
committed by copyberry
parent f941a5fb92
commit 5158e9780e
32 changed files with 1283 additions and 110 deletions
@@ -40,6 +40,10 @@ impl CodeModeToolOutput {
}
impl ToolOutput for CodeModeToolOutput {
fn output_budget_applied(&self) -> bool {
true
}
fn log_output(&self) -> String {
self.output.log_output()
}
+4
View File
@@ -178,6 +178,10 @@ pub struct McpToolOutput {
}
impl ToolOutput for McpToolOutput {
fn output_budget_applied(&self) -> bool {
true
}
fn log_output(&self) -> String {
// Logging has its own budget; do not first apply the model-context budget.
let output = self.result.log_output();
+7 -1
View File
@@ -112,6 +112,12 @@ impl ToolCallRuntime {
) -> impl std::future::Future<Output = Result<ResponseItemEnvelope, CodexErr>> {
let error_call = call.clone();
let source = call.direct_source();
let file_backed_tool_audio = self
.step_context
.turn
.config
.features
.enabled(codex_features::Feature::FileBackedToolAudio);
let recorder = self.session.services.executed_tool_calls.clone();
let item_id_generator = Arc::clone(&self.session.item_id_generator);
let recorded_call = recorder.prepare_direct_call(&call, &source, &self.step_context);
@@ -135,7 +141,7 @@ impl ToolCallRuntime {
{
call.set_tool_result_metadata(ToolResultMetadata::new(metadata));
}
result.into_response()
result.into_response(file_backed_tool_audio)
}
Err(FunctionCallError::Fatal(message)) => return Err(CodexErr::Fatal(message)),
Err(other) => {
+14 -6
View File
@@ -35,6 +35,7 @@ use codex_history::CodexHarnessMetadata;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::EventMsg;
use codex_rollout::state_db;
@@ -327,7 +328,7 @@ pub(crate) struct AnyToolResult {
}
impl AnyToolResult {
pub(crate) fn into_response(self) -> ResponseItemEnvelope {
pub(crate) fn into_response(self, file_backed_tool_audio: bool) -> ResponseItemEnvelope {
let Self {
call_id,
payload,
@@ -335,12 +336,19 @@ impl AnyToolResult {
..
} = self;
let history_truncation_token_limit = result.fallback_token_limit_override();
let item = result.to_response_item(&call_id, &payload).into();
let producer_bounded_tool_audio = file_backed_tool_audio
&& result.output_budget_applied()
&& matches!(&item, ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } if output.contains_audio());
ResponseItemEnvelope {
item: result.to_response_item(&call_id, &payload).into(),
metadata: history_truncation_token_limit.map(|limit| CodexHarnessMetadata {
history_truncation_token_limit: Some(limit),
..Default::default()
}),
item,
metadata: (producer_bounded_tool_audio || history_truncation_token_limit.is_some())
.then_some(CodexHarnessMetadata {
producer_bounded_tool_audio,
history_truncation_token_limit,
..Default::default()
}),
}
}
+41 -2
View File
@@ -640,7 +640,7 @@ fn post_tool_use_feedback_output_preserves_fallback_token_limit_override(
};
assert_eq!(
result.into_response(),
result.into_response(/*file_backed_tool_audio*/ false),
ResponseItemEnvelope {
item: ResponseItem::from(ResponseInputItem::FunctionCallOutput {
call_id: "call-1".to_string(),
@@ -654,6 +654,45 @@ fn post_tool_use_feedback_output_preserves_fallback_token_limit_override(
);
}
#[test_case::test_case(true; "file-backed audio enabled")]
#[test_case::test_case(false; "file-backed audio disabled")]
fn mcp_audio_producer_limit_is_transient_until_history_admission(file_backed_tool_audio: bool) {
let result = AnyToolResult {
call_id: "clip".into(),
payload: ToolPayload::Function {
arguments: "{}".into(),
},
result: Box::new(crate::tools::context::McpToolOutput {
result: codex_protocol::mcp::CallToolResult {
content: vec![serde_json::json!({
"type": "audio",
"mimeType": "audio/wav",
"data": "UklGRiYAAABXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YQIAAAAAAA==",
})],
structured_content: None,
is_error: None,
meta: None,
},
tool_input: serde_json::json!({}),
result_metadata_capture_allowed: false,
wall_time: Duration::ZERO,
original_image_detail_supported: false,
truncation_policy: TruncationPolicy::Tokens(100),
}),
post_tool_use_payload: None,
};
let metadata = result
.into_response(file_backed_tool_audio)
.metadata
.unwrap();
assert_eq!(metadata.producer_bounded_tool_audio, file_backed_tool_audio);
assert!(!metadata.tool_audio_budget_applied);
let serialized = serde_json::to_value(&metadata).unwrap();
assert!(serialized.get("producer_bounded_tool_audio").is_none());
assert!(serialized.get("tool_audio_budget_applied").is_none());
}
#[test]
fn post_tool_use_feedback_output_keeps_code_mode_result_typed() {
let result = AnyToolResult {
@@ -674,7 +713,7 @@ fn post_tool_use_feedback_output_keeps_code_mode_result_typed() {
};
assert_eq!(
result.into_response(),
result.into_response(/*file_backed_tool_audio*/ false),
ResponseItemEnvelope::new(
ResponseInputItem::FunctionCallOutput {
call_id: "call-1".to_string(),
+1 -1
View File
@@ -692,7 +692,7 @@ async fn extension_tool_executors_are_model_visible_and_dispatchable() -> anyhow
)
.await?;
let response = result.into_response().item;
let response = result.into_response(/*file_backed_tool_audio*/ false).item;
match response {
ResponseItem::FunctionCallOutput {
call_id, output, ..