Support audio input

This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-05-15 22:11:50 -07:00
parent 326e31ab65
commit b42b2ff9ec
37 changed files with 2112 additions and 110 deletions
+131 -20
View File
@@ -1,6 +1,7 @@
//! Helpers for truncating tool and exec output using [`TruncationPolicy`](codex_protocol::protocol::TruncationPolicy).
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::InputAudio;
pub use codex_utils_string::approx_bytes_for_tokens;
pub use codex_utils_string::approx_token_count;
pub use codex_utils_string::approx_tokens_from_byte_count;
@@ -9,6 +10,9 @@ use codex_utils_string::truncate_middle_with_token_budget;
pub use codex_protocol::protocol::TruncationPolicy;
const INPUT_AUDIO_JSON_OVERHEAD_BYTES: usize =
r#"{"type":"input_audio","input_audio":{"data":"","format":""}}"#.len();
pub fn formatted_truncate_text(content: &str, policy: TruncationPolicy) -> String {
if content.len() <= policy.byte_budget() {
return content.to_string();
@@ -34,12 +38,16 @@ pub fn formatted_truncate_text_content_items_with_policy(
.iter()
.filter_map(|item| match item {
FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()),
FunctionCallOutputContentItem::InputImage { .. } => None,
FunctionCallOutputContentItem::InputImage { .. }
| FunctionCallOutputContentItem::InputAudio { .. } => None,
})
.collect::<Vec<_>>();
if text_segments.is_empty() {
return (items.to_vec(), None);
return (
truncate_function_output_items_with_policy(items, policy),
None,
);
}
let mut combined = String::new();
@@ -50,22 +58,59 @@ pub fn formatted_truncate_text_content_items_with_policy(
combined.push_str(text);
}
if combined.len() <= policy.byte_budget() {
return (items.to_vec(), None);
let combined_cost = serialized_byte_cost_for_policy(combined.len(), policy);
let budget = budget_for_policy(policy);
if combined_cost <= budget {
let mut remaining_budget = budget.saturating_sub(combined_cost);
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
let mut omitted_audio_items = 0usize;
for item in items {
match item {
FunctionCallOutputContentItem::InputText { text } => {
out.push(FunctionCallOutputContentItem::InputText { text: text.clone() });
}
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
out.push(FunctionCallOutputContentItem::InputImage {
image_url: image_url.clone(),
detail: *detail,
});
}
FunctionCallOutputContentItem::InputAudio { input_audio } => {
push_audio_item_with_budget(
&mut out,
input_audio,
policy,
&mut remaining_budget,
&mut omitted_audio_items,
);
}
}
}
push_omitted_audio_summary(&mut out, omitted_audio_items);
return (out, None);
}
let mut out = vec![FunctionCallOutputContentItem::InputText {
text: formatted_truncate_text(&combined, policy),
}];
out.extend(items.iter().filter_map(|item| match item {
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
Some(FunctionCallOutputContentItem::InputImage {
image_url: image_url.clone(),
detail: *detail,
})
let mut omitted_audio_items = 0usize;
for item in items {
match item {
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
out.push(FunctionCallOutputContentItem::InputImage {
image_url: image_url.clone(),
detail: *detail,
});
}
FunctionCallOutputContentItem::InputAudio { .. } => {
omitted_audio_items += 1;
}
FunctionCallOutputContentItem::InputText { .. } => {}
}
FunctionCallOutputContentItem::InputText { .. } => None,
}));
}
push_omitted_audio_summary(&mut out, omitted_audio_items);
(out, Some(approx_token_count(&combined)))
}
@@ -75,11 +120,9 @@ pub fn truncate_function_output_items_with_policy(
policy: TruncationPolicy,
) -> Vec<FunctionCallOutputContentItem> {
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
let mut remaining_budget = match policy {
TruncationPolicy::Bytes(_) => policy.byte_budget(),
TruncationPolicy::Tokens(_) => policy.token_budget(),
};
let mut remaining_budget = budget_for_policy(policy);
let mut omitted_text_items = 0usize;
let mut omitted_audio_items = 0usize;
for item in items {
match item {
@@ -89,10 +132,7 @@ pub fn truncate_function_output_items_with_policy(
continue;
}
let cost = match policy {
TruncationPolicy::Bytes(_) => text.len(),
TruncationPolicy::Tokens(_) => approx_token_count(text),
};
let cost = serialized_byte_cost_for_policy(text.len(), policy);
if cost <= remaining_budget {
out.push(FunctionCallOutputContentItem::InputText { text: text.clone() });
@@ -117,6 +157,15 @@ pub fn truncate_function_output_items_with_policy(
detail: *detail,
});
}
FunctionCallOutputContentItem::InputAudio { input_audio } => {
push_audio_item_with_budget(
&mut out,
input_audio,
policy,
&mut remaining_budget,
&mut omitted_audio_items,
);
}
}
}
@@ -125,10 +174,72 @@ pub fn truncate_function_output_items_with_policy(
text: format!("[omitted {omitted_text_items} text items ...]"),
});
}
push_omitted_audio_summary(&mut out, omitted_audio_items);
out
}
fn budget_for_policy(policy: TruncationPolicy) -> usize {
match policy {
TruncationPolicy::Bytes(_) => policy.byte_budget(),
TruncationPolicy::Tokens(_) => policy.token_budget(),
}
}
fn serialized_byte_cost_for_policy(byte_count: usize, policy: TruncationPolicy) -> usize {
match policy {
TruncationPolicy::Bytes(_) => byte_count,
TruncationPolicy::Tokens(_) => {
usize::try_from(approx_tokens_from_byte_count(byte_count)).unwrap_or(usize::MAX)
}
}
}
fn push_audio_item_with_budget(
out: &mut Vec<FunctionCallOutputContentItem>,
input_audio: &InputAudio,
policy: TruncationPolicy,
remaining_budget: &mut usize,
omitted_audio_items: &mut usize,
) {
// Preserve audio only when the payload fits the remaining output budget.
let byte_count = INPUT_AUDIO_JSON_OVERHEAD_BYTES
.saturating_add(input_audio.data.len())
.saturating_add(input_audio.format.len());
let cost = serialized_byte_cost_for_policy(byte_count, policy);
if cost <= *remaining_budget {
out.push(FunctionCallOutputContentItem::InputAudio {
input_audio: input_audio.clone(),
});
*remaining_budget = remaining_budget.saturating_sub(cost);
} else {
*omitted_audio_items += 1;
}
}
fn push_omitted_audio_summary(
out: &mut Vec<FunctionCallOutputContentItem>,
omitted_audio_items: usize,
) {
if omitted_audio_items > 0 {
let item_word = if omitted_audio_items == 1 {
"item"
} else {
"items"
};
let owner = if omitted_audio_items == 1 {
"its"
} else {
"their"
};
out.push(FunctionCallOutputContentItem::InputText {
text: format!(
"[omitted {omitted_audio_items} audio {item_word} because {owner} size exceeds the output truncation budget]"
),
});
}
}
pub fn approx_tokens_from_byte_count_i64(bytes: i64) -> i64 {
if bytes <= 0 {
return 0;
@@ -7,8 +7,11 @@ use crate::truncate_function_output_items_with_policy;
use crate::truncate_text;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::InputAudio;
use pretty_assertions::assert_eq;
const SMALL_AUDIO_SERIALIZED_BYTES: usize = 71;
#[test]
fn truncate_bytes_less_than_placeholder_returns_placeholder() {
let content = "example output";
@@ -251,6 +254,141 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_ima
assert_eq!(original_token_count, Some(4));
}
#[test]
fn formatted_truncate_text_content_items_with_policy_preserves_audio_when_budget_allows() {
let items = vec![
FunctionCallOutputContentItem::InputText {
text: "abcd".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
input_audio: InputAudio {
data: "UklGRg==".to_string(),
format: "wav".to_string(),
},
},
FunctionCallOutputContentItem::InputText {
text: "efgh".to_string(),
},
];
let (output, original_token_count) = formatted_truncate_text_content_items_with_policy(
&items,
TruncationPolicy::Bytes(SMALL_AUDIO_SERIALIZED_BYTES + "abcd\nefgh".len()),
);
assert_eq!(output, items);
assert_eq!(original_token_count, None);
}
#[test]
fn formatted_truncate_text_content_items_with_policy_omits_audio_when_budget_is_spent() {
let items = vec![
FunctionCallOutputContentItem::InputText {
text: "abcd".to_string(),
},
FunctionCallOutputContentItem::InputAudio {
input_audio: InputAudio {
data: "UklGRg==".to_string(),
format: "wav".to_string(),
},
},
FunctionCallOutputContentItem::InputText {
text: "efgh".to_string(),
},
];
let (output, original_token_count) =
formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(4));
assert_eq!(
output,
vec![
FunctionCallOutputContentItem::InputText {
text: "Total output lines: 2\n\nab…5 chars truncated…gh".to_string(),
},
FunctionCallOutputContentItem::InputText {
text:
"[omitted 1 audio item because its size exceeds the output truncation budget]"
.to_string(),
},
]
);
assert_eq!(original_token_count, Some(3));
}
#[test]
fn formatted_truncate_text_content_items_with_policy_omits_audio_only_over_budget() {
let items = vec![FunctionCallOutputContentItem::InputAudio {
input_audio: InputAudio {
data: "A".repeat(200),
format: "wav".to_string(),
},
}];
let (output, original_token_count) =
formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(32));
assert_eq!(
output,
vec![FunctionCallOutputContentItem::InputText {
text: "[omitted 1 audio item because its size exceeds the output truncation budget]"
.to_string(),
}]
);
assert_eq!(original_token_count, None);
}
#[test]
fn truncate_function_output_items_with_policy_omits_audio_over_budget() {
let items = vec![FunctionCallOutputContentItem::InputAudio {
input_audio: InputAudio {
data: "A".repeat(200),
format: "wav".to_string(),
},
}];
let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(32));
assert_eq!(
output,
vec![FunctionCallOutputContentItem::InputText {
text: "[omitted 1 audio item because its size exceeds the output truncation budget]"
.to_string(),
}]
);
}
#[test]
fn truncate_function_output_items_with_policy_charges_preserved_audio_to_budget() {
let audio = FunctionCallOutputContentItem::InputAudio {
input_audio: InputAudio {
data: "UklGRg==".to_string(),
format: "wav".to_string(),
},
};
let items = vec![
audio.clone(),
FunctionCallOutputContentItem::InputText {
text: "tail".to_string(),
},
];
let output = truncate_function_output_items_with_policy(
&items,
TruncationPolicy::Bytes(SMALL_AUDIO_SERIALIZED_BYTES),
);
assert_eq!(
output,
vec![
audio,
FunctionCallOutputContentItem::InputText {
text: "[omitted 1 text items ...]".to_string(),
},
]
);
}
#[test]
fn formatted_truncate_text_content_items_with_policy_merges_all_text_for_token_budget() {
let items = vec![