mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Support audio input
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
//! Helpers for truncating tool and exec output using [`TruncationPolicy`](codex_protocol::protocol::TruncationPolicy).
|
||||
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::InputAudio;
|
||||
pub use codex_utils_string::approx_bytes_for_tokens;
|
||||
pub use codex_utils_string::approx_token_count;
|
||||
pub use codex_utils_string::approx_tokens_from_byte_count;
|
||||
@@ -9,6 +10,9 @@ use codex_utils_string::truncate_middle_with_token_budget;
|
||||
|
||||
pub use codex_protocol::protocol::TruncationPolicy;
|
||||
|
||||
const INPUT_AUDIO_JSON_OVERHEAD_BYTES: usize =
|
||||
r#"{"type":"input_audio","input_audio":{"data":"","format":""}}"#.len();
|
||||
|
||||
pub fn formatted_truncate_text(content: &str, policy: TruncationPolicy) -> String {
|
||||
if content.len() <= policy.byte_budget() {
|
||||
return content.to_string();
|
||||
@@ -34,12 +38,16 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()),
|
||||
FunctionCallOutputContentItem::InputImage { .. } => None,
|
||||
FunctionCallOutputContentItem::InputImage { .. }
|
||||
| FunctionCallOutputContentItem::InputAudio { .. } => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if text_segments.is_empty() {
|
||||
return (items.to_vec(), None);
|
||||
return (
|
||||
truncate_function_output_items_with_policy(items, policy),
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
let mut combined = String::new();
|
||||
@@ -50,22 +58,59 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
combined.push_str(text);
|
||||
}
|
||||
|
||||
if combined.len() <= policy.byte_budget() {
|
||||
return (items.to_vec(), None);
|
||||
let combined_cost = serialized_byte_cost_for_policy(combined.len(), policy);
|
||||
let budget = budget_for_policy(policy);
|
||||
if combined_cost <= budget {
|
||||
let mut remaining_budget = budget.saturating_sub(combined_cost);
|
||||
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
|
||||
let mut omitted_audio_items = 0usize;
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
FunctionCallOutputContentItem::InputText { text } => {
|
||||
out.push(FunctionCallOutputContentItem::InputText { text: text.clone() });
|
||||
}
|
||||
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
|
||||
out.push(FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.clone(),
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
FunctionCallOutputContentItem::InputAudio { input_audio } => {
|
||||
push_audio_item_with_budget(
|
||||
&mut out,
|
||||
input_audio,
|
||||
policy,
|
||||
&mut remaining_budget,
|
||||
&mut omitted_audio_items,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
push_omitted_audio_summary(&mut out, omitted_audio_items);
|
||||
return (out, None);
|
||||
}
|
||||
|
||||
let mut out = vec![FunctionCallOutputContentItem::InputText {
|
||||
text: formatted_truncate_text(&combined, policy),
|
||||
}];
|
||||
out.extend(items.iter().filter_map(|item| match item {
|
||||
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
|
||||
Some(FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.clone(),
|
||||
detail: *detail,
|
||||
})
|
||||
let mut omitted_audio_items = 0usize;
|
||||
for item in items {
|
||||
match item {
|
||||
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
|
||||
out.push(FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.clone(),
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
FunctionCallOutputContentItem::InputAudio { .. } => {
|
||||
omitted_audio_items += 1;
|
||||
}
|
||||
FunctionCallOutputContentItem::InputText { .. } => {}
|
||||
}
|
||||
FunctionCallOutputContentItem::InputText { .. } => None,
|
||||
}));
|
||||
}
|
||||
push_omitted_audio_summary(&mut out, omitted_audio_items);
|
||||
|
||||
(out, Some(approx_token_count(&combined)))
|
||||
}
|
||||
@@ -75,11 +120,9 @@ pub fn truncate_function_output_items_with_policy(
|
||||
policy: TruncationPolicy,
|
||||
) -> Vec<FunctionCallOutputContentItem> {
|
||||
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
|
||||
let mut remaining_budget = match policy {
|
||||
TruncationPolicy::Bytes(_) => policy.byte_budget(),
|
||||
TruncationPolicy::Tokens(_) => policy.token_budget(),
|
||||
};
|
||||
let mut remaining_budget = budget_for_policy(policy);
|
||||
let mut omitted_text_items = 0usize;
|
||||
let mut omitted_audio_items = 0usize;
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
@@ -89,10 +132,7 @@ pub fn truncate_function_output_items_with_policy(
|
||||
continue;
|
||||
}
|
||||
|
||||
let cost = match policy {
|
||||
TruncationPolicy::Bytes(_) => text.len(),
|
||||
TruncationPolicy::Tokens(_) => approx_token_count(text),
|
||||
};
|
||||
let cost = serialized_byte_cost_for_policy(text.len(), policy);
|
||||
|
||||
if cost <= remaining_budget {
|
||||
out.push(FunctionCallOutputContentItem::InputText { text: text.clone() });
|
||||
@@ -117,6 +157,15 @@ pub fn truncate_function_output_items_with_policy(
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
FunctionCallOutputContentItem::InputAudio { input_audio } => {
|
||||
push_audio_item_with_budget(
|
||||
&mut out,
|
||||
input_audio,
|
||||
policy,
|
||||
&mut remaining_budget,
|
||||
&mut omitted_audio_items,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,10 +174,72 @@ pub fn truncate_function_output_items_with_policy(
|
||||
text: format!("[omitted {omitted_text_items} text items ...]"),
|
||||
});
|
||||
}
|
||||
push_omitted_audio_summary(&mut out, omitted_audio_items);
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
fn budget_for_policy(policy: TruncationPolicy) -> usize {
|
||||
match policy {
|
||||
TruncationPolicy::Bytes(_) => policy.byte_budget(),
|
||||
TruncationPolicy::Tokens(_) => policy.token_budget(),
|
||||
}
|
||||
}
|
||||
|
||||
fn serialized_byte_cost_for_policy(byte_count: usize, policy: TruncationPolicy) -> usize {
|
||||
match policy {
|
||||
TruncationPolicy::Bytes(_) => byte_count,
|
||||
TruncationPolicy::Tokens(_) => {
|
||||
usize::try_from(approx_tokens_from_byte_count(byte_count)).unwrap_or(usize::MAX)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_audio_item_with_budget(
|
||||
out: &mut Vec<FunctionCallOutputContentItem>,
|
||||
input_audio: &InputAudio,
|
||||
policy: TruncationPolicy,
|
||||
remaining_budget: &mut usize,
|
||||
omitted_audio_items: &mut usize,
|
||||
) {
|
||||
// Preserve audio only when the payload fits the remaining output budget.
|
||||
let byte_count = INPUT_AUDIO_JSON_OVERHEAD_BYTES
|
||||
.saturating_add(input_audio.data.len())
|
||||
.saturating_add(input_audio.format.len());
|
||||
let cost = serialized_byte_cost_for_policy(byte_count, policy);
|
||||
if cost <= *remaining_budget {
|
||||
out.push(FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: input_audio.clone(),
|
||||
});
|
||||
*remaining_budget = remaining_budget.saturating_sub(cost);
|
||||
} else {
|
||||
*omitted_audio_items += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn push_omitted_audio_summary(
|
||||
out: &mut Vec<FunctionCallOutputContentItem>,
|
||||
omitted_audio_items: usize,
|
||||
) {
|
||||
if omitted_audio_items > 0 {
|
||||
let item_word = if omitted_audio_items == 1 {
|
||||
"item"
|
||||
} else {
|
||||
"items"
|
||||
};
|
||||
let owner = if omitted_audio_items == 1 {
|
||||
"its"
|
||||
} else {
|
||||
"their"
|
||||
};
|
||||
out.push(FunctionCallOutputContentItem::InputText {
|
||||
text: format!(
|
||||
"[omitted {omitted_audio_items} audio {item_word} because {owner} size exceeds the output truncation budget]"
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn approx_tokens_from_byte_count_i64(bytes: i64) -> i64 {
|
||||
if bytes <= 0 {
|
||||
return 0;
|
||||
|
||||
@@ -7,8 +7,11 @@ use crate::truncate_function_output_items_with_policy;
|
||||
use crate::truncate_text;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::InputAudio;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
const SMALL_AUDIO_SERIALIZED_BYTES: usize = 71;
|
||||
|
||||
#[test]
|
||||
fn truncate_bytes_less_than_placeholder_returns_placeholder() {
|
||||
let content = "example output";
|
||||
@@ -251,6 +254,141 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_ima
|
||||
assert_eq!(original_token_count, Some(4));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatted_truncate_text_content_items_with_policy_preserves_audio_when_budget_allows() {
|
||||
let items = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "abcd".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "UklGRg==".to_string(),
|
||||
format: "wav".to_string(),
|
||||
},
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "efgh".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let (output, original_token_count) = formatted_truncate_text_content_items_with_policy(
|
||||
&items,
|
||||
TruncationPolicy::Bytes(SMALL_AUDIO_SERIALIZED_BYTES + "abcd\nefgh".len()),
|
||||
);
|
||||
|
||||
assert_eq!(output, items);
|
||||
assert_eq!(original_token_count, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatted_truncate_text_content_items_with_policy_omits_audio_when_budget_is_spent() {
|
||||
let items = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "abcd".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "UklGRg==".to_string(),
|
||||
format: "wav".to_string(),
|
||||
},
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "efgh".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let (output, original_token_count) =
|
||||
formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(4));
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "Total output lines: 2\n\nab…5 chars truncated…gh".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text:
|
||||
"[omitted 1 audio item because its size exceeds the output truncation budget]"
|
||||
.to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(original_token_count, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatted_truncate_text_content_items_with_policy_omits_audio_only_over_budget() {
|
||||
let items = vec![FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "A".repeat(200),
|
||||
format: "wav".to_string(),
|
||||
},
|
||||
}];
|
||||
|
||||
let (output, original_token_count) =
|
||||
formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(32));
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 audio item because its size exceeds the output truncation budget]"
|
||||
.to_string(),
|
||||
}]
|
||||
);
|
||||
assert_eq!(original_token_count, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_function_output_items_with_policy_omits_audio_over_budget() {
|
||||
let items = vec![FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "A".repeat(200),
|
||||
format: "wav".to_string(),
|
||||
},
|
||||
}];
|
||||
|
||||
let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(32));
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 audio item because its size exceeds the output truncation budget]"
|
||||
.to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_function_output_items_with_policy_charges_preserved_audio_to_budget() {
|
||||
let audio = FunctionCallOutputContentItem::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "UklGRg==".to_string(),
|
||||
format: "wav".to_string(),
|
||||
},
|
||||
};
|
||||
let items = vec![
|
||||
audio.clone(),
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "tail".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let output = truncate_function_output_items_with_policy(
|
||||
&items,
|
||||
TruncationPolicy::Bytes(SMALL_AUDIO_SERIALIZED_BYTES),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
audio,
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 text items ...]".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatted_truncate_text_content_items_with_policy_merges_all_text_for_token_budget() {
|
||||
let items = vec![
|
||||
|
||||
Reference in New Issue
Block a user