Use measured durations to budget uploaded audio (#59268)

## Why

Uploaded audio always reserved 12,000 tokens per file, even for short clips. This could discard recordings during compaction that would fit within the retained-message budget.

## What changed

- Add optional audio duration metadata to attachments and cache measured durations per session, resolving metadata for new input and replayed history.
- Use measured durations for history estimates and compaction budgets at 10 tokens per second, rounded up and capped at 12,000 tokens. Keep the full budget for missing or unusable metadata.
- Bound metadata lookups to 250 milliseconds per batch with up to eight concurrent requests, and record accepted input before lookup so interruption preserves the batch.

## Testing

Add coverage for duration rounding and fallback estimates, mixed measured and unknown audio, retention through compaction and cold resume, and interruption during metadata lookup.

GitOrigin-RevId: 26fe320c7024944d9d4c371d5fdcda84442b5cc5
This commit is contained in:
Hung
2026-11-04 00:19:12 +00:00
committed by copyberry
parent b34808060a
commit 1685f2d621
21 changed files with 755 additions and 117 deletions
+18 -7
View File
@@ -4,7 +4,8 @@ use crate::compact_retained_tool_outputs::is_retained_tool_output;
use crate::context::ContextualUserFragment;
use crate::context::ImageResizeNotice;
use crate::context_manager::ContextManager;
use crate::context_manager::estimate_item_token_count;
use crate::context_manager::estimate_item_token_count_with_audio;
use crate::session::FileAudioDurations;
use crate::session::turn_context::TurnContext;
use codex_features::Feature;
use codex_history::ResponseItemEnvelope;
@@ -29,10 +30,16 @@ impl<T: Borrow<ResponseItem>> HistoryItemGroup<T> {
std::iter::once(self.source).chain(self.attached_notice)
}
pub(crate) fn estimated_token_count(&self) -> i128 {
let source_tokens = i128::from(estimate_item_token_count(self.source.borrow()));
pub(crate) fn estimated_token_count(&self, audio_durations: &FileAudioDurations) -> i128 {
let source_tokens = i128::from(estimate_item_token_count_with_audio(
self.source.borrow(),
audio_durations,
));
let notice_tokens = self.attached_notice.as_ref().map_or(0, |notice| {
i128::from(estimate_item_token_count(notice.borrow()))
i128::from(estimate_item_token_count_with_audio(
notice.borrow(),
audio_durations,
))
});
source_tokens.saturating_add(notice_tokens)
}
@@ -80,9 +87,10 @@ pub(crate) fn trim_function_call_history_to_fit_context_window(
// saturation in the normal history estimator.
let base_tokens =
i128::try_from(approx_token_count(&base_instructions.text)).unwrap_or(i128::MAX);
let audio_durations = &history.file_audio_durations;
let original_items = history.annotated_items();
let mut estimated_tokens = history_item_groups(original_items.iter().map(|item| &item.item))
.map(|group| group.estimated_token_count())
.map(|group| group.estimated_token_count(audio_durations))
.fold(base_tokens, i128::saturating_add);
let initial_estimated_tokens = i64::try_from(estimated_tokens).unwrap_or(i64::MAX);
let mut rewritten_items = Vec::new();
@@ -116,8 +124,11 @@ pub(crate) fn trim_function_call_history_to_fit_context_window(
break;
};
estimated_tokens = estimated_tokens
.saturating_sub(group.estimated_token_count())
.saturating_add(i128::from(estimate_item_token_count(&rewritten_item.item)));
.saturating_sub(group.estimated_token_count(audio_durations))
.saturating_add(i128::from(estimate_item_token_count_with_audio(
&rewritten_item.item,
audio_durations,
)));
consumed_items += group_item_count;
rewritten_items.push(rewritten_item);
}
+63 -16
View File
@@ -2,6 +2,7 @@ use std::sync::Arc;
use std::sync::OnceLock;
use crate::compact_retained_tool_outputs::is_retained_tool_output;
use crate::session::FileAudioDurations;
use crate::Prompt;
use crate::ResponseStream;
@@ -18,6 +19,7 @@ use crate::compact_remote_history::HistoryItemGroup;
use crate::compact_remote_history::history_item_groups;
use crate::context::world_state::WorldState;
use crate::context_manager::estimate_item_token_count;
use crate::context_manager::estimate_item_token_count_with_audio;
use crate::hook_runtime::PostCompactHookOutcome;
use crate::hook_runtime::PreCompactHookOutcome;
use crate::hook_runtime::run_post_compact_hooks;
@@ -493,6 +495,7 @@ fn build_v2_compacted_history(
compaction_output: ResponseItem,
features: &Features,
image_budget: RetainedImageBudget,
audio_durations: &FileAudioDurations,
) -> (Vec<ResponseItemEnvelope>, usize) {
debug_assert_eq!(prompt_input.len(), prompt_input_metadata.len());
let prompt_input = prompt_input
@@ -508,8 +511,12 @@ fn build_v2_compacted_history(
.filter(|group| is_retained_for_remote_compaction_v2(&group.source, features))
.flat_map(HistoryItemGroup::into_items)
.collect::<Vec<_>>();
let mut retained =
truncate_retained_messages(retained, RETAINED_MESSAGE_TOKEN_BUDGET, image_budget);
let mut retained = truncate_retained_messages(
retained,
RETAINED_MESSAGE_TOKEN_BUDGET,
image_budget,
audio_durations,
);
let retained_image_count = retained
.iter()
.map(|envelope| retained_input_image_count(&envelope.item))
@@ -620,14 +627,21 @@ fn retained_input_image_count(item: &ResponseItem) -> usize {
pub(crate) fn truncate_retained_messages_for_remote_compaction(
items: Vec<ResponseItemEnvelope>,
max_tokens: usize,
audio_durations: &FileAudioDurations,
) -> Vec<ResponseItemEnvelope> {
truncate_retained_messages(items, max_tokens, RetainedImageBudget::Disabled)
truncate_retained_messages(
items,
max_tokens,
RetainedImageBudget::Disabled,
audio_durations,
)
}
fn truncate_retained_messages(
items: Vec<ResponseItemEnvelope>,
max_tokens: usize,
image_budget: RetainedImageBudget,
audio_durations: &FileAudioDurations,
) -> Vec<ResponseItemEnvelope> {
let mut remaining = max_tokens;
let mut truncated_reversed = Vec::with_capacity(items.len());
@@ -656,13 +670,23 @@ fn truncate_retained_messages(
// modality still defines its own grouping and token cost in `media`.
let requires_media_boundary_truncation = has_budgeted_image || has_file_audio;
let notice_tokens = group.attached_notice.as_ref().map_or(0, |notice| {
retained_message_token_count(&notice.item, RetainedImageBudget::Disabled).max(1)
retained_message_token_count(
&notice.item,
RetainedImageBudget::Disabled,
audio_durations,
)
.max(1)
});
// Client-authored developer messages already charge non-text content via
// the serialized estimate. Preserve their text-only boundary correction.
let content_tokens = retained_message_token_count(&group.source.item, image_budget);
let content_tokens =
retained_message_token_count(&group.source.item, image_budget, audio_durations);
let source_tokens = if client_developer {
usize::try_from(estimate_item_token_count(&group.source.item)).unwrap_or(usize::MAX)
usize::try_from(estimate_item_token_count_with_audio(
&group.source.item,
audio_durations,
))
.unwrap_or(usize::MAX)
} else {
content_tokens.max(1)
};
@@ -689,7 +713,12 @@ fn truncate_retained_messages(
remaining = 0;
}
let truncated_item = if requires_media_boundary_truncation {
media::truncate_message_to_token_budget(group.source, content_budget, image_budget)
media::truncate_message_to_token_budget(
group.source,
content_budget,
image_budget,
audio_durations,
)
} else {
truncate_message_text_to_token_budget(group.source, content_budget)
};
@@ -697,8 +726,11 @@ fn truncate_retained_messages(
continue;
};
if client_developer {
let item_tokens = usize::try_from(estimate_item_token_count(&truncated_item.item))
.unwrap_or(usize::MAX);
let item_tokens = usize::try_from(estimate_item_token_count_with_audio(
&truncated_item.item,
audio_durations,
))
.unwrap_or(usize::MAX);
if item_tokens > available_tokens {
let adjusted_budget = content_budget
.saturating_sub(item_tokens - available_tokens)
@@ -708,6 +740,7 @@ fn truncate_retained_messages(
truncated_item,
adjusted_budget,
image_budget,
audio_durations,
)
} else {
truncate_message_text_to_token_budget(truncated_item, adjusted_budget)
@@ -715,8 +748,11 @@ fn truncate_retained_messages(
let Some(adjusted) = adjusted else {
continue;
};
if usize::try_from(estimate_item_token_count(&adjusted.item))
.unwrap_or(usize::MAX)
if usize::try_from(estimate_item_token_count_with_audio(
&adjusted.item,
audio_durations,
))
.unwrap_or(usize::MAX)
> available_tokens
{
continue;
@@ -737,14 +773,19 @@ fn truncate_retained_messages(
truncated_reversed
}
fn retained_message_token_count(item: &ResponseItem, image_budget: RetainedImageBudget) -> usize {
fn retained_message_token_count(
item: &ResponseItem,
image_budget: RetainedImageBudget,
audio_durations: &FileAudioDurations,
) -> usize {
let ResponseItem::Message { content, .. } = item else {
return usize::try_from(estimate_item_token_count(item)).unwrap_or(usize::MAX);
return usize::try_from(estimate_item_token_count_with_audio(item, audio_durations))
.unwrap_or(usize::MAX);
};
content
.iter()
.map(|part| media::content_item_token_count(part, image_budget))
.map(|part| media::content_item_token_count(part, image_budget, audio_durations))
.fold(0, usize::saturating_add)
}
@@ -867,6 +908,7 @@ mod tests {
output,
&Features::with_defaults(),
RetainedImageBudget::Disabled,
&FileAudioDurations::default(),
)
}
@@ -885,6 +927,7 @@ mod tests {
raw(truncate_retained_messages_for_remote_compaction(
annotated(items),
max_tokens,
&FileAudioDurations::default(),
))
}
@@ -992,6 +1035,7 @@ mod tests {
output.clone(),
&features,
RetainedImageBudget::Disabled,
&FileAudioDurations::default(),
);
let mut expected = vec![
ResponseItemEnvelope {
@@ -1030,8 +1074,11 @@ mod tests {
metadata: Some(CodexHarnessMetadata::default()),
};
let truncated =
truncate_retained_messages_for_remote_compaction(vec![item], /*max_tokens*/ 4);
let truncated = truncate_retained_messages_for_remote_compaction(
vec![item],
/*max_tokens*/ 4,
&FileAudioDurations::default(),
);
assert_eq!(truncated.len(), 1);
assert_eq!(truncated[0].metadata, Some(CodexHarnessMetadata::default()));
@@ -201,6 +201,7 @@ async fn build_and_install_compaction(
} else {
RetainedImageBudget::Disabled
},
crate::session::file_audio::snapshot(sess).await.as_ref(),
);
let reviewer_compaction_hash = reviewer_compaction_hash(sess, turn_context).await;
let installed = if let Some((prefix, model_info, cutoff_world_state)) = prefix {
+6 -4
View File
@@ -1,8 +1,8 @@
//! Counts retained media tokens and truncates each media part together with its labels.
use super::RetainedImageBudget;
use crate::context_manager::FILE_AUDIO_TOKENS_ESTIMATE;
use crate::context_manager::estimate_image_reference_bytes;
use crate::session::FileAudioDurations;
use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
@@ -19,6 +19,7 @@ use codex_utils_output_truncation::truncate_text;
pub(super) fn content_item_token_count(
item: &ContentItem,
image_budget: RetainedImageBudget,
audio_durations: &FileAudioDurations,
) -> usize {
match item {
ContentItem::InputText { text } | ContentItem::OutputText { text, .. } => {
@@ -30,8 +31,8 @@ pub(super) fn content_item_token_count(
)
.unwrap_or(usize::MAX),
ContentItem::InputAudio {
audio: AudioReference::File { .. },
} => FILE_AUDIO_TOKENS_ESTIMATE,
audio: AudioReference::File { file_id },
} => audio_durations.tokens(file_id),
ContentItem::InputAudio {
audio: AudioReference::Inline { .. },
} => 0,
@@ -61,6 +62,7 @@ pub(super) fn truncate_message_to_token_budget(
mut envelope: ResponseItemEnvelope,
max_tokens: usize,
image_budget: RetainedImageBudget,
audio_durations: &FileAudioDurations,
) -> Option<ResponseItemEnvelope> {
let mut content = to_annotated_content(&mut envelope.item)?;
let mut remaining = max_tokens;
@@ -69,7 +71,7 @@ pub(super) fn truncate_message_to_token_budget(
if let Some(start) = trailing_media_start(&content) {
let tokens = content[start..]
.iter()
.map(|part| content_item_token_count(part.content(), image_budget))
.map(|part| content_item_token_count(part.content(), image_budget, audio_durations))
.fold(0, usize::saturating_add);
if tokens > remaining {
break;
@@ -60,6 +60,7 @@ fn trim(items: Vec<ResponseItem>, max_tokens: usize) -> Vec<ResponseItem> {
items.into_iter().map(ResponseItemEnvelope::new).collect(),
max_tokens,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
)
.into_iter()
.map(ResponseItemEnvelope::into_item)
@@ -75,7 +76,11 @@ fn image_only_boundary_is_atomic_and_does_not_backfill_older_messages() {
message(vec![image.clone()]),
newest.clone(),
];
let image_tokens = media::content_item_token_count(&image, RetainedImageBudget::Enabled);
let image_tokens = media::content_item_token_count(
&image,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
);
for (max_tokens, expected) in [
(
image_tokens + 1,
@@ -92,7 +97,11 @@ fn image_only_boundary_is_atomic_and_does_not_backfill_older_messages() {
#[test]
fn file_audio_is_charged_and_atomic_in_both_image_budget_modes() {
let audio = file_audio();
let audio_tokens = media::content_item_token_count(&audio, RetainedImageBudget::Enabled);
let audio_tokens = media::content_item_token_count(
&audio,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
);
let newest = message(vec![text("new")]);
let parts = [text("before"), audio, text("after")];
let kinds = ["before", "audio", "after"];
@@ -118,13 +127,25 @@ fn file_audio_is_charged_and_atomic_in_both_image_budget_modes() {
item
};
let boundary = annotated_boundary(0);
let newest_tokens = retained_message_token_count(&newest, RetainedImageBudget::Disabled).max(1);
let after_tokens =
media::content_item_token_count(&text("after"), RetainedImageBudget::Enabled);
let newest_tokens = retained_message_token_count(
&newest,
RetainedImageBudget::Disabled,
&FileAudioDurations::default(),
)
.max(1);
let after_tokens = media::content_item_token_count(
&text("after"),
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
);
for image_budget in [RetainedImageBudget::Disabled, RetainedImageBudget::Enabled] {
for (budget, expected) in [
(
retained_message_token_count(&boundary, image_budget) + newest_tokens,
retained_message_token_count(
&boundary,
image_budget,
&FileAudioDurations::default(),
) + newest_tokens,
vec![boundary.clone(), newest.clone()],
),
(
@@ -147,6 +168,7 @@ fn file_audio_is_charged_and_atomic_in_both_image_budget_modes() {
.collect(),
budget,
image_budget,
&FileAudioDurations::default(),
)
.into_iter()
.map(ResponseItemEnvelope::into_item)
@@ -177,15 +199,22 @@ fn file_audio_cutoff_does_not_backfill_older_uncharged_media() {
file_audio(),
trailing_text.clone(),
]);
let max_tokens = media::content_item_token_count(&file_audio(), RetainedImageBudget::Enabled)
+ media::content_item_token_count(&trailing_text, RetainedImageBudget::Enabled)
- 1;
let max_tokens = media::content_item_token_count(
&file_audio(),
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
) + media::content_item_token_count(
&trailing_text,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
) - 1;
for image_budget in [RetainedImageBudget::Disabled, RetainedImageBudget::Enabled] {
let actual = truncate_retained_messages(
vec![ResponseItemEnvelope::new(boundary.clone())],
max_tokens,
image_budget,
&FileAudioDurations::default(),
)
.into_iter()
.map(ResponseItemEnvelope::into_item)
@@ -200,7 +229,11 @@ fn file_audio_cutoff_does_not_backfill_older_uncharged_media() {
#[test]
fn older_media_keeps_both_labels_or_is_dropped_after_file_audio_fits() {
let file = file_audio();
let file_tokens = media::content_item_token_count(&file, RetainedImageBudget::Enabled);
let file_tokens = media::content_item_token_count(
&file,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
);
for (open, media, close) in [
(
local_image_open_tag_text_with_path(
@@ -224,7 +257,8 @@ fn older_media_keeps_both_labels_or_is_dropped_after_file_audio_fits() {
let whole = message(parts.clone());
let source = message([vec![text("older text")], parts.clone()].concat());
for image_budget in [RetainedImageBudget::Disabled, RetainedImageBudget::Enabled] {
let whole_tokens = retained_message_token_count(&whole, image_budget);
let whole_tokens =
retained_message_token_count(&whole, image_budget, &FileAudioDurations::default());
for (budget, expected) in [
(file_tokens, vec![file.clone()]),
(whole_tokens - 1, vec![file.clone()]),
@@ -234,6 +268,7 @@ fn older_media_keeps_both_labels_or_is_dropped_after_file_audio_fits() {
vec![ResponseItemEnvelope::new(source.clone())],
budget,
image_budget,
&FileAudioDurations::default(),
);
let [
ResponseItemEnvelope {
@@ -285,7 +320,13 @@ fn later_image_parts_preserve_labels_audio_and_annotations() {
});
let image_tokens = parts[1..4]
.iter()
.map(|item| media::content_item_token_count(item, RetainedImageBudget::Enabled))
.map(|item| {
media::content_item_token_count(
item,
RetainedImageBudget::Enabled,
&FileAudioDurations::default(),
)
})
.sum::<usize>();
for (max_tokens, start) in [(image_tokens, 4), (image_tokens + 1, 1)] {
let mut expected = source.clone();
@@ -324,8 +365,13 @@ fn image_treatment_preserves_text_only_boundary_behavior() {
vec![source.clone()],
max_tokens,
RetainedImageBudget::Enabled,
&FileAudioDurations::default()
),
truncate_retained_messages_for_remote_compaction(
vec![source.clone()],
max_tokens,
&FileAudioDurations::default()
),
truncate_retained_messages_for_remote_compaction(vec![source.clone()], max_tokens),
);
}
}
@@ -349,9 +395,14 @@ fn image_treatment_preserves_client_developer_boundary_behavior() {
truncate_retained_messages(
vec![source.clone()],
max_tokens,
RetainedImageBudget::Enabled
RetainedImageBudget::Enabled,
&FileAudioDurations::default()
),
truncate_retained_messages_for_remote_compaction(
vec![source.clone()],
max_tokens,
&FileAudioDurations::default()
),
truncate_retained_messages_for_remote_compaction(vec![source.clone()], max_tokens),
);
}
}
+35 -17
View File
@@ -31,6 +31,7 @@ use crate::event_mapping::is_contextual_user_message_content;
use crate::event_mapping::parse_turn_item;
use crate::guardian::GUARDIAN_MAX_ROOT_MESSAGE_TOKENS;
use crate::guardian::guardian_truncate_text;
use crate::session::FileAudioDurations;
use crate::session::turn_context::TurnContext;
use crate::utils::json::serialized_json_bytes;
use base64::Engine;
@@ -91,11 +92,6 @@ use codex_features::Feature;
static GUARDIAN_REVIEW_CONTEXT_REVISION: AtomicU64 = AtomicU64::new(/*v*/ 1);
// Responses validates uploaded audio against its 20-minute duration limit.
// File durations are unavailable locally, so budget that maximum at the existing
// inline estimate of 10 audio tokens per second.
pub(crate) const FILE_AUDIO_TOKENS_ESTIMATE: usize = 20 * 60 * 10;
fn next_guardian_review_context_revision() -> u64 {
GUARDIAN_REVIEW_CONTEXT_REVISION.fetch_add(/*val*/ 1, Ordering::Relaxed)
}
@@ -113,6 +109,7 @@ pub(crate) struct ContextManager {
/// File IDs rejected by Responses. Retained history stays append-only; only
/// request projections omit these references.
rejected_audio_file_ids: Arc<BTreeSet<String>>,
pub(crate) file_audio_durations: Arc<FileAudioDurations>,
/// Reviewer policy travels with the history snapshot, independently of capture.
guardian_review_mode: GuardianContextMode,
guardian_include_additional_context: bool,
@@ -277,6 +274,7 @@ impl ContextManager {
review_history: None,
retained_context: Arc::default(),
rejected_audio_file_ids: Arc::default(),
file_audio_durations: Arc::default(),
guardian_review_mode: GuardianContextMode::ThreadOwned,
guardian_include_additional_context: Feature::GuardianIncludeAdditionalContext
.default_enabled(),
@@ -795,7 +793,9 @@ impl ContextManager {
let items_tokens = self
.items
.iter()
.map(|envelope| estimate_item_token_count(&envelope.item))
.map(|envelope| {
estimate_item_token_count_with_audio(&envelope.item, &self.file_audio_durations)
})
.fold(0i64, i64::saturating_add);
Some(base_tokens.saturating_add(items_tokens))
@@ -1068,7 +1068,9 @@ impl ContextManager {
}
)
})
.map(|envelope| estimate_item_token_count(&envelope.item))
.map(|envelope| {
estimate_item_token_count_with_audio(&envelope.item, &self.file_audio_durations)
})
.fold(0i64, i64::saturating_add)
}
@@ -1103,7 +1105,7 @@ impl ContextManager {
pub(crate) fn estimated_tokens_since_last_usage(&self) -> i64 {
self.items_since_last_usage()
.map(estimate_item_token_count)
.map(|item| estimate_item_token_count_with_audio(item, &self.file_audio_durations))
.fold(0i64, i64::saturating_add)
}
@@ -1236,8 +1238,17 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {
/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.
/// Original-detail file images use the maximum patch count.
pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {
let model_visible_bytes = estimate_response_item_model_visible_bytes(item);
approx_tokens_from_byte_count_i64(model_visible_bytes)
estimate_item_token_count_with_audio(item, &FileAudioDurations::default())
}
pub(crate) fn estimate_item_token_count_with_audio(
item: &ResponseItem,
audio_durations: &FileAudioDurations,
) -> i64 {
approx_tokens_from_byte_count_i64(estimate_response_item_model_visible_bytes(
item,
audio_durations,
))
}
/// Approximate model-visible byte cost for one image input.
@@ -1263,7 +1274,10 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<OriginalImageEstimateCache> = Laz
)
});
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
fn estimate_response_item_model_visible_bytes(
item: &ResponseItem,
audio_durations: &FileAudioDurations,
) -> i64 {
match item {
ResponseItem::Message { content, .. } => content
.iter()
@@ -1274,7 +1288,9 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
ContentItem::InputImage { image, detail } => {
estimate_image_reference_bytes(image, *detail)
}
ContentItem::InputAudio { audio } => estimate_audio_reference_bytes(audio),
ContentItem::InputAudio { audio } => {
estimate_audio_reference_bytes(audio, audio_durations)
}
})
.fold(0i64, i64::saturating_add),
ResponseItem::AgentMessage {
@@ -1487,13 +1503,15 @@ fn estimate_audio_bytes(audio_url: &str) -> i64 {
.unwrap_or(i64::MAX)
}
/// Audio estimate for callers that only have the reference. Uploaded file durations are not
/// available locally, so file references use the maximum duration accepted by Responses.
fn estimate_audio_reference_bytes(audio: &AudioReference) -> i64 {
fn estimate_audio_reference_bytes(
audio: &AudioReference,
audio_durations: &FileAudioDurations,
) -> i64 {
match audio {
AudioReference::Inline { audio_url } => estimate_audio_bytes(audio_url),
AudioReference::File { .. } => {
i64::try_from(approx_bytes_for_tokens(FILE_AUDIO_TOKENS_ESTIMATE)).unwrap_or(i64::MAX)
AudioReference::File { file_id } => {
i64::try_from(approx_bytes_for_tokens(audio_durations.tokens(file_id)))
.unwrap_or(i64::MAX)
}
}
}
@@ -150,6 +150,10 @@ fn raw_items(history: &ContextManager) -> Vec<ResponseItem> {
history.raw_items().cloned().collect()
}
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
super::estimate_response_item_model_visible_bytes(item, &FileAudioDurations::default())
}
#[test]
fn delivered_assistant_context_invalidates_reviews_without_changing_authorization() {
let revisions = |history: &ContextManager| {
+1 -1
View File
@@ -4,10 +4,10 @@ mod normalize;
pub(crate) mod updates;
pub(crate) use history::ContextManager;
pub(crate) use history::FILE_AUDIO_TOKENS_ESTIMATE;
pub(crate) use history::HistoryReplacement;
pub(crate) use history::estimate_image_reference_bytes;
pub(crate) use history::estimate_item_token_count;
pub(crate) use history::estimate_item_token_count_with_audio;
pub(crate) use history::is_user_turn_boundary;
pub(crate) use history::rejected_audio;
pub(crate) use image_availability::replace_unresolvable_image_files;
+100
View File
@@ -0,0 +1,100 @@
//! Caches trusted file-audio durations for one session. Metadata lookup is best effort
//! and time-bounded. Accepted input is recorded before lookup, and unmeasured
//! files reserve the full audio token budget.
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use codex_attachment_store::FormatSpecificMetadata;
use codex_attachment_store::ResolveRequest;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_utils_audio::estimate_file_audio_tokens;
use futures::StreamExt;
use futures::stream;
use crate::session::Session;
/// Thread-local metadata. It is never serialized into a rollout or a provider request.
#[derive(Clone, Debug, Default)]
pub(crate) struct FileAudioDurations(BTreeMap<String, Duration>);
impl FileAudioDurations {
pub(crate) fn tokens(&self, file_id: &str) -> usize {
estimate_file_audio_tokens(self.0.get(file_id).copied())
}
}
pub(crate) async fn snapshot(session: &Session) -> Arc<FileAudioDurations> {
Arc::clone(&session.state.lock().await.history.file_audio_durations)
}
pub(super) fn file_ids(items: &[ResponseItemEnvelope]) -> impl Iterator<Item = &String> {
items
.iter()
.filter_map(|envelope| match &envelope.item {
ResponseItem::Message { content, .. } => Some(content),
_ => None,
})
.flatten()
.filter_map(|part| match part {
ContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
})
}
pub(super) async fn resolve(session: &Session, mut file_ids: BTreeSet<String>) {
if file_ids.is_empty() {
return;
}
{
let state = session.state.lock().await;
file_ids.retain(|id| !state.history.file_audio_durations.0.contains_key(id));
}
let mut lookups = stream::iter(file_ids)
.map(|id| async move {
let result = session
.services
.image_store
.resolve(ResolveRequest {
file_id: &id,
tags: HashMap::new(),
download_url_ttl: None,
})
.await;
(id, result)
})
.buffer_unordered(8);
let mut resolved = BTreeMap::new();
// Metadata is optional. Bound the entire batch, keep completed lookups on timeout,
// and allow future attachments to retry a missing value.
let _ = tokio::time::timeout(Duration::from_millis(250), async {
while let Some((id, result)) = lookups.next().await {
if let Ok(metadata) = result
&& let Some(FormatSpecificMetadata::Audio(audio)) = metadata.format_specific
&& let Some(duration_ms) = audio.duration_ms
&& duration_ms > 0
{
resolved.insert(id, Duration::from_millis(duration_ms));
}
}
})
.await;
if !resolved.is_empty() {
let mut state = session.state.lock().await;
Arc::make_mut(&mut state.history.file_audio_durations)
.0
.extend(resolved);
}
}
#[cfg(test)]
#[path = "file_audio_tests.rs"]
mod tests;
@@ -0,0 +1,45 @@
//! Checks history estimates with and without trusted file-audio durations.
#![cfg(test)]
use super::*;
use crate::context_manager::ContextManager;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::user_input::UserInput;
use codex_utils_output_truncation::TruncationPolicy;
use pretty_assertions::assert_eq;
#[test]
fn history_sums_measured_and_unknown_file_audio() {
let mut history = ContextManager::new();
let item = ResponseInputItem::from(
["short", "unknown"]
.map(|file_id| UserInput::Audio {
audio: AudioReference::File {
file_id: file_id.into(),
},
})
.to_vec(),
);
history.record_items(
[&ResponseItem::from(item)],
TruncationPolicy::Tokens(10_000),
);
let base = BaseInstructions {
text: String::new(),
provenance: None,
};
assert_eq!(
history.estimate_token_count_with_base_instructions(&base),
Some(24_000)
);
history.file_audio_durations = Arc::new(FileAudioDurations(BTreeMap::from([(
"short".into(),
Duration::from_secs(30),
)])));
assert_eq!(
history.estimate_token_count_with_base_instructions(&base),
Some(12_300)
);
assert_eq!(history.estimated_tokens_since_last_usage(), 12_300);
}
+9 -1
View File
@@ -245,6 +245,9 @@ use codex_protocol::error::Result as CodexResult;
use codex_protocol::exec_output::StreamOutput;
pub(crate) mod async_compaction;
pub(crate) mod file_audio;
pub(crate) use file_audio::FileAudioDurations;
mod code_mode_warning;
mod config_refresh;
pub(crate) mod context_window;
@@ -1956,7 +1959,8 @@ impl Session {
.zip(metadata)
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
.collect();
// Ordinary replay needs no reviewer or network refresh. Only an opaque
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
// Ordinary replay needs no reviewer refresh. Only an opaque
// checkpoint can make reconstruction depend on reviewer compatibility.
let reviewer_compaction_hash = if codex_history::CompactionCheckpoint::latest(&history)
.is_some_and(codex_history::CompactionCheckpoint::is_usable)
@@ -4053,6 +4057,9 @@ impl Session {
});
}
}
state
.pending_audio_duration_ids
.extend(file_audio::file_ids(&items).cloned());
state
.history
.record_annotated_items(&mut items, model_info.truncation_policy.into());
@@ -5283,6 +5290,7 @@ impl Session {
.cloned()
.collect(),
crate::compact_remote_v2::RETAINED_MESSAGE_TOKEN_BUDGET,
&history.file_audio_durations,
);
let window = {
let mut state = self.state.lock().await;
+6
View File
@@ -1024,6 +1024,12 @@ pub(crate) async fn run_hooks_and_record_inputs(
.await;
}
}
// The batch has left the input queue. Persist every accepted item before the
// optional lookup can suspend this task long enough for an interrupt to abort it.
if !input.is_empty() {
let ids = std::mem::take(&mut sess.state.lock().await.pending_audio_duration_ids);
super::file_audio::resolve(sess, ids).await;
}
Ok(blocked_input && !accepted_user_input)
}
+4
View File
@@ -5,6 +5,7 @@ use codex_protocol::models::BaseInstructionsProvenance;
#[cfg(test)]
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ReasoningEffort;
use std::collections::BTreeSet;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::sync::Arc;
@@ -77,6 +78,8 @@ pub(crate) struct SessionState {
/// Persisted origin of the session base instructions, when known.
pub(crate) base_instructions_provenance: Option<BaseInstructionsProvenance>,
pub(crate) history: ContextManager,
/// Newly recorded files awaiting best-effort lookup after the input batch is durable.
pub(crate) pending_audio_duration_ids: BTreeSet<String>,
/// Cancels work bound to discarded history or a superseded Guardian evidence policy.
pub(crate) history_reset: CancellationToken,
pub(crate) latest_rate_limits: Option<RateLimitSnapshot>,
@@ -132,6 +135,7 @@ impl SessionState {
session_configuration,
base_instructions_provenance: None,
history,
pending_audio_duration_ids: BTreeSet::new(),
history_reset: CancellationToken::new(),
latest_rate_limits: None,
latest_token_usage_record: None,