Use measured durations to budget uploaded audio (#59268)

## Why

Uploaded audio always reserved 12,000 tokens per file, even for short clips. This could discard recordings during compaction that would fit within the retained-message budget.

## What changed

- Add optional audio duration metadata to attachments and cache measured durations per session, resolving metadata for new input and replayed history.
- Use measured durations for history estimates and compaction budgets at 10 tokens per second, rounded up and capped at 12,000 tokens. Keep the full budget for missing or unusable metadata.
- Bound metadata lookups to 250 milliseconds per batch with up to eight concurrent requests, and record accepted input before lookup so interruption preserves the batch.

## Testing

Add coverage for duration rounding and fallback estimates, mixed measured and unknown audio, retention through compaction and cold resume, and interruption during metadata lookup.

GitOrigin-RevId: 26fe320c7024944d9d4c371d5fdcda84442b5cc5
This commit is contained in:
Hung
2026-11-04 00:19:12 +00:00
committed by copyberry
parent b34808060a
commit 1685f2d621
21 changed files with 755 additions and 117 deletions
+100
View File
@@ -0,0 +1,100 @@
//! Caches trusted file-audio durations for one session. Metadata lookup is best effort
//! and time-bounded. Accepted input is recorded before lookup, and unmeasured
//! files reserve the full audio token budget.
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use codex_attachment_store::FormatSpecificMetadata;
use codex_attachment_store::ResolveRequest;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_utils_audio::estimate_file_audio_tokens;
use futures::StreamExt;
use futures::stream;
use crate::session::Session;
/// Thread-local metadata. It is never serialized into a rollout or a provider request.
#[derive(Clone, Debug, Default)]
pub(crate) struct FileAudioDurations(BTreeMap<String, Duration>);
impl FileAudioDurations {
pub(crate) fn tokens(&self, file_id: &str) -> usize {
estimate_file_audio_tokens(self.0.get(file_id).copied())
}
}
pub(crate) async fn snapshot(session: &Session) -> Arc<FileAudioDurations> {
Arc::clone(&session.state.lock().await.history.file_audio_durations)
}
pub(super) fn file_ids(items: &[ResponseItemEnvelope]) -> impl Iterator<Item = &String> {
items
.iter()
.filter_map(|envelope| match &envelope.item {
ResponseItem::Message { content, .. } => Some(content),
_ => None,
})
.flatten()
.filter_map(|part| match part {
ContentItem::InputAudio {
audio: AudioReference::File { file_id },
} => Some(file_id),
_ => None,
})
}
pub(super) async fn resolve(session: &Session, mut file_ids: BTreeSet<String>) {
if file_ids.is_empty() {
return;
}
{
let state = session.state.lock().await;
file_ids.retain(|id| !state.history.file_audio_durations.0.contains_key(id));
}
let mut lookups = stream::iter(file_ids)
.map(|id| async move {
let result = session
.services
.image_store
.resolve(ResolveRequest {
file_id: &id,
tags: HashMap::new(),
download_url_ttl: None,
})
.await;
(id, result)
})
.buffer_unordered(8);
let mut resolved = BTreeMap::new();
// Metadata is optional. Bound the entire batch, keep completed lookups on timeout,
// and allow future attachments to retry a missing value.
let _ = tokio::time::timeout(Duration::from_millis(250), async {
while let Some((id, result)) = lookups.next().await {
if let Ok(metadata) = result
&& let Some(FormatSpecificMetadata::Audio(audio)) = metadata.format_specific
&& let Some(duration_ms) = audio.duration_ms
&& duration_ms > 0
{
resolved.insert(id, Duration::from_millis(duration_ms));
}
}
})
.await;
if !resolved.is_empty() {
let mut state = session.state.lock().await;
Arc::make_mut(&mut state.history.file_audio_durations)
.0
.extend(resolved);
}
}
#[cfg(test)]
#[path = "file_audio_tests.rs"]
mod tests;
@@ -0,0 +1,45 @@
//! Checks history estimates with and without trusted file-audio durations.
#![cfg(test)]
use super::*;
use crate::context_manager::ContextManager;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::user_input::UserInput;
use codex_utils_output_truncation::TruncationPolicy;
use pretty_assertions::assert_eq;
#[test]
fn history_sums_measured_and_unknown_file_audio() {
let mut history = ContextManager::new();
let item = ResponseInputItem::from(
["short", "unknown"]
.map(|file_id| UserInput::Audio {
audio: AudioReference::File {
file_id: file_id.into(),
},
})
.to_vec(),
);
history.record_items(
[&ResponseItem::from(item)],
TruncationPolicy::Tokens(10_000),
);
let base = BaseInstructions {
text: String::new(),
provenance: None,
};
assert_eq!(
history.estimate_token_count_with_base_instructions(&base),
Some(24_000)
);
history.file_audio_durations = Arc::new(FileAudioDurations(BTreeMap::from([(
"short".into(),
Duration::from_secs(30),
)])));
assert_eq!(
history.estimate_token_count_with_base_instructions(&base),
Some(12_300)
);
assert_eq!(history.estimated_tokens_since_last_usage(), 12_300);
}
+9 -1
View File
@@ -245,6 +245,9 @@ use codex_protocol::error::Result as CodexResult;
use codex_protocol::exec_output::StreamOutput;
pub(crate) mod async_compaction;
pub(crate) mod file_audio;
pub(crate) use file_audio::FileAudioDurations;
mod code_mode_warning;
mod config_refresh;
pub(crate) mod context_window;
@@ -1956,7 +1959,8 @@ impl Session {
.zip(metadata)
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
.collect();
// Ordinary replay needs no reviewer or network refresh. Only an opaque
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
// Ordinary replay needs no reviewer refresh. Only an opaque
// checkpoint can make reconstruction depend on reviewer compatibility.
let reviewer_compaction_hash = if codex_history::CompactionCheckpoint::latest(&history)
.is_some_and(codex_history::CompactionCheckpoint::is_usable)
@@ -4053,6 +4057,9 @@ impl Session {
});
}
}
state
.pending_audio_duration_ids
.extend(file_audio::file_ids(&items).cloned());
state
.history
.record_annotated_items(&mut items, model_info.truncation_policy.into());
@@ -5283,6 +5290,7 @@ impl Session {
.cloned()
.collect(),
crate::compact_remote_v2::RETAINED_MESSAGE_TOKEN_BUDGET,
&history.file_audio_durations,
);
let window = {
let mut state = self.state.lock().await;
+6
View File
@@ -1024,6 +1024,12 @@ pub(crate) async fn run_hooks_and_record_inputs(
.await;
}
}
// The batch has left the input queue. Persist every accepted item before the
// optional lookup can suspend this task long enough for an interrupt to abort it.
if !input.is_empty() {
let ids = std::mem::take(&mut sess.state.lock().await.pending_audio_duration_ids);
super::file_audio::resolve(sess, ids).await;
}
Ok(blocked_input && !accepted_user_input)
}