mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Use measured durations to budget uploaded audio (#59268)
## Why Uploaded audio always reserved 12,000 tokens per file, even for short clips. This could discard recordings during compaction that would fit within the retained-message budget. ## What changed - Add optional audio duration metadata to attachments and cache measured durations per session, resolving metadata for new input and replayed history. - Use measured durations for history estimates and compaction budgets at 10 tokens per second, rounded up and capped at 12,000 tokens. Keep the full budget for missing or unusable metadata. - Bound metadata lookups to 250 milliseconds per batch with up to eight concurrent requests, and record accepted input before lookup so interruption preserves the batch. ## Testing Add coverage for duration rounding and fallback estimates, mixed measured and unknown audio, retention through compaction and cold resume, and interruption during metadata lookup. GitOrigin-RevId: 26fe320c7024944d9d4c371d5fdcda84442b5cc5
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
//! Caches trusted file-audio durations for one session. Metadata lookup is best effort
|
||||
//! and time-bounded. Accepted input is recorded before lookup, and unmeasured
|
||||
//! files reserve the full audio token budget.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_attachment_store::FormatSpecificMetadata;
|
||||
use codex_attachment_store::ResolveRequest;
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_utils_audio::estimate_file_audio_tokens;
|
||||
use futures::StreamExt;
|
||||
use futures::stream;
|
||||
|
||||
use crate::session::Session;
|
||||
|
||||
/// Thread-local metadata. It is never serialized into a rollout or a provider request.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct FileAudioDurations(BTreeMap<String, Duration>);
|
||||
|
||||
impl FileAudioDurations {
|
||||
pub(crate) fn tokens(&self, file_id: &str) -> usize {
|
||||
estimate_file_audio_tokens(self.0.get(file_id).copied())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn snapshot(session: &Session) -> Arc<FileAudioDurations> {
|
||||
Arc::clone(&session.state.lock().await.history.file_audio_durations)
|
||||
}
|
||||
|
||||
pub(super) fn file_ids(items: &[ResponseItemEnvelope]) -> impl Iterator<Item = &String> {
|
||||
items
|
||||
.iter()
|
||||
.filter_map(|envelope| match &envelope.item {
|
||||
ResponseItem::Message { content, .. } => Some(content),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.filter_map(|part| match part {
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File { file_id },
|
||||
} => Some(file_id),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn resolve(session: &Session, mut file_ids: BTreeSet<String>) {
|
||||
if file_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
{
|
||||
let state = session.state.lock().await;
|
||||
file_ids.retain(|id| !state.history.file_audio_durations.0.contains_key(id));
|
||||
}
|
||||
let mut lookups = stream::iter(file_ids)
|
||||
.map(|id| async move {
|
||||
let result = session
|
||||
.services
|
||||
.image_store
|
||||
.resolve(ResolveRequest {
|
||||
file_id: &id,
|
||||
tags: HashMap::new(),
|
||||
download_url_ttl: None,
|
||||
})
|
||||
.await;
|
||||
(id, result)
|
||||
})
|
||||
.buffer_unordered(8);
|
||||
let mut resolved = BTreeMap::new();
|
||||
// Metadata is optional. Bound the entire batch, keep completed lookups on timeout,
|
||||
// and allow future attachments to retry a missing value.
|
||||
let _ = tokio::time::timeout(Duration::from_millis(250), async {
|
||||
while let Some((id, result)) = lookups.next().await {
|
||||
if let Ok(metadata) = result
|
||||
&& let Some(FormatSpecificMetadata::Audio(audio)) = metadata.format_specific
|
||||
&& let Some(duration_ms) = audio.duration_ms
|
||||
&& duration_ms > 0
|
||||
{
|
||||
resolved.insert(id, Duration::from_millis(duration_ms));
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if !resolved.is_empty() {
|
||||
let mut state = session.state.lock().await;
|
||||
Arc::make_mut(&mut state.history.file_audio_durations)
|
||||
.0
|
||||
.extend(resolved);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "file_audio_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,45 @@
|
||||
//! Checks history estimates with and without trusted file-audio durations.
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use crate::context_manager::ContextManager;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_utils_output_truncation::TruncationPolicy;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn history_sums_measured_and_unknown_file_audio() {
|
||||
let mut history = ContextManager::new();
|
||||
let item = ResponseInputItem::from(
|
||||
["short", "unknown"]
|
||||
.map(|file_id| UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: file_id.into(),
|
||||
},
|
||||
})
|
||||
.to_vec(),
|
||||
);
|
||||
history.record_items(
|
||||
[&ResponseItem::from(item)],
|
||||
TruncationPolicy::Tokens(10_000),
|
||||
);
|
||||
let base = BaseInstructions {
|
||||
text: String::new(),
|
||||
provenance: None,
|
||||
};
|
||||
assert_eq!(
|
||||
history.estimate_token_count_with_base_instructions(&base),
|
||||
Some(24_000)
|
||||
);
|
||||
history.file_audio_durations = Arc::new(FileAudioDurations(BTreeMap::from([(
|
||||
"short".into(),
|
||||
Duration::from_secs(30),
|
||||
)])));
|
||||
assert_eq!(
|
||||
history.estimate_token_count_with_base_instructions(&base),
|
||||
Some(12_300)
|
||||
);
|
||||
assert_eq!(history.estimated_tokens_since_last_usage(), 12_300);
|
||||
}
|
||||
@@ -245,6 +245,9 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::exec_output::StreamOutput;
|
||||
|
||||
pub(crate) mod async_compaction;
|
||||
pub(crate) mod file_audio;
|
||||
pub(crate) use file_audio::FileAudioDurations;
|
||||
|
||||
mod code_mode_warning;
|
||||
mod config_refresh;
|
||||
pub(crate) mod context_window;
|
||||
@@ -1956,7 +1959,8 @@ impl Session {
|
||||
.zip(metadata)
|
||||
.map(|(item, metadata)| ResponseItemEnvelope { item, metadata })
|
||||
.collect();
|
||||
// Ordinary replay needs no reviewer or network refresh. Only an opaque
|
||||
file_audio::resolve(self, file_audio::file_ids(&history).cloned().collect()).await;
|
||||
// Ordinary replay needs no reviewer refresh. Only an opaque
|
||||
// checkpoint can make reconstruction depend on reviewer compatibility.
|
||||
let reviewer_compaction_hash = if codex_history::CompactionCheckpoint::latest(&history)
|
||||
.is_some_and(codex_history::CompactionCheckpoint::is_usable)
|
||||
@@ -4053,6 +4057,9 @@ impl Session {
|
||||
});
|
||||
}
|
||||
}
|
||||
state
|
||||
.pending_audio_duration_ids
|
||||
.extend(file_audio::file_ids(&items).cloned());
|
||||
state
|
||||
.history
|
||||
.record_annotated_items(&mut items, model_info.truncation_policy.into());
|
||||
@@ -5283,6 +5290,7 @@ impl Session {
|
||||
.cloned()
|
||||
.collect(),
|
||||
crate::compact_remote_v2::RETAINED_MESSAGE_TOKEN_BUDGET,
|
||||
&history.file_audio_durations,
|
||||
);
|
||||
let window = {
|
||||
let mut state = self.state.lock().await;
|
||||
|
||||
@@ -1024,6 +1024,12 @@ pub(crate) async fn run_hooks_and_record_inputs(
|
||||
.await;
|
||||
}
|
||||
}
|
||||
// The batch has left the input queue. Persist every accepted item before the
|
||||
// optional lookup can suspend this task long enough for an interrupt to abort it.
|
||||
if !input.is_empty() {
|
||||
let ids = std::mem::take(&mut sess.state.lock().await.pending_audio_duration_ids);
|
||||
super::file_audio::resolve(sess, ids).await;
|
||||
}
|
||||
Ok(blocked_input && !accepted_user_input)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user