diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 24d86a4836..fd16c4daed 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -909,6 +909,7 @@ "event-listener-strategy_0.5.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"event-listener\",\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"event-listener/loom\"],\"portable-atomic\":[\"event-listener/portable-atomic\"],\"std\":[\"event-listener/std\"]}}", "event-listener_5.4.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"concurrent-queue\",\"req\":\"^2.4.0\"},{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.2.0\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking\",\"optional\":true,\"req\":\"^2.0.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"portable-atomic-util\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"portable_atomic_crate\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"try-lock\",\"req\":\"^0.2.5\"},{\"kind\":\"dev\",\"name\":\"waker-fn\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"concurrent-queue/loom\",\"parking?/loom\",\"dep:loom\"],\"portable-atomic\":[\"portable-atomic-util\",\"portable_atomic_crate\",\"concurrent-queue/portable-atomic\"],\"std\":[\"concurrent-queue/std\",\"parking\"]}}", "eventsource-stream_0.2.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"nom\",\"req\":\"^7.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"stream\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.11\"},{\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"url\",\"req\":\"^2.2\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"futures-core/std\",\"nom/std\"]}}", + "extended_0.1.0": "{\"dependencies\":[],\"features\":{}}", "eyre_0.6.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.28\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.46\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"indenter\",\"req\":\"^0.3.0\"},{\"name\":\"once_cell\",\"req\":\"^1.18.0\"},{\"default_features\":false,\"name\":\"pyo3\",\"optional\":true,\"req\":\"^0.20\"},{\"default_features\":false,\"features\":[\"auto-initialize\"],\"kind\":\"dev\",\"name\":\"pyo3\",\"req\":\"^0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.19\"}],\"features\":{\"auto-install\":[],\"default\":[\"auto-install\",\"track-caller\"],\"track-caller\":[]}}", "fancy-regex_0.16.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bit-set\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"matches\",\"req\":\"^0.1.10\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"name\":\"regex-automata\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"unicode\",\"perf\",\"std\"],\"perf\":[\"regex-automata/perf\"],\"std\":[\"regex-automata/std\",\"regex-syntax/std\",\"bit-set/std\"],\"track_caller\":[],\"unicode\":[\"regex-automata/unicode\",\"regex-syntax/unicode\"]}}", "faster-hex_0.10.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1.4.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"heapless\",\"req\":\"^0.8\",\"target\":\"cfg(not(feature = \\\"alloc\\\"))\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustc-hex\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"serde\"],\"serde\":[\"dep:serde\",\"alloc\"],\"std\":[\"alloc\",\"serde?/std\"]}}", @@ -1553,6 +1554,15 @@ "subtle_2.6.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"core_hint_black_box\":[],\"default\":[\"std\",\"i128\"],\"i128\":[],\"nightly\":[],\"std\":[]}}", "supports-color_2.1.0": "{\"dependencies\":[{\"name\":\"is-terminal\",\"req\":\"^0.4.0\"},{\"name\":\"is_ci\",\"req\":\"^1.1.1\"}],\"features\":{}}", "supports-color_3.0.2": "{\"dependencies\":[{\"name\":\"is_ci\",\"req\":\"^1.2.0\"}],\"features\":{}}", + "symphonia-bundle-mp3_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{\"default\":[\"mp1\",\"mp2\",\"mp3\"],\"mp1\":[],\"mp2\":[],\"mp3\":[]}}", + "symphonia-common_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"features\":[\"flac\"],\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}", + "symphonia-core_0.6.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.2\"},{\"name\":\"bytemuck\",\"req\":\"^1.7\"},{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"num-complex\",\"req\":\"^0.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"rustfft\",\"optional\":true,\"req\":\"^6.1.0\"},{\"name\":\"smallvec\",\"req\":\"^1.13.1\"}],\"features\":{\"default\":[],\"exp-subtitle-codecs\":[],\"exp-video-codecs\":[],\"opt-simd\":[\"opt-simd-sse\",\"opt-simd-avx\",\"opt-simd-neon\"],\"opt-simd-avx\":[\"rustfft/avx\"],\"opt-simd-neon\":[\"rustfft/neon\"],\"opt-simd-sse\":[\"rustfft/sse\"]}}", + "symphonia-format-isomp4_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}", + "symphonia-format-mkv_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{}}", + "symphonia-format-ogg_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"features\":[\"vorbis\"],\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}", + "symphonia-format-riff_0.6.0": "{\"dependencies\":[{\"name\":\"extended\",\"req\":\"^0.1.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{\"aiff\":[\"symphonia-metadata/riff-id3\"],\"default\":[\"aiff\",\"wav\"],\"wav\":[\"symphonia-metadata/riff-info\"]}}", + "symphonia-metadata_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"regex-lite\",\"req\":\"^0.1.6\"},{\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{\"ape\":[],\"default\":[\"ape\",\"id3v1\",\"id3v2\"],\"flac\":[\"vorbis\"],\"id3v1\":[],\"id3v2\":[],\"riff\":[\"riff-id3\",\"riff-info\"],\"riff-id3\":[\"id3v2\"],\"riff-info\":[],\"vorbis\":[\"flac\"]}}", + "symphonia_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"symphonia-bundle-flac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-bundle-mp3\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-aac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-adpcm\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-alac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-pcm\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-vorbis\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-caf\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-isomp4\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-mkv\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-ogg\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-format-riff\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{\"aac\":[\"dep:symphonia-codec-aac\"],\"adpcm\":[\"dep:symphonia-codec-adpcm\"],\"aiff\":[\"dep:symphonia-format-riff\",\"symphonia-format-riff/aiff\"],\"alac\":[\"dep:symphonia-codec-alac\"],\"all\":[\"all-codecs\",\"all-formats\",\"all-meta\"],\"all-codecs\":[\"aac\",\"adpcm\",\"alac\",\"flac\",\"mp1\",\"mp2\",\"mp3\",\"pcm\",\"vorbis\"],\"all-formats\":[\"caf\",\"isomp4\",\"mkv\",\"ogg\",\"aiff\",\"wav\"],\"all-meta\":[\"ape\",\"id3v1\",\"id3v2\"],\"ape\":[\"symphonia-metadata/ape\"],\"caf\":[\"dep:symphonia-format-caf\"],\"default\":[\"opt-simd\",\"all-meta\",\"adpcm\",\"flac\",\"mkv\",\"ogg\",\"pcm\",\"vorbis\",\"wav\"],\"exp-subtitle-codecs\":[\"symphonia-core/exp-subtitle-codecs\"],\"exp-video-codecs\":[\"symphonia-core/exp-video-codecs\"],\"flac\":[\"dep:symphonia-bundle-flac\"],\"id3v1\":[\"symphonia-metadata/id3v1\"],\"id3v2\":[\"symphonia-metadata/id3v2\"],\"isomp4\":[\"dep:symphonia-format-isomp4\"],\"mkv\":[\"dep:symphonia-format-mkv\"],\"mp1\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp1\"],\"mp2\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp2\"],\"mp3\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp3\"],\"mpa\":[\"mp1\",\"mp2\",\"mp3\"],\"ogg\":[\"dep:symphonia-format-ogg\"],\"opt-simd\":[\"opt-simd-sse\",\"opt-simd-avx\",\"opt-simd-neon\"],\"opt-simd-avx\":[\"symphonia-core/opt-simd-avx\"],\"opt-simd-neon\":[\"symphonia-core/opt-simd-neon\"],\"opt-simd-sse\":[\"symphonia-core/opt-simd-sse\"],\"pcm\":[\"dep:symphonia-codec-pcm\"],\"vorbis\":[\"dep:symphonia-codec-vorbis\"],\"wav\":[\"dep:symphonia-format-riff\",\"symphonia-format-riff/wav\"]}}", "syn_1.0.109": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.46\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1.0\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.1\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 02de9b4c05..aaeeb3328c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2715,6 +2715,7 @@ dependencies = [ "sha1 0.10.6", "shlex", "similar", + "symphonia", "tempfile", "test-case", "test-log", @@ -5882,6 +5883,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "eyre" version = "0.6.12" @@ -12772,6 +12779,119 @@ dependencies = [ "is_ci", ] +[[package]] +name = "symphonia" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1" +dependencies = [ + "bitflags 2.10.0", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 501a4106a8..f43faa319e 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -398,6 +398,13 @@ scopeguard = "1.2.0" sha2 = "0.10" shlex = "1.3.0" similar = "2.7.0" +symphonia = { version = "0.6.0", default-features = false, features = [ + "isomp4", + "mkv", + "mp3", + "ogg", + "wav", +] } socket2 = "0.6.1" sqlx = { version = "0.9.0", default-features = false, features = [ "chrono", diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index c5d559b77b..eccdd75092 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -1489,6 +1489,16 @@ impl ThreadHistoryBuilder { detail: payload.local_image_details.get(idx).copied().flatten(), }); } + if let Some(audio) = &payload.audio { + content.extend(audio.iter().cloned().map(|url| UserInput::Audio { url })); + } + content.extend( + payload + .local_audio + .iter() + .cloned() + .map(|path| UserInput::LocalAudio { path }), + ); content } } @@ -1862,16 +1872,19 @@ mod tests { } #[test] - fn rebuilds_user_message_image_details_from_legacy_events() { - let local_path = PathBuf::from("/tmp/local.png"); + fn rebuilds_user_message_attachments_from_legacy_events() { + let local_image_path = PathBuf::from("/tmp/local.png"); + let local_audio_path = PathBuf::from("/tmp/local.wav"); let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage( UserMessageEvent { client_id: None, message: "inspect these".into(), images: Some(vec!["https://example.com/image.png".into()]), image_details: vec![Some(ImageDetail::Original)], - local_images: vec![local_path.clone()], + local_images: vec![local_image_path.clone()], local_image_details: vec![Some(ImageDetail::Original)], + audio: Some(vec!["https://example.com/audio.mp3".into()]), + local_audio: vec![local_audio_path.clone()], text_elements: Vec::new(), }, ))]; @@ -1894,9 +1907,15 @@ mod tests { detail: Some(ImageDetail::Original), }, UserInput::LocalImage { - path: local_path, + path: local_image_path, detail: Some(ImageDetail::Original), }, + UserInput::Audio { + url: "https://example.com/audio.mp3".into(), + }, + UserInput::LocalAudio { + path: local_audio_path, + }, ], } ); diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index c41be5be71..48a983ca08 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -29,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::openai_models::InputModality; use core_test_support::load_default_config_for_test; use core_test_support::responses; use pretty_assertions::assert_eq; @@ -597,7 +598,9 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result Result<()> { audio_url: INLINE_AUDIO_DATA_URL.to_string(), }, ]; - // Audio remains available on the app-server item, but the base truncation policy intentionally - // omits it from the model-visible payload until the history/accounting layer. let model_content_items = vec![ FunctionCallOutputContentItem::InputText { text: "dynamic-ok".to_string(), @@ -717,6 +718,9 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> { image_url: TINY_PNG_DATA_URL.to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: INLINE_AUDIO_DATA_URL.to_string(), + }, ]; let response = DynamicToolCallResponse { content_items: response_content_items, @@ -776,6 +780,10 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> { "type": "input_image", "image_url": TINY_PNG_DATA_URL, "detail": "high" + }, + { + "type": "input_audio", + "audio_url": INLINE_AUDIO_DATA_URL } ]) ); diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 14efbd72f0..1a44dcecbb 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -826,6 +826,24 @@ async fn thread_read_loaded_include_turns_reads_store_history_without_rollout_pa let ThreadReadResponse { thread, .. } = serde_json::from_value(result)?; assert_eq!(turn_user_texts(&thread.turns), vec!["history from store"]); + let [ThreadItem::UserMessage { content, .. }] = thread.turns[0].items.as_slice() else { + panic!("expected one user message item"); + }; + assert_eq!( + content, + &vec![ + UserInput::Text { + text: "history from store".to_string(), + text_elements: Vec::new(), + }, + UserInput::Audio { + url: "https://example.com/recording.mp3".to_string(), + }, + UserInput::LocalAudio { + path: "recording.wav".into(), + }, + ] + ); client.shutdown().await?; Ok(()) @@ -2276,6 +2294,8 @@ fn store_history_items() -> Vec { message: "history from store".to_string(), images: None, local_images: Vec::new(), + audio: Some(vec!["https://example.com/recording.mp3".to_string()]), + local_audio: vec!["recording.wav".into()], text_elements: Vec::new(), ..Default::default() }, diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index de9d118e9c..88f5369b7c 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -104,6 +104,7 @@ serde_json = { workspace = true } sha1 = { workspace = true } shlex = { workspace = true } similar = { workspace = true } +symphonia = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ diff --git a/codex-rs/core/src/audio_preparation.rs b/codex-rs/core/src/audio_preparation.rs index e0ed11fb93..9f44420470 100644 --- a/codex-rs/core/src/audio_preparation.rs +++ b/codex-rs/core/src/audio_preparation.rs @@ -3,6 +3,17 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; +use codex_utils_cache::BlockingLruCache; +use codex_utils_cache::sha1_digest; +use codex_utils_output_truncation::approx_token_count; +use std::io::Cursor; +use std::num::NonZeroUsize; +use std::sync::LazyLock; +use symphonia::core::formats::FormatOptions; +use symphonia::core::formats::TrackType; +use symphonia::core::formats::probe::Hint; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::meta::MetadataOptions; use tracing::warn; const AUDIO_PROCESSING_ERROR_PLACEHOLDER: &str = @@ -17,6 +28,15 @@ const UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER: &str = /// This matches the Responses API audio input limit. const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; const MAX_PROMPT_AUDIO_BASE64_BYTES: usize = MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4; +const AUDIO_TOKEN_ESTIMATE_CACHE_SIZE: usize = 32; +const AUDIO_TOKENS_PER_SECOND: f64 = 10.0; + +static AUDIO_TOKEN_ESTIMATE_CACHE: LazyLock> = + LazyLock::new(|| { + BlockingLruCache::new( + NonZeroUsize::new(AUDIO_TOKEN_ESTIMATE_CACHE_SIZE).unwrap_or(NonZeroUsize::MIN), + ) + }); #[derive(Debug, thiserror::Error)] enum AudioPreparationError { @@ -121,6 +141,65 @@ fn canonical_audio_mime(mime: &str) -> Option<&'static str> { } } +pub(crate) fn estimate_audio_token_count(audio_url: &str) -> usize { + let key = sha1_digest(audio_url.as_bytes()); + AUDIO_TOKEN_ESTIMATE_CACHE.get_or_insert_with(key, || { + let Some(duration_seconds) = audio_duration_seconds(audio_url) else { + return approx_token_count(audio_url); + }; + let token_count = (duration_seconds * AUDIO_TOKENS_PER_SECOND).ceil(); + if token_count >= usize::MAX as f64 { + usize::MAX + } else { + token_count as usize + } + }) +} + +fn audio_duration_seconds(audio_url: &str) -> Option { + let (metadata, payload) = audio_url.split_once(',')?; + let metadata = metadata.get("data:".len()..)?; + let mut metadata_parts = metadata.split(';'); + let canonical_mime = canonical_audio_mime(metadata_parts.next()?)?; + if !metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")) { + return None; + } + + let bytes = match BASE64_STANDARD.decode(payload) { + Ok(bytes) => bytes, + Err(error) => { + tracing::trace!(%error, "failed to decode audio payload for token estimation"); + return None; + } + }; + let media_source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default()); + let mut hint = Hint::new(); + hint.mime_type(canonical_mime); + let format = match symphonia::default::get_probe().probe( + &hint, + media_source, + FormatOptions::default(), + MetadataOptions::default(), + ) { + Ok(format) => format, + Err(error) => { + tracing::trace!(%error, "failed to read audio duration for token estimation"); + return None; + } + }; + let track = format.default_track(TrackType::Audio)?; + let timing = track.time_base.zip(track.duration).or_else(|| { + format + .media_info() + .time_base + .zip(format.media_info().duration) + }); + let (time_base, duration) = timing?; + let duration_seconds = + duration.get() as f64 * f64::from(time_base.numer.get()) / f64::from(time_base.denom.get()); + duration_seconds.is_finite().then_some(duration_seconds) +} + fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> { if !is_data_url(audio_url) { return Err(AudioPreparationError::InvalidDataUrl { diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 785fd0ac7a..fa5fca3822 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,3 +1,4 @@ +use crate::audio_preparation::estimate_audio_token_count; use crate::context::ContextualUserFragment; use crate::context::world_state::WorldState; use crate::context::world_state::WorldStateSnapshot; @@ -437,7 +438,7 @@ pub(crate) fn truncate_function_output_payload( FunctionCallOutputBody::Text(truncate_text(content, policy)) } FunctionCallOutputBody::ContentItems(items) => FunctionCallOutputBody::ContentItems( - truncate_function_output_items_with_policy(items, policy), + truncate_function_output_items_with_policy(items, policy, estimate_audio_token_count), ), }; @@ -531,14 +532,18 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { .unwrap_or_default(); let (image_payload_bytes, image_replacement_bytes) = image_data_url_estimate_adjustment(item); + let (audio_payload_bytes, audio_replacement_bytes) = + audio_data_url_estimate_adjustment(item); let (encrypted_payload_bytes, encrypted_replacement_bytes) = encrypted_function_output_estimate_adjustment(item); - // Replace raw base64 payload bytes with a per-image estimate. + // Replace raw base64 payload bytes with per-modality estimates. // We intentionally preserve the data URL prefix and JSON // wrapper bytes already included in `raw`. let raw = raw .saturating_sub(image_payload_bytes) - .saturating_add(image_replacement_bytes); + .saturating_add(image_replacement_bytes) + .saturating_sub(audio_payload_bytes) + .saturating_add(audio_replacement_bytes); raw.saturating_sub(encrypted_payload_bytes) .saturating_add(encrypted_replacement_bytes) } @@ -551,6 +556,16 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { /// We only discount payloads for `data:image/...;base64,...` URLs (case /// insensitive markers) and leave everything else at raw serialized size. fn parse_base64_image_data_url(url: &str) -> Option<&str> { + parse_base64_data_url(url, "image/") +} + +/// Returns the base64 payload for inline audio data URLs that are eligible for +/// token-estimation discounting. +fn parse_base64_audio_data_url(url: &str) -> Option<&str> { + parse_base64_data_url(url, "audio/") +} + +fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> { if !url .get(.."data:".len()) .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) @@ -561,15 +576,15 @@ fn parse_base64_image_data_url(url: &str) -> Option<&str> { let metadata = &url[..comma_index]; let payload = &url[comma_index + 1..]; // Parse the media type and parameters without decoding. This keeps the - // estimator cheap while ensuring we only apply the fixed-cost image - // heuristic to image-typed base64 data URLs. + // estimator cheap while ensuring we only apply modality heuristics to + // appropriately typed base64 data URLs. let metadata_without_scheme = &metadata["data:".len()..]; let mut metadata_parts = metadata_without_scheme.split(';'); let mime_type = metadata_parts.next().unwrap_or_default(); let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")); if !mime_type - .get(.."image/".len()) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) + .get(..media_type_prefix.len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix)) { return None; } @@ -662,6 +677,50 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { (payload_bytes, replacement_bytes) } +/// Scans one response item for inline base64 audio data URLs and returns: +/// - total base64 payload bytes to subtract from raw serialized size +/// - total replacement byte estimate for those audio inputs +fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { + let mut payload_bytes = 0i64; + let mut replacement_bytes = 0i64; + + let mut accumulate = |audio_url: &str| { + if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) { + payload_bytes = + payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX)); + replacement_bytes = replacement_bytes.saturating_add( + i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count( + audio_url, + ))) + .unwrap_or(i64::MAX), + ); + } + }; + + match item { + ResponseItem::Message { content, .. } => { + for content_item in content { + if let ContentItem::InputAudio { audio_url } = content_item { + accumulate(audio_url); + } + } + } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let FunctionCallOutputBody::ContentItems(items) = &output.body { + for content_item in items { + if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item { + accumulate(audio_url); + } + } + } + } + _ => {} + } + + (payload_bytes, replacement_bytes) +} + fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { let ResponseItem::FunctionCallOutput { output, .. } = item else { return (0, 0); diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 3c934f2775..86f9cea51c 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -40,6 +40,31 @@ use regex_lite::Regex; const EXEC_FORMAT_MAX_BYTES: usize = 10_000; const EXEC_FORMAT_MAX_TOKENS: usize = 2_500; +const TEST_WAV_SAMPLE_RATE: u32 = 8_000; + +fn pcm_wav_data_url(sample_count: u32) -> (String, usize) { + let padding = sample_count % 2; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&sample_count.to_le_bytes()); + bytes.resize( + bytes.len() + sample_count as usize + padding as usize, + /*value*/ 0, + ); + let payload = BASE64_STANDARD.encode(bytes); + let payload_len = payload.len(); + (format!("data:audio/wav;base64,{payload}"), payload_len) +} fn assistant_msg(text: &str) -> ResponseItem { ResponseItem::Message { @@ -513,6 +538,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { image_url: "https://example.com/result.png".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(), + }, ]), internal_chat_message_metadata_passthrough: None, }, @@ -537,10 +565,14 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { image_url: "https://example.com/js-repl-result.png".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + }, ]), internal_chat_message_metadata_passthrough: None, }, ]; + let fully_supported_items = items.clone(); let history = create_history_with_items(items); let text_only_modalities = vec![InputModality::Text]; let stripped = history.for_prompt(&text_only_modalities); @@ -587,6 +619,10 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "image content omitted because you do not support image input" .to_string(), }, + FunctionCallOutputContentItem::InputText { + text: "audio content omitted because you do not support audio input" + .to_string(), + }, ]), internal_chat_message_metadata_passthrough: None, }, @@ -611,11 +647,23 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { text: "image content omitted because you do not support image input" .to_string(), }, + FunctionCallOutputContentItem::InputText { + text: "audio content omitted because you do not support audio input" + .to_string(), + }, ]), internal_chat_message_metadata_passthrough: None, }, ]; assert_eq!(stripped, expected); + assert_eq!( + create_history_with_items(fully_supported_items.clone()).for_prompt(&[ + InputModality::Text, + InputModality::Image, + InputModality::Audio, + ]), + fully_supported_items + ); // With image support, images are preserved let modalities = default_input_modalities(); @@ -1949,6 +1997,121 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { assert!(estimated < raw_len); } +#[test] +fn audio_data_url_payload_does_not_dominate_message_estimate() { + let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801); + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputAudio { audio_url }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64; + + assert_eq!(estimated, expected); + assert!(estimated < raw_len); +} + +#[test] +fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { + let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800); + let item = ResponseItem::FunctionCallOutput { + id: None, + call_id: "call-audio".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputAudio { audio_url }, + ]), + internal_chat_message_metadata_passthrough: None, + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64; + + assert_eq!(estimated, expected); + assert!(estimated < raw_len); +} + +#[test] +fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { + let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000); + let item = ResponseItem::CustomToolCallOutput { + id: None, + call_id: "call-custom-audio".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputAudio { audio_url }, + ]), + internal_chat_message_metadata_passthrough: None, + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64; + + assert_eq!(estimated, expected); + assert!(estimated < raw_len); +} + +#[test] +fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() { + let payload = "A".repeat(/*n*/ 100_000); + let audio_url = format!("data:audio/wav;base64,{payload}"); + let fallback_bytes = approx_bytes_for_tokens(approx_token_count(&audio_url)) as i64; + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputAudio { audio_url }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + + assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes); +} + +#[test] +fn record_items_omits_audio_that_exceeds_the_output_budget() { + let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000); + let item = ResponseItem::FunctionCallOutput { + id: None, + call_id: "call-audio".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputAudio { audio_url }, + ]), + success: Some(true), + }, + internal_chat_message_metadata_passthrough: None, + }; + let mut history = ContextManager::new(); + + history.record_items([&item], TruncationPolicy::Tokens(50)); + + assert_eq!( + history.items, + vec![ResponseItem::FunctionCallOutput { + id: None, + call_id: "call-audio".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "[omitted 1 audio items ...]".to_string(), + }, + ]), + success: Some(true), + }, + internal_chat_message_metadata_passthrough: None, + }] + ); +} + #[test] fn non_base64_image_urls_are_unchanged() { let message_item = ResponseItem::Message { diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs index a98c47efd8..1f7d32aaae 100644 --- a/codex-rs/core/src/context_manager/normalize.rs +++ b/codex-rs/core/src/context_manager/normalize.rs @@ -369,7 +369,7 @@ pub(crate) fn strip_images_when_unsupported( } } -/// Strip audio content from messages when the model does not support audio. +/// Strip audio content from messages and tool outputs when the model does not support audio. /// When `input_modalities` contains `InputModality::Audio`, no stripping is performed. pub(crate) fn strip_audio_when_unsupported( input_modalities: &[InputModality], @@ -380,14 +380,32 @@ pub(crate) fn strip_audio_when_unsupported( } for item in items.iter_mut() { - if let ResponseItem::Message { content, .. } = item { - for content_item in content.iter_mut() { - if matches!(content_item, ContentItem::InputAudio { .. }) { - *content_item = ContentItem::InputText { - text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), - }; + match item { + ResponseItem::Message { content, .. } => { + for content_item in content.iter_mut() { + if matches!(content_item, ContentItem::InputAudio { .. }) { + *content_item = ContentItem::InputText { + text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), + }; + } } } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content_items) = output.content_items_mut() { + for content_item in content_items.iter_mut() { + if matches!( + content_item, + FunctionCallOutputContentItem::InputAudio { .. } + ) { + *content_item = FunctionCallOutputContentItem::InputText { + text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), + }; + } + } + } + } + _ => {} } } } diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index bde373de94..3d9b12e9b0 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -21,6 +21,7 @@ use serde_json::Value as JsonValue; use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; +use crate::audio_preparation::estimate_audio_token_count; use crate::function_tool::FunctionCallError; use crate::original_image_detail::can_request_original_image_detail; use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items; @@ -269,7 +270,7 @@ fn truncate_code_mode_result( return truncated_items; } - truncate_function_output_items_with_policy(&items, policy) + truncate_function_output_items_with_policy(&items, policy, estimate_audio_token_count) } async fn call_nested_tool( @@ -427,6 +428,20 @@ mod tests { ); } + #[test] + fn over_budget_audio_output_is_omitted() { + let items = vec![FunctionCallOutputContentItem::InputAudio { + audio_url: format!("data:audio/wav;base64,{}", "A".repeat(100)), + }]; + + assert_eq!( + truncate_code_mode_result(items, Some(5)), + vec![FunctionCallOutputContentItem::InputText { + text: "[omitted 1 audio items ...]".to_string(), + }] + ); + } + #[tokio::test] async fn missing_process_host_falls_back_to_in_process_session() { let service = CodeModeService::new(Arc::new( diff --git a/codex-rs/core/tests/suite/audio_truncation.rs b/codex-rs/core/tests/suite/audio_truncation.rs new file mode 100644 index 0000000000..747b65fde3 --- /dev/null +++ b/codex-rs/core/tests/suite/audio_truncation.rs @@ -0,0 +1,158 @@ +use anyhow::Result; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; +use codex_protocol::dynamic_tools::DynamicToolFunctionSpec; +use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec; +use codex_protocol::dynamic_tools::DynamicToolNamespaceTool; +use codex_protocol::dynamic_tools::DynamicToolResponse; +use codex_protocol::dynamic_tools::DynamicToolSpec; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::openai_models::InputModality; +use codex_protocol::openai_models::TruncationPolicyConfig; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; +use core_test_support::responses; +use core_test_support::responses::sse; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; + +const TEST_WAV_SAMPLE_RATE: u32 = 8_000; +const OMITTED_AUDIO_MARKER: &str = "[omitted 1 audio items ...]"; + +fn pcm_wav_data_url(sample_count: u32) -> String { + let padding = sample_count % 2; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&sample_count.to_le_bytes()); + bytes.resize( + bytes.len() + sample_count as usize + padding as usize, + /*value*/ 0, + ); + format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(bytes)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dynamic_tool_audio_exceeding_the_output_budget_is_omitted() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let call_id = "audio-call"; + let tool_name = "recording"; + let responses_mock = responses::mount_sse_sequence( + &server, + vec![ + sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace(call_id, "codex_app", tool_name, "{}"), + responses::ev_completed("resp-1"), + ]), + sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-2"), + ]), + ], + ) + .await; + + let mut builder = test_codex().with_model_info_override("gpt-5.5", |model_info| { + model_info.input_modalities.push(InputModality::Audio); + model_info.truncation_policy = TruncationPolicyConfig::tokens(/*limit*/ 50); + }); + let base_test = builder.build_with_auto_env(&server).await?; + let dynamic_tool = DynamicToolSpec::Namespace(DynamicToolNamespaceSpec { + name: "codex_app".to_string(), + description: "Audio tools.".to_string(), + tools: vec![DynamicToolNamespaceTool::Function( + DynamicToolFunctionSpec { + name: tool_name.to_string(), + description: "Returns a recording.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false, + }), + defer_loading: false, + }, + )], + }); + let new_thread = base_test + .thread_manager + .start_thread_with_tools(base_test.config.clone(), vec![dynamic_tool]) + .await?; + let mut test = base_test; + test.codex = new_thread.thread; + test.session_configured = new_thread.session_configured; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "Return a recording".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + let EventMsg::DynamicToolCallRequest(request) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::DynamicToolCallRequest(_)) + }) + .await + else { + unreachable!("event guard guarantees DynamicToolCallRequest"); + }; + test.codex + .submit(Op::DynamicToolResponse { + id: request.call_id, + response: DynamicToolResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputAudio { + audio_url: pcm_wav_data_url(/*sample_count*/ 80_000), + }], + success: true, + }, + }) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = responses_mock.requests(); + assert_eq!(requests.len(), 2); + let output = requests[1] + .function_call_output(call_id) + .get("output") + .cloned() + .expect("follow-up request should contain the dynamic tool output"); + assert_eq!( + serde_json::from_value::(output)?, + FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: OMITTED_AUDIO_MARKER.to_string(), + }, + ]), + success: None, + } + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 47d373375c..36e9d7696e 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -2,6 +2,8 @@ use core_test_support::test_codex::local_selections; use std::fs; use anyhow::Result; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_core::compact::SUMMARY_PREFIX; use codex_features::Feature; use codex_login::CodexAuth; @@ -9,13 +11,19 @@ use codex_login::auth::AgentIdentityAuth; use codex_login::auth::AgentIdentityAuthRecord; use codex_protocol::account::PlanType as AccountPlanType; use codex_protocol::config_types::ServiceTier; +use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; use codex_protocol::dynamic_tools::DynamicToolFunctionSpec; use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec; use codex_protocol::dynamic_tools::DynamicToolNamespaceTool; +use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ItemCompletedEvent; @@ -52,6 +60,29 @@ use wiremock::ResponseTemplate; const CONTEXT_WINDOW_TRUNCATED_OUTPUT_MESSAGE: &str = "Output exceeded the available model context and was truncated"; +const TEST_WAV_SAMPLE_RATE: u32 = 8_000; + +fn pcm_wav_data_url(sample_count: u32) -> String { + let padding = sample_count % 2; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&sample_count.to_le_bytes()); + bytes.resize( + bytes.len() + sample_count as usize + padding as usize, + /*value*/ 0, + ); + format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(bytes)) +} fn approx_token_count(text: &str) -> i64 { i64::try_from(text.len().saturating_add(3) / 4).unwrap_or(i64::MAX) @@ -1315,6 +1346,123 @@ async fn remote_compact_filters_deferred_dynamic_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_compact_does_not_charge_inline_audio_payload_as_text() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let call_id = "audio-call"; + let tool_name = "recording"; + let audio_url = pcm_wav_data_url(/*sample_count*/ 300_000); + let responses_mock = responses::mount_sse_sequence( + &server, + vec![ + sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace(call_id, "codex_app", tool_name, "{}"), + responses::ev_completed_with_tokens("resp-1", /*total_tokens*/ 100), + ]), + sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed_with_tokens("resp-2", /*total_tokens*/ 200), + ]), + ], + ) + .await; + let compact_mock = responses::mount_compact_json_once( + &server, + json!({ "output": compacted_summary_only_output("compact summary") }), + ) + .await; + + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model_info_override("gpt-5.5", |model_info| { + model_info.input_modalities.push(InputModality::Audio); + }) + .with_config(|config| { + config.model_context_window = Some(50_000); + }); + let mut test = builder.build(&server).await?; + let dynamic_tool = DynamicToolSpec::Namespace(DynamicToolNamespaceSpec { + name: "codex_app".to_string(), + description: "Audio tools.".to_string(), + tools: vec![DynamicToolNamespaceTool::Function( + DynamicToolFunctionSpec { + name: tool_name.to_string(), + description: "Returns a recording.".to_string(), + input_schema: json!({ + "type": "object", + "properties": {}, + "additionalProperties": false, + }), + defer_loading: false, + }, + )], + }); + let new_thread = test + .thread_manager + .start_thread_with_tools(test.config.clone(), vec![dynamic_tool]) + .await?; + test.codex = new_thread.thread; + test.session_configured = new_thread.session_configured; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "Return a recording".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + let EventMsg::DynamicToolCallRequest(request) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::DynamicToolCallRequest(_)) + }) + .await + else { + unreachable!("event guard guarantees DynamicToolCallRequest"); + }; + test.codex + .submit(Op::DynamicToolResponse { + id: request.call_id, + response: DynamicToolResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputAudio { + audio_url: audio_url.clone(), + }], + success: true, + }, + }) + .await?; + wait_for_turn_complete(&test.codex).await; + + test.codex.submit(Op::Compact).await?; + wait_for_turn_complete(&test.codex).await; + + assert_eq!(responses_mock.requests().len(), 2); + let output = compact_mock + .single_request() + .function_call_output(call_id) + .get("output") + .cloned() + .expect("compact request should retain the dynamic tool output"); + assert_eq!( + serde_json::from_value::(output)?, + FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputAudio { audio_url }, + ]), + success: None, + } + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_runs_automatically() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index b83fc9c2a0..42695e9bc2 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -37,6 +37,7 @@ mod agents_md; mod apply_patch_cli; #[cfg(not(target_os = "windows"))] mod approvals; +mod audio_truncation; mod auto_review; mod catalog_permission_messages; mod cli_stream; diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index aac2849e64..cf8155c380 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -85,6 +85,8 @@ impl UserMessageItem { image_details: self.image_details(), local_images: self.local_image_paths(), local_image_details: self.local_image_details(), + audio: Some(self.audio_urls()), + local_audio: self.local_audio_paths(), text_elements: self.text_elements(), } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 201b3c4834..292326a4e0 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2345,6 +2345,15 @@ pub struct UserMessageEvent { /// imply default image detail behavior. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub local_image_details: Vec>, + /// Audio URLs sourced from `UserInput::Audio`. These are safe to replay in + /// legacy UI history events and correspond to audio sent to the model. + #[serde(skip_serializing_if = "Option::is_none")] + pub audio: Option>, + /// Local file paths sourced from `UserInput::LocalAudio`. These are kept so + /// clients can reattach audio when editing history and should not be + /// treated as API-ready URLs. + #[serde(default)] + pub local_audio: Vec, /// UI-defined spans within `message` used to render or persist special elements. #[serde(default)] pub text_elements: Vec, @@ -2364,6 +2373,9 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option { { return Some("[Image]".to_string()); } + if user.audio.as_ref().is_some_and(|audio| !audio.is_empty()) || !user.local_audio.is_empty() { + return Some("[Audio]".to_string()); + } None } @@ -5771,6 +5783,7 @@ mod tests { json!({ "message": "hello", "local_images": [], + "local_audio": [], "text_elements": [], }) ); @@ -5795,14 +5808,17 @@ mod tests { assert_eq!(event.image_details, Vec::>::new()); assert_eq!(event.local_images, vec![PathBuf::from("/tmp/local.png")]); assert_eq!(event.local_image_details, Vec::>::new()); + assert_eq!(event.audio, None); + assert_eq!(event.local_audio, Vec::::new()); assert_eq!(event.text_elements, Vec::new()); Ok(()) } #[test] - fn user_message_item_legacy_event_preserves_image_details() { + fn user_message_item_legacy_event_preserves_attachments() { let local_path = PathBuf::from("/tmp/local.png"); + let local_audio_path = PathBuf::from("/tmp/local.wav"); let mut item = UserMessageItem::new(&[ crate::user_input::UserInput::Image { image_url: "https://example.com/first.png".to_string(), @@ -5816,6 +5832,12 @@ mod tests { path: local_path.clone(), detail: Some(ImageDetail::Original), }, + crate::user_input::UserInput::Audio { + audio_url: "https://example.com/remote.mp3".to_string(), + }, + crate::user_input::UserInput::LocalAudio { + path: local_audio_path.clone(), + }, ]); item.client_id = Some("client-message-1".to_string()); @@ -5834,6 +5856,21 @@ mod tests { assert_eq!(event.image_details, vec![Some(ImageDetail::Original)]); assert_eq!(event.local_images, vec![local_path]); assert_eq!(event.local_image_details, vec![Some(ImageDetail::Original)]); + assert_eq!( + event.audio, + Some(vec!["https://example.com/remote.mp3".to_string()]) + ); + assert_eq!(event.local_audio, vec![local_audio_path]); + } + + #[test] + fn audio_only_user_message_has_placeholder_preview() { + let event = UserMessageEvent { + audio: Some(vec!["https://example.com/remote.mp3".to_string()]), + ..Default::default() + }; + + assert_eq!(user_message_preview(&event), Some("[Audio]".to_string())); } #[test] diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index f2d2b3c09a..06dfff5a73 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -287,6 +287,27 @@ mod tests { assert_eq!(metadata.title, ""); } + #[test] + fn event_msg_audio_only_user_message_sets_audio_placeholder_preview() { + let mut metadata = metadata_for_test(); + let item = RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent { + client_id: None, + message: String::new(), + images: None, + local_images: vec![], + audio: Some(vec!["https://example.com/audio.mp3".to_string()]), + local_audio: vec![], + text_elements: vec![], + ..Default::default() + })); + + apply_rollout_item(&mut metadata, &item, "test-provider"); + + assert_eq!(metadata.first_user_message.as_deref(), Some("[Audio]")); + assert_eq!(metadata.preview.as_deref(), Some("[Audio]")); + assert_eq!(metadata.title, ""); + } + #[test] fn event_msg_blank_user_message_without_images_keeps_first_user_message_empty() { let mut metadata = metadata_for_test(); diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index e2b4a45c88..ed8193d175 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -42,16 +42,9 @@ pub fn formatted_truncate_text_content_items_with_policy( | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>(); - let without_audio = || { - items - .iter() - .filter(|item| !matches!(item, FunctionCallOutputContentItem::InputAudio { .. })) - .cloned() - .collect() - }; if text_segments.is_empty() { - return (without_audio(), None); + return (items.to_vec(), None); } let mut combined = String::new(); @@ -63,7 +56,7 @@ pub fn formatted_truncate_text_content_items_with_policy( } if combined.len() <= policy.byte_budget() { - return (without_audio(), None); + return (items.to_vec(), None); } let original_token_count = approx_token_count(&combined); @@ -77,7 +70,11 @@ pub fn formatted_truncate_text_content_items_with_policy( detail: *detail, }) } - FunctionCallOutputContentItem::InputAudio { .. } => None, + FunctionCallOutputContentItem::InputAudio { audio_url } => { + Some(FunctionCallOutputContentItem::InputAudio { + audio_url: audio_url.clone(), + }) + } FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { Some(FunctionCallOutputContentItem::EncryptedContent { encrypted_content: encrypted_content.clone(), @@ -92,6 +89,7 @@ pub fn formatted_truncate_text_content_items_with_policy( pub fn truncate_function_output_items_with_policy( items: &[FunctionCallOutputContentItem], policy: TruncationPolicy, + estimate_audio_token_count: impl Fn(&str) -> usize, ) -> Vec { let mut out: Vec = Vec::with_capacity(items.len()); let mut remaining_budget = match policy { @@ -99,6 +97,7 @@ pub fn truncate_function_output_items_with_policy( TruncationPolicy::Tokens(_) => policy.token_budget(), }; let mut omitted_text_items = 0usize; + let mut omitted_audio_items = 0usize; for item in items { match item { @@ -136,7 +135,21 @@ pub fn truncate_function_output_items_with_policy( detail: *detail, }); } - FunctionCallOutputContentItem::InputAudio { .. } => {} + FunctionCallOutputContentItem::InputAudio { audio_url } => { + let token_cost = estimate_audio_token_count(audio_url); + let cost = match policy { + TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(token_cost), + TruncationPolicy::Tokens(_) => token_cost, + }; + if cost <= remaining_budget { + out.push(FunctionCallOutputContentItem::InputAudio { + audio_url: audio_url.clone(), + }); + remaining_budget = remaining_budget.saturating_sub(cost); + } else { + omitted_audio_items += 1; + } + } FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { out.push(FunctionCallOutputContentItem::EncryptedContent { encrypted_content: encrypted_content.clone(), @@ -150,6 +163,11 @@ pub fn truncate_function_output_items_with_policy( text: format!("[omitted {omitted_text_items} text items ...]"), }); } + if omitted_audio_items > 0 { + out.push(FunctionCallOutputContentItem::InputText { + text: format!("[omitted {omitted_audio_items} audio items ...]"), + }); + } out } diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index 7f8a7cdf60..cc81521f58 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -123,7 +123,7 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { ]; let output = - truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit)); + truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit), |_| 0); assert_eq!(output.len(), 5); @@ -208,7 +208,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_empty_leading_tex } #[test] -fn formatted_truncate_text_content_items_with_policy_merges_text_and_drops_audio() { +fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_media() { let items = vec![ FunctionCallOutputContentItem::InputText { text: "abcd".to_string(), @@ -245,6 +245,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_drops_audio image_url: "img:one".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "audio:one".to_string(), + }, FunctionCallOutputContentItem::InputImage { image_url: "img:two".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), @@ -283,7 +286,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_encrypted_content } #[test] -fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_drops_audio() { +fn truncate_function_output_items_with_policy_omits_audio_over_budget() { let items = vec![ FunctionCallOutputContentItem::InputText { text: "abcdefgh".to_string(), @@ -296,7 +299,8 @@ fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_dr }, ]; - let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2)); + let output = + truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2), |_| 1); assert_eq!( output, @@ -307,6 +311,35 @@ fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_dr FunctionCallOutputContentItem::EncryptedContent { encrypted_content: "enc_opaque".to_string(), }, + FunctionCallOutputContentItem::InputText { + text: "[omitted 1 audio items ...]".to_string(), + }, + ] + ); +} + +#[test] +fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() { + let audio = FunctionCallOutputContentItem::InputAudio { + audio_url: "audio:one".to_string(), + }; + let items = vec![ + audio.clone(), + FunctionCallOutputContentItem::InputText { + text: "abcdefgh".to_string(), + }, + ]; + + let output = + truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(5), |_| 1); + + assert_eq!( + output, + vec![ + audio, + FunctionCallOutputContentItem::InputText { + text: truncate_text("abcdefgh", TruncationPolicy::Bytes(1)), + }, ] ); }