mirror of
https://github.com/openai/codex.git
synced 2026-10-30 19:18:49 +00:00
Preserve audio across history and tool outputs (#34385)
## Why Audio attachments could be dropped from tool outputs and reconstructed thread history, while inline base64 payload size could overstate their model context cost. ## What changed - Preserve remote and local audio attachments in legacy user-message events and thread history. - Keep audio from function and custom tool outputs when the model supports it and the output fits the truncation budget; otherwise replace it with an omission marker. - Estimate inline audio cost from its duration, with a serialized-size fallback for malformed or unsupported data URLs, so compaction and truncation use model-visible cost instead of raw base64 size. - Replace audio in messages and tool outputs with an explanatory placeholder for models without audio input support. ## Testing Add coverage for history round trips, dynamic tool responses, modality filtering, duration-based context estimates, output-budget truncation, and remote compaction. GitOrigin-RevId: 5ed4628850a668f8383b7cd73db71b144cc02cd1
This commit is contained in:
nhamidi-oai
Generated
+10
@@ -909,6 +909,7 @@
|
||||
"event-listener-strategy_0.5.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"event-listener\",\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"event-listener/loom\"],\"portable-atomic\":[\"event-listener/portable-atomic\"],\"std\":[\"event-listener/std\"]}}",
|
||||
"event-listener_5.4.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"concurrent-queue\",\"req\":\"^2.4.0\"},{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.2.0\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking\",\"optional\":true,\"req\":\"^2.0.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"portable-atomic-util\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"portable_atomic_crate\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"try-lock\",\"req\":\"^0.2.5\"},{\"kind\":\"dev\",\"name\":\"waker-fn\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"concurrent-queue/loom\",\"parking?/loom\",\"dep:loom\"],\"portable-atomic\":[\"portable-atomic-util\",\"portable_atomic_crate\",\"concurrent-queue/portable-atomic\"],\"std\":[\"concurrent-queue/std\",\"parking\"]}}",
|
||||
"eventsource-stream_0.2.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"nom\",\"req\":\"^7.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"stream\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.11\"},{\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"url\",\"req\":\"^2.2\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"futures-core/std\",\"nom/std\"]}}",
|
||||
"extended_0.1.0": "{\"dependencies\":[],\"features\":{}}",
|
||||
"eyre_0.6.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.28\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.46\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"indenter\",\"req\":\"^0.3.0\"},{\"name\":\"once_cell\",\"req\":\"^1.18.0\"},{\"default_features\":false,\"name\":\"pyo3\",\"optional\":true,\"req\":\"^0.20\"},{\"default_features\":false,\"features\":[\"auto-initialize\"],\"kind\":\"dev\",\"name\":\"pyo3\",\"req\":\"^0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.19\"}],\"features\":{\"auto-install\":[],\"default\":[\"auto-install\",\"track-caller\"],\"track-caller\":[]}}",
|
||||
"fancy-regex_0.16.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bit-set\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"matches\",\"req\":\"^0.1.10\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"name\":\"regex-automata\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"unicode\",\"perf\",\"std\"],\"perf\":[\"regex-automata/perf\"],\"std\":[\"regex-automata/std\",\"regex-syntax/std\",\"bit-set/std\"],\"track_caller\":[],\"unicode\":[\"regex-automata/unicode\",\"regex-syntax/unicode\"]}}",
|
||||
"faster-hex_0.10.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1.4.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"heapless\",\"req\":\"^0.8\",\"target\":\"cfg(not(feature = \\\"alloc\\\"))\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustc-hex\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"serde\"],\"serde\":[\"dep:serde\",\"alloc\"],\"std\":[\"alloc\",\"serde?/std\"]}}",
|
||||
@@ -1553,6 +1554,15 @@
|
||||
"subtle_2.6.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"core_hint_black_box\":[],\"default\":[\"std\",\"i128\"],\"i128\":[],\"nightly\":[],\"std\":[]}}",
|
||||
"supports-color_2.1.0": "{\"dependencies\":[{\"name\":\"is-terminal\",\"req\":\"^0.4.0\"},{\"name\":\"is_ci\",\"req\":\"^1.1.1\"}],\"features\":{}}",
|
||||
"supports-color_3.0.2": "{\"dependencies\":[{\"name\":\"is_ci\",\"req\":\"^1.2.0\"}],\"features\":{}}",
|
||||
"symphonia-bundle-mp3_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{\"default\":[\"mp1\",\"mp2\",\"mp3\"],\"mp1\":[],\"mp2\":[],\"mp3\":[]}}",
|
||||
"symphonia-common_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"features\":[\"flac\"],\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}",
|
||||
"symphonia-core_0.6.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.2\"},{\"name\":\"bytemuck\",\"req\":\"^1.7\"},{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"num-complex\",\"req\":\"^0.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"rustfft\",\"optional\":true,\"req\":\"^6.1.0\"},{\"name\":\"smallvec\",\"req\":\"^1.13.1\"}],\"features\":{\"default\":[],\"exp-subtitle-codecs\":[],\"exp-video-codecs\":[],\"opt-simd\":[\"opt-simd-sse\",\"opt-simd-avx\",\"opt-simd-neon\"],\"opt-simd-avx\":[\"rustfft/avx\"],\"opt-simd-neon\":[\"rustfft/neon\"],\"opt-simd-sse\":[\"rustfft/sse\"]}}",
|
||||
"symphonia-format-isomp4_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}",
|
||||
"symphonia-format-mkv_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{}}",
|
||||
"symphonia-format-ogg_0.6.0": "{\"dependencies\":[{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-common\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"features\":[\"vorbis\"],\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{}}",
|
||||
"symphonia-format-riff_0.6.0": "{\"dependencies\":[{\"name\":\"extended\",\"req\":\"^0.1.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{\"aiff\":[\"symphonia-metadata/riff-id3\"],\"default\":[\"aiff\",\"wav\"],\"wav\":[\"symphonia-metadata/riff-info\"]}}",
|
||||
"symphonia-metadata_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"regex-lite\",\"req\":\"^0.1.6\"},{\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"}],\"features\":{\"ape\":[],\"default\":[\"ape\",\"id3v1\",\"id3v2\"],\"flac\":[\"vorbis\"],\"id3v1\":[],\"id3v2\":[],\"riff\":[\"riff-id3\",\"riff-info\"],\"riff-id3\":[\"id3v2\"],\"riff-info\":[],\"vorbis\":[\"flac\"]}}",
|
||||
"symphonia_0.6.0": "{\"dependencies\":[{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"symphonia-bundle-flac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-bundle-mp3\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-aac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-adpcm\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-alac\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-pcm\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-codec-vorbis\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-core\",\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-caf\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-isomp4\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-mkv\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"symphonia-format-ogg\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-format-riff\",\"optional\":true,\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"symphonia-metadata\",\"req\":\"^0.6.0\"}],\"features\":{\"aac\":[\"dep:symphonia-codec-aac\"],\"adpcm\":[\"dep:symphonia-codec-adpcm\"],\"aiff\":[\"dep:symphonia-format-riff\",\"symphonia-format-riff/aiff\"],\"alac\":[\"dep:symphonia-codec-alac\"],\"all\":[\"all-codecs\",\"all-formats\",\"all-meta\"],\"all-codecs\":[\"aac\",\"adpcm\",\"alac\",\"flac\",\"mp1\",\"mp2\",\"mp3\",\"pcm\",\"vorbis\"],\"all-formats\":[\"caf\",\"isomp4\",\"mkv\",\"ogg\",\"aiff\",\"wav\"],\"all-meta\":[\"ape\",\"id3v1\",\"id3v2\"],\"ape\":[\"symphonia-metadata/ape\"],\"caf\":[\"dep:symphonia-format-caf\"],\"default\":[\"opt-simd\",\"all-meta\",\"adpcm\",\"flac\",\"mkv\",\"ogg\",\"pcm\",\"vorbis\",\"wav\"],\"exp-subtitle-codecs\":[\"symphonia-core/exp-subtitle-codecs\"],\"exp-video-codecs\":[\"symphonia-core/exp-video-codecs\"],\"flac\":[\"dep:symphonia-bundle-flac\"],\"id3v1\":[\"symphonia-metadata/id3v1\"],\"id3v2\":[\"symphonia-metadata/id3v2\"],\"isomp4\":[\"dep:symphonia-format-isomp4\"],\"mkv\":[\"dep:symphonia-format-mkv\"],\"mp1\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp1\"],\"mp2\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp2\"],\"mp3\":[\"dep:symphonia-bundle-mp3\",\"symphonia-bundle-mp3/mp3\"],\"mpa\":[\"mp1\",\"mp2\",\"mp3\"],\"ogg\":[\"dep:symphonia-format-ogg\"],\"opt-simd\":[\"opt-simd-sse\",\"opt-simd-avx\",\"opt-simd-neon\"],\"opt-simd-avx\":[\"symphonia-core/opt-simd-avx\"],\"opt-simd-neon\":[\"symphonia-core/opt-simd-neon\"],\"opt-simd-sse\":[\"symphonia-core/opt-simd-sse\"],\"pcm\":[\"dep:symphonia-codec-pcm\"],\"vorbis\":[\"dep:symphonia-codec-vorbis\"],\"wav\":[\"dep:symphonia-format-riff\",\"symphonia-format-riff/wav\"]}}",
|
||||
"syn_1.0.109": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.46\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1.0\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.1\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}",
|
||||
"syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}",
|
||||
"sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}",
|
||||
|
||||
Generated
+120
@@ -2715,6 +2715,7 @@ dependencies = [
|
||||
"sha1 0.10.6",
|
||||
"shlex",
|
||||
"similar",
|
||||
"symphonia",
|
||||
"tempfile",
|
||||
"test-case",
|
||||
"test-log",
|
||||
@@ -5882,6 +5883,12 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "extended"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
|
||||
|
||||
[[package]]
|
||||
name = "eyre"
|
||||
version = "0.6.12"
|
||||
@@ -12772,6 +12779,119 @@ dependencies = [
|
||||
"is_ci",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"symphonia-bundle-mp3",
|
||||
"symphonia-core",
|
||||
"symphonia-format-isomp4",
|
||||
"symphonia-format-mkv",
|
||||
"symphonia-format-ogg",
|
||||
"symphonia-format-riff",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-bundle-mp3"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-common"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-core"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"bytemuck",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"num-complex",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-isomp4"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-common",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-mkv"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-common",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-ogg"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-common",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-riff"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
|
||||
dependencies = [
|
||||
"extended",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-metadata"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"regex-lite",
|
||||
"smallvec",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
||||
@@ -398,6 +398,13 @@ scopeguard = "1.2.0"
|
||||
sha2 = "0.10"
|
||||
shlex = "1.3.0"
|
||||
similar = "2.7.0"
|
||||
symphonia = { version = "0.6.0", default-features = false, features = [
|
||||
"isomp4",
|
||||
"mkv",
|
||||
"mp3",
|
||||
"ogg",
|
||||
"wav",
|
||||
] }
|
||||
socket2 = "0.6.1"
|
||||
sqlx = { version = "0.9.0", default-features = false, features = [
|
||||
"chrono",
|
||||
|
||||
@@ -1489,6 +1489,16 @@ impl ThreadHistoryBuilder {
|
||||
detail: payload.local_image_details.get(idx).copied().flatten(),
|
||||
});
|
||||
}
|
||||
if let Some(audio) = &payload.audio {
|
||||
content.extend(audio.iter().cloned().map(|url| UserInput::Audio { url }));
|
||||
}
|
||||
content.extend(
|
||||
payload
|
||||
.local_audio
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|path| UserInput::LocalAudio { path }),
|
||||
);
|
||||
content
|
||||
}
|
||||
}
|
||||
@@ -1862,16 +1872,19 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilds_user_message_image_details_from_legacy_events() {
|
||||
let local_path = PathBuf::from("/tmp/local.png");
|
||||
fn rebuilds_user_message_attachments_from_legacy_events() {
|
||||
let local_image_path = PathBuf::from("/tmp/local.png");
|
||||
let local_audio_path = PathBuf::from("/tmp/local.wav");
|
||||
let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
client_id: None,
|
||||
message: "inspect these".into(),
|
||||
images: Some(vec!["https://example.com/image.png".into()]),
|
||||
image_details: vec![Some(ImageDetail::Original)],
|
||||
local_images: vec![local_path.clone()],
|
||||
local_images: vec![local_image_path.clone()],
|
||||
local_image_details: vec![Some(ImageDetail::Original)],
|
||||
audio: Some(vec!["https://example.com/audio.mp3".into()]),
|
||||
local_audio: vec![local_audio_path.clone()],
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
))];
|
||||
@@ -1894,9 +1907,15 @@ mod tests {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: local_path,
|
||||
path: local_image_path,
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "https://example.com/audio.mp3".into(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: local_audio_path,
|
||||
},
|
||||
],
|
||||
}
|
||||
);
|
||||
|
||||
@@ -29,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use core_test_support::load_default_config_for_test;
|
||||
use core_test_support::responses;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -597,7 +598,9 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result<PendingDynami
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
let config = load_default_config_for_test(&codex_home).await;
|
||||
let model_info = codex_core::test_support::construct_model_info_offline("mock-model", &config);
|
||||
let mut model_info =
|
||||
codex_core::test_support::construct_model_info_offline("mock-model", &config);
|
||||
model_info.input_modalities.push(InputModality::Audio);
|
||||
write_models_cache_with_models(codex_home.path(), vec![model_info])?;
|
||||
|
||||
let mut mcp = TestAppServer::builder()
|
||||
@@ -707,8 +710,6 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
|
||||
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
|
||||
},
|
||||
];
|
||||
// Audio remains available on the app-server item, but the base truncation policy intentionally
|
||||
// omits it from the model-visible payload until the history/accounting layer.
|
||||
let model_content_items = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "dynamic-ok".to_string(),
|
||||
@@ -717,6 +718,9 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
|
||||
image_url: TINY_PNG_DATA_URL.to_string(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
|
||||
},
|
||||
];
|
||||
let response = DynamicToolCallResponse {
|
||||
content_items: response_content_items,
|
||||
@@ -776,6 +780,10 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
|
||||
"type": "input_image",
|
||||
"image_url": TINY_PNG_DATA_URL,
|
||||
"detail": "high"
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"audio_url": INLINE_AUDIO_DATA_URL
|
||||
}
|
||||
])
|
||||
);
|
||||
|
||||
@@ -826,6 +826,24 @@ async fn thread_read_loaded_include_turns_reads_store_history_without_rollout_pa
|
||||
let ThreadReadResponse { thread, .. } = serde_json::from_value(result)?;
|
||||
|
||||
assert_eq!(turn_user_texts(&thread.turns), vec!["history from store"]);
|
||||
let [ThreadItem::UserMessage { content, .. }] = thread.turns[0].items.as_slice() else {
|
||||
panic!("expected one user message item");
|
||||
};
|
||||
assert_eq!(
|
||||
content,
|
||||
&vec![
|
||||
UserInput::Text {
|
||||
text: "history from store".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "https://example.com/recording.mp3".to_string(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: "recording.wav".into(),
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
client.shutdown().await?;
|
||||
Ok(())
|
||||
@@ -2276,6 +2294,8 @@ fn store_history_items() -> Vec<RolloutItem> {
|
||||
message: "history from store".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
audio: Some(vec!["https://example.com/recording.mp3".to_string()]),
|
||||
local_audio: vec!["recording.wav".into()],
|
||||
text_elements: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -104,6 +104,7 @@ serde_json = { workspace = true }
|
||||
sha1 = { workspace = true }
|
||||
shlex = { workspace = true }
|
||||
similar = { workspace = true }
|
||||
symphonia = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = [
|
||||
|
||||
@@ -3,6 +3,17 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_utils_cache::BlockingLruCache;
|
||||
use codex_utils_cache::sha1_digest;
|
||||
use codex_utils_output_truncation::approx_token_count;
|
||||
use std::io::Cursor;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::LazyLock;
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::formats::TrackType;
|
||||
use symphonia::core::formats::probe::Hint;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use tracing::warn;
|
||||
|
||||
const AUDIO_PROCESSING_ERROR_PLACEHOLDER: &str =
|
||||
@@ -17,6 +28,15 @@ const UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER: &str =
|
||||
/// This matches the Responses API audio input limit.
|
||||
const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024;
|
||||
const MAX_PROMPT_AUDIO_BASE64_BYTES: usize = MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4;
|
||||
const AUDIO_TOKEN_ESTIMATE_CACHE_SIZE: usize = 32;
|
||||
const AUDIO_TOKENS_PER_SECOND: f64 = 10.0;
|
||||
|
||||
static AUDIO_TOKEN_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], usize>> =
|
||||
LazyLock::new(|| {
|
||||
BlockingLruCache::new(
|
||||
NonZeroUsize::new(AUDIO_TOKEN_ESTIMATE_CACHE_SIZE).unwrap_or(NonZeroUsize::MIN),
|
||||
)
|
||||
});
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum AudioPreparationError {
|
||||
@@ -121,6 +141,65 @@ fn canonical_audio_mime(mime: &str) -> Option<&'static str> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn estimate_audio_token_count(audio_url: &str) -> usize {
|
||||
let key = sha1_digest(audio_url.as_bytes());
|
||||
AUDIO_TOKEN_ESTIMATE_CACHE.get_or_insert_with(key, || {
|
||||
let Some(duration_seconds) = audio_duration_seconds(audio_url) else {
|
||||
return approx_token_count(audio_url);
|
||||
};
|
||||
let token_count = (duration_seconds * AUDIO_TOKENS_PER_SECOND).ceil();
|
||||
if token_count >= usize::MAX as f64 {
|
||||
usize::MAX
|
||||
} else {
|
||||
token_count as usize
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn audio_duration_seconds(audio_url: &str) -> Option<f64> {
|
||||
let (metadata, payload) = audio_url.split_once(',')?;
|
||||
let metadata = metadata.get("data:".len()..)?;
|
||||
let mut metadata_parts = metadata.split(';');
|
||||
let canonical_mime = canonical_audio_mime(metadata_parts.next()?)?;
|
||||
if !metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let bytes = match BASE64_STANDARD.decode(payload) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
tracing::trace!(%error, "failed to decode audio payload for token estimation");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let media_source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
|
||||
let mut hint = Hint::new();
|
||||
hint.mime_type(canonical_mime);
|
||||
let format = match symphonia::default::get_probe().probe(
|
||||
&hint,
|
||||
media_source,
|
||||
FormatOptions::default(),
|
||||
MetadataOptions::default(),
|
||||
) {
|
||||
Ok(format) => format,
|
||||
Err(error) => {
|
||||
tracing::trace!(%error, "failed to read audio duration for token estimation");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let track = format.default_track(TrackType::Audio)?;
|
||||
let timing = track.time_base.zip(track.duration).or_else(|| {
|
||||
format
|
||||
.media_info()
|
||||
.time_base
|
||||
.zip(format.media_info().duration)
|
||||
});
|
||||
let (time_base, duration) = timing?;
|
||||
let duration_seconds =
|
||||
duration.get() as f64 * f64::from(time_base.numer.get()) / f64::from(time_base.denom.get());
|
||||
duration_seconds.is_finite().then_some(duration_seconds)
|
||||
}
|
||||
|
||||
fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> {
|
||||
if !is_data_url(audio_url) {
|
||||
return Err(AudioPreparationError::InvalidDataUrl {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::audio_preparation::estimate_audio_token_count;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context::world_state::WorldState;
|
||||
use crate::context::world_state::WorldStateSnapshot;
|
||||
@@ -437,7 +438,7 @@ pub(crate) fn truncate_function_output_payload(
|
||||
FunctionCallOutputBody::Text(truncate_text(content, policy))
|
||||
}
|
||||
FunctionCallOutputBody::ContentItems(items) => FunctionCallOutputBody::ContentItems(
|
||||
truncate_function_output_items_with_policy(items, policy),
|
||||
truncate_function_output_items_with_policy(items, policy, estimate_audio_token_count),
|
||||
),
|
||||
};
|
||||
|
||||
@@ -531,14 +532,18 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
|
||||
.unwrap_or_default();
|
||||
let (image_payload_bytes, image_replacement_bytes) =
|
||||
image_data_url_estimate_adjustment(item);
|
||||
let (audio_payload_bytes, audio_replacement_bytes) =
|
||||
audio_data_url_estimate_adjustment(item);
|
||||
let (encrypted_payload_bytes, encrypted_replacement_bytes) =
|
||||
encrypted_function_output_estimate_adjustment(item);
|
||||
// Replace raw base64 payload bytes with a per-image estimate.
|
||||
// Replace raw base64 payload bytes with per-modality estimates.
|
||||
// We intentionally preserve the data URL prefix and JSON
|
||||
// wrapper bytes already included in `raw`.
|
||||
let raw = raw
|
||||
.saturating_sub(image_payload_bytes)
|
||||
.saturating_add(image_replacement_bytes);
|
||||
.saturating_add(image_replacement_bytes)
|
||||
.saturating_sub(audio_payload_bytes)
|
||||
.saturating_add(audio_replacement_bytes);
|
||||
raw.saturating_sub(encrypted_payload_bytes)
|
||||
.saturating_add(encrypted_replacement_bytes)
|
||||
}
|
||||
@@ -551,6 +556,16 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
|
||||
/// We only discount payloads for `data:image/...;base64,...` URLs (case
|
||||
/// insensitive markers) and leave everything else at raw serialized size.
|
||||
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
|
||||
parse_base64_data_url(url, "image/")
|
||||
}
|
||||
|
||||
/// Returns the base64 payload for inline audio data URLs that are eligible for
|
||||
/// token-estimation discounting.
|
||||
fn parse_base64_audio_data_url(url: &str) -> Option<&str> {
|
||||
parse_base64_data_url(url, "audio/")
|
||||
}
|
||||
|
||||
fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> {
|
||||
if !url
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
@@ -561,15 +576,15 @@ fn parse_base64_image_data_url(url: &str) -> Option<&str> {
|
||||
let metadata = &url[..comma_index];
|
||||
let payload = &url[comma_index + 1..];
|
||||
// Parse the media type and parameters without decoding. This keeps the
|
||||
// estimator cheap while ensuring we only apply the fixed-cost image
|
||||
// heuristic to image-typed base64 data URLs.
|
||||
// estimator cheap while ensuring we only apply modality heuristics to
|
||||
// appropriately typed base64 data URLs.
|
||||
let metadata_without_scheme = &metadata["data:".len()..];
|
||||
let mut metadata_parts = metadata_without_scheme.split(';');
|
||||
let mime_type = metadata_parts.next().unwrap_or_default();
|
||||
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
|
||||
if !mime_type
|
||||
.get(.."image/".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
|
||||
.get(..media_type_prefix.len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -662,6 +677,50 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
(payload_bytes, replacement_bytes)
|
||||
}
|
||||
|
||||
/// Scans one response item for inline base64 audio data URLs and returns:
|
||||
/// - total base64 payload bytes to subtract from raw serialized size
|
||||
/// - total replacement byte estimate for those audio inputs
|
||||
fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
let mut payload_bytes = 0i64;
|
||||
let mut replacement_bytes = 0i64;
|
||||
|
||||
let mut accumulate = |audio_url: &str| {
|
||||
if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) {
|
||||
payload_bytes =
|
||||
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
|
||||
replacement_bytes = replacement_bytes.saturating_add(
|
||||
i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
|
||||
audio_url,
|
||||
)))
|
||||
.unwrap_or(i64::MAX),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => {
|
||||
for content_item in content {
|
||||
if let ContentItem::InputAudio { audio_url } = content_item {
|
||||
accumulate(audio_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
|
||||
for content_item in items {
|
||||
if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item {
|
||||
accumulate(audio_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
(payload_bytes, replacement_bytes)
|
||||
}
|
||||
|
||||
fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
let ResponseItem::FunctionCallOutput { output, .. } = item else {
|
||||
return (0, 0);
|
||||
|
||||
@@ -40,6 +40,31 @@ use regex_lite::Regex;
|
||||
|
||||
const EXEC_FORMAT_MAX_BYTES: usize = 10_000;
|
||||
const EXEC_FORMAT_MAX_TOKENS: usize = 2_500;
|
||||
const TEST_WAV_SAMPLE_RATE: u32 = 8_000;
|
||||
|
||||
fn pcm_wav_data_url(sample_count: u32) -> (String, usize) {
|
||||
let padding = sample_count % 2;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(b"RIFF");
|
||||
bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes());
|
||||
bytes.extend_from_slice(b"WAVEfmt ");
|
||||
bytes.extend_from_slice(&16u32.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&8u16.to_le_bytes());
|
||||
bytes.extend_from_slice(b"data");
|
||||
bytes.extend_from_slice(&sample_count.to_le_bytes());
|
||||
bytes.resize(
|
||||
bytes.len() + sample_count as usize + padding as usize,
|
||||
/*value*/ 0,
|
||||
);
|
||||
let payload = BASE64_STANDARD.encode(bytes);
|
||||
let payload_len = payload.len();
|
||||
(format!("data:audio/wav;base64,{payload}"), payload_len)
|
||||
}
|
||||
|
||||
fn assistant_msg(text: &str) -> ResponseItem {
|
||||
ResponseItem::Message {
|
||||
@@ -513,6 +538,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
image_url: "https://example.com/result.png".to_string(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
@@ -537,10 +565,14 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
image_url: "https://example.com/js-repl-result.png".to_string(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
];
|
||||
let fully_supported_items = items.clone();
|
||||
let history = create_history_with_items(items);
|
||||
let text_only_modalities = vec![InputModality::Text];
|
||||
let stripped = history.for_prompt(&text_only_modalities);
|
||||
@@ -587,6 +619,10 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
text: "image content omitted because you do not support image input"
|
||||
.to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "audio content omitted because you do not support audio input"
|
||||
.to_string(),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
@@ -611,11 +647,23 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
text: "image content omitted because you do not support image input"
|
||||
.to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "audio content omitted because you do not support audio input"
|
||||
.to_string(),
|
||||
},
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
},
|
||||
];
|
||||
assert_eq!(stripped, expected);
|
||||
assert_eq!(
|
||||
create_history_with_items(fully_supported_items.clone()).for_prompt(&[
|
||||
InputModality::Text,
|
||||
InputModality::Image,
|
||||
InputModality::Audio,
|
||||
]),
|
||||
fully_supported_items
|
||||
);
|
||||
|
||||
// With image support, images are preserved
|
||||
let modalities = default_input_modalities();
|
||||
@@ -1949,6 +1997,121 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
|
||||
assert!(estimated < raw_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_data_url_payload_does_not_dominate_message_estimate() {
|
||||
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801);
|
||||
let item = ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio { audio_url }],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
|
||||
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800);
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
id: None,
|
||||
call_id: "call-audio".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url },
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
|
||||
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000);
|
||||
let item = ResponseItem::CustomToolCallOutput {
|
||||
id: None,
|
||||
call_id: "call-custom-audio".to_string(),
|
||||
name: None,
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url },
|
||||
]),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() {
|
||||
let payload = "A".repeat(/*n*/ 100_000);
|
||||
let audio_url = format!("data:audio/wav;base64,{payload}");
|
||||
let fallback_bytes = approx_bytes_for_tokens(approx_token_count(&audio_url)) as i64;
|
||||
let item = ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio { audio_url }],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
|
||||
assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_items_omits_audio_that_exceeds_the_output_budget() {
|
||||
let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000);
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
id: None,
|
||||
call_id: "call-audio".to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url },
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
let mut history = ContextManager::new();
|
||||
|
||||
history.record_items([&item], TruncationPolicy::Tokens(50));
|
||||
|
||||
assert_eq!(
|
||||
history.items,
|
||||
vec![ResponseItem::FunctionCallOutput {
|
||||
id: None,
|
||||
call_id: "call-audio".to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 audio items ...]".to_string(),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_base64_image_urls_are_unchanged() {
|
||||
let message_item = ResponseItem::Message {
|
||||
|
||||
@@ -369,7 +369,7 @@ pub(crate) fn strip_images_when_unsupported(
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip audio content from messages when the model does not support audio.
|
||||
/// Strip audio content from messages and tool outputs when the model does not support audio.
|
||||
/// When `input_modalities` contains `InputModality::Audio`, no stripping is performed.
|
||||
pub(crate) fn strip_audio_when_unsupported(
|
||||
input_modalities: &[InputModality],
|
||||
@@ -380,14 +380,32 @@ pub(crate) fn strip_audio_when_unsupported(
|
||||
}
|
||||
|
||||
for item in items.iter_mut() {
|
||||
if let ResponseItem::Message { content, .. } = item {
|
||||
for content_item in content.iter_mut() {
|
||||
if matches!(content_item, ContentItem::InputAudio { .. }) {
|
||||
*content_item = ContentItem::InputText {
|
||||
text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
|
||||
};
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => {
|
||||
for content_item in content.iter_mut() {
|
||||
if matches!(content_item, ContentItem::InputAudio { .. }) {
|
||||
*content_item = ContentItem::InputText {
|
||||
text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
if let Some(content_items) = output.content_items_mut() {
|
||||
for content_item in content_items.iter_mut() {
|
||||
if matches!(
|
||||
content_item,
|
||||
FunctionCallOutputContentItem::InputAudio { .. }
|
||||
) {
|
||||
*content_item = FunctionCallOutputContentItem::InputText {
|
||||
text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ use serde_json::Value as JsonValue;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::audio_preparation::estimate_audio_token_count;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::original_image_detail::can_request_original_image_detail;
|
||||
use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items;
|
||||
@@ -269,7 +270,7 @@ fn truncate_code_mode_result(
|
||||
return truncated_items;
|
||||
}
|
||||
|
||||
truncate_function_output_items_with_policy(&items, policy)
|
||||
truncate_function_output_items_with_policy(&items, policy, estimate_audio_token_count)
|
||||
}
|
||||
|
||||
async fn call_nested_tool(
|
||||
@@ -427,6 +428,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn over_budget_audio_output_is_omitted() {
|
||||
let items = vec![FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: format!("data:audio/wav;base64,{}", "A".repeat(100)),
|
||||
}];
|
||||
|
||||
assert_eq!(
|
||||
truncate_code_mode_result(items, Some(5)),
|
||||
vec![FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 audio items ...]".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_process_host_falls_back_to_in_process_session() {
|
||||
let service = CodeModeService::new(Arc::new(
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
use anyhow::Result;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
|
||||
use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;
|
||||
use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
|
||||
use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
|
||||
use codex_protocol::dynamic_tools::DynamicToolResponse;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::openai_models::TruncationPolicyConfig;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
const TEST_WAV_SAMPLE_RATE: u32 = 8_000;
|
||||
const OMITTED_AUDIO_MARKER: &str = "[omitted 1 audio items ...]";
|
||||
|
||||
fn pcm_wav_data_url(sample_count: u32) -> String {
|
||||
let padding = sample_count % 2;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(b"RIFF");
|
||||
bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes());
|
||||
bytes.extend_from_slice(b"WAVEfmt ");
|
||||
bytes.extend_from_slice(&16u32.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&8u16.to_le_bytes());
|
||||
bytes.extend_from_slice(b"data");
|
||||
bytes.extend_from_slice(&sample_count.to_le_bytes());
|
||||
bytes.resize(
|
||||
bytes.len() + sample_count as usize + padding as usize,
|
||||
/*value*/ 0,
|
||||
);
|
||||
format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(bytes))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dynamic_tool_audio_exceeding_the_output_budget_is_omitted() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let call_id = "audio-call";
|
||||
let tool_name = "recording";
|
||||
let responses_mock = responses::mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(call_id, "codex_app", tool_name, "{}"),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
sse(vec![
|
||||
responses::ev_response_created("resp-2"),
|
||||
responses::ev_assistant_message("msg-1", "done"),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut builder = test_codex().with_model_info_override("gpt-5.5", |model_info| {
|
||||
model_info.input_modalities.push(InputModality::Audio);
|
||||
model_info.truncation_policy = TruncationPolicyConfig::tokens(/*limit*/ 50);
|
||||
});
|
||||
let base_test = builder.build_with_auto_env(&server).await?;
|
||||
let dynamic_tool = DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
||||
name: "codex_app".to_string(),
|
||||
description: "Audio tools.".to_string(),
|
||||
tools: vec![DynamicToolNamespaceTool::Function(
|
||||
DynamicToolFunctionSpec {
|
||||
name: tool_name.to_string(),
|
||||
description: "Returns a recording.".to_string(),
|
||||
input_schema: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": false,
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
)],
|
||||
});
|
||||
let new_thread = base_test
|
||||
.thread_manager
|
||||
.start_thread_with_tools(base_test.config.clone(), vec![dynamic_tool])
|
||||
.await?;
|
||||
let mut test = base_test;
|
||||
test.codex = new_thread.thread;
|
||||
test.session_configured = new_thread.session_configured;
|
||||
|
||||
test.codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "Return a recording".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
let EventMsg::DynamicToolCallRequest(request) = wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::DynamicToolCallRequest(_))
|
||||
})
|
||||
.await
|
||||
else {
|
||||
unreachable!("event guard guarantees DynamicToolCallRequest");
|
||||
};
|
||||
test.codex
|
||||
.submit(Op::DynamicToolResponse {
|
||||
id: request.call_id,
|
||||
response: DynamicToolResponse {
|
||||
content_items: vec![DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url: pcm_wav_data_url(/*sample_count*/ 80_000),
|
||||
}],
|
||||
success: true,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let requests = responses_mock.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
let output = requests[1]
|
||||
.function_call_output(call_id)
|
||||
.get("output")
|
||||
.cloned()
|
||||
.expect("follow-up request should contain the dynamic tool output");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<FunctionCallOutputPayload>(output)?,
|
||||
FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: OMITTED_AUDIO_MARKER.to_string(),
|
||||
},
|
||||
]),
|
||||
success: None,
|
||||
}
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -2,6 +2,8 @@ use core_test_support::test_codex::local_selections;
|
||||
use std::fs;
|
||||
|
||||
use anyhow::Result;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_core::compact::SUMMARY_PREFIX;
|
||||
use codex_features::Feature;
|
||||
use codex_login::CodexAuth;
|
||||
@@ -9,13 +11,19 @@ use codex_login::auth::AgentIdentityAuth;
|
||||
use codex_login::auth::AgentIdentityAuthRecord;
|
||||
use codex_protocol::account::PlanType as AccountPlanType;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
|
||||
use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;
|
||||
use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
|
||||
use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
|
||||
use codex_protocol::dynamic_tools::DynamicToolResponse;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::ConversationStartParams;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
@@ -52,6 +60,29 @@ use wiremock::ResponseTemplate;
|
||||
|
||||
const CONTEXT_WINDOW_TRUNCATED_OUTPUT_MESSAGE: &str =
|
||||
"Output exceeded the available model context and was truncated";
|
||||
const TEST_WAV_SAMPLE_RATE: u32 = 8_000;
|
||||
|
||||
fn pcm_wav_data_url(sample_count: u32) -> String {
|
||||
let padding = sample_count % 2;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(b"RIFF");
|
||||
bytes.extend_from_slice(&(36 + sample_count + padding).to_le_bytes());
|
||||
bytes.extend_from_slice(b"WAVEfmt ");
|
||||
bytes.extend_from_slice(&16u32.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&TEST_WAV_SAMPLE_RATE.to_le_bytes());
|
||||
bytes.extend_from_slice(&1u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&8u16.to_le_bytes());
|
||||
bytes.extend_from_slice(b"data");
|
||||
bytes.extend_from_slice(&sample_count.to_le_bytes());
|
||||
bytes.resize(
|
||||
bytes.len() + sample_count as usize + padding as usize,
|
||||
/*value*/ 0,
|
||||
);
|
||||
format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(bytes))
|
||||
}
|
||||
|
||||
fn approx_token_count(text: &str) -> i64 {
|
||||
i64::try_from(text.len().saturating_add(3) / 4).unwrap_or(i64::MAX)
|
||||
@@ -1315,6 +1346,123 @@ async fn remote_compact_filters_deferred_dynamic_tools() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn remote_compact_does_not_charge_inline_audio_payload_as_text() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let call_id = "audio-call";
|
||||
let tool_name = "recording";
|
||||
let audio_url = pcm_wav_data_url(/*sample_count*/ 300_000);
|
||||
let responses_mock = responses::mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(call_id, "codex_app", tool_name, "{}"),
|
||||
responses::ev_completed_with_tokens("resp-1", /*total_tokens*/ 100),
|
||||
]),
|
||||
sse(vec![
|
||||
responses::ev_response_created("resp-2"),
|
||||
responses::ev_assistant_message("msg-1", "done"),
|
||||
responses::ev_completed_with_tokens("resp-2", /*total_tokens*/ 200),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let compact_mock = responses::mount_compact_json_once(
|
||||
&server,
|
||||
json!({ "output": compacted_summary_only_output("compact summary") }),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut builder = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_model_info_override("gpt-5.5", |model_info| {
|
||||
model_info.input_modalities.push(InputModality::Audio);
|
||||
})
|
||||
.with_config(|config| {
|
||||
config.model_context_window = Some(50_000);
|
||||
});
|
||||
let mut test = builder.build(&server).await?;
|
||||
let dynamic_tool = DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
||||
name: "codex_app".to_string(),
|
||||
description: "Audio tools.".to_string(),
|
||||
tools: vec![DynamicToolNamespaceTool::Function(
|
||||
DynamicToolFunctionSpec {
|
||||
name: tool_name.to_string(),
|
||||
description: "Returns a recording.".to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": false,
|
||||
}),
|
||||
defer_loading: false,
|
||||
},
|
||||
)],
|
||||
});
|
||||
let new_thread = test
|
||||
.thread_manager
|
||||
.start_thread_with_tools(test.config.clone(), vec![dynamic_tool])
|
||||
.await?;
|
||||
test.codex = new_thread.thread;
|
||||
test.session_configured = new_thread.session_configured;
|
||||
|
||||
test.codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "Return a recording".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
let EventMsg::DynamicToolCallRequest(request) = wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::DynamicToolCallRequest(_))
|
||||
})
|
||||
.await
|
||||
else {
|
||||
unreachable!("event guard guarantees DynamicToolCallRequest");
|
||||
};
|
||||
test.codex
|
||||
.submit(Op::DynamicToolResponse {
|
||||
id: request.call_id,
|
||||
response: DynamicToolResponse {
|
||||
content_items: vec![DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url: audio_url.clone(),
|
||||
}],
|
||||
success: true,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
wait_for_turn_complete(&test.codex).await;
|
||||
|
||||
test.codex.submit(Op::Compact).await?;
|
||||
wait_for_turn_complete(&test.codex).await;
|
||||
|
||||
assert_eq!(responses_mock.requests().len(), 2);
|
||||
let output = compact_mock
|
||||
.single_request()
|
||||
.function_call_output(call_id)
|
||||
.get("output")
|
||||
.cloned()
|
||||
.expect("compact request should retain the dynamic tool output");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<FunctionCallOutputPayload>(output)?,
|
||||
FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url },
|
||||
]),
|
||||
success: None,
|
||||
}
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn remote_compact_runs_automatically() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
@@ -37,6 +37,7 @@ mod agents_md;
|
||||
mod apply_patch_cli;
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
mod approvals;
|
||||
mod audio_truncation;
|
||||
mod auto_review;
|
||||
mod catalog_permission_messages;
|
||||
mod cli_stream;
|
||||
|
||||
@@ -85,6 +85,8 @@ impl UserMessageItem {
|
||||
image_details: self.image_details(),
|
||||
local_images: self.local_image_paths(),
|
||||
local_image_details: self.local_image_details(),
|
||||
audio: Some(self.audio_urls()),
|
||||
local_audio: self.local_audio_paths(),
|
||||
text_elements: self.text_elements(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2345,6 +2345,15 @@ pub struct UserMessageEvent {
|
||||
/// imply default image detail behavior.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub local_image_details: Vec<Option<ImageDetail>>,
|
||||
/// Audio URLs sourced from `UserInput::Audio`. These are safe to replay in
|
||||
/// legacy UI history events and correspond to audio sent to the model.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio: Option<Vec<String>>,
|
||||
/// Local file paths sourced from `UserInput::LocalAudio`. These are kept so
|
||||
/// clients can reattach audio when editing history and should not be
|
||||
/// treated as API-ready URLs.
|
||||
#[serde(default)]
|
||||
pub local_audio: Vec<std::path::PathBuf>,
|
||||
/// UI-defined spans within `message` used to render or persist special elements.
|
||||
#[serde(default)]
|
||||
pub text_elements: Vec<crate::user_input::TextElement>,
|
||||
@@ -2364,6 +2373,9 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
|
||||
{
|
||||
return Some("[Image]".to_string());
|
||||
}
|
||||
if user.audio.as_ref().is_some_and(|audio| !audio.is_empty()) || !user.local_audio.is_empty() {
|
||||
return Some("[Audio]".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -5771,6 +5783,7 @@ mod tests {
|
||||
json!({
|
||||
"message": "hello",
|
||||
"local_images": [],
|
||||
"local_audio": [],
|
||||
"text_elements": [],
|
||||
})
|
||||
);
|
||||
@@ -5795,14 +5808,17 @@ mod tests {
|
||||
assert_eq!(event.image_details, Vec::<Option<ImageDetail>>::new());
|
||||
assert_eq!(event.local_images, vec![PathBuf::from("/tmp/local.png")]);
|
||||
assert_eq!(event.local_image_details, Vec::<Option<ImageDetail>>::new());
|
||||
assert_eq!(event.audio, None);
|
||||
assert_eq!(event.local_audio, Vec::<PathBuf>::new());
|
||||
assert_eq!(event.text_elements, Vec::new());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_message_item_legacy_event_preserves_image_details() {
|
||||
fn user_message_item_legacy_event_preserves_attachments() {
|
||||
let local_path = PathBuf::from("/tmp/local.png");
|
||||
let local_audio_path = PathBuf::from("/tmp/local.wav");
|
||||
let mut item = UserMessageItem::new(&[
|
||||
crate::user_input::UserInput::Image {
|
||||
image_url: "https://example.com/first.png".to_string(),
|
||||
@@ -5816,6 +5832,12 @@ mod tests {
|
||||
path: local_path.clone(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
crate::user_input::UserInput::Audio {
|
||||
audio_url: "https://example.com/remote.mp3".to_string(),
|
||||
},
|
||||
crate::user_input::UserInput::LocalAudio {
|
||||
path: local_audio_path.clone(),
|
||||
},
|
||||
]);
|
||||
item.client_id = Some("client-message-1".to_string());
|
||||
|
||||
@@ -5834,6 +5856,21 @@ mod tests {
|
||||
assert_eq!(event.image_details, vec![Some(ImageDetail::Original)]);
|
||||
assert_eq!(event.local_images, vec![local_path]);
|
||||
assert_eq!(event.local_image_details, vec![Some(ImageDetail::Original)]);
|
||||
assert_eq!(
|
||||
event.audio,
|
||||
Some(vec!["https://example.com/remote.mp3".to_string()])
|
||||
);
|
||||
assert_eq!(event.local_audio, vec![local_audio_path]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_only_user_message_has_placeholder_preview() {
|
||||
let event = UserMessageEvent {
|
||||
audio: Some(vec!["https://example.com/remote.mp3".to_string()]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(user_message_preview(&event), Some("[Audio]".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -287,6 +287,27 @@ mod tests {
|
||||
assert_eq!(metadata.title, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_msg_audio_only_user_message_sets_audio_placeholder_preview() {
|
||||
let mut metadata = metadata_for_test();
|
||||
let item = RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
|
||||
client_id: None,
|
||||
message: String::new(),
|
||||
images: None,
|
||||
local_images: vec![],
|
||||
audio: Some(vec!["https://example.com/audio.mp3".to_string()]),
|
||||
local_audio: vec![],
|
||||
text_elements: vec![],
|
||||
..Default::default()
|
||||
}));
|
||||
|
||||
apply_rollout_item(&mut metadata, &item, "test-provider");
|
||||
|
||||
assert_eq!(metadata.first_user_message.as_deref(), Some("[Audio]"));
|
||||
assert_eq!(metadata.preview.as_deref(), Some("[Audio]"));
|
||||
assert_eq!(metadata.title, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_msg_blank_user_message_without_images_keeps_first_user_message_empty() {
|
||||
let mut metadata = metadata_for_test();
|
||||
|
||||
@@ -42,16 +42,9 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
| FunctionCallOutputContentItem::EncryptedContent { .. } => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let without_audio = || {
|
||||
items
|
||||
.iter()
|
||||
.filter(|item| !matches!(item, FunctionCallOutputContentItem::InputAudio { .. }))
|
||||
.cloned()
|
||||
.collect()
|
||||
};
|
||||
|
||||
if text_segments.is_empty() {
|
||||
return (without_audio(), None);
|
||||
return (items.to_vec(), None);
|
||||
}
|
||||
|
||||
let mut combined = String::new();
|
||||
@@ -63,7 +56,7 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
}
|
||||
|
||||
if combined.len() <= policy.byte_budget() {
|
||||
return (without_audio(), None);
|
||||
return (items.to_vec(), None);
|
||||
}
|
||||
|
||||
let original_token_count = approx_token_count(&combined);
|
||||
@@ -77,7 +70,11 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
detail: *detail,
|
||||
})
|
||||
}
|
||||
FunctionCallOutputContentItem::InputAudio { .. } => None,
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url } => {
|
||||
Some(FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: audio_url.clone(),
|
||||
})
|
||||
}
|
||||
FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {
|
||||
Some(FunctionCallOutputContentItem::EncryptedContent {
|
||||
encrypted_content: encrypted_content.clone(),
|
||||
@@ -92,6 +89,7 @@ pub fn formatted_truncate_text_content_items_with_policy(
|
||||
pub fn truncate_function_output_items_with_policy(
|
||||
items: &[FunctionCallOutputContentItem],
|
||||
policy: TruncationPolicy,
|
||||
estimate_audio_token_count: impl Fn(&str) -> usize,
|
||||
) -> Vec<FunctionCallOutputContentItem> {
|
||||
let mut out: Vec<FunctionCallOutputContentItem> = Vec::with_capacity(items.len());
|
||||
let mut remaining_budget = match policy {
|
||||
@@ -99,6 +97,7 @@ pub fn truncate_function_output_items_with_policy(
|
||||
TruncationPolicy::Tokens(_) => policy.token_budget(),
|
||||
};
|
||||
let mut omitted_text_items = 0usize;
|
||||
let mut omitted_audio_items = 0usize;
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
@@ -136,7 +135,21 @@ pub fn truncate_function_output_items_with_policy(
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
FunctionCallOutputContentItem::InputAudio { .. } => {}
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url } => {
|
||||
let token_cost = estimate_audio_token_count(audio_url);
|
||||
let cost = match policy {
|
||||
TruncationPolicy::Bytes(_) => approx_bytes_for_tokens(token_cost),
|
||||
TruncationPolicy::Tokens(_) => token_cost,
|
||||
};
|
||||
if cost <= remaining_budget {
|
||||
out.push(FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: audio_url.clone(),
|
||||
});
|
||||
remaining_budget = remaining_budget.saturating_sub(cost);
|
||||
} else {
|
||||
omitted_audio_items += 1;
|
||||
}
|
||||
}
|
||||
FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {
|
||||
out.push(FunctionCallOutputContentItem::EncryptedContent {
|
||||
encrypted_content: encrypted_content.clone(),
|
||||
@@ -150,6 +163,11 @@ pub fn truncate_function_output_items_with_policy(
|
||||
text: format!("[omitted {omitted_text_items} text items ...]"),
|
||||
});
|
||||
}
|
||||
if omitted_audio_items > 0 {
|
||||
out.push(FunctionCallOutputContentItem::InputText {
|
||||
text: format!("[omitted {omitted_audio_items} audio items ...]"),
|
||||
});
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() {
|
||||
];
|
||||
|
||||
let output =
|
||||
truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit));
|
||||
truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit), |_| 0);
|
||||
|
||||
assert_eq!(output.len(), 5);
|
||||
|
||||
@@ -208,7 +208,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_empty_leading_tex
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatted_truncate_text_content_items_with_policy_merges_text_and_drops_audio() {
|
||||
fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_media() {
|
||||
let items = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "abcd".to_string(),
|
||||
@@ -245,6 +245,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_drops_audio
|
||||
image_url: "img:one".to_string(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "audio:one".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "img:two".to_string(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
@@ -283,7 +286,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_encrypted_content
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_drops_audio() {
|
||||
fn truncate_function_output_items_with_policy_omits_audio_over_budget() {
|
||||
let items = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "abcdefgh".to_string(),
|
||||
@@ -296,7 +299,8 @@ fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_dr
|
||||
},
|
||||
];
|
||||
|
||||
let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2));
|
||||
let output =
|
||||
truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2), |_| 1);
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
@@ -307,6 +311,35 @@ fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_dr
|
||||
FunctionCallOutputContentItem::EncryptedContent {
|
||||
encrypted_content: "enc_opaque".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "[omitted 1 audio items ...]".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_function_output_items_with_policy_charges_audio_against_byte_budget() {
|
||||
let audio = FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "audio:one".to_string(),
|
||||
};
|
||||
let items = vec![
|
||||
audio.clone(),
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "abcdefgh".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let output =
|
||||
truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(5), |_| 1);
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
audio,
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: truncate_text("abcdefgh", TruncationPolicy::Bytes(1)),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user