From db9778b73cf0291da3c025709cc55c4a6eb97c30 Mon Sep 17 00:00:00 2001 From: Sean Huang Date: Tue, 17 Nov 2026 06:11:48 +0000 Subject: [PATCH] Distinguish image and tool audio attachment uploads (#62047) ## What changed Introduce `UploadContent::Image` and `UploadContent::ToolAudio` in `UploadRequest`. Keep image filenames and policy tags on the image variant, and carry tool audio's declared MIME type explicitly instead of encoding its type and MIME metadata in tags. Update image preparation, image generation, and tool audio callers to use the typed content. Preserve inline bytes for both variants and redact their payloads from debug output. Document that persistent backends must verify the declared audio MIME type against the uploaded bytes. ## Testing Add coverage for inline tool audio byte preservation and audio debug redaction. Update integration assertions for typed image and audio uploads, and verify that Code Mode does not upload audio when `FileBackedToolAudio` is disabled. GitOrigin-RevId: 7dbd69922b377aead2dc28d1e5b3938cce6343b7 --- codex-rs/attachment-store/src/lib.rs | 62 +++++++++++++++---- codex-rs/attachment-store/src/lib_tests.rs | 51 ++++++++++++--- codex-rs/core/src/image_preparation.rs | 9 ++- codex-rs/core/src/session/file_audio.rs | 13 ++-- .../core/tests/suite/audio_duration_store.rs | 11 ++-- codex-rs/core/tests/suite/code_mode.rs | 8 +++ .../core/tests/suite/extension_sandbox.rs | 25 ++++++-- codex-rs/core/tests/suite/image_rollout.rs | 16 +++-- codex-rs/core/tests/suite/scenarios.rs | 20 +++++- .../tests/suite/scenarios_audio_file_tests.rs | 19 ++++-- codex-rs/ext/image-generation/src/tool.rs | 9 ++- 11 files changed, 188 insertions(+), 55 deletions(-) diff --git a/codex-rs/attachment-store/src/lib.rs b/codex-rs/attachment-store/src/lib.rs index 9bc70388e2..56f44d15f2 100644 --- a/codex-rs/attachment-store/src/lib.rs +++ b/codex-rs/attachment-store/src/lib.rs @@ -46,6 +46,49 @@ pub trait AttachmentStore: Send + Sync { } } +/// Typed attachment content supplied to [`AttachmentStore::upload`]. +#[derive(Clone, Eq, PartialEq)] +pub enum UploadContent { + /// An image that may be shown to the model or retained for the UI. + Image { + /// Optional name associated with the image. + file_name: Option, + /// Caller-supplied tags used by storage backends for image policy decisions. + tags: HashMap, + /// Encoded image bytes. + data: Vec, + }, + /// Audio produced by a tool. + ToolAudio { + /// IANA media type a persistent backend must verify against the uploaded bytes. + declared_mime_type: String, + /// Encoded audio bytes. + data: Vec, + }, +} + +impl fmt::Debug for UploadContent { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Image { + file_name, tags, .. + } => formatter + .debug_struct("Image") + .field("file_name", file_name) + .field("tags", tags) + .field("data", &"") + .finish(), + Self::ToolAudio { + declared_mime_type, .. + } => formatter + .debug_struct("ToolAudio") + .field("declared_mime_type", declared_mime_type) + .field("data", &"") + .finish(), + } + } +} + /// Attachment data supplied to [`AttachmentStore::upload`]. #[derive(Clone, Eq, PartialEq)] pub struct UploadRequest { @@ -53,12 +96,8 @@ pub struct UploadRequest { pub thread_id: String, /// Whether the originating thread intentionally skips durable persistence. pub ephemeral: bool, - /// Optional name associated with the attachment. - pub file_name: Option, - /// Caller-supplied tags used by storage backends for policy decisions. - pub tags: HashMap, - /// Attachment bytes to persist. - pub data: Vec, + /// Type-specific content to upload. + pub content: UploadContent, } impl fmt::Debug for UploadRequest { @@ -67,9 +106,7 @@ impl fmt::Debug for UploadRequest { .debug_struct("UploadRequest") .field("thread_id", &self.thread_id) .field("ephemeral", &self.ephemeral) - .field("file_name", &self.file_name) - .field("tags", &self.tags) - .field("data", &"") + .field("content", &self.content) .finish() } } @@ -195,9 +232,10 @@ impl AttachmentStore for InlineAttachmentStore { #[tracing::instrument(level = "trace", skip_all)] fn upload(&self, request: UploadRequest) -> UploadFuture<'_> { Box::pin(async move { - Ok(UploadResult::Inline { - bytes: request.data, - }) + let bytes = match request.content { + UploadContent::Image { data, .. } | UploadContent::ToolAudio { data, .. } => data, + }; + Ok(UploadResult::Inline { bytes }) }) } diff --git a/codex-rs/attachment-store/src/lib_tests.rs b/codex-rs/attachment-store/src/lib_tests.rs index 8e9ef750ed..3faca9bc5e 100644 --- a/codex-rs/attachment-store/src/lib_tests.rs +++ b/codex-rs/attachment-store/src/lib_tests.rs @@ -8,18 +8,29 @@ fn attachment_debug_output_redacts_bytes() { let request = UploadRequest { thread_id: "thread-1".to_string(), ephemeral: false, - file_name: Some("image.png".to_string()), - tags: HashMap::new(), - data: b"secret".to_vec(), + content: UploadContent::Image { + file_name: Some("image.png".to_string()), + tags: HashMap::new(), + data: b"secret".to_vec(), + }, }; let result = UploadResult::Inline { bytes: b"secret".to_vec(), }; + let tool_audio = UploadContent::ToolAudio { + declared_mime_type: "audio/wav".to_string(), + data: b"secret".to_vec(), + }; assert_eq!( - (format!("{request:?}"), format!("{result:?}")), ( - r#"UploadRequest { thread_id: "thread-1", ephemeral: false, file_name: Some("image.png"), tags: {}, data: "" }"#.to_string(), + format!("{request:?}"), + format!("{tool_audio:?}"), + format!("{result:?}"), + ), + ( + r#"UploadRequest { thread_id: "thread-1", ephemeral: false, content: Image { file_name: Some("image.png"), tags: {}, data: "" } }"#.to_string(), + r#"ToolAudio { declared_mime_type: "audio/wav", data: "" }"#.to_string(), r#"Inline { bytes: "" }"#.to_string(), ) ); @@ -52,9 +63,11 @@ async fn inline_store_preserves_image_bytes() { .upload(UploadRequest { thread_id: "thread-1".to_string(), ephemeral: false, - file_name: Some(file_name.to_string()), - tags: HashMap::new(), - data: data.to_vec(), + content: UploadContent::Image { + file_name: Some(file_name.to_string()), + tags: HashMap::new(), + data: data.to_vec(), + }, }) .await .expect("inline attachment"); @@ -66,6 +79,28 @@ async fn inline_store_preserves_image_bytes() { } } +/// The inline store preserves tool audio bytes without interpreting them. +#[tokio::test] +async fn inline_store_preserves_tool_audio_bytes() { + let data = b"RIFF\x04\x00\x00\x00WAVE"; + let attachment = InlineAttachmentStore + .upload(UploadRequest { + thread_id: "thread-1".to_string(), + ephemeral: false, + content: UploadContent::ToolAudio { + declared_mime_type: "audio/wav".to_string(), + data: data.to_vec(), + }, + }) + .await + .expect("inline attachment"); + let UploadResult::Inline { bytes } = attachment else { + panic!("inline store returned a file reference"); + }; + + assert_eq!(bytes, data); +} + /// The inline store preserves opaque file references even though it cannot resolve them. #[tokio::test] async fn inline_store_cannot_resolve_file_references() { diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index a014bc2113..6252316b92 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -10,6 +10,7 @@ use crate::original_image_detail::can_request_original_image_detail; use codex_analytics::ImageDetailSetting; use codex_analytics::ImagePreparationMetadata; use codex_attachment_store::AttachmentStore; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_context_fragments::AnnotatedContent; @@ -449,9 +450,11 @@ async fn prepare_image( let request = UploadRequest { thread_id: origin.thread_id.to_owned(), ephemeral: origin.ephemeral, - file_name: None, - tags: HashMap::new(), - data: prepared.encoded.bytes.to_vec(), + content: UploadContent::Image { + file_name: None, + tags: HashMap::new(), + data: prepared.encoded.bytes.to_vec(), + }, }; let prepared_image = match preparation.image_store.upload(request).await { Ok(UploadResult::Inline { bytes }) => ImageReference::Inline { diff --git a/codex-rs/core/src/session/file_audio.rs b/codex-rs/core/src/session/file_audio.rs index 15b46fcfd2..704f9403dc 100644 --- a/codex-rs/core/src/session/file_audio.rs +++ b/codex-rs/core/src/session/file_audio.rs @@ -11,6 +11,7 @@ use std::time::Duration; use codex_attachment_store::AttachmentStore; use codex_attachment_store::FormatSpecificMetadata; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_history::ResponseItemEnvelope; @@ -184,18 +185,14 @@ async fn store_tool_audio( }; } }; - // These tags come from Core. MCP servers and Code Mode never supply attachment IDs or policy tags. - let tags = HashMap::from([ - ("usage".to_string(), "tool_audio".to_string()), - ("mime_type".to_string(), mime.to_string()), - ]); let result = store .upload(UploadRequest { thread_id: thread_id.to_owned(), ephemeral, - file_name: None, - tags, - data, + content: UploadContent::ToolAudio { + declared_mime_type: mime.to_owned(), + data, + }, }) .await; tool_audio_after_upload(result, mime, inline_url) diff --git a/codex-rs/core/tests/suite/audio_duration_store.rs b/codex-rs/core/tests/suite/audio_duration_store.rs index 1678de6e26..0a4172bfa0 100644 --- a/codex-rs/core/tests/suite/audio_duration_store.rs +++ b/codex-rs/core/tests/suite/audio_duration_store.rs @@ -10,6 +10,7 @@ use codex_attachment_store::ImageMetadata; use codex_attachment_store::InlineAttachmentStore; use codex_attachment_store::ResolveFuture; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadFuture; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; @@ -29,11 +30,11 @@ pub(super) struct TestAudioDurationStore { impl AttachmentStore for TestAudioDurationStore { fn upload(&self, request: UploadRequest) -> UploadFuture<'_> { - if request - .tags - .get("usage") - .is_some_and(|usage| usage == "tool_audio") - { + let is_tool_audio = match &request.content { + UploadContent::Image { .. } => false, + UploadContent::ToolAudio { .. } => true, + }; + if is_tool_audio { self.tool_uploads .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index c2f01448b2..2ea57626f0 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -7,6 +7,7 @@ mod dynamic_tools; #[path = "code_mode_host_recovery_tests.rs"] mod host_recovery; +use super::image_rollout::RecordingFileAttachmentStore; use anyhow::Context; use anyhow::Result; use base64::Engine; @@ -6604,12 +6605,18 @@ text("after"); ); let server = responses::start_mock_server().await; + let store = Arc::new(RecordingFileAttachmentStore::default()); let mut builder = test_codex() + .with_image_store(store.clone()) .with_model_info_override("gpt-5.4", |model| { model.input_modalities.push(InputModality::Audio); }) .with_config(|config| { let _ = config.features.enable(Feature::CodeMode); + config + .features + .disable(Feature::FileBackedToolAudio) + .unwrap(); }); let test = builder.build_with_auto_env(&server).await?; responses::mount_sse_once( @@ -6633,6 +6640,7 @@ text("after"); test.submit_turn("use exec to return short and valid audio with text") .await?; + assert!(store.uploads.lock().unwrap().is_empty()); let req = second_mock.single_request(); let items = custom_tool_output_items(&req, "call-1"); let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); diff --git a/codex-rs/core/tests/suite/extension_sandbox.rs b/codex-rs/core/tests/suite/extension_sandbox.rs index 1d31674af2..f62a4f66ea 100644 --- a/codex-rs/core/tests/suite/extension_sandbox.rs +++ b/codex-rs/core/tests/suite/extension_sandbox.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use anyhow::Context; use anyhow::Result; +use codex_attachment_store::UploadContent; use codex_core::TurnInputRequest; use codex_core::config::Config; use codex_core::config::Constrained; @@ -172,12 +173,28 @@ async fn generated_image_ui_item_is_file_backed_in_rollout() -> Result<()> { test.session_configured.thread_id.to_string() ); assert!(!item_upload.ephemeral); + let UploadContent::Image { + file_name, + tags, + data, + } = &item_upload.content + else { + panic!("expected generated image upload"); + }; + assert!(file_name.is_none()); assert_eq!( - item_upload.tags, - HashMap::from([("usage".to_string(), "image_gen".to_string())]) + tags, + &HashMap::from([("usage".to_string(), "image_gen".to_string())]) ); - assert_eq!(item_upload.data, TINY_PNG_BYTES); - assert_eq!(model_upload.tags, HashMap::new()); + assert_eq!(data, TINY_PNG_BYTES); + let UploadContent::Image { + file_name, tags, .. + } = &model_upload.content + else { + panic!("expected model image upload"); + }; + assert!(file_name.is_none()); + assert_eq!(tags, &HashMap::new()); } let rollout_path = test.codex.rollout_path().context("rollout path")?; diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index dc27b7e0c3..c61355669c 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -8,6 +8,7 @@ use codex_attachment_store::AttachmentStoreErrorKind; use codex_attachment_store::IsResolvableFuture; use codex_attachment_store::ResolveFuture; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadFuture; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; @@ -726,10 +727,17 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R upload.thread_id, resumed.session_configured.thread_id.to_string(), ); - assert_eq!( - image::load_from_memory(&upload.data)?.dimensions(), - (6000, 240) - ); + let UploadContent::Image { + file_name, + tags, + data, + } = &upload.content + else { + panic!("expected image upload"); + }; + assert!(file_name.is_none()); + assert!(tags.is_empty()); + assert_eq!(image::load_from_memory(data)?.dimensions(), (6000, 240)); } let request = resumed_mock.single_request(); diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index 2cebcc55bb..e9ea6837e5 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -9,6 +9,7 @@ mod multi_environment_mcp; use super::image_rollout::RecordingFileAttachmentStore; use super::permissions_messages::PermissionPathScenario; use codex_attachment_store::AttachmentStoreErrorKind; +use codex_attachment_store::UploadContent; use codex_core::RequestCountry; use codex_protocol_test_support::ContentItemTestExt; use std::collections::HashMap; @@ -492,10 +493,23 @@ async fn gif_generation_preserves_animation_and_replays_a_still_preview() -> Res let [animation, preview] = uploads.as_slice() else { panic!("expected an animated artifact and a separate model preview"); }; - assert_eq!(animation.data, BASE64_STANDARD.decode(gif)?); - assert_eq!(image::guess_format(&preview.data)?, image::ImageFormat::Png); + let UploadContent::Image { + data: animation_data, + .. + } = &animation.content + else { + panic!("expected animation image upload"); + }; + let UploadContent::Image { + data: preview_data, .. + } = &preview.content + else { + panic!("expected preview image upload"); + }; + assert_eq!(*animation_data, BASE64_STANDARD.decode(gif)?); + assert_eq!(image::guess_format(preview_data)?, image::ImageFormat::Png); assert_eq!( - image::load_from_memory(&preview.data)? + image::load_from_memory(preview_data)? .to_rgba8() .get_pixel(0, 0) .0, diff --git a/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs index 93b62e9ea9..32a23a55cb 100644 --- a/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs +++ b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs @@ -3,6 +3,8 @@ use super::super::audio_duration_store::TestAudioDurationStore; use super::*; use anyhow::Context as _; +use codex_attachment_store::UploadContent; +use codex_attachment_store::UploadRequest; use codex_protocol::models::AudioReference; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::ThreadHistoryMode; @@ -328,11 +330,18 @@ async fn tool_audio_store_id_survives_rollout_and_next_request( .tool_uploads .lock() .map_err(|_| anyhow::anyhow!("upload recorder poisoned"))?; - let [upload] = uploads.as_slice() else { - panic!("expected one upload across execution and resume"); - }; - assert_eq!(upload.tags["usage"], "tool_audio"); - assert_eq!(upload.data, codex_utils_audio::decode_audio(&wav)?.1); + assert_eq!( + uploads.as_slice(), + &[UploadRequest { + thread_id: test.session_configured.thread_id.to_string(), + ephemeral: false, + content: UploadContent::ToolAudio { + declared_mime_type: "audio/wav".into(), + data: codex_utils_audio::decode_audio(&wav)?.1, + }, + }], + "expected one upload across execution and resume", + ); } insta::assert_snapshot!( format!("file_backed_{output}_trace"), diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 162b421115..96b394df04 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -9,6 +9,7 @@ use codex_api::ImageEditRequest; use codex_api::ImageGenerationRequest; use codex_api::ImageOutput; use codex_api::ImageQuality; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_exec_server::CreateDirectoryOptions; @@ -278,9 +279,11 @@ impl ImageGenerationTool { let request = UploadRequest { thread_id: self.thread_id.clone(), ephemeral: call.ephemeral, - file_name: None, - tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]), - data, + content: UploadContent::Image { + file_name: None, + tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]), + data, + }, }; match call.attachment_store.upload(request).await { Ok(UploadResult::File { file_id }) => GeneratedImageRef::File { file_id },