diff --git a/codex-rs/attachment-store/src/lib.rs b/codex-rs/attachment-store/src/lib.rs index 9bc70388e2..56f44d15f2 100644 --- a/codex-rs/attachment-store/src/lib.rs +++ b/codex-rs/attachment-store/src/lib.rs @@ -46,6 +46,49 @@ pub trait AttachmentStore: Send + Sync { } } +/// Typed attachment content supplied to [`AttachmentStore::upload`]. +#[derive(Clone, Eq, PartialEq)] +pub enum UploadContent { + /// An image that may be shown to the model or retained for the UI. + Image { + /// Optional name associated with the image. + file_name: Option, + /// Caller-supplied tags used by storage backends for image policy decisions. + tags: HashMap, + /// Encoded image bytes. + data: Vec, + }, + /// Audio produced by a tool. + ToolAudio { + /// IANA media type a persistent backend must verify against the uploaded bytes. + declared_mime_type: String, + /// Encoded audio bytes. + data: Vec, + }, +} + +impl fmt::Debug for UploadContent { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Image { + file_name, tags, .. + } => formatter + .debug_struct("Image") + .field("file_name", file_name) + .field("tags", tags) + .field("data", &"") + .finish(), + Self::ToolAudio { + declared_mime_type, .. + } => formatter + .debug_struct("ToolAudio") + .field("declared_mime_type", declared_mime_type) + .field("data", &"") + .finish(), + } + } +} + /// Attachment data supplied to [`AttachmentStore::upload`]. #[derive(Clone, Eq, PartialEq)] pub struct UploadRequest { @@ -53,12 +96,8 @@ pub struct UploadRequest { pub thread_id: String, /// Whether the originating thread intentionally skips durable persistence. pub ephemeral: bool, - /// Optional name associated with the attachment. - pub file_name: Option, - /// Caller-supplied tags used by storage backends for policy decisions. - pub tags: HashMap, - /// Attachment bytes to persist. - pub data: Vec, + /// Type-specific content to upload. + pub content: UploadContent, } impl fmt::Debug for UploadRequest { @@ -67,9 +106,7 @@ impl fmt::Debug for UploadRequest { .debug_struct("UploadRequest") .field("thread_id", &self.thread_id) .field("ephemeral", &self.ephemeral) - .field("file_name", &self.file_name) - .field("tags", &self.tags) - .field("data", &"") + .field("content", &self.content) .finish() } } @@ -195,9 +232,10 @@ impl AttachmentStore for InlineAttachmentStore { #[tracing::instrument(level = "trace", skip_all)] fn upload(&self, request: UploadRequest) -> UploadFuture<'_> { Box::pin(async move { - Ok(UploadResult::Inline { - bytes: request.data, - }) + let bytes = match request.content { + UploadContent::Image { data, .. } | UploadContent::ToolAudio { data, .. } => data, + }; + Ok(UploadResult::Inline { bytes }) }) } diff --git a/codex-rs/attachment-store/src/lib_tests.rs b/codex-rs/attachment-store/src/lib_tests.rs index 8e9ef750ed..3faca9bc5e 100644 --- a/codex-rs/attachment-store/src/lib_tests.rs +++ b/codex-rs/attachment-store/src/lib_tests.rs @@ -8,18 +8,29 @@ fn attachment_debug_output_redacts_bytes() { let request = UploadRequest { thread_id: "thread-1".to_string(), ephemeral: false, - file_name: Some("image.png".to_string()), - tags: HashMap::new(), - data: b"secret".to_vec(), + content: UploadContent::Image { + file_name: Some("image.png".to_string()), + tags: HashMap::new(), + data: b"secret".to_vec(), + }, }; let result = UploadResult::Inline { bytes: b"secret".to_vec(), }; + let tool_audio = UploadContent::ToolAudio { + declared_mime_type: "audio/wav".to_string(), + data: b"secret".to_vec(), + }; assert_eq!( - (format!("{request:?}"), format!("{result:?}")), ( - r#"UploadRequest { thread_id: "thread-1", ephemeral: false, file_name: Some("image.png"), tags: {}, data: "" }"#.to_string(), + format!("{request:?}"), + format!("{tool_audio:?}"), + format!("{result:?}"), + ), + ( + r#"UploadRequest { thread_id: "thread-1", ephemeral: false, content: Image { file_name: Some("image.png"), tags: {}, data: "" } }"#.to_string(), + r#"ToolAudio { declared_mime_type: "audio/wav", data: "" }"#.to_string(), r#"Inline { bytes: "" }"#.to_string(), ) ); @@ -52,9 +63,11 @@ async fn inline_store_preserves_image_bytes() { .upload(UploadRequest { thread_id: "thread-1".to_string(), ephemeral: false, - file_name: Some(file_name.to_string()), - tags: HashMap::new(), - data: data.to_vec(), + content: UploadContent::Image { + file_name: Some(file_name.to_string()), + tags: HashMap::new(), + data: data.to_vec(), + }, }) .await .expect("inline attachment"); @@ -66,6 +79,28 @@ async fn inline_store_preserves_image_bytes() { } } +/// The inline store preserves tool audio bytes without interpreting them. +#[tokio::test] +async fn inline_store_preserves_tool_audio_bytes() { + let data = b"RIFF\x04\x00\x00\x00WAVE"; + let attachment = InlineAttachmentStore + .upload(UploadRequest { + thread_id: "thread-1".to_string(), + ephemeral: false, + content: UploadContent::ToolAudio { + declared_mime_type: "audio/wav".to_string(), + data: data.to_vec(), + }, + }) + .await + .expect("inline attachment"); + let UploadResult::Inline { bytes } = attachment else { + panic!("inline store returned a file reference"); + }; + + assert_eq!(bytes, data); +} + /// The inline store preserves opaque file references even though it cannot resolve them. #[tokio::test] async fn inline_store_cannot_resolve_file_references() { diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index a014bc2113..6252316b92 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -10,6 +10,7 @@ use crate::original_image_detail::can_request_original_image_detail; use codex_analytics::ImageDetailSetting; use codex_analytics::ImagePreparationMetadata; use codex_attachment_store::AttachmentStore; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_context_fragments::AnnotatedContent; @@ -449,9 +450,11 @@ async fn prepare_image( let request = UploadRequest { thread_id: origin.thread_id.to_owned(), ephemeral: origin.ephemeral, - file_name: None, - tags: HashMap::new(), - data: prepared.encoded.bytes.to_vec(), + content: UploadContent::Image { + file_name: None, + tags: HashMap::new(), + data: prepared.encoded.bytes.to_vec(), + }, }; let prepared_image = match preparation.image_store.upload(request).await { Ok(UploadResult::Inline { bytes }) => ImageReference::Inline { diff --git a/codex-rs/core/src/session/file_audio.rs b/codex-rs/core/src/session/file_audio.rs index 15b46fcfd2..704f9403dc 100644 --- a/codex-rs/core/src/session/file_audio.rs +++ b/codex-rs/core/src/session/file_audio.rs @@ -11,6 +11,7 @@ use std::time::Duration; use codex_attachment_store::AttachmentStore; use codex_attachment_store::FormatSpecificMetadata; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_history::ResponseItemEnvelope; @@ -184,18 +185,14 @@ async fn store_tool_audio( }; } }; - // These tags come from Core. MCP servers and Code Mode never supply attachment IDs or policy tags. - let tags = HashMap::from([ - ("usage".to_string(), "tool_audio".to_string()), - ("mime_type".to_string(), mime.to_string()), - ]); let result = store .upload(UploadRequest { thread_id: thread_id.to_owned(), ephemeral, - file_name: None, - tags, - data, + content: UploadContent::ToolAudio { + declared_mime_type: mime.to_owned(), + data, + }, }) .await; tool_audio_after_upload(result, mime, inline_url) diff --git a/codex-rs/core/tests/suite/audio_duration_store.rs b/codex-rs/core/tests/suite/audio_duration_store.rs index 1678de6e26..0a4172bfa0 100644 --- a/codex-rs/core/tests/suite/audio_duration_store.rs +++ b/codex-rs/core/tests/suite/audio_duration_store.rs @@ -10,6 +10,7 @@ use codex_attachment_store::ImageMetadata; use codex_attachment_store::InlineAttachmentStore; use codex_attachment_store::ResolveFuture; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadFuture; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; @@ -29,11 +30,11 @@ pub(super) struct TestAudioDurationStore { impl AttachmentStore for TestAudioDurationStore { fn upload(&self, request: UploadRequest) -> UploadFuture<'_> { - if request - .tags - .get("usage") - .is_some_and(|usage| usage == "tool_audio") - { + let is_tool_audio = match &request.content { + UploadContent::Image { .. } => false, + UploadContent::ToolAudio { .. } => true, + }; + if is_tool_audio { self.tool_uploads .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index c2f01448b2..2ea57626f0 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -7,6 +7,7 @@ mod dynamic_tools; #[path = "code_mode_host_recovery_tests.rs"] mod host_recovery; +use super::image_rollout::RecordingFileAttachmentStore; use anyhow::Context; use anyhow::Result; use base64::Engine; @@ -6604,12 +6605,18 @@ text("after"); ); let server = responses::start_mock_server().await; + let store = Arc::new(RecordingFileAttachmentStore::default()); let mut builder = test_codex() + .with_image_store(store.clone()) .with_model_info_override("gpt-5.4", |model| { model.input_modalities.push(InputModality::Audio); }) .with_config(|config| { let _ = config.features.enable(Feature::CodeMode); + config + .features + .disable(Feature::FileBackedToolAudio) + .unwrap(); }); let test = builder.build_with_auto_env(&server).await?; responses::mount_sse_once( @@ -6633,6 +6640,7 @@ text("after"); test.submit_turn("use exec to return short and valid audio with text") .await?; + assert!(store.uploads.lock().unwrap().is_empty()); let req = second_mock.single_request(); let items = custom_tool_output_items(&req, "call-1"); let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); diff --git a/codex-rs/core/tests/suite/extension_sandbox.rs b/codex-rs/core/tests/suite/extension_sandbox.rs index 1d31674af2..f62a4f66ea 100644 --- a/codex-rs/core/tests/suite/extension_sandbox.rs +++ b/codex-rs/core/tests/suite/extension_sandbox.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use anyhow::Context; use anyhow::Result; +use codex_attachment_store::UploadContent; use codex_core::TurnInputRequest; use codex_core::config::Config; use codex_core::config::Constrained; @@ -172,12 +173,28 @@ async fn generated_image_ui_item_is_file_backed_in_rollout() -> Result<()> { test.session_configured.thread_id.to_string() ); assert!(!item_upload.ephemeral); + let UploadContent::Image { + file_name, + tags, + data, + } = &item_upload.content + else { + panic!("expected generated image upload"); + }; + assert!(file_name.is_none()); assert_eq!( - item_upload.tags, - HashMap::from([("usage".to_string(), "image_gen".to_string())]) + tags, + &HashMap::from([("usage".to_string(), "image_gen".to_string())]) ); - assert_eq!(item_upload.data, TINY_PNG_BYTES); - assert_eq!(model_upload.tags, HashMap::new()); + assert_eq!(data, TINY_PNG_BYTES); + let UploadContent::Image { + file_name, tags, .. + } = &model_upload.content + else { + panic!("expected model image upload"); + }; + assert!(file_name.is_none()); + assert_eq!(tags, &HashMap::new()); } let rollout_path = test.codex.rollout_path().context("rollout path")?; diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index dc27b7e0c3..c61355669c 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -8,6 +8,7 @@ use codex_attachment_store::AttachmentStoreErrorKind; use codex_attachment_store::IsResolvableFuture; use codex_attachment_store::ResolveFuture; use codex_attachment_store::ResolveRequest; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadFuture; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; @@ -726,10 +727,17 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R upload.thread_id, resumed.session_configured.thread_id.to_string(), ); - assert_eq!( - image::load_from_memory(&upload.data)?.dimensions(), - (6000, 240) - ); + let UploadContent::Image { + file_name, + tags, + data, + } = &upload.content + else { + panic!("expected image upload"); + }; + assert!(file_name.is_none()); + assert!(tags.is_empty()); + assert_eq!(image::load_from_memory(data)?.dimensions(), (6000, 240)); } let request = resumed_mock.single_request(); diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index 2cebcc55bb..e9ea6837e5 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -9,6 +9,7 @@ mod multi_environment_mcp; use super::image_rollout::RecordingFileAttachmentStore; use super::permissions_messages::PermissionPathScenario; use codex_attachment_store::AttachmentStoreErrorKind; +use codex_attachment_store::UploadContent; use codex_core::RequestCountry; use codex_protocol_test_support::ContentItemTestExt; use std::collections::HashMap; @@ -492,10 +493,23 @@ async fn gif_generation_preserves_animation_and_replays_a_still_preview() -> Res let [animation, preview] = uploads.as_slice() else { panic!("expected an animated artifact and a separate model preview"); }; - assert_eq!(animation.data, BASE64_STANDARD.decode(gif)?); - assert_eq!(image::guess_format(&preview.data)?, image::ImageFormat::Png); + let UploadContent::Image { + data: animation_data, + .. + } = &animation.content + else { + panic!("expected animation image upload"); + }; + let UploadContent::Image { + data: preview_data, .. + } = &preview.content + else { + panic!("expected preview image upload"); + }; + assert_eq!(*animation_data, BASE64_STANDARD.decode(gif)?); + assert_eq!(image::guess_format(preview_data)?, image::ImageFormat::Png); assert_eq!( - image::load_from_memory(&preview.data)? + image::load_from_memory(preview_data)? .to_rgba8() .get_pixel(0, 0) .0, diff --git a/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs index 93b62e9ea9..32a23a55cb 100644 --- a/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs +++ b/codex-rs/core/tests/suite/scenarios_audio_file_tests.rs @@ -3,6 +3,8 @@ use super::super::audio_duration_store::TestAudioDurationStore; use super::*; use anyhow::Context as _; +use codex_attachment_store::UploadContent; +use codex_attachment_store::UploadRequest; use codex_protocol::models::AudioReference; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::ThreadHistoryMode; @@ -328,11 +330,18 @@ async fn tool_audio_store_id_survives_rollout_and_next_request( .tool_uploads .lock() .map_err(|_| anyhow::anyhow!("upload recorder poisoned"))?; - let [upload] = uploads.as_slice() else { - panic!("expected one upload across execution and resume"); - }; - assert_eq!(upload.tags["usage"], "tool_audio"); - assert_eq!(upload.data, codex_utils_audio::decode_audio(&wav)?.1); + assert_eq!( + uploads.as_slice(), + &[UploadRequest { + thread_id: test.session_configured.thread_id.to_string(), + ephemeral: false, + content: UploadContent::ToolAudio { + declared_mime_type: "audio/wav".into(), + data: codex_utils_audio::decode_audio(&wav)?.1, + }, + }], + "expected one upload across execution and resume", + ); } insta::assert_snapshot!( format!("file_backed_{output}_trace"), diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 162b421115..96b394df04 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -9,6 +9,7 @@ use codex_api::ImageEditRequest; use codex_api::ImageGenerationRequest; use codex_api::ImageOutput; use codex_api::ImageQuality; +use codex_attachment_store::UploadContent; use codex_attachment_store::UploadRequest; use codex_attachment_store::UploadResult; use codex_exec_server::CreateDirectoryOptions; @@ -278,9 +279,11 @@ impl ImageGenerationTool { let request = UploadRequest { thread_id: self.thread_id.clone(), ephemeral: call.ephemeral, - file_name: None, - tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]), - data, + content: UploadContent::Image { + file_name: None, + tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]), + data, + }, }; match call.attachment_store.upload(request).await { Ok(UploadResult::File { file_id }) => GeneratedImageRef::File { file_id },