Distinguish image and tool audio attachment uploads (#62047)

## What changed

Introduce `UploadContent::Image` and `UploadContent::ToolAudio` in `UploadRequest`. Keep image filenames and policy tags on the image variant, and carry tool audio's declared MIME type explicitly instead of encoding its type and MIME metadata in tags.

Update image preparation, image generation, and tool audio callers to use the typed content. Preserve inline bytes for both variants and redact their payloads from debug output. Document that persistent backends must verify the declared audio MIME type against the uploaded bytes.

## Testing

Add coverage for inline tool audio byte preservation and audio debug redaction. Update integration assertions for typed image and audio uploads, and verify that Code Mode does not upload audio when `FileBackedToolAudio` is disabled.

GitOrigin-RevId: 7dbd69922b377aead2dc28d1e5b3938cce6343b7
This commit is contained in:
Sean Huang
2026-11-17 13:38:34 +00:00
committed by copyberry
parent d75ea7b0fa
commit db9778b73c
11 changed files with 188 additions and 55 deletions
+50 -12
View File
@@ -46,6 +46,49 @@ pub trait AttachmentStore: Send + Sync {
}
}
/// Typed attachment content supplied to [`AttachmentStore::upload`].
#[derive(Clone, Eq, PartialEq)]
pub enum UploadContent {
/// An image that may be shown to the model or retained for the UI.
Image {
/// Optional name associated with the image.
file_name: Option<String>,
/// Caller-supplied tags used by storage backends for image policy decisions.
tags: HashMap<String, String>,
/// Encoded image bytes.
data: Vec<u8>,
},
/// Audio produced by a tool.
ToolAudio {
/// IANA media type a persistent backend must verify against the uploaded bytes.
declared_mime_type: String,
/// Encoded audio bytes.
data: Vec<u8>,
},
}
impl fmt::Debug for UploadContent {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Image {
file_name, tags, ..
} => formatter
.debug_struct("Image")
.field("file_name", file_name)
.field("tags", tags)
.field("data", &"<redacted>")
.finish(),
Self::ToolAudio {
declared_mime_type, ..
} => formatter
.debug_struct("ToolAudio")
.field("declared_mime_type", declared_mime_type)
.field("data", &"<redacted>")
.finish(),
}
}
}
/// Attachment data supplied to [`AttachmentStore::upload`].
#[derive(Clone, Eq, PartialEq)]
pub struct UploadRequest {
@@ -53,12 +96,8 @@ pub struct UploadRequest {
pub thread_id: String,
/// Whether the originating thread intentionally skips durable persistence.
pub ephemeral: bool,
/// Optional name associated with the attachment.
pub file_name: Option<String>,
/// Caller-supplied tags used by storage backends for policy decisions.
pub tags: HashMap<String, String>,
/// Attachment bytes to persist.
pub data: Vec<u8>,
/// Type-specific content to upload.
pub content: UploadContent,
}
impl fmt::Debug for UploadRequest {
@@ -67,9 +106,7 @@ impl fmt::Debug for UploadRequest {
.debug_struct("UploadRequest")
.field("thread_id", &self.thread_id)
.field("ephemeral", &self.ephemeral)
.field("file_name", &self.file_name)
.field("tags", &self.tags)
.field("data", &"<redacted>")
.field("content", &self.content)
.finish()
}
}
@@ -195,9 +232,10 @@ impl AttachmentStore for InlineAttachmentStore {
#[tracing::instrument(level = "trace", skip_all)]
fn upload(&self, request: UploadRequest) -> UploadFuture<'_> {
Box::pin(async move {
Ok(UploadResult::Inline {
bytes: request.data,
})
let bytes = match request.content {
UploadContent::Image { data, .. } | UploadContent::ToolAudio { data, .. } => data,
};
Ok(UploadResult::Inline { bytes })
})
}
+43 -8
View File
@@ -8,18 +8,29 @@ fn attachment_debug_output_redacts_bytes() {
let request = UploadRequest {
thread_id: "thread-1".to_string(),
ephemeral: false,
file_name: Some("image.png".to_string()),
tags: HashMap::new(),
data: b"secret".to_vec(),
content: UploadContent::Image {
file_name: Some("image.png".to_string()),
tags: HashMap::new(),
data: b"secret".to_vec(),
},
};
let result = UploadResult::Inline {
bytes: b"secret".to_vec(),
};
let tool_audio = UploadContent::ToolAudio {
declared_mime_type: "audio/wav".to_string(),
data: b"secret".to_vec(),
};
assert_eq!(
(format!("{request:?}"), format!("{result:?}")),
(
r#"UploadRequest { thread_id: "thread-1", ephemeral: false, file_name: Some("image.png"), tags: {}, data: "<redacted>" }"#.to_string(),
format!("{request:?}"),
format!("{tool_audio:?}"),
format!("{result:?}"),
),
(
r#"UploadRequest { thread_id: "thread-1", ephemeral: false, content: Image { file_name: Some("image.png"), tags: {}, data: "<redacted>" } }"#.to_string(),
r#"ToolAudio { declared_mime_type: "audio/wav", data: "<redacted>" }"#.to_string(),
r#"Inline { bytes: "<redacted>" }"#.to_string(),
)
);
@@ -52,9 +63,11 @@ async fn inline_store_preserves_image_bytes() {
.upload(UploadRequest {
thread_id: "thread-1".to_string(),
ephemeral: false,
file_name: Some(file_name.to_string()),
tags: HashMap::new(),
data: data.to_vec(),
content: UploadContent::Image {
file_name: Some(file_name.to_string()),
tags: HashMap::new(),
data: data.to_vec(),
},
})
.await
.expect("inline attachment");
@@ -66,6 +79,28 @@ async fn inline_store_preserves_image_bytes() {
}
}
/// The inline store preserves tool audio bytes without interpreting them.
#[tokio::test]
async fn inline_store_preserves_tool_audio_bytes() {
let data = b"RIFF\x04\x00\x00\x00WAVE";
let attachment = InlineAttachmentStore
.upload(UploadRequest {
thread_id: "thread-1".to_string(),
ephemeral: false,
content: UploadContent::ToolAudio {
declared_mime_type: "audio/wav".to_string(),
data: data.to_vec(),
},
})
.await
.expect("inline attachment");
let UploadResult::Inline { bytes } = attachment else {
panic!("inline store returned a file reference");
};
assert_eq!(bytes, data);
}
/// The inline store preserves opaque file references even though it cannot resolve them.
#[tokio::test]
async fn inline_store_cannot_resolve_file_references() {
+6 -3
View File
@@ -10,6 +10,7 @@ use crate::original_image_detail::can_request_original_image_detail;
use codex_analytics::ImageDetailSetting;
use codex_analytics::ImagePreparationMetadata;
use codex_attachment_store::AttachmentStore;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
use codex_context_fragments::AnnotatedContent;
@@ -449,9 +450,11 @@ async fn prepare_image(
let request = UploadRequest {
thread_id: origin.thread_id.to_owned(),
ephemeral: origin.ephemeral,
file_name: None,
tags: HashMap::new(),
data: prepared.encoded.bytes.to_vec(),
content: UploadContent::Image {
file_name: None,
tags: HashMap::new(),
data: prepared.encoded.bytes.to_vec(),
},
};
let prepared_image = match preparation.image_store.upload(request).await {
Ok(UploadResult::Inline { bytes }) => ImageReference::Inline {
+5 -8
View File
@@ -11,6 +11,7 @@ use std::time::Duration;
use codex_attachment_store::AttachmentStore;
use codex_attachment_store::FormatSpecificMetadata;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
use codex_history::ResponseItemEnvelope;
@@ -184,18 +185,14 @@ async fn store_tool_audio(
};
}
};
// These tags come from Core. MCP servers and Code Mode never supply attachment IDs or policy tags.
let tags = HashMap::from([
("usage".to_string(), "tool_audio".to_string()),
("mime_type".to_string(), mime.to_string()),
]);
let result = store
.upload(UploadRequest {
thread_id: thread_id.to_owned(),
ephemeral,
file_name: None,
tags,
data,
content: UploadContent::ToolAudio {
declared_mime_type: mime.to_owned(),
data,
},
})
.await;
tool_audio_after_upload(result, mime, inline_url)
@@ -10,6 +10,7 @@ use codex_attachment_store::ImageMetadata;
use codex_attachment_store::InlineAttachmentStore;
use codex_attachment_store::ResolveFuture;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadFuture;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
@@ -29,11 +30,11 @@ pub(super) struct TestAudioDurationStore {
impl AttachmentStore for TestAudioDurationStore {
fn upload(&self, request: UploadRequest) -> UploadFuture<'_> {
if request
.tags
.get("usage")
.is_some_and(|usage| usage == "tool_audio")
{
let is_tool_audio = match &request.content {
UploadContent::Image { .. } => false,
UploadContent::ToolAudio { .. } => true,
};
if is_tool_audio {
self.tool_uploads
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
+8
View File
@@ -7,6 +7,7 @@ mod dynamic_tools;
#[path = "code_mode_host_recovery_tests.rs"]
mod host_recovery;
use super::image_rollout::RecordingFileAttachmentStore;
use anyhow::Context;
use anyhow::Result;
use base64::Engine;
@@ -6604,12 +6605,18 @@ text("after");
);
let server = responses::start_mock_server().await;
let store = Arc::new(RecordingFileAttachmentStore::default());
let mut builder = test_codex()
.with_image_store(store.clone())
.with_model_info_override("gpt-5.4", |model| {
model.input_modalities.push(InputModality::Audio);
})
.with_config(|config| {
let _ = config.features.enable(Feature::CodeMode);
config
.features
.disable(Feature::FileBackedToolAudio)
.unwrap();
});
let test = builder.build_with_auto_env(&server).await?;
responses::mount_sse_once(
@@ -6633,6 +6640,7 @@ text("after");
test.submit_turn("use exec to return short and valid audio with text")
.await?;
assert!(store.uploads.lock().unwrap().is_empty());
let req = second_mock.single_request();
let items = custom_tool_output_items(&req, "call-1");
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
+21 -4
View File
@@ -5,6 +5,7 @@ use std::sync::Arc;
use anyhow::Context;
use anyhow::Result;
use codex_attachment_store::UploadContent;
use codex_core::TurnInputRequest;
use codex_core::config::Config;
use codex_core::config::Constrained;
@@ -172,12 +173,28 @@ async fn generated_image_ui_item_is_file_backed_in_rollout() -> Result<()> {
test.session_configured.thread_id.to_string()
);
assert!(!item_upload.ephemeral);
let UploadContent::Image {
file_name,
tags,
data,
} = &item_upload.content
else {
panic!("expected generated image upload");
};
assert!(file_name.is_none());
assert_eq!(
item_upload.tags,
HashMap::from([("usage".to_string(), "image_gen".to_string())])
tags,
&HashMap::from([("usage".to_string(), "image_gen".to_string())])
);
assert_eq!(item_upload.data, TINY_PNG_BYTES);
assert_eq!(model_upload.tags, HashMap::new());
assert_eq!(data, TINY_PNG_BYTES);
let UploadContent::Image {
file_name, tags, ..
} = &model_upload.content
else {
panic!("expected model image upload");
};
assert!(file_name.is_none());
assert_eq!(tags, &HashMap::new());
}
let rollout_path = test.codex.rollout_path().context("rollout path")?;
+12 -4
View File
@@ -8,6 +8,7 @@ use codex_attachment_store::AttachmentStoreErrorKind;
use codex_attachment_store::IsResolvableFuture;
use codex_attachment_store::ResolveFuture;
use codex_attachment_store::ResolveRequest;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadFuture;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
@@ -726,10 +727,17 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
upload.thread_id,
resumed.session_configured.thread_id.to_string(),
);
assert_eq!(
image::load_from_memory(&upload.data)?.dimensions(),
(6000, 240)
);
let UploadContent::Image {
file_name,
tags,
data,
} = &upload.content
else {
panic!("expected image upload");
};
assert!(file_name.is_none());
assert!(tags.is_empty());
assert_eq!(image::load_from_memory(data)?.dimensions(), (6000, 240));
}
let request = resumed_mock.single_request();
+17 -3
View File
@@ -9,6 +9,7 @@ mod multi_environment_mcp;
use super::image_rollout::RecordingFileAttachmentStore;
use super::permissions_messages::PermissionPathScenario;
use codex_attachment_store::AttachmentStoreErrorKind;
use codex_attachment_store::UploadContent;
use codex_core::RequestCountry;
use codex_protocol_test_support::ContentItemTestExt;
use std::collections::HashMap;
@@ -492,10 +493,23 @@ async fn gif_generation_preserves_animation_and_replays_a_still_preview() -> Res
let [animation, preview] = uploads.as_slice() else {
panic!("expected an animated artifact and a separate model preview");
};
assert_eq!(animation.data, BASE64_STANDARD.decode(gif)?);
assert_eq!(image::guess_format(&preview.data)?, image::ImageFormat::Png);
let UploadContent::Image {
data: animation_data,
..
} = &animation.content
else {
panic!("expected animation image upload");
};
let UploadContent::Image {
data: preview_data, ..
} = &preview.content
else {
panic!("expected preview image upload");
};
assert_eq!(*animation_data, BASE64_STANDARD.decode(gif)?);
assert_eq!(image::guess_format(preview_data)?, image::ImageFormat::Png);
assert_eq!(
image::load_from_memory(&preview.data)?
image::load_from_memory(preview_data)?
.to_rgba8()
.get_pixel(0, 0)
.0,
@@ -3,6 +3,8 @@
use super::super::audio_duration_store::TestAudioDurationStore;
use super::*;
use anyhow::Context as _;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadRequest;
use codex_protocol::models::AudioReference;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::ThreadHistoryMode;
@@ -328,11 +330,18 @@ async fn tool_audio_store_id_survives_rollout_and_next_request(
.tool_uploads
.lock()
.map_err(|_| anyhow::anyhow!("upload recorder poisoned"))?;
let [upload] = uploads.as_slice() else {
panic!("expected one upload across execution and resume");
};
assert_eq!(upload.tags["usage"], "tool_audio");
assert_eq!(upload.data, codex_utils_audio::decode_audio(&wav)?.1);
assert_eq!(
uploads.as_slice(),
&[UploadRequest {
thread_id: test.session_configured.thread_id.to_string(),
ephemeral: false,
content: UploadContent::ToolAudio {
declared_mime_type: "audio/wav".into(),
data: codex_utils_audio::decode_audio(&wav)?.1,
},
}],
"expected one upload across execution and resume",
);
}
insta::assert_snapshot!(
format!("file_backed_{output}_trace"),
+6 -3
View File
@@ -9,6 +9,7 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageOutput;
use codex_api::ImageQuality;
use codex_attachment_store::UploadContent;
use codex_attachment_store::UploadRequest;
use codex_attachment_store::UploadResult;
use codex_exec_server::CreateDirectoryOptions;
@@ -278,9 +279,11 @@ impl ImageGenerationTool {
let request = UploadRequest {
thread_id: self.thread_id.clone(),
ephemeral: call.ephemeral,
file_name: None,
tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]),
data,
content: UploadContent::Image {
file_name: None,
tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]),
data,
},
};
match call.attachment_store.upload(request).await {
Ok(UploadResult::File { file_id }) => GeneratedImageRef::File { file_id },