Capture local attachments before queuing turn input (#61962)

## Why

Queued input can outlive its local attachment files. Reading those files only when rendering a model message can lose media submitted while turn admission is pending.

## What changed

- Introduce `ResolvedUserInput` to carry captured image bytes, audio data URLs, or read errors through session queues and agent input.
- Resolve client attachments on the submitting host before queuing, using a blocking task for local file reads.
- Render messages from captured payloads while preserving original paths and text spans for display, along with image position mappings.

## Testing

Add coverage for rendering and serialization after source files are removed, steered audio retained during pending admission, and local audio errors and display content.

GitOrigin-RevId: 4bbb6bda44167439a8cfdae15d3c476a7b1d336b
This commit is contained in:
Adam Perry @ OpenAI
2026-11-17 04:08:00 +00:00
committed by copyberry
parent 65a4450886
commit d8d840d32a
50 changed files with 1060 additions and 573 deletions
+2
View File
@@ -15,6 +15,8 @@ workspace = true
[dependencies]
codex-protocol = { workspace = true }
codex-utils-image = { workspace = true }
derive_more = { workspace = true, features = ["debug"] }
serde = { workspace = true, features = ["derive"] }
[dev-dependencies]
anyhow = { workspace = true }
+3 -3
View File
@@ -1,12 +1,12 @@
//! Captures user attachments and renders resolved input for model requests.
mod local_media;
mod message;
mod resolved_user_input;
pub use local_media::audio_mime_for_path;
pub use local_media::snapshot_local_user_input;
pub use message::LocalImagePreparation;
pub use message::from_user_input;
pub use resolved_user_input::LocalImagePreparation;
pub use resolved_user_input::ResolvedUserInput;
#[cfg(test)]
#[path = "message_tests.rs"]
+14 -6
View File
@@ -26,7 +26,7 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
PromptImageMode::ResizeToFit
}
};
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
let file_bytes = read_local_image_bytes(path)?;
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
*input = UserInput::Image {
image: ImageReference::Inline {
@@ -36,13 +36,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
};
}
UserInput::LocalAudio { path } => {
let mime = audio_mime_for_path(path).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format")
})?;
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
*input = UserInput::Audio {
audio: AudioReference::Inline {
audio_url: data_url_from_bytes(mime, &file_bytes),
audio_url: read_local_audio_url(path)?,
},
};
}
@@ -51,6 +47,18 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
Ok(())
}
/// Reads a bounded client-local audio file into a data URL.
pub(super) fn read_local_audio_url(path: &Path) -> io::Result<String> {
let mime = audio_mime_for_path(path)
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format"))?;
let bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
Ok(data_url_from_bytes(mime, &bytes))
}
pub(super) fn read_local_image_bytes(path: &Path) -> io::Result<Vec<u8>> {
read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")
}
fn read_bounded_local_media(path: &Path, max_bytes: usize, kind: &str) -> io::Result<Vec<u8>> {
let file = std::fs::File::open(path)?;
if file.metadata()?.len() > max_bytes as u64 {
-160
View File
@@ -1,160 +0,0 @@
//! Converts user inputs to model content and records image positions for UI events.
use crate::local_media::audio_mime_for_path;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::audio_close_tag_text;
use codex_protocol::models::local_audio_open_tag_text_with_path;
use codex_protocol::models::local_image_content_items;
use codex_protocol::models::local_image_content_items_with_label_number;
use codex_protocol::user_input::UserInput;
use codex_utils_image::data_url_from_bytes;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LocalMediaKind {
Audio,
Image,
}
impl LocalMediaKind {
fn name(self) -> &'static str {
match self {
Self::Audio => "audio",
Self::Image => "image",
}
}
}
fn local_media_error_placeholder(
path: &std::path::Path,
error: impl std::fmt::Display,
media_kind: LocalMediaKind,
) -> ContentItem {
let media_name = media_kind.name();
let path = path.display();
ContentItem::InputText {
text: format!("Codex could not read the local {media_name} at `{path}`: {error}"),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LocalImagePreparation {
Process,
Defer,
}
fn unsupported_audio_error_placeholder(path: &std::path::Path) -> ContentItem {
ContentItem::InputText {
text: format!(
"Codex cannot attach audio at `{}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.",
path.display()
),
}
}
fn local_audio_content_items(
path: &std::path::Path,
file_bytes: &[u8],
label_number: usize,
) -> Vec<ContentItem> {
let Some(mime) = audio_mime_for_path(path) else {
return vec![unsupported_audio_error_placeholder(path)];
};
vec![
ContentItem::InputText {
text: local_audio_open_tag_text_with_path(label_number, path),
},
ContentItem::InputAudio {
audio: AudioReference::Inline {
audio_url: data_url_from_bytes(mime, file_bytes),
},
},
ContentItem::InputText {
text: audio_close_tag_text(),
},
]
}
pub fn from_user_input(
items: Vec<UserInput>,
local_image_preparation: LocalImagePreparation,
user_image_content_indices: &mut HashMap<usize, usize>,
) -> ResponseInputItem {
user_image_content_indices.clear();
let mut image_index = 0;
let mut audio_index = 0;
let mut content = Vec::new();
for (input_index, input) in items.into_iter().enumerate() {
let generated = match input {
UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],
UserInput::Image { image, detail, .. } => {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
vec![ContentItem::InputImage {
image,
detail: Some(detail),
}]
}
UserInput::LocalImage { path, detail, .. } => {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
match std::fs::read(&path) {
Ok(file_bytes) => match local_image_preparation {
LocalImagePreparation::Process => {
local_image_content_items_with_label_number(
&path,
file_bytes,
Some(image_index),
detail,
)
}
LocalImagePreparation::Defer => local_image_content_items(
&path,
data_url_from_bytes("application/octet-stream", &file_bytes),
Some(image_index),
detail,
),
},
Err(err) => vec![local_media_error_placeholder(
&path,
err,
LocalMediaKind::Image,
)],
}
}
UserInput::Audio { audio } => {
audio_index += 1;
vec![ContentItem::InputAudio { audio }]
}
UserInput::LocalAudio { path } => {
audio_index += 1;
match std::fs::read(&path) {
Ok(file_bytes) => local_audio_content_items(&path, &file_bytes, audio_index),
Err(err) => vec![local_media_error_placeholder(
&path,
err,
LocalMediaKind::Audio,
)],
}
}
_ => Vec::new(), // Skills and mentions are injected later in core.
};
// Local images add framing text, while mentions and skills add no content.
// Capture the image's actual slot before preparation replaces its reference.
for (offset, item) in generated.iter().enumerate() {
if matches!(item, ContentItem::InputImage { .. }) {
user_image_content_indices.insert(input_index, content.len() + offset);
}
}
content.extend(generated);
}
ResponseInputItem::Message {
role: "user".to_string(),
content,
phase: None,
}
}
+11 -4
View File
@@ -1,7 +1,7 @@
#![cfg(test)]
use crate::LocalImagePreparation;
use crate::from_user_input;
use crate::ResolvedUserInput;
use anyhow::Result;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
@@ -29,7 +29,11 @@ const TINY_PNG_BYTES: &[u8] = &[
];
fn user_message(items: Vec<UserInput>) -> ResponseInputItem {
from_user_input(items, LocalImagePreparation::Process, &mut HashMap::new())
let items: Vec<_> = items
.into_iter()
.map(ResolvedUserInput::from_client)
.collect();
ResolvedUserInput::into_message(&items, LocalImagePreparation::Process).0
}
#[test]
@@ -294,9 +298,12 @@ fn user_input_image_indices_survive_content_expansion() -> Result<()> {
inline_image.clone(),
inline_image,
];
let inputs: Vec<_> = inputs
.into_iter()
.map(ResolvedUserInput::from_client)
.collect();
for preparation in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
let mut indices = HashMap::new();
let item = from_user_input(inputs.clone(), preparation, &mut indices);
let (item, indices) = ResolvedUserInput::into_message(&inputs, preparation);
assert_eq!(indices, HashMap::from([(4, 3), (5, 5), (6, 6)]));
let ResponseInputItem::Message { content, .. } = item else {
panic!("expected a user message");
@@ -0,0 +1,279 @@
//! Resolves one client input at a time. A local attachment carries its captured contents
//! or its read error. Its path is retained for display, labels, and diagnostics.
use crate::local_media::audio_mime_for_path;
use crate::local_media::read_local_audio_url;
use crate::local_media::read_local_image_bytes;
use codex_protocol::models::AudioReference;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::audio_close_tag_text;
use codex_protocol::models::local_audio_open_tag_text_with_path;
use codex_protocol::models::local_image_content_items;
use codex_protocol::models::local_image_content_items_with_label_number;
use codex_protocol::user_input::TextElement;
use codex_protocol::user_input::UserInput;
use codex_utils_image::data_url_from_bytes;
use serde::Deserialize;
use serde::Serialize;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LocalImagePreparation {
Process,
Defer,
}
/// One input admitted to the harness. Clones share captured attachment payloads.
#[derive(Clone, PartialEq, derive_more::Debug, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResolvedUserInput {
Text {
#[debug("{:?} <{} bytes>", &text[..text.floor_char_boundary(/*index*/ 512)], text.len())]
text: String,
#[serde(default)]
text_elements: Vec<TextElement>,
},
Image {
#[debug(skip)]
#[serde(flatten)]
image: Arc<ImageReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<ImageDetail>,
},
LocalImage {
path: PathBuf,
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<ImageDetail>,
#[debug(skip)]
bytes: Result<Arc<[u8]>, String>,
},
Audio {
#[debug(skip)]
#[serde(flatten)]
audio: Arc<AudioReference>,
},
LocalAudio {
path: PathBuf,
#[debug(skip)]
audio_url: Result<Arc<str>, String>,
},
Skill {
name: String,
path: PathBuf,
},
Mention {
name: String,
path: String,
},
}
impl ResolvedUserInput {
/// Reads a client-selected attachment on this host. Executor-owned paths must be resolved there.
pub fn from_client(input: UserInput) -> Self {
match input {
UserInput::Text {
text,
text_elements,
} => Self::Text {
text,
text_elements,
},
UserInput::Image { image, detail } => Self::Image {
image: Arc::new(image),
detail,
},
UserInput::LocalImage { path, detail } => {
let bytes = read_local_image_bytes(&path).map(Arc::from).map_err(|err| {
format!(
"Codex could not read the local image at `{}`: {err}",
path.display()
)
});
Self::LocalImage {
path,
detail,
bytes,
}
}
UserInput::Audio { audio } => Self::Audio {
audio: Arc::new(audio),
},
UserInput::LocalAudio { path } => {
let audio_url = read_local_audio_url(&path).map(Arc::from).map_err(|err| {
let name = path.display();
if audio_mime_for_path(&path).is_none() {
format!("Codex cannot attach audio at `{name}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.")
} else {
format!("Codex could not read the local audio at `{name}`: {err}")
}
});
Self::LocalAudio { path, audio_url }
}
UserInput::Skill { name, path } => Self::Skill { name, path },
UserInput::Mention { name, path } => Self::Mention { name, path },
_ => Self::text("Codex cannot prepare this input type.".into()),
}
}
pub fn text(text: String) -> Self {
Self::Text {
text,
text_elements: Vec::new(),
}
}
/// Projects the original client-facing input, including UI text spans and local display paths.
pub fn to_user_input(&self) -> UserInput {
match self {
Self::Text {
text,
text_elements,
} => UserInput::Text {
text: text.clone(),
text_elements: text_elements.clone(),
},
Self::Image { image, detail } => UserInput::Image {
image: image.as_ref().clone(),
detail: *detail,
},
Self::LocalImage { path, detail, .. } => UserInput::LocalImage {
path: path.clone(),
detail: *detail,
},
Self::Audio { audio } => UserInput::Audio {
audio: audio.as_ref().clone(),
},
Self::LocalAudio { path, .. } => UserInput::LocalAudio { path: path.clone() },
Self::Skill { name, path } => UserInput::Skill {
name: name.clone(),
path: path.clone(),
},
Self::Mention { name, path } => UserInput::Mention {
name: name.clone(),
path: path.clone(),
},
}
}
pub fn as_text(&self) -> Option<&str> {
match self {
Self::Text { text, .. } => Some(text),
_ => None,
}
}
pub fn is_uploaded_audio_file(&self) -> bool {
matches!(self, Self::Audio { audio } if matches!(audio.as_ref(), AudioReference::File { .. }))
}
pub fn is_image(&self) -> bool {
matches!(self, Self::Image { .. } | Self::LocalImage { .. })
}
/// Renders the captured inputs and maps the supplied slice's image indices to message slots.
pub fn into_message(
items: &[Self],
images: LocalImagePreparation,
) -> (ResponseInputItem, HashMap<usize, usize>) {
let mut image_positions = HashMap::new();
let mut content = Vec::new();
let mut image_number = 0;
let mut audio_number = 0;
for (index, item) in items.iter().enumerate() {
let start = content.len();
match item {
Self::Text { text, .. } => {
content.push(ContentItem::InputText { text: text.clone() })
}
Self::Image { image, detail } => {
image_number += 1;
content.push(ContentItem::InputImage {
image: image.as_ref().clone(),
detail: Some(detail.unwrap_or(DEFAULT_IMAGE_DETAIL)),
});
}
Self::LocalImage {
path,
bytes,
detail,
} => {
image_number += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
match bytes {
Ok(bytes) => content.extend(match images {
LocalImagePreparation::Process => {
local_image_content_items_with_label_number(
path,
bytes.to_vec(),
Some(image_number),
detail,
)
}
LocalImagePreparation::Defer => local_image_content_items(
path,
data_url_from_bytes("application/octet-stream", bytes),
Some(image_number),
detail,
),
}),
Err(message) => content.push(ContentItem::InputText {
text: message.clone(),
}),
}
}
Self::Audio { audio } => {
audio_number += 1;
content.push(ContentItem::InputAudio {
audio: audio.as_ref().clone(),
});
}
Self::LocalAudio { path, audio_url } => {
audio_number += 1;
match audio_url {
Ok(audio_url) => content.extend([
ContentItem::InputText {
text: local_audio_open_tag_text_with_path(audio_number, path),
},
ContentItem::InputAudio {
audio: AudioReference::Inline {
audio_url: audio_url.to_string(),
},
},
ContentItem::InputText {
text: audio_close_tag_text(),
},
]),
Err(message) => content.push(ContentItem::InputText {
text: message.clone(),
}),
}
}
Self::Skill { .. } | Self::Mention { .. } => continue,
}
if let Some(offset) = content[start..]
.iter()
.position(|item| matches!(item, ContentItem::InputImage { .. }))
{
image_positions.insert(index, start + offset);
}
}
(
ResponseInputItem::Message {
role: "user".into(),
content,
phase: None,
},
image_positions,
)
}
}
#[cfg(test)]
#[path = "resolved_user_input_tests.rs"]
mod tests;
@@ -0,0 +1,189 @@
#![cfg(test)]
use super::*;
use anyhow::Result;
use codex_protocol::models::image_close_tag_text;
use codex_protocol::models::local_image_open_tag_text_with_path;
use pretty_assertions::assert_eq;
// A tiny valid PNG (1x1) so image conversion tests don't depend on cross-crate
// file paths, which break under Bazel sandboxing.
const TINY_PNG_BYTES: &[u8] = &[
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
];
#[test]
fn resolved_media_renders_after_source_files_are_removed() -> Result<()> {
let dir = tempfile::tempdir()?;
let missing = dir.path().join("missing.png");
let path = dir.path().join("image.png");
let audio_path = dir.path().join("clip.wav");
std::fs::write(&path, TINY_PNG_BYTES)?;
std::fs::write(&audio_path, b"audio")?;
let image = ImageReference::File {
file_id: "file_image".into(),
};
let audio = AudioReference::File {
file_id: "file_audio".into(),
};
let inline = ImageReference::Inline {
image_url: data_url_from_bytes("image/png", TINY_PNG_BYTES),
};
let inputs = vec![
UserInput::Text {
text: "compare".into(),
text_elements: vec![TextElement::new((0..7).into(), Some("selection".into()))],
},
UserInput::Mention {
name: "app".into(),
path: "app://example".into(),
},
UserInput::Skill {
name: "skill".into(),
path: dir.path().join("SKILL.md"),
},
UserInput::LocalImage {
path: missing.clone(),
detail: None,
},
UserInput::Image {
image: image.clone(),
detail: None,
},
UserInput::Audio {
audio: audio.clone(),
},
UserInput::LocalAudio {
path: audio_path.clone(),
},
UserInput::LocalImage {
path: path.clone(),
detail: Some(ImageDetail::Original),
},
UserInput::Image {
image: inline.clone(),
detail: Some(ImageDetail::Original),
},
];
let resolved: Vec<_> = inputs
.iter()
.cloned()
.map(ResolvedUserInput::from_client)
.collect();
for (original, item) in inputs.iter().zip(&resolved) {
let wire = serde_json::to_value(original)?;
if matches!(
original,
UserInput::LocalImage { .. } | UserInput::LocalAudio { .. }
) {
// A path alone cannot rehydrate an already-captured attachment.
assert!(serde_json::from_value::<ResolvedUserInput>(wire).is_err());
} else {
assert_eq!(serde_json::to_value(item)?, wire);
}
}
let serialized = serde_json::to_vec(&resolved)?;
dir.close()?;
let restored: Vec<ResolvedUserInput> = serde_json::from_slice(&serialized)?;
assert_eq!(restored, resolved);
assert_eq!(
restored
.iter()
.map(ResolvedUserInput::to_user_input)
.collect::<Vec<_>>(),
inputs
);
// Filtering a turn changes both label numbers and which UI positions own images.
let (subset, positions) =
ResolvedUserInput::into_message(&restored[6..8], LocalImagePreparation::Defer);
let ResponseInputItem::Message { content, .. } = subset else {
panic!("expected a user message");
};
assert_eq!(positions, HashMap::from([(1, 4)]));
assert_eq!(
content[0],
ContentItem::InputText {
text: local_audio_open_tag_text_with_path(/*label_number*/ 1, &audio_path)
}
);
assert_eq!(
content[3],
ContentItem::InputText {
text: local_image_open_tag_text_with_path(/*label_number*/ 1, &path)
}
);
for mode in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
let (message, positions) = ResolvedUserInput::into_message(&restored, mode);
let ResponseInputItem::Message { content, .. } = message else {
panic!("expected a user message");
};
assert_eq!(positions, HashMap::from([(4, 2), (7, 8), (8, 10)]));
assert_eq!(
content[0],
ContentItem::InputText {
text: "compare".into()
}
);
let ContentItem::InputText { text } = &content[1] else {
panic!("expected a read error");
};
assert!(text.starts_with(&format!(
"Codex could not read the local image at `{}`:",
missing.display()
)));
let image_url = match mode {
LocalImagePreparation::Defer => {
data_url_from_bytes("application/octet-stream", TINY_PNG_BYTES)
}
LocalImagePreparation::Process => codex_utils_image::load_for_prompt_bytes(
&path,
TINY_PNG_BYTES.to_vec(),
codex_utils_image::PromptImageMode::Original,
)?
.into_data_url(),
};
assert_eq!(
&content[2..],
&[
ContentItem::InputImage {
image: image.clone(),
detail: Some(DEFAULT_IMAGE_DETAIL)
},
ContentItem::InputAudio {
audio: audio.clone()
},
ContentItem::InputText {
text: local_audio_open_tag_text_with_path(/*label_number*/ 2, &audio_path)
},
ContentItem::InputAudio {
audio: AudioReference::Inline {
audio_url: "data:audio/wav;base64,YXVkaW8=".into()
}
},
ContentItem::InputText {
text: audio_close_tag_text()
},
ContentItem::InputText {
text: local_image_open_tag_text_with_path(/*label_number*/ 3, &path)
},
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original)
},
ContentItem::InputText {
text: image_close_tag_text()
},
ContentItem::InputImage {
image: inline.clone(),
detail: Some(ImageDetail::Original)
},
]
);
}
Ok(())
}