mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Capture local attachments before queuing turn input (#61962)
## Why Queued input can outlive its local attachment files. Reading those files only when rendering a model message can lose media submitted while turn admission is pending. ## What changed - Introduce `ResolvedUserInput` to carry captured image bytes, audio data URLs, or read errors through session queues and agent input. - Resolve client attachments on the submitting host before queuing, using a blocking task for local file reads. - Render messages from captured payloads while preserving original paths and text spans for display, along with image position mappings. ## Testing Add coverage for rendering and serialization after source files are removed, steered audio retained during pending admission, and local audio errors and display content. GitOrigin-RevId: 4bbb6bda44167439a8cfdae15d3c476a7b1d336b
This commit is contained in:
committed by
copyberry
parent
65a4450886
commit
d8d840d32a
@@ -15,6 +15,8 @@ workspace = true
|
||||
[dependencies]
|
||||
codex-protocol = { workspace = true }
|
||||
codex-utils-image = { workspace = true }
|
||||
derive_more = { workspace = true, features = ["debug"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
//! Captures user attachments and renders resolved input for model requests.
|
||||
|
||||
mod local_media;
|
||||
mod message;
|
||||
mod resolved_user_input;
|
||||
|
||||
pub use local_media::audio_mime_for_path;
|
||||
pub use local_media::snapshot_local_user_input;
|
||||
pub use message::LocalImagePreparation;
|
||||
pub use message::from_user_input;
|
||||
pub use resolved_user_input::LocalImagePreparation;
|
||||
pub use resolved_user_input::ResolvedUserInput;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "message_tests.rs"]
|
||||
|
||||
@@ -26,7 +26,7 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
PromptImageMode::ResizeToFit
|
||||
}
|
||||
};
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
|
||||
let file_bytes = read_local_image_bytes(path)?;
|
||||
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
|
||||
*input = UserInput::Image {
|
||||
image: ImageReference::Inline {
|
||||
@@ -36,13 +36,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
};
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
let mime = audio_mime_for_path(path).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format")
|
||||
})?;
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
*input = UserInput::Audio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, &file_bytes),
|
||||
audio_url: read_local_audio_url(path)?,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -51,6 +47,18 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads a bounded client-local audio file into a data URL.
|
||||
pub(super) fn read_local_audio_url(path: &Path) -> io::Result<String> {
|
||||
let mime = audio_mime_for_path(path)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format"))?;
|
||||
let bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
Ok(data_url_from_bytes(mime, &bytes))
|
||||
}
|
||||
|
||||
pub(super) fn read_local_image_bytes(path: &Path) -> io::Result<Vec<u8>> {
|
||||
read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")
|
||||
}
|
||||
|
||||
fn read_bounded_local_media(path: &Path, max_bytes: usize, kind: &str) -> io::Result<Vec<u8>> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
if file.metadata()?.len() > max_bytes as u64 {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
//! Converts user inputs to model content and records image positions for UI events.
|
||||
|
||||
use crate::local_media::audio_mime_for_path;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::audio_close_tag_text;
|
||||
use codex_protocol::models::local_audio_open_tag_text_with_path;
|
||||
use codex_protocol::models::local_image_content_items;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LocalMediaKind {
|
||||
Audio,
|
||||
Image,
|
||||
}
|
||||
|
||||
impl LocalMediaKind {
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Audio => "audio",
|
||||
Self::Image => "image",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn local_media_error_placeholder(
|
||||
path: &std::path::Path,
|
||||
error: impl std::fmt::Display,
|
||||
media_kind: LocalMediaKind,
|
||||
) -> ContentItem {
|
||||
let media_name = media_kind.name();
|
||||
let path = path.display();
|
||||
ContentItem::InputText {
|
||||
text: format!("Codex could not read the local {media_name} at `{path}`: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalImagePreparation {
|
||||
Process,
|
||||
Defer,
|
||||
}
|
||||
|
||||
fn unsupported_audio_error_placeholder(path: &std::path::Path) -> ContentItem {
|
||||
ContentItem::InputText {
|
||||
text: format!(
|
||||
"Codex cannot attach audio at `{}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_audio_content_items(
|
||||
path: &std::path::Path,
|
||||
file_bytes: &[u8],
|
||||
label_number: usize,
|
||||
) -> Vec<ContentItem> {
|
||||
let Some(mime) = audio_mime_for_path(path) else {
|
||||
return vec![unsupported_audio_error_placeholder(path)];
|
||||
};
|
||||
|
||||
vec![
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(label_number, path),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, file_bytes),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
pub fn from_user_input(
|
||||
items: Vec<UserInput>,
|
||||
local_image_preparation: LocalImagePreparation,
|
||||
user_image_content_indices: &mut HashMap<usize, usize>,
|
||||
) -> ResponseInputItem {
|
||||
user_image_content_indices.clear();
|
||||
let mut image_index = 0;
|
||||
let mut audio_index = 0;
|
||||
let mut content = Vec::new();
|
||||
for (input_index, input) in items.into_iter().enumerate() {
|
||||
let generated = match input {
|
||||
UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],
|
||||
UserInput::Image { image, detail, .. } => {
|
||||
image_index += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
vec![ContentItem::InputImage {
|
||||
image,
|
||||
detail: Some(detail),
|
||||
}]
|
||||
}
|
||||
UserInput::LocalImage { path, detail, .. } => {
|
||||
image_index += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
match std::fs::read(&path) {
|
||||
Ok(file_bytes) => match local_image_preparation {
|
||||
LocalImagePreparation::Process => {
|
||||
local_image_content_items_with_label_number(
|
||||
&path,
|
||||
file_bytes,
|
||||
Some(image_index),
|
||||
detail,
|
||||
)
|
||||
}
|
||||
LocalImagePreparation::Defer => local_image_content_items(
|
||||
&path,
|
||||
data_url_from_bytes("application/octet-stream", &file_bytes),
|
||||
Some(image_index),
|
||||
detail,
|
||||
),
|
||||
},
|
||||
Err(err) => vec![local_media_error_placeholder(
|
||||
&path,
|
||||
err,
|
||||
LocalMediaKind::Image,
|
||||
)],
|
||||
}
|
||||
}
|
||||
UserInput::Audio { audio } => {
|
||||
audio_index += 1;
|
||||
vec![ContentItem::InputAudio { audio }]
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
audio_index += 1;
|
||||
match std::fs::read(&path) {
|
||||
Ok(file_bytes) => local_audio_content_items(&path, &file_bytes, audio_index),
|
||||
Err(err) => vec![local_media_error_placeholder(
|
||||
&path,
|
||||
err,
|
||||
LocalMediaKind::Audio,
|
||||
)],
|
||||
}
|
||||
}
|
||||
_ => Vec::new(), // Skills and mentions are injected later in core.
|
||||
};
|
||||
// Local images add framing text, while mentions and skills add no content.
|
||||
// Capture the image's actual slot before preparation replaces its reference.
|
||||
for (offset, item) in generated.iter().enumerate() {
|
||||
if matches!(item, ContentItem::InputImage { .. }) {
|
||||
user_image_content_indices.insert(input_index, content.len() + offset);
|
||||
}
|
||||
}
|
||||
content.extend(generated);
|
||||
}
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content,
|
||||
phase: None,
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use crate::LocalImagePreparation;
|
||||
use crate::from_user_input;
|
||||
use crate::ResolvedUserInput;
|
||||
use anyhow::Result;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
@@ -29,7 +29,11 @@ const TINY_PNG_BYTES: &[u8] = &[
|
||||
];
|
||||
|
||||
fn user_message(items: Vec<UserInput>) -> ResponseInputItem {
|
||||
from_user_input(items, LocalImagePreparation::Process, &mut HashMap::new())
|
||||
let items: Vec<_> = items
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
ResolvedUserInput::into_message(&items, LocalImagePreparation::Process).0
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -294,9 +298,12 @@ fn user_input_image_indices_survive_content_expansion() -> Result<()> {
|
||||
inline_image.clone(),
|
||||
inline_image,
|
||||
];
|
||||
let inputs: Vec<_> = inputs
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
for preparation in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
|
||||
let mut indices = HashMap::new();
|
||||
let item = from_user_input(inputs.clone(), preparation, &mut indices);
|
||||
let (item, indices) = ResolvedUserInput::into_message(&inputs, preparation);
|
||||
assert_eq!(indices, HashMap::from([(4, 3), (5, 5), (6, 6)]));
|
||||
let ResponseInputItem::Message { content, .. } = item else {
|
||||
panic!("expected a user message");
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Resolves one client input at a time. A local attachment carries its captured contents
|
||||
//! or its read error. Its path is retained for display, labels, and diagnostics.
|
||||
|
||||
use crate::local_media::audio_mime_for_path;
|
||||
use crate::local_media::read_local_audio_url;
|
||||
use crate::local_media::read_local_image_bytes;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::audio_close_tag_text;
|
||||
use codex_protocol::models::local_audio_open_tag_text_with_path;
|
||||
use codex_protocol::models::local_image_content_items;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::user_input::TextElement;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalImagePreparation {
|
||||
Process,
|
||||
Defer,
|
||||
}
|
||||
|
||||
/// One input admitted to the harness. Clones share captured attachment payloads.
|
||||
#[derive(Clone, PartialEq, derive_more::Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResolvedUserInput {
|
||||
Text {
|
||||
#[debug("{:?} <{} bytes>", &text[..text.floor_char_boundary(/*index*/ 512)], text.len())]
|
||||
text: String,
|
||||
#[serde(default)]
|
||||
text_elements: Vec<TextElement>,
|
||||
},
|
||||
Image {
|
||||
#[debug(skip)]
|
||||
#[serde(flatten)]
|
||||
image: Arc<ImageReference>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<ImageDetail>,
|
||||
},
|
||||
LocalImage {
|
||||
path: PathBuf,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<ImageDetail>,
|
||||
#[debug(skip)]
|
||||
bytes: Result<Arc<[u8]>, String>,
|
||||
},
|
||||
Audio {
|
||||
#[debug(skip)]
|
||||
#[serde(flatten)]
|
||||
audio: Arc<AudioReference>,
|
||||
},
|
||||
LocalAudio {
|
||||
path: PathBuf,
|
||||
#[debug(skip)]
|
||||
audio_url: Result<Arc<str>, String>,
|
||||
},
|
||||
Skill {
|
||||
name: String,
|
||||
path: PathBuf,
|
||||
},
|
||||
Mention {
|
||||
name: String,
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ResolvedUserInput {
|
||||
/// Reads a client-selected attachment on this host. Executor-owned paths must be resolved there.
|
||||
pub fn from_client(input: UserInput) -> Self {
|
||||
match input {
|
||||
UserInput::Text {
|
||||
text,
|
||||
text_elements,
|
||||
} => Self::Text {
|
||||
text,
|
||||
text_elements,
|
||||
},
|
||||
UserInput::Image { image, detail } => Self::Image {
|
||||
image: Arc::new(image),
|
||||
detail,
|
||||
},
|
||||
UserInput::LocalImage { path, detail } => {
|
||||
let bytes = read_local_image_bytes(&path).map(Arc::from).map_err(|err| {
|
||||
format!(
|
||||
"Codex could not read the local image at `{}`: {err}",
|
||||
path.display()
|
||||
)
|
||||
});
|
||||
Self::LocalImage {
|
||||
path,
|
||||
detail,
|
||||
bytes,
|
||||
}
|
||||
}
|
||||
UserInput::Audio { audio } => Self::Audio {
|
||||
audio: Arc::new(audio),
|
||||
},
|
||||
UserInput::LocalAudio { path } => {
|
||||
let audio_url = read_local_audio_url(&path).map(Arc::from).map_err(|err| {
|
||||
let name = path.display();
|
||||
if audio_mime_for_path(&path).is_none() {
|
||||
format!("Codex cannot attach audio at `{name}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.")
|
||||
} else {
|
||||
format!("Codex could not read the local audio at `{name}`: {err}")
|
||||
}
|
||||
});
|
||||
Self::LocalAudio { path, audio_url }
|
||||
}
|
||||
UserInput::Skill { name, path } => Self::Skill { name, path },
|
||||
UserInput::Mention { name, path } => Self::Mention { name, path },
|
||||
_ => Self::text("Codex cannot prepare this input type.".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn text(text: String) -> Self {
|
||||
Self::Text {
|
||||
text,
|
||||
text_elements: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Projects the original client-facing input, including UI text spans and local display paths.
|
||||
pub fn to_user_input(&self) -> UserInput {
|
||||
match self {
|
||||
Self::Text {
|
||||
text,
|
||||
text_elements,
|
||||
} => UserInput::Text {
|
||||
text: text.clone(),
|
||||
text_elements: text_elements.clone(),
|
||||
},
|
||||
Self::Image { image, detail } => UserInput::Image {
|
||||
image: image.as_ref().clone(),
|
||||
detail: *detail,
|
||||
},
|
||||
Self::LocalImage { path, detail, .. } => UserInput::LocalImage {
|
||||
path: path.clone(),
|
||||
detail: *detail,
|
||||
},
|
||||
Self::Audio { audio } => UserInput::Audio {
|
||||
audio: audio.as_ref().clone(),
|
||||
},
|
||||
Self::LocalAudio { path, .. } => UserInput::LocalAudio { path: path.clone() },
|
||||
Self::Skill { name, path } => UserInput::Skill {
|
||||
name: name.clone(),
|
||||
path: path.clone(),
|
||||
},
|
||||
Self::Mention { name, path } => UserInput::Mention {
|
||||
name: name.clone(),
|
||||
path: path.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_text(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Text { text, .. } => Some(text),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_uploaded_audio_file(&self) -> bool {
|
||||
matches!(self, Self::Audio { audio } if matches!(audio.as_ref(), AudioReference::File { .. }))
|
||||
}
|
||||
|
||||
pub fn is_image(&self) -> bool {
|
||||
matches!(self, Self::Image { .. } | Self::LocalImage { .. })
|
||||
}
|
||||
|
||||
/// Renders the captured inputs and maps the supplied slice's image indices to message slots.
|
||||
pub fn into_message(
|
||||
items: &[Self],
|
||||
images: LocalImagePreparation,
|
||||
) -> (ResponseInputItem, HashMap<usize, usize>) {
|
||||
let mut image_positions = HashMap::new();
|
||||
let mut content = Vec::new();
|
||||
let mut image_number = 0;
|
||||
let mut audio_number = 0;
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
let start = content.len();
|
||||
match item {
|
||||
Self::Text { text, .. } => {
|
||||
content.push(ContentItem::InputText { text: text.clone() })
|
||||
}
|
||||
Self::Image { image, detail } => {
|
||||
image_number += 1;
|
||||
content.push(ContentItem::InputImage {
|
||||
image: image.as_ref().clone(),
|
||||
detail: Some(detail.unwrap_or(DEFAULT_IMAGE_DETAIL)),
|
||||
});
|
||||
}
|
||||
Self::LocalImage {
|
||||
path,
|
||||
bytes,
|
||||
detail,
|
||||
} => {
|
||||
image_number += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
match bytes {
|
||||
Ok(bytes) => content.extend(match images {
|
||||
LocalImagePreparation::Process => {
|
||||
local_image_content_items_with_label_number(
|
||||
path,
|
||||
bytes.to_vec(),
|
||||
Some(image_number),
|
||||
detail,
|
||||
)
|
||||
}
|
||||
LocalImagePreparation::Defer => local_image_content_items(
|
||||
path,
|
||||
data_url_from_bytes("application/octet-stream", bytes),
|
||||
Some(image_number),
|
||||
detail,
|
||||
),
|
||||
}),
|
||||
Err(message) => content.push(ContentItem::InputText {
|
||||
text: message.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
Self::Audio { audio } => {
|
||||
audio_number += 1;
|
||||
content.push(ContentItem::InputAudio {
|
||||
audio: audio.as_ref().clone(),
|
||||
});
|
||||
}
|
||||
Self::LocalAudio { path, audio_url } => {
|
||||
audio_number += 1;
|
||||
match audio_url {
|
||||
Ok(audio_url) => content.extend([
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(audio_number, path),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
},
|
||||
]),
|
||||
Err(message) => content.push(ContentItem::InputText {
|
||||
text: message.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
Self::Skill { .. } | Self::Mention { .. } => continue,
|
||||
}
|
||||
if let Some(offset) = content[start..]
|
||||
.iter()
|
||||
.position(|item| matches!(item, ContentItem::InputImage { .. }))
|
||||
{
|
||||
image_positions.insert(index, start + offset);
|
||||
}
|
||||
}
|
||||
(
|
||||
ResponseInputItem::Message {
|
||||
role: "user".into(),
|
||||
content,
|
||||
phase: None,
|
||||
},
|
||||
image_positions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "resolved_user_input_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,189 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
use codex_protocol::models::image_close_tag_text;
|
||||
use codex_protocol::models::local_image_open_tag_text_with_path;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
// A tiny valid PNG (1x1) so image conversion tests don't depend on cross-crate
|
||||
// file paths, which break under Bazel sandboxing.
|
||||
const TINY_PNG_BYTES: &[u8] = &[
|
||||
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
|
||||
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
|
||||
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn resolved_media_renders_after_source_files_are_removed() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let missing = dir.path().join("missing.png");
|
||||
let path = dir.path().join("image.png");
|
||||
let audio_path = dir.path().join("clip.wav");
|
||||
std::fs::write(&path, TINY_PNG_BYTES)?;
|
||||
std::fs::write(&audio_path, b"audio")?;
|
||||
|
||||
let image = ImageReference::File {
|
||||
file_id: "file_image".into(),
|
||||
};
|
||||
let audio = AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
};
|
||||
let inline = ImageReference::Inline {
|
||||
image_url: data_url_from_bytes("image/png", TINY_PNG_BYTES),
|
||||
};
|
||||
let inputs = vec![
|
||||
UserInput::Text {
|
||||
text: "compare".into(),
|
||||
text_elements: vec![TextElement::new((0..7).into(), Some("selection".into()))],
|
||||
},
|
||||
UserInput::Mention {
|
||||
name: "app".into(),
|
||||
path: "app://example".into(),
|
||||
},
|
||||
UserInput::Skill {
|
||||
name: "skill".into(),
|
||||
path: dir.path().join("SKILL.md"),
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: missing.clone(),
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Image {
|
||||
image: image.clone(),
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: audio.clone(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: audio_path.clone(),
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: path.clone(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Image {
|
||||
image: inline.clone(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
];
|
||||
let resolved: Vec<_> = inputs
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
for (original, item) in inputs.iter().zip(&resolved) {
|
||||
let wire = serde_json::to_value(original)?;
|
||||
if matches!(
|
||||
original,
|
||||
UserInput::LocalImage { .. } | UserInput::LocalAudio { .. }
|
||||
) {
|
||||
// A path alone cannot rehydrate an already-captured attachment.
|
||||
assert!(serde_json::from_value::<ResolvedUserInput>(wire).is_err());
|
||||
} else {
|
||||
assert_eq!(serde_json::to_value(item)?, wire);
|
||||
}
|
||||
}
|
||||
let serialized = serde_json::to_vec(&resolved)?;
|
||||
dir.close()?;
|
||||
let restored: Vec<ResolvedUserInput> = serde_json::from_slice(&serialized)?;
|
||||
assert_eq!(restored, resolved);
|
||||
assert_eq!(
|
||||
restored
|
||||
.iter()
|
||||
.map(ResolvedUserInput::to_user_input)
|
||||
.collect::<Vec<_>>(),
|
||||
inputs
|
||||
);
|
||||
|
||||
// Filtering a turn changes both label numbers and which UI positions own images.
|
||||
let (subset, positions) =
|
||||
ResolvedUserInput::into_message(&restored[6..8], LocalImagePreparation::Defer);
|
||||
let ResponseInputItem::Message { content, .. } = subset else {
|
||||
panic!("expected a user message");
|
||||
};
|
||||
assert_eq!(positions, HashMap::from([(1, 4)]));
|
||||
assert_eq!(
|
||||
content[0],
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(/*label_number*/ 1, &audio_path)
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
content[3],
|
||||
ContentItem::InputText {
|
||||
text: local_image_open_tag_text_with_path(/*label_number*/ 1, &path)
|
||||
}
|
||||
);
|
||||
|
||||
for mode in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
|
||||
let (message, positions) = ResolvedUserInput::into_message(&restored, mode);
|
||||
let ResponseInputItem::Message { content, .. } = message else {
|
||||
panic!("expected a user message");
|
||||
};
|
||||
assert_eq!(positions, HashMap::from([(4, 2), (7, 8), (8, 10)]));
|
||||
assert_eq!(
|
||||
content[0],
|
||||
ContentItem::InputText {
|
||||
text: "compare".into()
|
||||
}
|
||||
);
|
||||
let ContentItem::InputText { text } = &content[1] else {
|
||||
panic!("expected a read error");
|
||||
};
|
||||
assert!(text.starts_with(&format!(
|
||||
"Codex could not read the local image at `{}`:",
|
||||
missing.display()
|
||||
)));
|
||||
let image_url = match mode {
|
||||
LocalImagePreparation::Defer => {
|
||||
data_url_from_bytes("application/octet-stream", TINY_PNG_BYTES)
|
||||
}
|
||||
LocalImagePreparation::Process => codex_utils_image::load_for_prompt_bytes(
|
||||
&path,
|
||||
TINY_PNG_BYTES.to_vec(),
|
||||
codex_utils_image::PromptImageMode::Original,
|
||||
)?
|
||||
.into_data_url(),
|
||||
};
|
||||
assert_eq!(
|
||||
&content[2..],
|
||||
&[
|
||||
ContentItem::InputImage {
|
||||
image: image.clone(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL)
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: audio.clone()
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(/*label_number*/ 2, &audio_path)
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".into()
|
||||
}
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text()
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: local_image_open_tag_text_with_path(/*label_number*/ 3, &path)
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image: ImageReference::Inline { image_url },
|
||||
detail: Some(ImageDetail::Original)
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: image_close_tag_text()
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image: inline.clone(),
|
||||
detail: Some(ImageDetail::Original)
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user