mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Add durable user-message queue dispatch (#37204)
## What changed - Add a storage-neutral queue extension for listing, adding, editing, reordering, deleting, and explicitly starting queued user messages. - Dispatch queued messages in FIFO order when a thread becomes idle after a completed turn, while leaving them queued after interruptions or failures. - Wait for user input to be persisted before removing its queue item, and report hook rejection, task termination, and persistence failures distinctly. - Snapshot local image and audio attachments into validated data URLs before storing them, enforce input limits, and emit `ThreadQueueChanged` events after mutations. ## Testing - Cover queue mutations, automatic and explicit dispatch, interruption and failure behavior, hook rejection, invalid records, input limits, and local attachment snapshotting. GitOrigin-RevId: 14f39d7c8cbaccb767a6da84b82cced0edfcb4b9
This commit is contained in:
@@ -20,6 +20,7 @@ pub mod error;
|
||||
pub mod exec_output;
|
||||
pub mod items;
|
||||
mod legacy_events;
|
||||
pub mod local_media;
|
||||
pub mod mcp;
|
||||
pub mod mcp_approval_meta;
|
||||
pub mod memory_citation;
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
use std::io;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use codex_utils_image::MAX_PROMPT_IMAGE_INPUT_BYTES;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use codex_utils_image::load_for_prompt_bytes;
|
||||
|
||||
use crate::models::ImageDetail;
|
||||
use crate::user_input::UserInput;
|
||||
|
||||
/// Maximum accepted decoded byte length for prompt audio inputs.
|
||||
///
|
||||
/// This matches the Responses API audio input limit.
|
||||
pub const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024;
|
||||
|
||||
/// Snapshots local image and audio input into portable, validated data URLs.
|
||||
pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
match input {
|
||||
UserInput::LocalImage { path, detail } => {
|
||||
let image_detail = *detail;
|
||||
let mode = match image_detail {
|
||||
Some(ImageDetail::Original) => PromptImageMode::Original,
|
||||
Some(ImageDetail::Auto | ImageDetail::Low | ImageDetail::High) | None => {
|
||||
PromptImageMode::ResizeToFit
|
||||
}
|
||||
};
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
|
||||
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
|
||||
*input = UserInput::Image {
|
||||
image_url: image.into_data_url(),
|
||||
detail: image_detail,
|
||||
};
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
let mime = audio_mime_for_path(path).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format")
|
||||
})?;
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
*input = UserInput::Audio {
|
||||
audio_url: data_url_from_bytes(mime, &file_bytes),
|
||||
};
|
||||
}
|
||||
UserInput::Text { .. }
|
||||
| UserInput::Image { .. }
|
||||
| UserInput::Audio { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_bounded_local_media(path: &Path, max_bytes: usize, kind: &str) -> io::Result<Vec<u8>> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
if file.metadata()?.len() > max_bytes as u64 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("{kind} input exceeds {max_bytes} bytes"),
|
||||
));
|
||||
}
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
file.take(max_bytes as u64 + 1).read_to_end(&mut bytes)?;
|
||||
if bytes.len() > max_bytes {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("{kind} input exceeds {max_bytes} bytes"),
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn audio_mime_for_path(path: &Path) -> Option<&'static str> {
|
||||
let extension = path.extension()?.to_str()?;
|
||||
if extension.eq_ignore_ascii_case("wav") {
|
||||
Some("audio/wav")
|
||||
} else if extension.eq_ignore_ascii_case("mp3") {
|
||||
Some("audio/mpeg")
|
||||
} else if extension.eq_ignore_ascii_case("m4a") {
|
||||
Some("audio/mp4")
|
||||
} else if extension.eq_ignore_ascii_case("webm") {
|
||||
Some("audio/webm")
|
||||
} else if extension.eq_ignore_ascii_case("ogg") {
|
||||
Some("audio/ogg")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "local_media_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,128 @@
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tempfile::tempdir;
|
||||
|
||||
const TINY_PNG_BYTES: &[u8] = &[
|
||||
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
|
||||
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
|
||||
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn snapshots_local_image_user_input_with_requested_detail() -> Result<()> {
|
||||
let temp_dir = tempdir()?;
|
||||
let image_path = temp_dir.path().join("sample.png");
|
||||
std::fs::write(&image_path, TINY_PNG_BYTES)?;
|
||||
|
||||
for detail in [
|
||||
None,
|
||||
Some(ImageDetail::Auto),
|
||||
Some(ImageDetail::Low),
|
||||
Some(ImageDetail::High),
|
||||
Some(ImageDetail::Original),
|
||||
] {
|
||||
let mut input = UserInput::LocalImage {
|
||||
path: image_path.clone(),
|
||||
detail,
|
||||
};
|
||||
|
||||
snapshot_local_user_input(&mut input)?;
|
||||
|
||||
assert_eq!(
|
||||
UserInput::Image {
|
||||
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="
|
||||
.to_string(),
|
||||
detail,
|
||||
},
|
||||
input
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshots_local_audio_user_input_with_supported_mime_types() -> Result<()> {
|
||||
let temp_dir = tempdir()?;
|
||||
|
||||
for (extension, mime) in [
|
||||
("wav", "audio/wav"),
|
||||
("MP3", "audio/mpeg"),
|
||||
("m4a", "audio/mp4"),
|
||||
("webm", "audio/webm"),
|
||||
("ogg", "audio/ogg"),
|
||||
] {
|
||||
let audio_path = temp_dir.path().join(format!("sample.{extension}"));
|
||||
std::fs::write(&audio_path, b"audio")?;
|
||||
let mut input = UserInput::LocalAudio { path: audio_path };
|
||||
|
||||
snapshot_local_user_input(&mut input)?;
|
||||
|
||||
assert_eq!(
|
||||
UserInput::Audio {
|
||||
audio_url: format!("data:{mime};base64,YXVkaW8="),
|
||||
},
|
||||
input
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_local_media_without_changing_user_input() -> Result<()> {
|
||||
let temp_dir = tempdir()?;
|
||||
let invalid_image_path = temp_dir.path().join("invalid.png");
|
||||
let unsupported_audio_path = temp_dir.path().join("unsupported.flac");
|
||||
std::fs::write(&invalid_image_path, b"invalid image")?;
|
||||
std::fs::write(&unsupported_audio_path, b"audio")?;
|
||||
|
||||
let mut invalid_image = UserInput::LocalImage {
|
||||
path: invalid_image_path,
|
||||
detail: Some(ImageDetail::Original),
|
||||
};
|
||||
let original_image = invalid_image.clone();
|
||||
let image_error = snapshot_local_user_input(&mut invalid_image)
|
||||
.expect_err("an invalid local image must be rejected");
|
||||
assert_eq!(io::ErrorKind::Other, image_error.kind());
|
||||
assert_eq!(original_image, invalid_image);
|
||||
|
||||
let mut unsupported_audio = UserInput::LocalAudio {
|
||||
path: unsupported_audio_path,
|
||||
};
|
||||
let original_audio = unsupported_audio.clone();
|
||||
let audio_error = snapshot_local_user_input(&mut unsupported_audio)
|
||||
.expect_err("an unsupported local audio format must be rejected");
|
||||
assert_eq!(io::ErrorKind::InvalidData, audio_error.kind());
|
||||
assert_eq!(original_audio, unsupported_audio);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversized_local_media_without_reading_it() -> Result<()> {
|
||||
let temp_dir = tempdir()?;
|
||||
|
||||
for (name, limit, is_image) in [
|
||||
("oversized.png", MAX_PROMPT_IMAGE_INPUT_BYTES, true),
|
||||
("oversized.mp3", MAX_PROMPT_AUDIO_INPUT_BYTES, false),
|
||||
] {
|
||||
let path = temp_dir.path().join(name);
|
||||
std::fs::File::create(&path)?.set_len(limit as u64 + 1)?;
|
||||
let mut input = if is_image {
|
||||
UserInput::LocalImage { path, detail: None }
|
||||
} else {
|
||||
UserInput::LocalAudio { path }
|
||||
};
|
||||
let original = input.clone();
|
||||
|
||||
let error = snapshot_local_user_input(&mut input)
|
||||
.expect_err("oversized local media must be rejected before it is read");
|
||||
|
||||
assert_eq!(io::ErrorKind::InvalidData, error.kind());
|
||||
assert_eq!(original, input);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -12,6 +12,7 @@ use serde::Serialize;
|
||||
use serde::ser::Serializer;
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::local_media::audio_mime_for_path;
|
||||
use crate::permissions::FileSystemAccessMode;
|
||||
use crate::permissions::FileSystemPath;
|
||||
use crate::permissions::FileSystemSandboxEntry;
|
||||
@@ -30,6 +31,8 @@ use crate::mcp::CallToolResult;
|
||||
|
||||
mod executed_tool_calls;
|
||||
|
||||
pub use crate::local_media::MAX_PROMPT_AUDIO_INPUT_BYTES;
|
||||
pub use crate::local_media::snapshot_local_user_input;
|
||||
pub use executed_tool_calls::ExecutedToolCall;
|
||||
pub use executed_tool_calls::ExecutedToolCallArguments;
|
||||
pub use executed_tool_calls::ExecutedToolCallTruncation;
|
||||
@@ -1529,23 +1532,6 @@ pub enum LocalImagePreparation {
|
||||
Defer,
|
||||
}
|
||||
|
||||
fn audio_mime_for_path(path: &std::path::Path) -> Option<&'static str> {
|
||||
let extension = path.extension()?.to_str()?;
|
||||
if extension.eq_ignore_ascii_case("wav") {
|
||||
Some("audio/wav")
|
||||
} else if extension.eq_ignore_ascii_case("mp3") {
|
||||
Some("audio/mpeg")
|
||||
} else if extension.eq_ignore_ascii_case("m4a") {
|
||||
Some("audio/mp4")
|
||||
} else if extension.eq_ignore_ascii_case("webm") {
|
||||
Some("audio/webm")
|
||||
} else if extension.eq_ignore_ascii_case("ogg") {
|
||||
Some("audio/ogg")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn unsupported_audio_error_placeholder(path: &std::path::Path) -> ContentItem {
|
||||
ContentItem::InputText {
|
||||
text: format!(
|
||||
|
||||
@@ -1371,6 +1371,9 @@ pub enum EventMsg {
|
||||
/// Updated long-running goal metadata for the thread.
|
||||
ThreadGoalUpdated(ThreadGoalUpdatedEvent),
|
||||
|
||||
/// A durable thread-scoped user-message queue changed.
|
||||
ThreadQueueChanged(ThreadQueueChangedEvent),
|
||||
|
||||
/// Incremental MCP startup progress updates.
|
||||
McpStartupUpdate(McpStartupUpdateEvent),
|
||||
|
||||
@@ -2332,7 +2335,7 @@ pub struct AgentMessageEvent {
|
||||
pub memory_citation: Option<MemoryCitation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema, TS)]
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
|
||||
pub struct UserMessageEvent {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub client_id: Option<String>,
|
||||
@@ -4117,6 +4120,13 @@ pub struct ThreadGoalUpdatedEvent {
|
||||
pub goal: ThreadGoal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "protocol/")]
|
||||
pub struct ThreadQueueChangedEvent {
|
||||
pub thread_id: ThreadId,
|
||||
}
|
||||
|
||||
/// User's decision in response to an ExecApprovalRequest.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Display, JsonSchema, TS)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
|
||||
Reference in New Issue
Block a user