diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 61addafe3b..5df4f68445 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,6 +6,7 @@ /codex-rs/exec-server-protocol/ @openai/codex-core-agent-team /codex-rs/ext/extension-api/ @openai/codex-core-agent-team /codex-rs/prompts/ @openai/codex-core-agent-team +/codex-rs/user-input/ @openai/codex-core-agent-team /codex-rs/utils/path-uri/ @openai/codex-core-agent-team # Keep macOS AKV signing changes reviewed by Codex maintainers. diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5b3393b593..759a75a9d4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3093,6 +3093,7 @@ dependencies = [ "codex-test-binary-support", "codex-thread-store", "codex-tools", + "codex-user-input", "codex-utils-absolute-path", "codex-utils-audio", "codex-utils-cache", @@ -4765,6 +4766,7 @@ dependencies = [ "codex-protocol", "codex-state", "codex-thread-store", + "codex-user-input", "codex-utils-absolute-path", "core_test_support", "pretty_assertions", @@ -5294,6 +5296,7 @@ dependencies = [ "codex-state", "codex-terminal-detection", "codex-uds", + "codex-user-input", "codex-utils-absolute-path", "codex-utils-approval-presets", "codex-utils-cargo-bin", @@ -5391,6 +5394,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "codex-user-input" +version = "0.0.0" +dependencies = [ + "anyhow", + "codex-protocol", + "codex-utils-image", + "pretty_assertions", + "serde_json", + "tempfile", +] + [[package]] name = "codex-user-verification" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index c7f4180ddc..4bccad35f3 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -118,6 +118,7 @@ members = [ "otel-trace-websocket", "tcp-tunnel", "tui", + "user-input", "user-verification", "tools", "v8-poc", @@ -339,6 +340,7 @@ codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" } codex-utils-stream-parser = { path = "utils/stream-parser" } codex-utils-string = { path = "utils/string" } codex-utils-template = { path = "utils/template" } +codex-user-input = { path = "user-input" } codex-user-verification = { path = "user-verification" } codex-workload-identity = { path = "workload-identity" } codex-windows-sandbox = { path = "windows-sandbox-rs" } diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index b968a3613a..e8d10d7451 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c485813d02..7e78f7fb3d 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -73,6 +73,7 @@ codex-state = { workspace = true } codex-terminal-detection = { workspace = true } codex-thread-store = { workspace = true } codex-tools = { workspace = true } +codex-user-input = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-audio = { workspace = true } codex-utils-cache = { workspace = true } diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 84a8b920d3..5316f85434 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -56,11 +56,12 @@ use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::user_input::UserInput; use codex_rollout_trace::InferenceTraceContext; +use codex_user_input::LocalImagePreparation; +use codex_user_input::from_user_input; use codex_utils_output_truncation::approx_token_count; use tracing::error; @@ -286,7 +287,11 @@ async fn run_compact_task_inner_impl( sess.emit_turn_item_started(&turn_context, &compaction_item) .await; } - let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); + let initial_input_for_turn = from_user_input( + input, + LocalImagePreparation::Process, + &mut std::collections::HashMap::new(), + ); let mut history = sess.clone_history().await; let input_goal_ids = UserGoalUpdate::message_ids(history.raw_items()); diff --git a/codex-rs/core/src/session/file_audio_tests.rs b/codex-rs/core/src/session/file_audio_tests.rs index 5871dbf048..c9d0020f8a 100644 --- a/codex-rs/core/src/session/file_audio_tests.rs +++ b/codex-rs/core/src/session/file_audio_tests.rs @@ -4,8 +4,8 @@ use super::*; use crate::context_manager::ContextManager; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; -use codex_protocol::user_input::UserInput; use codex_protocol_test_support::uuid_v7_item_id_generator_for_test; use codex_utils_output_truncation::TruncationPolicy; use pretty_assertions::assert_eq; @@ -13,15 +13,17 @@ use pretty_assertions::assert_eq; #[test] fn history_sums_measured_and_unknown_file_audio() { let mut history = ContextManager::new(uuid_v7_item_id_generator_for_test()); - let item = ResponseInputItem::from( - ["short", "unknown"] - .map(|file_id| UserInput::Audio { + let item = ResponseInputItem::Message { + role: "user".to_string(), + content: ["short", "unknown"] + .map(|file_id| ContentItem::InputAudio { audio: AudioReference::File { file_id: file_id.into(), }, }) - .to_vec(), - ); + .into(), + phase: None, + }; history.record_items( [&ResponseItem::from(item)], TruncationPolicy::Tokens(10_000), diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 84408e7d19..a20b470a30 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -390,8 +390,6 @@ use codex_protocol::config_types::Settings; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::mcp::OPENAI_STANDARD_FORM_INPUT_EXTENSION_ID; -use codex_protocol::models::LocalImagePreparation; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; @@ -430,6 +428,8 @@ use codex_protocol::turn_input::TurnStartOptions; use codex_protocol::user_input::UserInput; use codex_skills_extension::HostSkillsService; use codex_tools::ToolName; +use codex_user_input::LocalImagePreparation; +use codex_user_input::from_user_input; use codex_utils_absolute_path::AbsolutePathBuf; #[cfg(test)] use codex_utils_stream_parser::ProposedPlanSegment; @@ -4016,7 +4016,7 @@ impl Session { input: Vec, user_image_content_indices: &mut HashMap, ) -> ResponseItem { - let mut item = ResponseItem::from(ResponseInputItem::from_user_input( + let mut item = ResponseItem::from(from_user_input( input, LocalImagePreparation::Defer, user_image_content_indices, diff --git a/codex-rs/core/src/session/turn_start_mcp_tests.rs b/codex-rs/core/src/session/turn_start_mcp_tests.rs index 1861172f00..c0f4084622 100644 --- a/codex-rs/core/src/session/turn_start_mcp_tests.rs +++ b/codex-rs/core/src/session/turn_start_mcp_tests.rs @@ -4,6 +4,7 @@ use super::*; use crate::McpRefreshReason; use codex_mcp::McpResourceClient; use codex_mcp::McpResourceClientCacheKey; +use codex_protocol::models::ResponseInputItem; use pretty_assertions::assert_eq; struct Recorder { diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index 8ebacae138..9b2fd2211d 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -2092,11 +2092,15 @@ async fn responses_websocket_uses_incremental_create_on_prefix() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); - let audio = ResponseItem::from(ResponseInputItem::from(vec![UserInput::Audio { - audio: AudioReference::File { - file_id: "file_audio".to_string(), - }, - }])); + let audio = ResponseItem::from(ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, + }], + phase: None, + }); let prompt_one = prompt_with_input(vec![audio.clone()]); let prompt_two = prompt_with_input(vec![ audio, diff --git a/codex-rs/ext/queue/Cargo.toml b/codex-rs/ext/queue/Cargo.toml index 4b5e5d04f4..8d2754113d 100644 --- a/codex-rs/ext/queue/Cargo.toml +++ b/codex-rs/ext/queue/Cargo.toml @@ -18,6 +18,7 @@ codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-protocol = { workspace = true } codex-thread-store = { workspace = true } +codex-user-input = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] } diff --git a/codex-rs/ext/queue/src/service.rs b/codex-rs/ext/queue/src/service.rs index 12671a0ef7..420181a04c 100644 --- a/codex-rs/ext/queue/src/service.rs +++ b/codex-rs/ext/queue/src/service.rs @@ -19,7 +19,6 @@ use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadResumeInput; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; -use codex_protocol::models::snapshot_local_user_input; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -31,6 +30,7 @@ use codex_thread_store::MAX_QUEUE_ITEMS; use codex_thread_store::QueueStore; use codex_thread_store::QueuedUserSubmissionRecord; use codex_thread_store::ThreadStoreError; +use codex_user_input::snapshot_local_user_input; use thiserror::Error; use tokio::sync::Mutex; use tokio::sync::OwnedMutexGuard; diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 095ed6a9e2..c4d2f9c11c 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -43,7 +43,6 @@ pub mod exec_output; pub mod hooks; pub mod items; mod legacy_events; -pub mod local_media; pub mod mcp; pub mod mcp_approval_meta; pub mod mcp_policy; diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 05273a0cac..1df16bc298 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -5,7 +5,6 @@ use std::path::Path; use std::sync::Arc; use codex_utils_image::PromptImageMode; -use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; use serde::Deserialize; use serde::Deserializer; @@ -14,7 +13,6 @@ use serde::ser::Serializer; use serde_with::serde_as; use ts_rs::TS; -use crate::local_media::audio_mime_for_path; use crate::permissions::FileSystemAccessMode; use crate::permissions::FileSystemPath; use crate::permissions::FileSystemSandboxEntry; @@ -25,7 +23,6 @@ use crate::permissions::LegacySandboxPolicyProjection; use crate::permissions::NetworkSandboxPolicy; use crate::permissions::RawFileSystemSandboxEntry; use crate::protocol::SandboxPolicy; -use crate::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_image::ImageProcessingError; use schemars::JsonSchema; @@ -43,8 +40,6 @@ mod mcp_content; use mcp_content::convert_mcp_content_to_items; -pub use crate::local_media::MAX_PROMPT_AUDIO_INPUT_BYTES; -pub use crate::local_media::snapshot_local_user_input; pub use crate::permission_profile_snapshot::PermissionProfileSnapshot; pub use crate::permission_profile_snapshot::ProfileWorkspaceRoot; pub use configuration_update::ConfigurationReasoning; @@ -1900,33 +1895,21 @@ fn should_serialize_reasoning_content(content: &Option } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum LocalMediaKind { - Audio, - Image, -} - -impl LocalMediaKind { - fn name(self) -> &'static str { - match self { - Self::Audio => "audio", - Self::Image => "image", - } - } -} - -fn local_media_error_placeholder( +fn local_image_error_placeholder( path: &std::path::Path, error: impl std::fmt::Display, - media_kind: LocalMediaKind, ) -> ContentItem { - let media_name = media_kind.name(); let path = path.display(); ContentItem::InputText { - text: format!("Codex could not read the local {media_name} at `{path}`: {error}"), + text: format!("Codex could not read the local image at `{path}`: {error}"), } } +/// Maximum accepted decoded byte length for prompt audio inputs. +/// +/// This matches the Responses API audio input limit. +pub const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; + pub const VIEW_IMAGE_TOOL_NAME: &str = "view_image"; const IMAGE_OPEN_TAG: &str = ""; @@ -1938,7 +1921,6 @@ const AUDIO_OPEN_TAG: &str = ""; const LOCAL_AUDIO_OPEN_TAG_PREFIX: &str = "