From d8d840d32a8ebf9ec75603e6aa9cd88a58a444a2 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 17 Nov 2026 03:51:12 +0000 Subject: [PATCH] Capture local attachments before queuing turn input (#61962) ## Why Queued input can outlive its local attachment files. Reading those files only when rendering a model message can lose media submitted while turn admission is pending. ## What changed - Introduce `ResolvedUserInput` to carry captured image bytes, audio data URLs, or read errors through session queues and agent input. - Resolve client attachments on the submitting host before queuing, using a blocking task for local file reads. - Render messages from captured payloads while preserving original paths and text spans for display, along with image position mappings. ## Testing Add coverage for rendering and serialization after source files are removed, steered audio retained during pending admission, and local audio errors and display content. GitOrigin-RevId: 4bbb6bda44167439a8cfdae15d3c476a7b1d336b --- codex-rs/Cargo.lock | 2 + codex-rs/core/src/agent/api.rs | 4 +- codex-rs/core/src/agent/control.rs | 7 +- codex-rs/core/src/agent/control/spawn.rs | 4 +- codex-rs/core/src/agent/control_tests.rs | 29 +- codex-rs/core/src/codex_delegate.rs | 6 +- codex-rs/core/src/codex_delegate_tests.rs | 10 +- codex-rs/core/src/codex_thread.rs | 80 ++++- codex-rs/core/src/compact.rs | 22 +- codex-rs/core/src/compact_tests.rs | 7 +- codex-rs/core/src/guardian/input_budget.rs | 43 ++- .../core/src/guardian/input_budget_tests.rs | 29 +- codex-rs/core/src/guardian/review_session.rs | 5 +- .../core/src/guardian/review_session_tests.rs | 15 +- codex-rs/core/src/hook_runtime.rs | 9 +- codex-rs/core/src/prompt_debug.rs | 8 +- .../realtime_conversation/transcript_tail.rs | 11 +- codex-rs/core/src/session/handlers.rs | 39 +-- codex-rs/core/src/session/input_queue.rs | 16 +- codex-rs/core/src/session/mod.rs | 57 ++-- .../core/src/session/notification_tests.rs | 9 +- codex-rs/core/src/session/review.rs | 6 +- .../core/src/session/step_activation_tests.rs | 3 +- codex-rs/core/src/session/submission.rs | 52 +++- codex-rs/core/src/session/tests.rs | 92 +++--- codex-rs/core/src/session/turn.rs | 28 +- codex-rs/core/src/session/turn_input.rs | 20 +- codex-rs/core/src/session/turn_input_agent.rs | 3 +- codex-rs/core/src/session/turn_input_tests.rs | 91 ++---- codex-rs/core/src/tasks/compact.rs | 11 +- codex-rs/core/src/tasks/mod_tests.rs | 5 +- codex-rs/core/src/tasks/review.rs | 4 +- .../thread_manager/helper_daybreak_tests.rs | 3 +- codex-rs/core/src/thread_manager_tests.rs | 4 +- .../tools/handlers/multi_agents/send_input.rs | 5 + .../src/tools/handlers/multi_agents/spawn.rs | 5 + .../src/tools/handlers/multi_agents_tests.rs | 31 +- codex-rs/core/tests/suite/agent_mailbox.rs | 23 +- codex-rs/core/tests/suite/client.rs | 72 ++++- .../core/tests/suite/host_turn_admission.rs | 53 ++++ .../tests/suite/scenarios_compaction_tests.rs | 14 +- codex-rs/protocol/src/protocol.rs | 11 - codex-rs/protocol/src/turn_input.rs | 14 +- codex-rs/user-input/Cargo.toml | 2 + codex-rs/user-input/src/lib.rs | 6 +- codex-rs/user-input/src/local_media.rs | 20 +- codex-rs/user-input/src/message.rs | 160 ---------- codex-rs/user-input/src/message_tests.rs | 15 +- .../user-input/src/resolved_user_input.rs | 279 ++++++++++++++++++ .../src/resolved_user_input_tests.rs | 189 ++++++++++++ 50 files changed, 1060 insertions(+), 573 deletions(-) delete mode 100644 codex-rs/user-input/src/message.rs create mode 100644 codex-rs/user-input/src/resolved_user_input.rs create mode 100644 codex-rs/user-input/src/resolved_user_input_tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 759a75a9d4..cf8d794bf9 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -5401,7 +5401,9 @@ dependencies = [ "anyhow", "codex-protocol", "codex-utils-image", + "derive_more 2.1.1", "pretty_assertions", + "serde", "serde_json", "tempfile", ] diff --git a/codex-rs/core/src/agent/api.rs b/codex-rs/core/src/agent/api.rs index 0f46d15de9..b4c0a8c5d7 100644 --- a/codex-rs/core/src/agent/api.rs +++ b/codex-rs/core/src/agent/api.rs @@ -24,8 +24,8 @@ use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TokenUsage; use codex_protocol::turn_input::TurnStartOptions; -use codex_protocol::user_input::UserInput; use codex_rollout_trace::ThreadTraceContext; +use codex_user_input::ResolvedUserInput; use futures::future::BoxFuture; // Keep dynamic dispatch a compile-time property of the contract. @@ -209,7 +209,7 @@ impl AgentInfo { /// User input starts or steers a turn; agent messages retain their sender and wake mode. pub enum AgentInput { - UserInput(Vec), + UserInput(Vec), Message { message: AgentMessage, mode: MessageDeliveryMode, diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index f6c37735c8..6d6ae79c8d 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -47,6 +47,7 @@ use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::user_input::UserInput; use codex_thread_store::ReadThreadParams; +use codex_user_input::ResolvedUserInput; use futures::StreamExt; use std::collections::VecDeque; use std::sync::Arc; @@ -137,13 +138,15 @@ impl LocalAgentControl { pub(crate) async fn send_input( &self, agent_id: ThreadId, - input: Vec, + input: Vec, start_options: TurnStartOptions, ) -> CodexResult { let state = self.runtime.upgrade()?; let thread = state.get_thread(agent_id).await?; let result = match thread - .start_or_steer_turn(TurnInputRequest::user_input(input).on_start(start_options)) + .start_or_steer_resolved_turn( + TurnInputRequest::::user_input(input).on_start(start_options), + ) .await { Ok(TurnInputSubmission::Started { turn_id, .. }) => Ok(turn_id), diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index bb3e8f1494..dae6dfb2a3 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -38,7 +38,7 @@ const AGENT_NAMES: &str = include_str!("../../../assets/agent/agent_names.txt"); /// unrepresentable. #[allow(clippy::large_enum_variant)] pub(super) enum SpawnInitialInput { - UserInput(Vec), + UserInput(Vec), InterAgentCommunication(InterAgentCommunication, AgentCommunicationContext), } @@ -208,7 +208,7 @@ impl LocalAgentControl { pub(crate) async fn spawn_agent( &self, config: Config, - initial_input: Vec, + initial_input: Vec, session_source: Option, ) -> CodexResult { let (spawned_agent, _) = Box::pin(self.spawn_agent_internal( diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 8634e21fbd..b22f68fced 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -141,7 +141,7 @@ impl LocalAgentControl { pub(crate) async fn spawn_agent_with_metadata( &self, config: Config, - initial_input: Vec, + initial_input: Vec, source: SessionSource, options: SpawnAgentOptions, ) -> CodexResult { @@ -181,8 +181,8 @@ async fn test_config() -> (TempDir, Config) { test_config_with_cli_overrides(Vec::new()).await } -fn text_input(text: &str) -> Vec { - vec![UserInput::Text { +fn text_input(text: &str) -> Vec { + vec![ResolvedUserInput::Text { text: text.to_string(), text_elements: Vec::new(), }] @@ -584,14 +584,7 @@ async fn assert_thread_not_loaded(manager: &ThreadManager, thread_id: ThreadId) async fn send_input_errors_when_manager_dropped() { let control = LocalAgentControl::default(); let err = control - .send_input( - ThreadId::new(), - vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], - Default::default(), - ) + .send_input(ThreadId::new(), text_input("hello"), Default::default()) .await .expect_err("send_input should fail without a manager"); assert_eq!( @@ -971,14 +964,7 @@ async fn send_input_errors_when_thread_missing() { let thread_id = ThreadId::new(); let err = harness .control - .send_input( - thread_id, - vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], - Default::default(), - ) + .send_input(thread_id, text_input("hello"), Default::default()) .await .expect_err("send_input should fail for missing thread"); assert_matches!( @@ -1063,10 +1049,7 @@ async fn send_input_submits_user_message() { caller: ThreadId::new(), target: AgentTarget::Id(thread_id), resume_config: harness.config.clone(), - input: AgentInput::UserInput(vec![UserInput::Text { - text: "hello from tests".to_string(), - text_elements: Vec::new(), - }]), + input: AgentInput::UserInput(text_input("hello from tests")), start_options: Default::default(), }) .await diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index fc08c00706..8fc45e1e89 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -12,7 +12,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnEnvironmentSelection; -use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use serde_json::Value; use std::time::Duration; use tokio::time::timeout; @@ -239,7 +239,7 @@ pub(crate) async fn run_codex_thread_one_shot( config: Config, auth_manager: Arc, models_manager: SharedModelsManager, - input: Vec, + input: Vec, parent_session: Arc, parent_ctx: Arc, cancel_token: CancellationToken, @@ -308,7 +308,7 @@ pub(crate) async fn run_codex_thread_one_shot( let _ = ops_tx .send(Box::new(Submission { id: "shutdown".to_string(), - op: Op::Shutdown {}, + op: Op::Shutdown.into(), turn_extension_init: None, trace: None, parent_turn_id: None, diff --git a/codex-rs/core/src/codex_delegate_tests.rs b/codex-rs/core/src/codex_delegate_tests.rs index 3f0e4a4906..49ca8630a0 100644 --- a/codex-rs/core/src/codex_delegate_tests.rs +++ b/codex-rs/core/src/codex_delegate_tests.rs @@ -127,11 +127,13 @@ async fn forward_events_filters_private_events_before_blocked_send_is_cancelled( ops.push(sub.op); } assert!( - ops.iter().any(|op| matches!(op, Op::Interrupt)), + ops.iter() + .any(|op| matches!(op.as_protocol(), Some(Op::Interrupt))), "expected Interrupt op after cancellation" ); assert!( - ops.iter().any(|op| matches!(op, Op::Shutdown)), + ops.iter() + .any(|op| matches!(op.as_protocol(), Some(Op::Shutdown))), "expected Shutdown op after cancellation" ); } @@ -153,7 +155,7 @@ async fn forward_ops_preserves_submission_trace_context() { let submission = Submission { id: "sub-1".to_string(), - op: Op::Interrupt, + op: Op::Interrupt.into(), turn_extension_init: None, trace: Some(codex_protocol::protocol::W3cTraceContext { traceparent: Some( @@ -174,7 +176,7 @@ async fn forward_ops_preserves_submission_trace_context() { .expect("forward_ops hung") .expect("forwarded submission missing"); assert_eq!("sub-1", forwarded.id); - assert!(matches!(forwarded.op, Op::Interrupt)); + assert!(matches!(forwarded.op.as_protocol(), Some(Op::Interrupt))); assert_eq!( forwarded.trace, Some(codex_protocol::protocol::W3cTraceContext { diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index b17ebccbbe..67287994a4 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -71,16 +71,19 @@ use codex_protocol::turn_input::RecoverTurnRequest; use codex_protocol::turn_input::StartIfIdleSubmission; use codex_protocol::turn_input::SteerSubmission; use codex_protocol::turn_input::SuspendTurnOutcome; +use codex_protocol::turn_input::TurnInput; use codex_protocol::turn_input::TurnInputMode; use codex_protocol::turn_input::TurnInputRequest; use codex_protocol::turn_input::TurnInputSubmission; use codex_protocol::turn_input::TurnStartOptions; +use codex_protocol::user_input::UserInput; use codex_thread_store::PersistContext; use codex_thread_store::StoredThread; use codex_thread_store::StoredThreadHistory; use codex_thread_store::ThreadMetadataPatch; use codex_thread_store::ThreadStoreError; use codex_thread_store::ThreadStoreResult; +use codex_user_input::ResolvedUserInput; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::LegacyAppPathString; use codex_utils_path_uri::PathUri; @@ -232,6 +235,54 @@ pub struct BackgroundTerminalInfo { pub cwd: PathUri, } +/// Captures client attachment paths before an ordered request enters the session. +pub(crate) async fn resolve_client_turn_input( + request: TurnInputRequest, +) -> CodexResult> { + let input = match request.input { + TurnInput::UserInput { content, client_id } => TurnInput::UserInput { + content: resolve_client_inputs(content).await?, + client_id, + }, + TurnInput::ResponseItem(item) => TurnInput::ResponseItem(item), + TurnInput::AnnotatedResponseItem(item) => TurnInput::AnnotatedResponseItem(item), + TurnInput::InterAgentCommunication(message) => TurnInput::InterAgentCommunication(message), + }; + Ok(TurnInputRequest { + input, + user_item_uuid: request.user_item_uuid, + user_input_extension_data: request.user_input_extension_data, + thread_settings: request.thread_settings, + start: request.start, + additional_context: request.additional_context, + responsesapi_client_metadata: request.responsesapi_client_metadata, + trace: request.trace, + }) +} + +/// Captures local media off the async executor. Inline inputs do not need a blocking task. +pub(crate) async fn resolve_client_inputs( + content: Vec, +) -> CodexResult> { + let has_local_media = content.iter().any(|item| { + matches!( + item, + UserInput::LocalImage { .. } | UserInput::LocalAudio { .. } + ) + }); + let resolve = move || { + content + .into_iter() + .map(ResolvedUserInput::from_client) + .collect() + }; + if has_local_media { + Ok(tokio::task::spawn_blocking(resolve).await?) + } else { + Ok(resolve()) + } +} + /// Conduit for the bidirectional stream of messages that compose a thread /// (formerly called a conversation) in Codex. impl CodexThread { @@ -691,7 +742,7 @@ impl CodexThread { .tx_sub .send(Box::new(Submission { id: new_submission_id(), - op: Op::SuspendTurnAndShutdown { reply }, + op: Op::SuspendTurnAndShutdown { reply }.into(), turn_extension_init: None, trace: current_span_w3c_trace_context(), parent_turn_id: None, @@ -736,6 +787,33 @@ impl CodexThread { &self, request: impl Into>, mode: TurnInputMode, + ) -> CodexResult { + let WithTurnExtensionData { + request, + turn_extension_init, + } = request.into(); + self.submit_resolved_turn_input_with_mode( + WithTurnExtensionData { + request: resolve_client_turn_input(request).await?, + turn_extension_init, + }, + mode, + ) + .await + } + + pub(crate) async fn start_or_steer_resolved_turn( + &self, + request: TurnInputRequest, + ) -> CodexResult { + self.submit_resolved_turn_input_with_mode(request.into(), TurnInputMode::StartOrSteer) + .await + } + + async fn submit_resolved_turn_input_with_mode( + &self, + request: WithTurnExtensionData>, + mode: TurnInputMode, ) -> CodexResult { if !matches!(mode, TurnInputMode::Steer { .. }) { self.ensure_execution_capacity_for_turn_start( diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 5316f85434..497a8f5fa0 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -58,10 +58,9 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; -use codex_protocol::user_input::UserInput; use codex_rollout_trace::InferenceTraceContext; use codex_user_input::LocalImagePreparation; -use codex_user_input::from_user_input; +use codex_user_input::ResolvedUserInput; use codex_utils_output_truncation::approx_token_count; use tracing::error; @@ -136,11 +135,7 @@ pub(crate) async fn run_inline_auto_compact_task( .as_deref() .unwrap_or(SUMMARIZATION_PROMPT) .to_string(); - let input = vec![UserInput::Text { - text: prompt, - // Compaction prompt is synthesized; no UI element ranges to preserve. - text_elements: Vec::new(), - }]; + let input = vec![ResolvedUserInput::text(prompt)]; run_compact_task_inner( sess, @@ -162,7 +157,7 @@ pub(crate) async fn run_compact_task( sess: Arc, step_context: Arc, world_state: Arc, - input: Vec, + input: Vec, ) -> CodexResult<()> { run_compact_task_inner( sess.clone(), @@ -185,7 +180,7 @@ async fn run_compact_task_inner( sess: Arc, step_context: Arc, replacement_step_context: Arc, - input: Vec, + input: Vec, world_state: Arc, compaction_metadata: CompactionTurnMetadata, ) -> CodexResult { @@ -273,7 +268,7 @@ async fn run_compact_task_inner_impl( sess: Arc, step_context: Arc, replacement_step_context: Arc, - input: Vec, + input: Vec, world_state: Arc, compaction_metadata: CompactionTurnMetadata, ) -> CodexResult { @@ -287,11 +282,8 @@ async fn run_compact_task_inner_impl( sess.emit_turn_item_started(&turn_context, &compaction_item) .await; } - let initial_input_for_turn = from_user_input( - input, - LocalImagePreparation::Process, - &mut std::collections::HashMap::new(), - ); + let (initial_input_for_turn, _) = + ResolvedUserInput::into_message(&input, LocalImagePreparation::Process); let mut history = sess.clone_history().await; let input_goal_ids = UserGoalUpdate::message_ids(history.raw_items()); diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index 32e8026225..9efe51eaa8 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -185,10 +185,9 @@ async fn local_compaction_retries_preserve_instructions_and_tool_metadata( Arc::clone(&session), step, world_state, - vec![UserInput::Text { - text: "Summarize the conversation.".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text( + "Summarize the conversation.".to_string(), + )], ) .await?; diff --git a/codex-rs/core/src/guardian/input_budget.rs b/codex-rs/core/src/guardian/input_budget.rs index 2a3de9508e..ca2768efea 100644 --- a/codex-rs/core/src/guardian/input_budget.rs +++ b/codex-rs/core/src/guardian/input_budget.rs @@ -22,6 +22,7 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use crate::context::ContextualUserFragment; use crate::context::GuardianBudgetOmission; @@ -259,7 +260,7 @@ pub(crate) async fn finalize( // Reserve the real input annotations as well as the shared content cost. // Removing optional items can only reduce this metadata. The remaining // margin covers IDs and timestamps assigned when recording the message. - let mut framing = session.response_item_from_user_input(content.clone()); + let mut framing = session.response_item_from_user_input(content); if let ResponseItem::Message { content, .. } = &mut framing { content.clear(); } @@ -314,9 +315,28 @@ pub(crate) async fn finalize( let observations = std::mem::take(&mut context.truncations); match context.clone().into_annotated_user_inputs() { Ok((user_input, metadata)) => { + let user_input = user_input + .into_iter() + .map(|item| match item { + UserInput::Text { + text, + text_elements, + } => Ok(ResolvedUserInput::Text { + text, + text_elements, + }), + UserInput::Image { image, detail } => Ok(ResolvedUserInput::Image { + image: image.into(), + detail, + }), + _ => Err(CodexErr::InvalidRequest( + "Unsupported Guardian input content".to_owned(), + )), + }) + .collect::>>()?; if metadata.is_some() { input[0] = TurnInput::ResponseItem(codex_history::ResponseItemEnvelope { - item: session.response_item_from_user_input(user_input), + item: session.response_item_from_user_input(&user_input), metadata, }); } else if let TurnInput::UserInput { content, .. } = &mut input[0] { @@ -338,14 +358,15 @@ pub(crate) async fn finalize( let user_input = content .iter() .map(|item| match item { - ContentItem::InputText { text } => Ok(UserInput::Text { - text: text.clone(), - text_elements: Vec::new(), - }), - ContentItem::InputImage { image, detail } => Ok(UserInput::Image { - image: image.clone(), - detail: *detail, - }), + ContentItem::InputText { text } => { + Ok(ResolvedUserInput::text(text.clone())) + } + ContentItem::InputImage { image, detail } => { + Ok(ResolvedUserInput::Image { + image: std::sync::Arc::new(image.clone()), + detail: *detail, + }) + } ContentItem::OutputText { .. } | ContentItem::InputAudio { .. } => { Err(CodexErr::InvalidRequest( "Unsupported Guardian input content".to_owned(), @@ -353,7 +374,7 @@ pub(crate) async fn finalize( } }) .collect::>>()?; - envelope.item = session.response_item_from_user_input(user_input); + envelope.item = session.response_item_from_user_input(&user_input); } Ok(TurnInput::ResponseItem(envelope)) }) diff --git a/codex-rs/core/src/guardian/input_budget_tests.rs b/codex-rs/core/src/guardian/input_budget_tests.rs index 2f4496bde5..f0f678a4b7 100644 --- a/codex-rs/core/src/guardian/input_budget_tests.rs +++ b/codex-rs/core/src/guardian/input_budget_tests.rs @@ -57,7 +57,13 @@ async fn finalization_exports_loss_totals_only_for_accepted_input(window: i64) { .collect(); let mut input = vec![TurnInput::UserInput { metadata: Default::default(), - content: context.clone().into_user_inputs().unwrap(), + content: context + .clone() + .into_user_inputs() + .unwrap() + .into_iter() + .map(ResolvedUserInput::from_client) + .collect(), client_id: None, }]; session @@ -188,7 +194,13 @@ fn required_context(text: String) -> ComposedContext { async fn cancelled_startup_does_not_record_unselected_review_evidence() { let (session, turn, events) = crate::session::tests::make_session_and_context_with_rx().await; let context = required_context("unselected review evidence ".repeat(/*n*/ 10_000)); - let content = context.clone().into_user_inputs().unwrap(); + let content = context + .clone() + .into_user_inputs() + .unwrap() + .into_iter() + .map(ResolvedUserInput::from_client) + .collect(); session .services .thread_extension_data @@ -272,7 +284,13 @@ async fn finalization_overflow_marks_the_reviewer_exhausted() { let context = required_context("required action".to_owned()); let mut input = vec![TurnInput::UserInput { metadata: Default::default(), - content: context.clone().into_user_inputs().unwrap(), + content: context + .clone() + .into_user_inputs() + .unwrap() + .into_iter() + .map(ResolvedUserInput::from_client) + .collect(), client_id: None, }]; session @@ -318,10 +336,7 @@ async fn compacted_review_restores_originals_once_and_persists_the_request_prefi .record_user_prompt_and_emit_turn_item( &step.turn, &step.settings.model_info, - &[codex_protocol::user_input::UserInput::Text { - text: instruction.clone(), - text_elements: vec![], - }], + &[ResolvedUserInput::text(instruction.clone())], /*client_id*/ None, crate::session::UserInputMetadata { acceptance_order: Some(0), diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index bdb84ff2fe..d99aab1e55 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -836,7 +836,10 @@ impl codex_guardian_reviewer::ReviewerRuntime for GuardianReviewSession { async fn submit_turn(&self, request: TurnInputRequest) -> anyhow::Result { Ok(self .io - .submit_turn_input(request, TurnInputMode::StartIfIdle) + .submit_turn_input( + crate::codex_thread::resolve_client_turn_input(request).await?, + TurnInputMode::StartIfIdle, + ) .await?) } diff --git a/codex-rs/core/src/guardian/review_session_tests.rs b/codex-rs/core/src/guardian/review_session_tests.rs index 8d130b3fa9..11e3fc606d 100644 --- a/codex-rs/core/src/guardian/review_session_tests.rs +++ b/codex-rs/core/src/guardian/review_session_tests.rs @@ -3,6 +3,7 @@ use crate::agents_md_manager::AgentsMdManager; use crate::context::ContextualUserFragment; use crate::context_manager::ContextManager; use crate::session::Submission; +use crate::session::submission::SubmissionOp; use codex_guardian_reviewer::GuardianReviewResponse; use codex_guardian_reviewer::ReviewerRequest; use codex_guardian_reviewer::guardian_output_contract_prompt; @@ -137,7 +138,7 @@ async fn run_review_preserves_evidence_during_parent_compaction() { Some(http::HeaderValue::from_static("__oailb=original-parent")) ); let id = submission.id; - let Op::TurnInput { request, reply, .. } = submission.op else { + let SubmissionOp::TurnInput { request, reply, .. } = submission.op else { panic!("expected reviewer prompt"); }; let codex_protocol::turn_input::TurnInput::UserInput { content, .. } = request.input else { @@ -711,7 +712,7 @@ async fn run_review_on_reused_session_waits_for_submitted_turn() { }); let submission = rx_sub.recv().await.expect("guardian submission"); let id = submission.id; - let Op::TurnInput { reply, .. } = submission.op else { + let SubmissionOp::TurnInput { reply, .. } = submission.op else { panic!("expected turn-input submission"); }; reply @@ -779,7 +780,7 @@ async fn run_review_removes_trunk_when_event_stream_is_broken() { tokio::spawn(async move { run_guardian_review_session(manager_for_review, params).await }); let submission = rx_sub.recv().await.expect("guardian submission"); let id = submission.id; - let Op::TurnInput { reply, .. } = submission.op else { + let SubmissionOp::TurnInput { reply, .. } = submission.op else { panic!("expected turn-input submission"); }; reply @@ -950,7 +951,7 @@ async fn wait_for_guardian_review_timeout_drains_expected_turn_after_stale_termi let tx_interrupt_event = tx_event.clone(); let interrupt_response = tokio::spawn(async move { let submission = rx_sub.recv().await.expect("interrupt submission"); - assert!(matches!(submission.op, Op::Interrupt)); + assert!(matches!(submission.op.as_protocol(), Some(Op::Interrupt))); tx_interrupt_event .send(turn_aborted_event("current-turn")) .await @@ -991,7 +992,7 @@ async fn wait_for_guardian_review_cancel_drains_expected_turn_after_stale_termin let tx_interrupt_event = tx_event.clone(); let interrupt_response = tokio::spawn(async move { let submission = rx_sub.recv().await.expect("interrupt submission"); - assert!(matches!(submission.op, Op::Interrupt)); + assert!(matches!(submission.op.as_protocol(), Some(Op::Interrupt))); tx_interrupt_event .send(turn_aborted_event("current-turn")) .await @@ -1165,13 +1166,13 @@ async fn compaction_and_review_submissions_share_execution_observer() { .expect("child submission deadline") .expect("child submission"); if index == 0 { - assert!(matches!(submission.op, Op::Compact)); + assert!(matches!(submission.op.as_protocol(), Some(Op::Compact))); assert_eq!( child_session.thread_config_snapshot().await.request_country, expected_country ); } else { - let Op::TurnInput { reply, .. } = submission.op else { + let SubmissionOp::TurnInput { reply, .. } = submission.op else { panic!("expected review input after compaction"); }; reply diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 776952eade..c0e7fe3d18 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -31,7 +31,6 @@ use codex_otel::HOOK_RUN_DURATION_METRIC; use codex_otel::HOOK_RUN_METRIC; use codex_protocol::items::FunctionCallOutputItem; use codex_protocol::items::TurnItem; -use codex_protocol::items::UserMessageItem; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; @@ -54,6 +53,7 @@ use codex_protocol::protocol::WarningEvent; use codex_rollout::state_db; use codex_thread_store::PersistContext; use codex_thread_store::ReadThreadParams; +use codex_user_input::ResolvedUserInput; use codex_utils_absolute_path::AbsolutePathBuf; use serde_json::Map; use serde_json::Value; @@ -722,7 +722,10 @@ pub(crate) async fn inspect_pending_input( transcript_path: sess.hook_transcript_path().await, model: turn_context.model_info().slug.clone(), permission_mode: hook_permission_mode(turn_context.approval_policy()), - prompt: UserMessageItem::new(content).message(), + prompt: content + .iter() + .filter_map(ResolvedUserInput::as_text) + .collect(), }; let hooks = with_environment_hooks( sess, @@ -763,8 +766,6 @@ pub(crate) async fn record_pending_input( client_id, metadata, } => { - // Prompt telemetry can contain full text. Emit it only for prompts selected for recording. - turn_context.session_telemetry.user_prompt(&content); sess.record_user_prompt_and_emit_turn_item( turn_context.as_ref(), model_info, diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index 9a8b6d8278..16912fe62c 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -11,6 +11,7 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use tokio_util::sync::CancellationToken; use crate::ThreadStoreBuilder; @@ -31,6 +32,7 @@ pub async fn build_prompt_input( extensions: Arc>, user_instructions_provider: Arc, ) -> CodexResult> { + let input = crate::codex_thread::resolve_client_inputs(input).await?; config.ephemeral = true; let auth_manager = @@ -86,9 +88,9 @@ pub async fn build_prompt_input( output } -pub(crate) async fn build_prompt_input_from_session( +async fn build_prompt_input_from_session( sess: &Arc, - input: Vec, + input: Vec, ) -> CodexResult> { let turn_context = sess.new_default_turn().await; // Prompt debugging builds a standalone request without entering run_turn. @@ -99,7 +101,7 @@ pub(crate) async fn build_prompt_input_from_session( .await?; if !input.is_empty() { - let response_item = sess.response_item_from_user_input(input); + let response_item = sess.response_item_from_user_input(&input); sess.record_conversation_items( turn_context.as_ref(), &step_context.settings.model_info, diff --git a/codex-rs/core/src/realtime_conversation/transcript_tail.rs b/codex-rs/core/src/realtime_conversation/transcript_tail.rs index 24d2d9671c..c9e8b62850 100644 --- a/codex-rs/core/src/realtime_conversation/transcript_tail.rs +++ b/codex-rs/core/src/realtime_conversation/transcript_tail.rs @@ -9,9 +9,9 @@ use codex_protocol::protocol::HasLegacyEvent; use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; -use codex_protocol::user_input::UserInput; use codex_rollout::RolloutItem; use codex_thread_store::PersistContext; +use codex_user_input::ResolvedUserInput; use std::sync::Arc; #[expect( @@ -30,10 +30,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<( .map(|task| Arc::clone(&task.turn_context)); let recording_turn = running_turn.is_none(); let turn = running_turn.unwrap_or(recording_context); - let content = vec![UserInput::Text { - text, - text_elements: Vec::new(), - }]; + let input = ResolvedUserInput::text(text); let now = chrono::Utc::now(); if recording_turn { let started = EventMsg::TurnStarted(TurnStartedEvent { @@ -52,7 +49,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<( .record_started_turn(&turn.sub_id, /*attribution*/ None) .await; } - let item = session.response_item_from_user_input(content.clone()); + let item = session.response_item_from_user_input(std::slice::from_ref(&input)); session .record_conversation_items(&turn, turn.model_info(), &[item]) .await; @@ -63,7 +60,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<( content_token_count: None, thread_id: session.thread_id, turn_id: turn.sub_id.clone(), - item: TurnItem::UserMessage(UserMessageItem::new(&content)), + item: TurnItem::UserMessage(UserMessageItem::new(&[input.to_user_input()])), started_at_ms: Some(now.timestamp_millis()), completed_at_ms: now.timestamp_millis(), }); diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 7f0bbbd612..40b9fad8c3 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -1,5 +1,6 @@ use super::Submission; use super::sampling_interrupt::SamplingInterrupt; +use super::submission::SubmissionOp; use crate::McpRefreshReason; use crate::WithTurnExtensionData; use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; @@ -549,7 +550,8 @@ pub(super) async fn submission_loop( }, async { while let Ok(sub) = rx_sub.recv().await { - if let Op::InterAgentCommunication { communication, start_options } = sub.op + if let SubmissionOp::Protocol(op) = sub.op + && let Op::InterAgentCommunication { communication, start_options } = *op && sub.message_delivery.as_ref().is_some_and(crate::agent_communication::MessageDelivery::awaits_result_write) { let mail = crate::PendingAgentMessage::new(communication, sub.message_delivery); @@ -611,7 +613,23 @@ pub(super) async fn submission_loop( ); let dispatch_span = submission_dispatch_span(&sub); let should_exit = async { - match sub.op { + let op = match sub.op { + SubmissionOp::TurnInput { + request, + mode, + reply, + } => { + let request = WithTurnExtensionData { + request: *request, + turn_extension_init: sub.turn_extension_init, + }; + let result = turn_input::handle(&sess, request, mode, sub.id.clone()).await; + let _ = reply.send(result); + return false; + } + SubmissionOp::Protocol(op) => *op, + }; + match op { Op::Interrupt => { interrupt(&sess).await; false @@ -702,19 +720,6 @@ pub(super) async fn submission_loop( realtime_conversation_list_voices(&sess, sub.id.clone()).await; false } - Op::TurnInput { - request, - mode, - reply, - } => { - let request = WithTurnExtensionData { - request: *request, - turn_extension_init: sub.turn_extension_init, - }; - let result = turn_input::handle(&sess, request, mode, sub.id.clone()).await; - let _ = reply.send(result); - false - } Op::RecoverTurn { thread_settings, start_options, @@ -971,8 +976,8 @@ async fn approve_guardian_denied_action(sess: &Arc, event: GuardianAsse pub(super) fn submission_dispatch_span(sub: &Submission) -> tracing::Span { let op_name = sub.op.kind(); let span_name = format!("op.dispatch.{op_name}"); - let dispatch_span = match &sub.op { - Op::RealtimeConversationAudio(_) => { + let dispatch_span = match sub.op.as_protocol() { + Some(Op::RealtimeConversationAudio(_)) => { debug_span!( "submission_dispatch", otel.name = span_name.as_str(), diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index daf5ad5839..e395ec74c8 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -8,7 +8,7 @@ use codex_history::ResponseItemEnvelope; use codex_protocol::ThreadId; use codex_protocol::models::ResponseItem; use codex_protocol::turn_input::TurnStartOptions; -use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -43,7 +43,7 @@ pub struct UserInputMetadata { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum TurnInput { UserInput { - content: Vec, + content: Vec, client_id: Option, #[serde(flatten)] metadata: UserInputMetadata, @@ -569,7 +569,7 @@ mod tests { use codex_history::CodexHarnessMetadata; use codex_protocol::AgentPath; use codex_protocol::protocol::InterAgentCommunication; - use codex_protocol::user_input::UserInput; + use codex_user_input::ResolvedUserInput; use pretty_assertions::assert_eq; #[test] @@ -697,10 +697,7 @@ mod tests { &turn_state, vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "steer".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("steer".to_string())], client_id: None, }], ) @@ -789,10 +786,7 @@ mod tests { &turn_state, vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "already pending".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("already pending".to_string())], client_id: None, }], ) diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index a20b470a30..7b822ad61c 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -263,9 +263,10 @@ mod guardian_checkpoint; mod handlers; pub(crate) mod inject; mod reasoning_effort; -mod submission; +pub(crate) mod submission; pub(crate) use reasoning_effort::RequestEffortUsage; pub(crate) use submission::Submission; +use submission::SubmissionOp; mod cloud_plugins; mod input_queue; mod mcp; @@ -429,7 +430,7 @@ use codex_protocol::user_input::UserInput; use codex_skills_extension::HostSkillsService; use codex_tools::ToolName; use codex_user_input::LocalImagePreparation; -use codex_user_input::from_user_input; +use codex_user_input::ResolvedUserInput; use codex_utils_absolute_path::AbsolutePathBuf; #[cfg(test)] use codex_utils_stream_parser::ProposedPlanSegment; @@ -1203,7 +1204,7 @@ impl SessionIo { } = op.into(); let sub = Submission { id: id.clone(), - op, + op: op.into(), turn_extension_init, trace, parent_turn_id, @@ -1233,7 +1234,7 @@ impl SessionIo { /// session loop exits before replying, the caller gets `InternalAgentDied`. pub(crate) async fn submit_turn_input( &self, - request: impl Into>, + request: impl Into>>, mode: TurnInputMode, ) -> CodexResult { let id = new_submission_id(); @@ -1245,7 +1246,7 @@ impl SessionIo { let trace = request.trace.take(); self.submit_with_id(Box::new(Submission { id, - op: Op::TurnInput { + op: SubmissionOp::TurnInput { request: Box::new(request), mode, reply: reply_tx, @@ -1276,7 +1277,8 @@ impl SessionIo { thread_settings, start_options, reply: reply_tx, - }, + } + .into(), trace, turn_extension_init, parent_turn_id: None, @@ -4007,20 +4009,21 @@ impl Session { assign_missing_item_id(item, &self.item_id_generator); } - pub(crate) fn response_item_from_user_input(&self, input: Vec) -> ResponseItem { - self.response_item_from_user_input_with_image_positions(input, &mut HashMap::new()) + pub(crate) fn response_item_from_user_input( + &self, + input: &[ResolvedUserInput], + ) -> ResponseItem { + self.response_item_from_user_input_with_image_positions(input) + .0 } fn response_item_from_user_input_with_image_positions( &self, - input: Vec, - user_image_content_indices: &mut HashMap, - ) -> ResponseItem { - let mut item = ResponseItem::from(from_user_input( - input, - LocalImagePreparation::Defer, - user_image_content_indices, - )); + input: &[ResolvedUserInput], + ) -> (ResponseItem, HashMap) { + let (message, image_positions) = + ResolvedUserInput::into_message(input, LocalImagePreparation::Defer); + let mut item = ResponseItem::from(message); if let ResponseItem::Message { content, internal_chat_message_metadata_passthrough, @@ -4049,7 +4052,7 @@ impl Session { ..Default::default() }); } - item + (item, image_positions) } /// Appends to history, persists the prepared items, then notifies raw-item observers. @@ -5944,19 +5947,20 @@ impl Session { &self, turn_context: &TurnContext, model_info: &ModelInfo, - input: &[UserInput], + input: &[ResolvedUserInput], client_id: Option, metadata: UserInputMetadata, persist_context: PersistContext, ) { - // Persist the user message to history, but emit the turn item from `UserInput` so - // UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry - // those spans, and `record_response_item_and_emit_turn_item` would drop them. - let mut user_image_content_indices = HashMap::new(); - let mut response_item = self.response_item_from_user_input_with_image_positions( - input.to_vec(), - &mut user_image_content_indices, - ); + let display_input = input + .iter() + .map(ResolvedUserInput::to_user_input) + .collect::>(); + let mut user_message_item = UserMessageItem::new(&display_input); + // Emit full prompt telemetry only for inputs selected for recording. + turn_context.session_telemetry.user_prompt(&display_input); + let (mut response_item, user_image_content_indices) = + self.response_item_from_user_input_with_image_positions(input); if let Some(uuid) = metadata.response_item_uuid { response_item.set_id(Some(ResponseItemId::for_admitted_user_input(uuid))); } @@ -5985,7 +5989,6 @@ impl Session { }], ) .await; - let mut user_message_item = UserMessageItem::new(input); apply_prepared_image_file_ids( &mut user_message_item, &prepared_items, diff --git a/codex-rs/core/src/session/notification_tests.rs b/codex-rs/core/src/session/notification_tests.rs index eb8dfed84b..76d9483b05 100644 --- a/codex-rs/core/src/session/notification_tests.rs +++ b/codex-rs/core/src/session/notification_tests.rs @@ -34,10 +34,9 @@ async fn task_finish_preserves_notification_budget_and_queued_user_input() { }]) .await .unwrap(); - let user_input = vec![UserInput::Text { - text: "keep this queued user input".to_string(), - text_elements: Vec::new(), - }]; + let user_input = vec![ResolvedUserInput::text( + "keep this queued user input".to_string(), + )]; assert!(matches!( submit_steer_only(&session, user_input.clone(), &turn.sub_id).await, TurnInputSubmission::Steered { .. } @@ -78,7 +77,7 @@ async fn task_finish_preserves_notification_budget_and_queued_user_input() { &text, codex_utils_output_truncation::TruncationPolicy::Tokens(120), )), - session.response_item_from_user_input(user_input), + session.response_item_from_user_input(&user_input), late[0].clone(), ]; let history = session.clone_history().await; diff --git a/codex-rs/core/src/session/review.rs b/codex-rs/core/src/session/review.rs index f73f895efd..40dba8df88 100644 --- a/codex-rs/core/src/session/review.rs +++ b/codex-rs/core/src/session/review.rs @@ -143,11 +143,7 @@ pub(super) async fn spawn_review_thread( // Seed the child task with the review prompt as the initial user message. let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: review_prompt, - // Review prompt is synthesized; no UI element ranges to preserve. - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text(review_prompt)], client_id: None, }]; let tc = Arc::new(review_turn_context); diff --git a/codex-rs/core/src/session/step_activation_tests.rs b/codex-rs/core/src/session/step_activation_tests.rs index 2366295b76..3aba936d32 100644 --- a/codex-rs/core/src/session/step_activation_tests.rs +++ b/codex-rs/core/src/session/step_activation_tests.rs @@ -518,7 +518,8 @@ fn settings_submission( turn_id: turn_id.to_string(), update, reply, - }, + } + .into(), trace: None, parent_turn_id: None, root_turn_id: None, diff --git a/codex-rs/core/src/session/submission.rs b/codex-rs/core/src/session/submission.rs index 01f52556ff..2bb05bfe48 100644 --- a/codex-rs/core/src/session/submission.rs +++ b/codex-rs/core/src/session/submission.rs @@ -1,14 +1,64 @@ //! Core-owned queue metadata; forwarding transfers the residency guard with the operation. use codex_extension_api::ExtensionDataInit; +use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::Op; use codex_protocol::protocol::W3cTraceContext; +use codex_protocol::turn_input::TurnInputMode; +use codex_protocol::turn_input::TurnInputRequest; +use codex_protocol::turn_input::TurnInputSubmission; +use codex_user_input::ResolvedUserInput; use tokio::sync::OwnedRwLockReadGuard; +use tokio::sync::oneshot; + +/// Turn input crosses this queue only after the submitting host has resolved its attachments. +pub(crate) enum SubmissionOp { + Protocol(Box), + TurnInput { + request: Box>, + mode: TurnInputMode, + reply: oneshot::Sender>, + }, +} + +impl From for SubmissionOp { + fn from(op: Op) -> Self { + Self::Protocol(Box::new(op)) + } +} + +impl SubmissionOp { + pub(crate) fn as_protocol(&self) -> Option<&Op> { + match self { + Self::Protocol(op) => Some(op), + Self::TurnInput { .. } => None, + } + } + + pub(super) fn kind(&self) -> &'static str { + match self { + Self::Protocol(op) => op.kind(), + Self::TurnInput { .. } => "turn_input", + } + } +} + +impl std::fmt::Debug for SubmissionOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Protocol(op) => std::fmt::Debug::fmt(op, f), + Self::TurnInput { request, .. } => f + .debug_struct("TurnInput") + .field("requested_settings", &request.thread_settings) + .finish(), + } + } +} #[derive(Debug)] pub(crate) struct Submission { pub id: String, - pub op: Op, + pub op: SubmissionOp, /// Accepted with settings, never separately from the request that supplied them. pub turn_extension_init: Option, /// Optional W3C trace carrier propagated across async submission handoffs. diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 362bfef633..fa91c913e6 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -6155,7 +6155,8 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc op: Op::ThreadSettings { thread_settings: ThreadSettingsOverrides::default(), reply: Some(reply), - }, + } + .into(), trace: None, parent_turn_id: None, root_turn_id: None, @@ -8250,7 +8251,7 @@ fn submission_dispatch_span_prefers_submission_trace_context() { submission_dispatch_span(&Submission { turn_extension_init: None, id: "sub-1".into(), - op: Op::Interrupt, + op: Op::Interrupt.into(), parent_turn_id: None, root_turn_id: None, residency_guard: None, @@ -8281,7 +8282,8 @@ fn submission_dispatch_span_uses_debug_for_realtime_audio() { samples_per_channel: Some(160), item_id: None, }, - }), + }) + .into(), parent_turn_id: None, root_turn_id: None, residency_guard: None, @@ -8644,7 +8646,7 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() { let dispatch_span = submission_dispatch_span(&Submission { turn_extension_init: None, id: "sub-1".into(), - op: Op::Interrupt, + op: Op::Interrupt.into(), parent_turn_id: None, root_turn_id: None, residency_guard: None, @@ -8660,10 +8662,7 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() { Arc::clone(&tc), vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }], TraceCaptureTask { @@ -9016,7 +9015,7 @@ async fn shutdown_and_wait_allows_multiple_waiters() { let (_tx_event, rx_event) = async_channel::unbounded(); let session_loop_handle = tokio::spawn(async move { let shutdown = rx_sub.recv().await.expect("shutdown submission"); - assert!(matches!(shutdown.op, Op::Shutdown)); + assert!(matches!(shutdown.op.as_protocol(), Some(Op::Shutdown))); tokio::time::sleep(StdDuration::from_millis(50)).await; }); let io = Arc::new(SessionIo { @@ -10419,10 +10418,7 @@ async fn spawn_task_does_not_update_previous_turn_settings_for_non_run_turn_task .await; let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }]; @@ -12373,10 +12369,9 @@ async fn extension_interrupt_survives_the_calling_runtime() { let (sess, tc, rx) = make_session_and_context_with_rx().await; let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "keep turn active for extension interruption".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text( + "keep turn active for extension interruption".to_string(), + )], client_id: None, }]; sess.spawn_task( @@ -12438,10 +12433,7 @@ async fn turn_complete_flushes_terminal_event_after_delivery() { let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "complete normally".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("complete normally".to_string())], client_id: None, }]; sess.spawn_task(Arc::clone(&tc), input, CompletingTask) @@ -12466,10 +12458,7 @@ async fn turn_aborted_flushes_terminal_event_after_delivery() { let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "interrupt me".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("interrupt me".to_string())], client_id: None, }]; sess.spawn_task( @@ -12509,10 +12498,7 @@ async fn abort_regular_task_emits_marker_before_turn_aborted() { let (sess, tc, rx) = make_session_and_context_with_rx().await; let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }]; sess.spawn_task( @@ -12551,10 +12537,7 @@ async fn abort_gracefully_emits_marker_before_turn_aborted() { let (sess, tc, rx) = make_session_and_context_with_rx().await; let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }]; sess.spawn_task( @@ -12590,7 +12573,7 @@ async fn abort_gracefully_emits_marker_before_turn_aborted() { async fn submit_steer_only( sess: &Arc, - input: Vec, + input: Vec, expected_turn_id: &str, ) -> TurnInputSubmission { super::turn_input::handle( @@ -12628,10 +12611,7 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() ); let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }]; sess.spawn_task( @@ -12667,7 +12647,16 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() detail: Some(ImageDetail::High), }, ]; - let submission = submit_steer_only(&sess, pending_user_input.clone(), &tc.sub_id).await; + let submission = submit_steer_only( + &sess, + pending_user_input + .iter() + .cloned() + .map(ResolvedUserInput::from_client) + .collect(), + &tc.sub_id, + ) + .await; assert!(matches!(submission, TurnInputSubmission::Steered { .. })); let mut current = tc.initial_settings.as_ref().clone(); @@ -13094,10 +13083,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() { .await; let submission = submit_steer_only( &sess, - vec![UserInput::Text { - text: "follow up".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("follow up".to_string())], &tc.sub_id, ) .await; @@ -13111,10 +13097,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() { acceptance_order: Some(0), ..Default::default() }, - content: vec![UserInput::Text { - text: "follow up".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("follow up".to_string())], client_id: None, }, TurnInput::InterAgentCommunication(communication.into()), @@ -13150,10 +13133,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() { .await; let submission = submit_steer_only( &sess, - vec![UserInput::Text { - text: "follow up".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("follow up".to_string())], &tc.sub_id, ) .await; @@ -13171,10 +13151,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() { acceptance_order: Some(0), ..Default::default() }, - content: vec![UserInput::Text { - text: "follow up".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("follow up".to_string())], client_id: None, }, TurnInput::InterAgentCommunication(communication.into()), @@ -13247,10 +13224,7 @@ async fn abort_review_task_emits_exited_then_aborted_and_records_history() { let (sess, tc, rx) = make_session_and_context_with_rx().await; let input = vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "start review".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("start review".to_string())], client_id: None, }]; sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new()) diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 6b9e6f5cf2..fd555b57ec 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -102,7 +102,6 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::items::PlanItem; use codex_protocol::items::TurnItem; use codex_protocol::items::build_hook_prompt_message; -use codex_protocol::models::AudioReference; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::MessagePhase; @@ -136,6 +135,7 @@ use codex_tools::DiscoverableTool; use codex_tools::ToolName; use codex_tools::ToolSpec; use codex_tools::filter_request_plugin_install_discoverable_tools_for_client; +use codex_user_input::ResolvedUserInput; use codex_utils_path_uri::PathUri; use codex_utils_stream_parser::AssistantTextChunk; use codex_utils_stream_parser::AssistantTextStreamParser; @@ -303,7 +303,7 @@ pub(crate) async fn run_turn( return Ok(None); } - let user_input = turn_user_input(&input); + let user_input = input_for_extensions(&input); let allow_plugin_mentions = !crate::guardian::is_basic_session_source(&turn_context.session_source); let McpStartupRequirements { @@ -594,7 +594,7 @@ pub(crate) async fn run_turn( .await? } Some(_) | None => { - let pending_user_input = turn_user_input(&pending_input); + let pending_user_input = input_for_extensions(&pending_input); if allow_plugin_mentions { required_plugins.extend(crate::plugins::collect_explicit_plugin_ids( &pending_user_input, @@ -1169,7 +1169,7 @@ pub(crate) fn recordable_inputs<'a>( && turn_input_has_uploaded_audio_file(&item) && let TurnInput::UserInput { content, .. } = item.to_mut() { - content.retain(|part| !is_uploaded_audio_file(part)); + content.retain(|part| !part.is_uploaded_audio_file()); if content.is_empty() { return None; } @@ -1180,7 +1180,7 @@ pub(crate) fn recordable_inputs<'a>( } fn turn_input_has_uploaded_audio_file(input: &TurnInput) -> bool { - matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file)) + matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(ResolvedUserInput::is_uploaded_audio_file)) } fn unsupported_uploaded_audio_error() -> CodexErr { @@ -1189,16 +1189,8 @@ fn unsupported_uploaded_audio_error() -> CodexErr { ) } -fn is_uploaded_audio_file(input: &UserInput) -> bool { - matches!( - input, - UserInput::Audio { - audio: AudioReference::File { .. } - } - ) -} - -pub(crate) fn turn_user_input(input: &[TurnInput]) -> Vec { +/// Extensions and telemetry consume the protocol view, including the original mention and path labels. +fn input_for_extensions(input: &[TurnInput]) -> Vec { input .iter() .filter_map(|item| match item { @@ -1208,7 +1200,7 @@ pub(crate) fn turn_user_input(input: &[TurnInput]) -> Vec { | TurnInput::InterAgentCommunication(_) => None, }) .flatten() - .cloned() + .map(ResolvedUserInput::to_user_input) .collect() } @@ -1568,9 +1560,7 @@ async fn track_turn_resolved_config_analytics( | TurnInput::InterAgentCommunication(_) => None, }) .flatten() - .filter(|item| { - matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) - }) + .filter(|item| item.is_image()) .count(), submission_type: None, ephemeral: thread_config.ephemeral, diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index 46c39d6b89..353432d43f 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -51,12 +51,12 @@ use codex_protocol::protocol::ThreadSettingsOverrides; use codex_protocol::turn_input::AnnotatedResponseItem; use codex_protocol::turn_input::NotSubmittedReason; use codex_protocol::turn_input::ResponsesApiClientMetadataUpdate; -use codex_protocol::turn_input::TurnInput as SubmittedTurnInput; +type SubmittedTurnInput = codex_protocol::turn_input::TurnInput; use codex_protocol::turn_input::TurnInputMode; -use codex_protocol::turn_input::TurnInputRequest; +type TurnInputRequest = codex_protocol::turn_input::TurnInputRequest; use codex_protocol::turn_input::TurnInputSubmission; use codex_protocol::turn_input::TurnStartOptions; -use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use serde_json::Value; use std::collections::BTreeMap; use std::sync::Arc; @@ -926,14 +926,12 @@ impl Session { let submission_id = Uuid::now_v7().to_string(); let submission = handle( self, - TurnInputRequest::user_input(vec![UserInput::Text { - text, - text_elements: Vec::new(), - }]) - .on_start(TurnStartOptions { - turn_trigger: Some("realtime".to_string()), - ..Default::default() - }), + TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start( + TurnStartOptions { + turn_trigger: Some("realtime".to_string()), + ..Default::default() + }, + ), TurnInputMode::StartOrSteer, submission_id.clone(), ) diff --git a/codex-rs/core/src/session/turn_input_agent.rs b/codex-rs/core/src/session/turn_input_agent.rs index 44c59cd0f1..ad87139568 100644 --- a/codex-rs/core/src/session/turn_input_agent.rs +++ b/codex-rs/core/src/session/turn_input_agent.rs @@ -11,7 +11,8 @@ use crate::session::session::Session; use codex_protocol::error::Result as CodexResult; use codex_protocol::turn_input::NotSubmittedReason; use codex_protocol::turn_input::TurnInput; -use codex_protocol::turn_input::TurnInputRequest; +type TurnInputRequest = + codex_protocol::turn_input::TurnInputRequest; use codex_protocol::turn_input::TurnInputSubmission; use std::sync::Arc; diff --git a/codex-rs/core/src/session/turn_input_tests.rs b/codex-rs/core/src/session/turn_input_tests.rs index cef4009cae..3d37a02ecb 100644 --- a/codex-rs/core/src/session/turn_input_tests.rs +++ b/codex-rs/core/src/session/turn_input_tests.rs @@ -22,8 +22,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::ThreadSettingsOverrides; use codex_protocol::protocol::TurnAbortReason; -use codex_protocol::turn_input::TurnInput as SubmittedTurnInput; -use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use codex_utils_absolute_path::AbsolutePathBuf; use core_test_support::test_codex::local_selections; use pretty_assertions::assert_eq; @@ -93,7 +92,7 @@ async fn submit_start_only( async fn submit_steer_only( session: &Arc, - input: Vec, + input: Vec, expected_turn_id: &str, ) -> TurnInputSubmission { handle( @@ -148,10 +147,9 @@ async fn steering_does_not_wait_for_realtime_history() { std::time::Duration::from_secs(5), handle( &session, - TurnInputRequest::user_input(vec![UserInput::Text { - text: "steer without waiting for persistence".to_string(), - text_elements: Vec::new(), - }]), + TurnInputRequest::user_input(vec![ResolvedUserInput::text( + "steer without waiting for persistence".to_string(), + )]), mode, "steer-submission".to_string(), ), @@ -177,27 +175,24 @@ async fn accepted_input_applies_thread_settings() { let config = session.get_config().await; handle( &session, - TurnInputRequest::user_input(vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }]) - .with_thread_settings(ThreadSettingsOverrides { - environments: Some(local_selections(config.cwd.clone()).into_requests()), - approval_policy: Some(config.permissions.approval_policy.value()), - approvals_reviewer: Some(codex_config::types::ApprovalsReviewer::AutoReview), - permission_profile: Some(config.permissions.effective_permission_profile()), - summary: config.model_reasoning_summary, - personality: config.personality, - collaboration_mode: Some(CollaborationMode { - mode: ModeKind::Default, - settings: Settings { - model: turn_context.model_info().slug.clone(), - reasoning_effort: config.model_reasoning_effort.clone(), - developer_instructions: None, - }, + TurnInputRequest::user_input(vec![ResolvedUserInput::text("hello".to_string())]) + .with_thread_settings(ThreadSettingsOverrides { + environments: Some(local_selections(config.cwd.clone()).into_requests()), + approval_policy: Some(config.permissions.approval_policy.value()), + approvals_reviewer: Some(codex_config::types::ApprovalsReviewer::AutoReview), + permission_profile: Some(config.permissions.effective_permission_profile()), + summary: config.model_reasoning_summary, + personality: config.personality, + collaboration_mode: Some(CollaborationMode { + mode: ModeKind::Default, + settings: Settings { + model: turn_context.model_info().slug.clone(), + reasoning_effort: config.model_reasoning_effort.clone(), + developer_instructions: None, + }, + }), + ..Default::default() }), - ..Default::default() - }), TurnInputMode::StartOrSteer, "sub-1".to_string(), ) @@ -787,10 +782,7 @@ async fn start_only_accepts_user_input_in_plan_mode() { let submission = submit_start_only( &session, SubmittedTurnInput::UserInput { - content: vec![UserInput::Text { - text: "queued user input".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("queued user input".to_string())], client_id: Some("queued-user-message".to_string()), }, ) @@ -928,10 +920,7 @@ async fn steer_only_requires_active_turn() { let (session, _turn_context, _rx) = make_session_and_context_with_rx().await; let submission = submit_steer_only( &session, - vec![UserInput::Text { - text: "steer".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("steer".to_string())], "missing-turn-id", ) .await; @@ -955,10 +944,7 @@ async fn steer_only_enforces_expected_turn_id() { Arc::clone(&turn_context), vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }], NeverEndingTask { @@ -970,10 +956,7 @@ async fn steer_only_enforces_expected_turn_id() { let submission = submit_steer_only( &session, - vec![UserInput::Text { - text: "steer".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("steer".to_string())], "different-turn-id", ) .await; @@ -1059,10 +1042,7 @@ async fn rejects_non_regular_turns() { Arc::clone(&turn_context), vec![TurnInput::UserInput { metadata: Default::default(), - content: vec![UserInput::Text { - text: "hello".to_string(), - text_elements: Vec::new(), - }], + content: vec![ResolvedUserInput::text("hello".to_string())], client_id: None, }], NeverEndingTask { @@ -1072,10 +1052,7 @@ async fn rejects_non_regular_turns() { ) .await; - let steer_input = vec![UserInput::Text { - text: "steer".to_string(), - text_elements: Vec::new(), - }]; + let steer_input = vec![ResolvedUserInput::text("steer".to_string())]; let steer_submission = submit_steer_only(&session, steer_input.clone(), "turn").await; assert_eq!( TurnInputSubmission::NotSubmitted { @@ -1128,10 +1105,9 @@ async fn steer_preserves_request_origin( }, ) .await; - let content = vec![UserInput::Text { - text: "Create the worktree now.".to_owned(), - text_elements: Vec::new(), - }]; + let content = vec![ResolvedUserInput::text( + "Create the worktree now.".to_owned(), + )]; handle( &session, TurnInputRequest::user_input(content.clone()).on_start(TurnStartOptions { @@ -1210,10 +1186,7 @@ async fn scored_prompt_keeps_its_submission_item_id() { // A legacy steer can present the same UI client ID as an admitted steer. for id in [Some(first), None, Some(second)] { let mut request = TurnInputRequest::new(SubmittedTurnInput::UserInput { - content: vec![UserInput::Text { - text: "identical input".into(), - text_elements: vec![], - }], + content: vec![ResolvedUserInput::text("identical input".into())], client_id: Some("same-client-id".into()), }); request.user_item_uuid = id; diff --git a/codex-rs/core/src/tasks/compact.rs b/codex-rs/core/src/tasks/compact.rs index 3bb0c61458..60213da452 100644 --- a/codex-rs/core/src/tasks/compact.rs +++ b/codex-rs/core/src/tasks/compact.rs @@ -10,7 +10,7 @@ use crate::state::TaskKind; use codex_features::Feature; use codex_model_provider::RemoteCompactionSupport; use codex_protocol::protocol::CodexErrorInfo; -use codex_protocol::user_input::UserInput; +use codex_user_input::ResolvedUserInput; use tokio_util::sync::CancellationToken; #[derive(Clone, Copy, Default)] @@ -82,16 +82,13 @@ impl SessionTask for CompactTask { "local", /*manual*/ true, ); - let input = vec![UserInput::Text { - text: ctx - .config + let input = vec![ResolvedUserInput::text( + ctx.config .compact_prompt .as_deref() .unwrap_or(crate::compact::SUMMARIZATION_PROMPT) .to_string(), - // Compaction prompt is synthesized; no UI element ranges to preserve. - text_elements: Vec::new(), - }]; + )]; crate::compact::run_compact_task(session.clone(), step_context, world_state, input) .await } diff --git a/codex-rs/core/src/tasks/mod_tests.rs b/codex-rs/core/src/tasks/mod_tests.rs index 244e6b0405..049de5196b 100644 --- a/codex-rs/core/src/tasks/mod_tests.rs +++ b/codex-rs/core/src/tasks/mod_tests.rs @@ -95,7 +95,10 @@ async fn task_finish_rejects_uploaded_audio_without_losing_other_pending_input() file_id: "file_disabled_at_finish".to_string(), }, }, - ], + ] + .into_iter() + .map(codex_user_input::ResolvedUserInput::from_client) + .collect(), client_id: None, metadata: Default::default(), }], diff --git a/codex-rs/core/src/tasks/review.rs b/codex-rs/core/src/tasks/review.rs index b65b67fbb3..0943f2816e 100644 --- a/codex-rs/core/src/tasks/review.rs +++ b/codex-rs/core/src/tasks/review.rs @@ -26,8 +26,8 @@ use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::state::TaskKind; use codex_features::Feature; -use codex_protocol::user_input::UserInput; use codex_thread_store::PersistContext; +use codex_user_input::ResolvedUserInput; use super::SessionTask; use super::SessionTaskResult; @@ -98,7 +98,7 @@ impl SessionTask for ReviewTask { async fn start_review_conversation( session: Arc, ctx: Arc, - input: Vec, + input: Vec, cancellation_token: CancellationToken, ) -> Option> { let config = ctx.config.clone(); diff --git a/codex-rs/core/src/thread_manager/helper_daybreak_tests.rs b/codex-rs/core/src/thread_manager/helper_daybreak_tests.rs index d342e74527..ed1fb4aac5 100644 --- a/codex-rs/core/src/thread_manager/helper_daybreak_tests.rs +++ b/codex-rs/core/src/thread_manager/helper_daybreak_tests.rs @@ -29,6 +29,7 @@ use codex_protocol::turn_input::TurnInputRequest; use codex_protocol::turn_input::TurnStartOptions; use codex_protocol::user_input::UserInput; use codex_thread_store::ThreadMetadataPatch; +use codex_user_input::ResolvedUserInput; use core_test_support::PathExt; use pretty_assertions::assert_eq; use std::sync::Arc; @@ -97,7 +98,7 @@ async fn spawn_child( .control(parent.session.session_id()) .spawn_agent_with_metadata( config.clone(), - text(), + vec![ResolvedUserInput::text("Continue".into())], source, SpawnAgentOptions { parent_thread_id: Some(parent_id), diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index a038e1e641..fd9a1cdc33 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -57,8 +57,8 @@ use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::protocol::UserMessageEvent; -use codex_protocol::user_input::UserInput; use codex_thread_store::InMemoryThreadStore; +use codex_user_input::ResolvedUserInput; use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::PathExt; @@ -794,7 +794,7 @@ async fn thread_id_generator_applies_to_roots_children_and_forks() { .control(root.thread.session.session_id()) .spawn_agent_with_metadata( config.clone(), - vec![UserInput::Text { + vec![ResolvedUserInput::Text { text: "child task".to_string(), text_elements: Vec::new(), }], diff --git a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs index 9998a17367..aabc073780 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs @@ -5,6 +5,7 @@ use crate::agent::control::render_input_preview; use crate::tools::handlers::multi_agents_spec::create_send_input_tool_v1; use codex_protocol::protocol::MultiAgentVersion; use codex_tools::ToolSpec; +use codex_user_input::ResolvedUserInput; pub(crate) struct Handler; @@ -49,6 +50,10 @@ impl Handler { let receiver_thread_id = parse_agent_id_target(&args.target)?; let input_items = parse_collab_input(args.message, args.items)?; let prompt = render_input_preview(&input_items); + let input_items = input_items + .into_iter() + .map(ResolvedUserInput::from_client) + .collect(); let local_agent_control = session .services .local_agent_runtime diff --git a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs index 28bf068798..5f46f1d1c8 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs @@ -16,6 +16,7 @@ use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions; use crate::tools::handlers::multi_agents_spec::create_spawn_agent_tool_v1; use codex_protocol::protocol::MultiAgentVersion; use codex_tools::ToolSpec; +use codex_user_input::ResolvedUserInput; #[derive(Default)] pub(crate) struct Handler { @@ -80,6 +81,10 @@ async fn handle_spawn_agent( "Agent depth limit reached. Solve the task yourself.".to_string(), )); } + let input_items = input_items + .into_iter() + .map(ResolvedUserInput::from_client) + .collect(); session .emit_turn_item_started( turn, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index 49d82f572f..03ff825bee 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -74,6 +74,7 @@ use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::user_input::UserInput; use codex_state::DirectionalThreadSpawnEdgeStatus; +use codex_user_input::ResolvedUserInput; use core_test_support::TempDirExt; use opentelemetry_sdk::metrics::InMemoryMetricExporter; use opentelemetry_sdk::metrics::data::AggregatedMetrics; @@ -1336,10 +1337,7 @@ async fn multi_agent_v2_send_message_accepts_root_target_from_child() { .control(session.session_id()) .spawn_agent_with_metadata( (*turn.config).clone(), - vec![UserInput::Text { - text: "inspect this repo".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("inspect this repo".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1417,10 +1415,7 @@ async fn multi_agent_v2_followup_task_rejects_root_target_from_child() { .control(session.session_id()) .spawn_agent_with_metadata( (*turn.config).clone(), - vec![UserInput::Text { - text: "inspect this repo".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("inspect this repo".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1584,10 +1579,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() { .control(session.session_id()) .spawn_agent_with_metadata( config.clone(), - vec![UserInput::Text { - text: "research".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("research".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1605,10 +1597,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() { .control(session.session_id()) .spawn_agent_with_metadata( config, - vec![UserInput::Text { - text: "build".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("build".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 2, @@ -4154,10 +4143,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_id() { .control(session.session_id()) .spawn_agent_with_metadata( (*turn.config).clone(), - vec![UserInput::Text { - text: "inspect this repo".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("inspect this repo".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -4222,10 +4208,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_task_name() { .control(session.session_id()) .spawn_agent_with_metadata( (*turn.config).clone(), - vec![UserInput::Text { - text: "inspect this repo".to_string(), - text_elements: Vec::new(), - }], + vec![ResolvedUserInput::text("inspect this repo".to_string())], SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 6484fdcaae..1f020a09ce 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -124,19 +124,14 @@ async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Resu .update_thread_settings(ThreadSettingsOverrides::default()) .await?; wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await; - let (reply, resumed) = oneshot::channel(); - child - .submit(Op::TurnInput { - request: Box::new(codex_core::TurnInputRequest::user_input(vec![ - codex_protocol::user_input::UserInput::Text { - text: "Resume.".into(), - text_elements: vec![], - }, - ])), - mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer, - reply, - }) - .await?; + let resumed = child.start_or_steer_turn(codex_core::TurnInputRequest::user_input(vec![ + codex_protocol::user_input::UserInput::Text { + text: "Resume.".into(), + text_elements: vec![], + }, + ])); + tokio::pin!(resumed); + assert!(futures::poll!(resumed.as_mut()).is_pending()); assert_eq!(child.agent_status().await, AgentStatus::Running); release_item.send(()).expect("release the sampled item"); let next = |event: &EventMsg| match event { @@ -153,7 +148,7 @@ async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Resu EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted )); assert!(matches!( - resumed.await??, + resumed.await?, codex_core::TurnInputSubmission::Started { .. } )); wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await; diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 69bc9bf80e..9627f407e8 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -38,6 +38,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::config_types::Verbosity; use codex_protocol::error::CodexErr; +use codex_protocol::items::TurnItem; use codex_protocol::models::AudioReference; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; @@ -56,6 +57,7 @@ use codex_protocol::models::WebSearchAction; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::Op; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; @@ -88,6 +90,7 @@ use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::local_requests; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; +use core_test_support::wait_for_event_match; use futures::StreamExt; use pretty_assertions::assert_eq; use serde_json::json; @@ -566,26 +569,63 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> { sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), ) .await; - let codex = test_codex() + let test = test_codex() .with_model_info_override("gpt-5.5", |model_info| { model_info.input_modalities.push(InputModality::Audio); }) - .build(&server) - .await? - .codex; + .build_with_auto_env(&server) + .await?; + // Client-selected attachments live on the app-server host, even with a remote executor. let temp_dir = tempfile::tempdir()?; let audio_path = temp_dir.path().join("recording.wav"); std::fs::write(&audio_path, b"audio")?; - codex - .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::LocalAudio { + let unsupported = temp_dir.path().join("clip.flac"); + std::fs::write(&unsupported, b"audio")?; + let missing = temp_dir.path().join("missing.wav"); + let inline = "data:audio/wav;base64,YXVkaW8="; + let input = vec![ + UserInput::Audio { + audio: AudioReference::Inline { + audio_url: inline.into(), + }, + }, + UserInput::LocalAudio { + path: unsupported.clone(), + }, + UserInput::LocalAudio { path: audio_path.clone(), - }])) + }, + UserInput::LocalAudio { + path: missing.clone(), + }, + ]; + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(input.clone())) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + let displayed_input = wait_for_event_match(&test.codex, |event| match event { + EventMsg::ItemCompleted(ItemCompletedEvent { + item: TurnItem::UserMessage(message), + .. + }) => Some(message.content.clone()), + _ => None, + }) + .await; + assert_eq!(displayed_input, input); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; let request = response_mock.single_request(); - assert!(request.has_content_kinds(&["user.text", "user.audio", "user.text"])); + assert!(request.has_content_kinds(&[ + "user.audio", + "user.text", + "user.text", + "user.audio", + "user.text", + "user.text" + ])); let user_message = request .input() .into_iter() @@ -593,12 +633,20 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> { .find(|item| item.get("role").and_then(serde_json::Value::as_str) == Some("user")) .expect("request should include a user message"); let audio_path = audio_path.display(); + let unsupported = unsupported.display(); + let missing_error = std::fs::File::open(&missing).unwrap_err(); + let missing = missing.display(); assert_eq!( user_message["content"], json!([ + {"type": "input_audio", "audio_url": inline}, { "type": "input_text", - "text": format!(r#"