mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Capture local attachments before queuing turn input (#61962)
## Why Queued input can outlive its local attachment files. Reading those files only when rendering a model message can lose media submitted while turn admission is pending. ## What changed - Introduce `ResolvedUserInput` to carry captured image bytes, audio data URLs, or read errors through session queues and agent input. - Resolve client attachments on the submitting host before queuing, using a blocking task for local file reads. - Render messages from captured payloads while preserving original paths and text spans for display, along with image position mappings. ## Testing Add coverage for rendering and serialization after source files are removed, steered audio retained during pending admission, and local audio errors and display content. GitOrigin-RevId: 4bbb6bda44167439a8cfdae15d3c476a7b1d336b
This commit is contained in:
committed by
copyberry
parent
65a4450886
commit
d8d840d32a
Generated
+2
@@ -5401,7 +5401,9 @@ dependencies = [
|
||||
"anyhow",
|
||||
"codex-protocol",
|
||||
"codex-utils-image",
|
||||
"derive_more 2.1.1",
|
||||
"pretty_assertions",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
@@ -24,8 +24,8 @@ use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_rollout_trace::ThreadTraceContext;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
// Keep dynamic dispatch a compile-time property of the contract.
|
||||
@@ -209,7 +209,7 @@ impl AgentInfo {
|
||||
|
||||
/// User input starts or steers a turn; agent messages retain their sender and wake mode.
|
||||
pub enum AgentInput {
|
||||
UserInput(Vec<UserInput>),
|
||||
UserInput(Vec<ResolvedUserInput>),
|
||||
Message {
|
||||
message: AgentMessage,
|
||||
mode: MessageDeliveryMode,
|
||||
|
||||
@@ -47,6 +47,7 @@ use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_protocol::protocol::ThreadSource;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use futures::StreamExt;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
@@ -137,13 +138,15 @@ impl LocalAgentControl {
|
||||
pub(crate) async fn send_input(
|
||||
&self,
|
||||
agent_id: ThreadId,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
start_options: TurnStartOptions,
|
||||
) -> CodexResult<String> {
|
||||
let state = self.runtime.upgrade()?;
|
||||
let thread = state.get_thread(agent_id).await?;
|
||||
let result = match thread
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(input).on_start(start_options))
|
||||
.start_or_steer_resolved_turn(
|
||||
TurnInputRequest::<ResolvedUserInput>::user_input(input).on_start(start_options),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(TurnInputSubmission::Started { turn_id, .. }) => Ok(turn_id),
|
||||
|
||||
@@ -38,7 +38,7 @@ const AGENT_NAMES: &str = include_str!("../../../assets/agent/agent_names.txt");
|
||||
/// unrepresentable.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub(super) enum SpawnInitialInput {
|
||||
UserInput(Vec<UserInput>),
|
||||
UserInput(Vec<ResolvedUserInput>),
|
||||
InterAgentCommunication(InterAgentCommunication, AgentCommunicationContext),
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ impl LocalAgentControl {
|
||||
pub(crate) async fn spawn_agent(
|
||||
&self,
|
||||
config: Config,
|
||||
initial_input: Vec<UserInput>,
|
||||
initial_input: Vec<ResolvedUserInput>,
|
||||
session_source: Option<SessionSource>,
|
||||
) -> CodexResult<ThreadId> {
|
||||
let (spawned_agent, _) = Box::pin(self.spawn_agent_internal(
|
||||
|
||||
@@ -141,7 +141,7 @@ impl LocalAgentControl {
|
||||
pub(crate) async fn spawn_agent_with_metadata(
|
||||
&self,
|
||||
config: Config,
|
||||
initial_input: Vec<UserInput>,
|
||||
initial_input: Vec<ResolvedUserInput>,
|
||||
source: SessionSource,
|
||||
options: SpawnAgentOptions,
|
||||
) -> CodexResult<LiveAgent> {
|
||||
@@ -181,8 +181,8 @@ async fn test_config() -> (TempDir, Config) {
|
||||
test_config_with_cli_overrides(Vec::new()).await
|
||||
}
|
||||
|
||||
fn text_input(text: &str) -> Vec<UserInput> {
|
||||
vec![UserInput::Text {
|
||||
fn text_input(text: &str) -> Vec<ResolvedUserInput> {
|
||||
vec![ResolvedUserInput::Text {
|
||||
text: text.to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}]
|
||||
@@ -584,14 +584,7 @@ async fn assert_thread_not_loaded(manager: &ThreadManager, thread_id: ThreadId)
|
||||
async fn send_input_errors_when_manager_dropped() {
|
||||
let control = LocalAgentControl::default();
|
||||
let err = control
|
||||
.send_input(
|
||||
ThreadId::new(),
|
||||
vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
Default::default(),
|
||||
)
|
||||
.send_input(ThreadId::new(), text_input("hello"), Default::default())
|
||||
.await
|
||||
.expect_err("send_input should fail without a manager");
|
||||
assert_eq!(
|
||||
@@ -971,14 +964,7 @@ async fn send_input_errors_when_thread_missing() {
|
||||
let thread_id = ThreadId::new();
|
||||
let err = harness
|
||||
.control
|
||||
.send_input(
|
||||
thread_id,
|
||||
vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
Default::default(),
|
||||
)
|
||||
.send_input(thread_id, text_input("hello"), Default::default())
|
||||
.await
|
||||
.expect_err("send_input should fail for missing thread");
|
||||
assert_matches!(
|
||||
@@ -1063,10 +1049,7 @@ async fn send_input_submits_user_message() {
|
||||
caller: ThreadId::new(),
|
||||
target: AgentTarget::Id(thread_id),
|
||||
resume_config: harness.config.clone(),
|
||||
input: AgentInput::UserInput(vec![UserInput::Text {
|
||||
text: "hello from tests".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}]),
|
||||
input: AgentInput::UserInput(text_input("hello from tests")),
|
||||
start_options: Default::default(),
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -12,7 +12,7 @@ use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::ThreadSource;
|
||||
use codex_protocol::protocol::TurnEnvironmentSelection;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use serde_json::Value;
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
@@ -239,7 +239,7 @@ pub(crate) async fn run_codex_thread_one_shot(
|
||||
config: Config,
|
||||
auth_manager: Arc<AuthManager>,
|
||||
models_manager: SharedModelsManager,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
parent_session: Arc<Session>,
|
||||
parent_ctx: Arc<TurnContext>,
|
||||
cancel_token: CancellationToken,
|
||||
@@ -308,7 +308,7 @@ pub(crate) async fn run_codex_thread_one_shot(
|
||||
let _ = ops_tx
|
||||
.send(Box::new(Submission {
|
||||
id: "shutdown".to_string(),
|
||||
op: Op::Shutdown {},
|
||||
op: Op::Shutdown.into(),
|
||||
turn_extension_init: None,
|
||||
trace: None,
|
||||
parent_turn_id: None,
|
||||
|
||||
@@ -127,11 +127,13 @@ async fn forward_events_filters_private_events_before_blocked_send_is_cancelled(
|
||||
ops.push(sub.op);
|
||||
}
|
||||
assert!(
|
||||
ops.iter().any(|op| matches!(op, Op::Interrupt)),
|
||||
ops.iter()
|
||||
.any(|op| matches!(op.as_protocol(), Some(Op::Interrupt))),
|
||||
"expected Interrupt op after cancellation"
|
||||
);
|
||||
assert!(
|
||||
ops.iter().any(|op| matches!(op, Op::Shutdown)),
|
||||
ops.iter()
|
||||
.any(|op| matches!(op.as_protocol(), Some(Op::Shutdown))),
|
||||
"expected Shutdown op after cancellation"
|
||||
);
|
||||
}
|
||||
@@ -153,7 +155,7 @@ async fn forward_ops_preserves_submission_trace_context() {
|
||||
|
||||
let submission = Submission {
|
||||
id: "sub-1".to_string(),
|
||||
op: Op::Interrupt,
|
||||
op: Op::Interrupt.into(),
|
||||
turn_extension_init: None,
|
||||
trace: Some(codex_protocol::protocol::W3cTraceContext {
|
||||
traceparent: Some(
|
||||
@@ -174,7 +176,7 @@ async fn forward_ops_preserves_submission_trace_context() {
|
||||
.expect("forward_ops hung")
|
||||
.expect("forwarded submission missing");
|
||||
assert_eq!("sub-1", forwarded.id);
|
||||
assert!(matches!(forwarded.op, Op::Interrupt));
|
||||
assert!(matches!(forwarded.op.as_protocol(), Some(Op::Interrupt)));
|
||||
assert_eq!(
|
||||
forwarded.trace,
|
||||
Some(codex_protocol::protocol::W3cTraceContext {
|
||||
|
||||
@@ -71,16 +71,19 @@ use codex_protocol::turn_input::RecoverTurnRequest;
|
||||
use codex_protocol::turn_input::StartIfIdleSubmission;
|
||||
use codex_protocol::turn_input::SteerSubmission;
|
||||
use codex_protocol::turn_input::SuspendTurnOutcome;
|
||||
use codex_protocol::turn_input::TurnInput;
|
||||
use codex_protocol::turn_input::TurnInputMode;
|
||||
use codex_protocol::turn_input::TurnInputRequest;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_thread_store::StoredThread;
|
||||
use codex_thread_store::StoredThreadHistory;
|
||||
use codex_thread_store::ThreadMetadataPatch;
|
||||
use codex_thread_store::ThreadStoreError;
|
||||
use codex_thread_store::ThreadStoreResult;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use codex_utils_path_uri::LegacyAppPathString;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
@@ -232,6 +235,54 @@ pub struct BackgroundTerminalInfo {
|
||||
pub cwd: PathUri,
|
||||
}
|
||||
|
||||
/// Captures client attachment paths before an ordered request enters the session.
|
||||
pub(crate) async fn resolve_client_turn_input(
|
||||
request: TurnInputRequest,
|
||||
) -> CodexResult<TurnInputRequest<ResolvedUserInput>> {
|
||||
let input = match request.input {
|
||||
TurnInput::UserInput { content, client_id } => TurnInput::UserInput {
|
||||
content: resolve_client_inputs(content).await?,
|
||||
client_id,
|
||||
},
|
||||
TurnInput::ResponseItem(item) => TurnInput::ResponseItem(item),
|
||||
TurnInput::AnnotatedResponseItem(item) => TurnInput::AnnotatedResponseItem(item),
|
||||
TurnInput::InterAgentCommunication(message) => TurnInput::InterAgentCommunication(message),
|
||||
};
|
||||
Ok(TurnInputRequest {
|
||||
input,
|
||||
user_item_uuid: request.user_item_uuid,
|
||||
user_input_extension_data: request.user_input_extension_data,
|
||||
thread_settings: request.thread_settings,
|
||||
start: request.start,
|
||||
additional_context: request.additional_context,
|
||||
responsesapi_client_metadata: request.responsesapi_client_metadata,
|
||||
trace: request.trace,
|
||||
})
|
||||
}
|
||||
|
||||
/// Captures local media off the async executor. Inline inputs do not need a blocking task.
|
||||
pub(crate) async fn resolve_client_inputs(
|
||||
content: Vec<UserInput>,
|
||||
) -> CodexResult<Vec<ResolvedUserInput>> {
|
||||
let has_local_media = content.iter().any(|item| {
|
||||
matches!(
|
||||
item,
|
||||
UserInput::LocalImage { .. } | UserInput::LocalAudio { .. }
|
||||
)
|
||||
});
|
||||
let resolve = move || {
|
||||
content
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect()
|
||||
};
|
||||
if has_local_media {
|
||||
Ok(tokio::task::spawn_blocking(resolve).await?)
|
||||
} else {
|
||||
Ok(resolve())
|
||||
}
|
||||
}
|
||||
|
||||
/// Conduit for the bidirectional stream of messages that compose a thread
|
||||
/// (formerly called a conversation) in Codex.
|
||||
impl CodexThread {
|
||||
@@ -691,7 +742,7 @@ impl CodexThread {
|
||||
.tx_sub
|
||||
.send(Box::new(Submission {
|
||||
id: new_submission_id(),
|
||||
op: Op::SuspendTurnAndShutdown { reply },
|
||||
op: Op::SuspendTurnAndShutdown { reply }.into(),
|
||||
turn_extension_init: None,
|
||||
trace: current_span_w3c_trace_context(),
|
||||
parent_turn_id: None,
|
||||
@@ -736,6 +787,33 @@ impl CodexThread {
|
||||
&self,
|
||||
request: impl Into<WithTurnExtensionData<TurnInputRequest>>,
|
||||
mode: TurnInputMode,
|
||||
) -> CodexResult<TurnInputSubmission> {
|
||||
let WithTurnExtensionData {
|
||||
request,
|
||||
turn_extension_init,
|
||||
} = request.into();
|
||||
self.submit_resolved_turn_input_with_mode(
|
||||
WithTurnExtensionData {
|
||||
request: resolve_client_turn_input(request).await?,
|
||||
turn_extension_init,
|
||||
},
|
||||
mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_or_steer_resolved_turn(
|
||||
&self,
|
||||
request: TurnInputRequest<ResolvedUserInput>,
|
||||
) -> CodexResult<TurnInputSubmission> {
|
||||
self.submit_resolved_turn_input_with_mode(request.into(), TurnInputMode::StartOrSteer)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn submit_resolved_turn_input_with_mode(
|
||||
&self,
|
||||
request: WithTurnExtensionData<TurnInputRequest<ResolvedUserInput>>,
|
||||
mode: TurnInputMode,
|
||||
) -> CodexResult<TurnInputSubmission> {
|
||||
if !matches!(mode, TurnInputMode::Steer { .. }) {
|
||||
self.ensure_execution_capacity_for_turn_start(
|
||||
|
||||
@@ -58,10 +58,9 @@ use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ContentItemKind;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_rollout_trace::InferenceTraceContext;
|
||||
use codex_user_input::LocalImagePreparation;
|
||||
use codex_user_input::from_user_input;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_output_truncation::approx_token_count;
|
||||
use tracing::error;
|
||||
|
||||
@@ -136,11 +135,7 @@ pub(crate) async fn run_inline_auto_compact_task(
|
||||
.as_deref()
|
||||
.unwrap_or(SUMMARIZATION_PROMPT)
|
||||
.to_string();
|
||||
let input = vec![UserInput::Text {
|
||||
text: prompt,
|
||||
// Compaction prompt is synthesized; no UI element ranges to preserve.
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
let input = vec![ResolvedUserInput::text(prompt)];
|
||||
|
||||
run_compact_task_inner(
|
||||
sess,
|
||||
@@ -162,7 +157,7 @@ pub(crate) async fn run_compact_task(
|
||||
sess: Arc<Session>,
|
||||
step_context: Arc<StepContext>,
|
||||
world_state: Arc<WorldState>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
) -> CodexResult<()> {
|
||||
run_compact_task_inner(
|
||||
sess.clone(),
|
||||
@@ -185,7 +180,7 @@ async fn run_compact_task_inner(
|
||||
sess: Arc<Session>,
|
||||
step_context: Arc<StepContext>,
|
||||
replacement_step_context: Arc<StepContext>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
world_state: Arc<WorldState>,
|
||||
compaction_metadata: CompactionTurnMetadata,
|
||||
) -> CodexResult<bool> {
|
||||
@@ -273,7 +268,7 @@ async fn run_compact_task_inner_impl(
|
||||
sess: Arc<Session>,
|
||||
step_context: Arc<StepContext>,
|
||||
replacement_step_context: Arc<StepContext>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
world_state: Arc<WorldState>,
|
||||
compaction_metadata: CompactionTurnMetadata,
|
||||
) -> CodexResult<String> {
|
||||
@@ -287,11 +282,8 @@ async fn run_compact_task_inner_impl(
|
||||
sess.emit_turn_item_started(&turn_context, &compaction_item)
|
||||
.await;
|
||||
}
|
||||
let initial_input_for_turn = from_user_input(
|
||||
input,
|
||||
LocalImagePreparation::Process,
|
||||
&mut std::collections::HashMap::new(),
|
||||
);
|
||||
let (initial_input_for_turn, _) =
|
||||
ResolvedUserInput::into_message(&input, LocalImagePreparation::Process);
|
||||
|
||||
let mut history = sess.clone_history().await;
|
||||
let input_goal_ids = UserGoalUpdate::message_ids(history.raw_items());
|
||||
|
||||
@@ -185,10 +185,9 @@ async fn local_compaction_retries_preserve_instructions_and_tool_metadata(
|
||||
Arc::clone(&session),
|
||||
step,
|
||||
world_state,
|
||||
vec![UserInput::Text {
|
||||
text: "Summarize the conversation.".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text(
|
||||
"Summarize the conversation.".to_string(),
|
||||
)],
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context::GuardianBudgetOmission;
|
||||
@@ -259,7 +260,7 @@ pub(crate) async fn finalize(
|
||||
// Reserve the real input annotations as well as the shared content cost.
|
||||
// Removing optional items can only reduce this metadata. The remaining
|
||||
// margin covers IDs and timestamps assigned when recording the message.
|
||||
let mut framing = session.response_item_from_user_input(content.clone());
|
||||
let mut framing = session.response_item_from_user_input(content);
|
||||
if let ResponseItem::Message { content, .. } = &mut framing {
|
||||
content.clear();
|
||||
}
|
||||
@@ -314,9 +315,28 @@ pub(crate) async fn finalize(
|
||||
let observations = std::mem::take(&mut context.truncations);
|
||||
match context.clone().into_annotated_user_inputs() {
|
||||
Ok((user_input, metadata)) => {
|
||||
let user_input = user_input
|
||||
.into_iter()
|
||||
.map(|item| match item {
|
||||
UserInput::Text {
|
||||
text,
|
||||
text_elements,
|
||||
} => Ok(ResolvedUserInput::Text {
|
||||
text,
|
||||
text_elements,
|
||||
}),
|
||||
UserInput::Image { image, detail } => Ok(ResolvedUserInput::Image {
|
||||
image: image.into(),
|
||||
detail,
|
||||
}),
|
||||
_ => Err(CodexErr::InvalidRequest(
|
||||
"Unsupported Guardian input content".to_owned(),
|
||||
)),
|
||||
})
|
||||
.collect::<CodexResult<Vec<_>>>()?;
|
||||
if metadata.is_some() {
|
||||
input[0] = TurnInput::ResponseItem(codex_history::ResponseItemEnvelope {
|
||||
item: session.response_item_from_user_input(user_input),
|
||||
item: session.response_item_from_user_input(&user_input),
|
||||
metadata,
|
||||
});
|
||||
} else if let TurnInput::UserInput { content, .. } = &mut input[0] {
|
||||
@@ -338,14 +358,15 @@ pub(crate) async fn finalize(
|
||||
let user_input = content
|
||||
.iter()
|
||||
.map(|item| match item {
|
||||
ContentItem::InputText { text } => Ok(UserInput::Text {
|
||||
text: text.clone(),
|
||||
text_elements: Vec::new(),
|
||||
}),
|
||||
ContentItem::InputImage { image, detail } => Ok(UserInput::Image {
|
||||
image: image.clone(),
|
||||
detail: *detail,
|
||||
}),
|
||||
ContentItem::InputText { text } => {
|
||||
Ok(ResolvedUserInput::text(text.clone()))
|
||||
}
|
||||
ContentItem::InputImage { image, detail } => {
|
||||
Ok(ResolvedUserInput::Image {
|
||||
image: std::sync::Arc::new(image.clone()),
|
||||
detail: *detail,
|
||||
})
|
||||
}
|
||||
ContentItem::OutputText { .. } | ContentItem::InputAudio { .. } => {
|
||||
Err(CodexErr::InvalidRequest(
|
||||
"Unsupported Guardian input content".to_owned(),
|
||||
@@ -353,7 +374,7 @@ pub(crate) async fn finalize(
|
||||
}
|
||||
})
|
||||
.collect::<CodexResult<Vec<_>>>()?;
|
||||
envelope.item = session.response_item_from_user_input(user_input);
|
||||
envelope.item = session.response_item_from_user_input(&user_input);
|
||||
}
|
||||
Ok(TurnInput::ResponseItem(envelope))
|
||||
})
|
||||
|
||||
@@ -57,7 +57,13 @@ async fn finalization_exports_loss_totals_only_for_accepted_input(window: i64) {
|
||||
.collect();
|
||||
let mut input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: context.clone().into_user_inputs().unwrap(),
|
||||
content: context
|
||||
.clone()
|
||||
.into_user_inputs()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect(),
|
||||
client_id: None,
|
||||
}];
|
||||
session
|
||||
@@ -188,7 +194,13 @@ fn required_context(text: String) -> ComposedContext {
|
||||
async fn cancelled_startup_does_not_record_unselected_review_evidence() {
|
||||
let (session, turn, events) = crate::session::tests::make_session_and_context_with_rx().await;
|
||||
let context = required_context("unselected review evidence ".repeat(/*n*/ 10_000));
|
||||
let content = context.clone().into_user_inputs().unwrap();
|
||||
let content = context
|
||||
.clone()
|
||||
.into_user_inputs()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
session
|
||||
.services
|
||||
.thread_extension_data
|
||||
@@ -272,7 +284,13 @@ async fn finalization_overflow_marks_the_reviewer_exhausted() {
|
||||
let context = required_context("required action".to_owned());
|
||||
let mut input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: context.clone().into_user_inputs().unwrap(),
|
||||
content: context
|
||||
.clone()
|
||||
.into_user_inputs()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect(),
|
||||
client_id: None,
|
||||
}];
|
||||
session
|
||||
@@ -318,10 +336,7 @@ async fn compacted_review_restores_originals_once_and_persists_the_request_prefi
|
||||
.record_user_prompt_and_emit_turn_item(
|
||||
&step.turn,
|
||||
&step.settings.model_info,
|
||||
&[codex_protocol::user_input::UserInput::Text {
|
||||
text: instruction.clone(),
|
||||
text_elements: vec![],
|
||||
}],
|
||||
&[ResolvedUserInput::text(instruction.clone())],
|
||||
/*client_id*/ None,
|
||||
crate::session::UserInputMetadata {
|
||||
acceptance_order: Some(0),
|
||||
|
||||
@@ -836,7 +836,10 @@ impl codex_guardian_reviewer::ReviewerRuntime for GuardianReviewSession {
|
||||
async fn submit_turn(&self, request: TurnInputRequest) -> anyhow::Result<TurnInputSubmission> {
|
||||
Ok(self
|
||||
.io
|
||||
.submit_turn_input(request, TurnInputMode::StartIfIdle)
|
||||
.submit_turn_input(
|
||||
crate::codex_thread::resolve_client_turn_input(request).await?,
|
||||
TurnInputMode::StartIfIdle,
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ use crate::agents_md_manager::AgentsMdManager;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context_manager::ContextManager;
|
||||
use crate::session::Submission;
|
||||
use crate::session::submission::SubmissionOp;
|
||||
use codex_guardian_reviewer::GuardianReviewResponse;
|
||||
use codex_guardian_reviewer::ReviewerRequest;
|
||||
use codex_guardian_reviewer::guardian_output_contract_prompt;
|
||||
@@ -137,7 +138,7 @@ async fn run_review_preserves_evidence_during_parent_compaction() {
|
||||
Some(http::HeaderValue::from_static("__oailb=original-parent"))
|
||||
);
|
||||
let id = submission.id;
|
||||
let Op::TurnInput { request, reply, .. } = submission.op else {
|
||||
let SubmissionOp::TurnInput { request, reply, .. } = submission.op else {
|
||||
panic!("expected reviewer prompt");
|
||||
};
|
||||
let codex_protocol::turn_input::TurnInput::UserInput { content, .. } = request.input else {
|
||||
@@ -711,7 +712,7 @@ async fn run_review_on_reused_session_waits_for_submitted_turn() {
|
||||
});
|
||||
let submission = rx_sub.recv().await.expect("guardian submission");
|
||||
let id = submission.id;
|
||||
let Op::TurnInput { reply, .. } = submission.op else {
|
||||
let SubmissionOp::TurnInput { reply, .. } = submission.op else {
|
||||
panic!("expected turn-input submission");
|
||||
};
|
||||
reply
|
||||
@@ -779,7 +780,7 @@ async fn run_review_removes_trunk_when_event_stream_is_broken() {
|
||||
tokio::spawn(async move { run_guardian_review_session(manager_for_review, params).await });
|
||||
let submission = rx_sub.recv().await.expect("guardian submission");
|
||||
let id = submission.id;
|
||||
let Op::TurnInput { reply, .. } = submission.op else {
|
||||
let SubmissionOp::TurnInput { reply, .. } = submission.op else {
|
||||
panic!("expected turn-input submission");
|
||||
};
|
||||
reply
|
||||
@@ -950,7 +951,7 @@ async fn wait_for_guardian_review_timeout_drains_expected_turn_after_stale_termi
|
||||
let tx_interrupt_event = tx_event.clone();
|
||||
let interrupt_response = tokio::spawn(async move {
|
||||
let submission = rx_sub.recv().await.expect("interrupt submission");
|
||||
assert!(matches!(submission.op, Op::Interrupt));
|
||||
assert!(matches!(submission.op.as_protocol(), Some(Op::Interrupt)));
|
||||
tx_interrupt_event
|
||||
.send(turn_aborted_event("current-turn"))
|
||||
.await
|
||||
@@ -991,7 +992,7 @@ async fn wait_for_guardian_review_cancel_drains_expected_turn_after_stale_termin
|
||||
let tx_interrupt_event = tx_event.clone();
|
||||
let interrupt_response = tokio::spawn(async move {
|
||||
let submission = rx_sub.recv().await.expect("interrupt submission");
|
||||
assert!(matches!(submission.op, Op::Interrupt));
|
||||
assert!(matches!(submission.op.as_protocol(), Some(Op::Interrupt)));
|
||||
tx_interrupt_event
|
||||
.send(turn_aborted_event("current-turn"))
|
||||
.await
|
||||
@@ -1165,13 +1166,13 @@ async fn compaction_and_review_submissions_share_execution_observer() {
|
||||
.expect("child submission deadline")
|
||||
.expect("child submission");
|
||||
if index == 0 {
|
||||
assert!(matches!(submission.op, Op::Compact));
|
||||
assert!(matches!(submission.op.as_protocol(), Some(Op::Compact)));
|
||||
assert_eq!(
|
||||
child_session.thread_config_snapshot().await.request_country,
|
||||
expected_country
|
||||
);
|
||||
} else {
|
||||
let Op::TurnInput { reply, .. } = submission.op else {
|
||||
let SubmissionOp::TurnInput { reply, .. } = submission.op else {
|
||||
panic!("expected review input after compaction");
|
||||
};
|
||||
reply
|
||||
|
||||
@@ -31,7 +31,6 @@ use codex_otel::HOOK_RUN_DURATION_METRIC;
|
||||
use codex_otel::HOOK_RUN_METRIC;
|
||||
use codex_protocol::items::FunctionCallOutputItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
@@ -54,6 +53,7 @@ use codex_protocol::protocol::WarningEvent;
|
||||
use codex_rollout::state_db;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value;
|
||||
@@ -722,7 +722,10 @@ pub(crate) async fn inspect_pending_input(
|
||||
transcript_path: sess.hook_transcript_path().await,
|
||||
model: turn_context.model_info().slug.clone(),
|
||||
permission_mode: hook_permission_mode(turn_context.approval_policy()),
|
||||
prompt: UserMessageItem::new(content).message(),
|
||||
prompt: content
|
||||
.iter()
|
||||
.filter_map(ResolvedUserInput::as_text)
|
||||
.collect(),
|
||||
};
|
||||
let hooks = with_environment_hooks(
|
||||
sess,
|
||||
@@ -763,8 +766,6 @@ pub(crate) async fn record_pending_input(
|
||||
client_id,
|
||||
metadata,
|
||||
} => {
|
||||
// Prompt telemetry can contain full text. Emit it only for prompts selected for recording.
|
||||
turn_context.session_telemetry.user_prompt(&content);
|
||||
sess.record_user_prompt_and_emit_turn_item(
|
||||
turn_context.as_ref(),
|
||||
model_info,
|
||||
|
||||
@@ -11,6 +11,7 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::ThreadStoreBuilder;
|
||||
@@ -31,6 +32,7 @@ pub async fn build_prompt_input(
|
||||
extensions: Arc<ExtensionRegistry<Config>>,
|
||||
user_instructions_provider: Arc<dyn UserInstructionsProvider>,
|
||||
) -> CodexResult<Vec<ResponseItem>> {
|
||||
let input = crate::codex_thread::resolve_client_inputs(input).await?;
|
||||
config.ephemeral = true;
|
||||
|
||||
let auth_manager =
|
||||
@@ -86,9 +88,9 @@ pub async fn build_prompt_input(
|
||||
output
|
||||
}
|
||||
|
||||
pub(crate) async fn build_prompt_input_from_session(
|
||||
async fn build_prompt_input_from_session(
|
||||
sess: &Arc<Session>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
) -> CodexResult<Vec<ResponseItem>> {
|
||||
let turn_context = sess.new_default_turn().await;
|
||||
// Prompt debugging builds a standalone request without entering run_turn.
|
||||
@@ -99,7 +101,7 @@ pub(crate) async fn build_prompt_input_from_session(
|
||||
.await?;
|
||||
|
||||
if !input.is_empty() {
|
||||
let response_item = sess.response_item_from_user_input(input);
|
||||
let response_item = sess.response_item_from_user_input(&input);
|
||||
sess.record_conversation_items(
|
||||
turn_context.as_ref(),
|
||||
&step_context.settings.model_info,
|
||||
|
||||
@@ -9,9 +9,9 @@ use codex_protocol::protocol::HasLegacyEvent;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_rollout::RolloutItem;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[expect(
|
||||
@@ -30,10 +30,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<(
|
||||
.map(|task| Arc::clone(&task.turn_context));
|
||||
let recording_turn = running_turn.is_none();
|
||||
let turn = running_turn.unwrap_or(recording_context);
|
||||
let content = vec![UserInput::Text {
|
||||
text,
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
let input = ResolvedUserInput::text(text);
|
||||
let now = chrono::Utc::now();
|
||||
if recording_turn {
|
||||
let started = EventMsg::TurnStarted(TurnStartedEvent {
|
||||
@@ -52,7 +49,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<(
|
||||
.record_started_turn(&turn.sub_id, /*attribution*/ None)
|
||||
.await;
|
||||
}
|
||||
let item = session.response_item_from_user_input(content.clone());
|
||||
let item = session.response_item_from_user_input(std::slice::from_ref(&input));
|
||||
session
|
||||
.record_conversation_items(&turn, turn.model_info(), &[item])
|
||||
.await;
|
||||
@@ -63,7 +60,7 @@ pub(super) async fn record(session: &Session, text: String) -> std::io::Result<(
|
||||
content_token_count: None,
|
||||
thread_id: session.thread_id,
|
||||
turn_id: turn.sub_id.clone(),
|
||||
item: TurnItem::UserMessage(UserMessageItem::new(&content)),
|
||||
item: TurnItem::UserMessage(UserMessageItem::new(&[input.to_user_input()])),
|
||||
started_at_ms: Some(now.timestamp_millis()),
|
||||
completed_at_ms: now.timestamp_millis(),
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use super::Submission;
|
||||
use super::sampling_interrupt::SamplingInterrupt;
|
||||
use super::submission::SubmissionOp;
|
||||
use crate::McpRefreshReason;
|
||||
use crate::WithTurnExtensionData;
|
||||
use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio;
|
||||
@@ -549,7 +550,8 @@ pub(super) async fn submission_loop(
|
||||
},
|
||||
async {
|
||||
while let Ok(sub) = rx_sub.recv().await {
|
||||
if let Op::InterAgentCommunication { communication, start_options } = sub.op
|
||||
if let SubmissionOp::Protocol(op) = sub.op
|
||||
&& let Op::InterAgentCommunication { communication, start_options } = *op
|
||||
&& sub.message_delivery.as_ref().is_some_and(crate::agent_communication::MessageDelivery::awaits_result_write)
|
||||
{
|
||||
let mail = crate::PendingAgentMessage::new(communication, sub.message_delivery);
|
||||
@@ -611,7 +613,23 @@ pub(super) async fn submission_loop(
|
||||
);
|
||||
let dispatch_span = submission_dispatch_span(&sub);
|
||||
let should_exit = async {
|
||||
match sub.op {
|
||||
let op = match sub.op {
|
||||
SubmissionOp::TurnInput {
|
||||
request,
|
||||
mode,
|
||||
reply,
|
||||
} => {
|
||||
let request = WithTurnExtensionData {
|
||||
request: *request,
|
||||
turn_extension_init: sub.turn_extension_init,
|
||||
};
|
||||
let result = turn_input::handle(&sess, request, mode, sub.id.clone()).await;
|
||||
let _ = reply.send(result);
|
||||
return false;
|
||||
}
|
||||
SubmissionOp::Protocol(op) => *op,
|
||||
};
|
||||
match op {
|
||||
Op::Interrupt => {
|
||||
interrupt(&sess).await;
|
||||
false
|
||||
@@ -702,19 +720,6 @@ pub(super) async fn submission_loop(
|
||||
realtime_conversation_list_voices(&sess, sub.id.clone()).await;
|
||||
false
|
||||
}
|
||||
Op::TurnInput {
|
||||
request,
|
||||
mode,
|
||||
reply,
|
||||
} => {
|
||||
let request = WithTurnExtensionData {
|
||||
request: *request,
|
||||
turn_extension_init: sub.turn_extension_init,
|
||||
};
|
||||
let result = turn_input::handle(&sess, request, mode, sub.id.clone()).await;
|
||||
let _ = reply.send(result);
|
||||
false
|
||||
}
|
||||
Op::RecoverTurn {
|
||||
thread_settings,
|
||||
start_options,
|
||||
@@ -971,8 +976,8 @@ async fn approve_guardian_denied_action(sess: &Arc<Session>, event: GuardianAsse
|
||||
pub(super) fn submission_dispatch_span(sub: &Submission) -> tracing::Span {
|
||||
let op_name = sub.op.kind();
|
||||
let span_name = format!("op.dispatch.{op_name}");
|
||||
let dispatch_span = match &sub.op {
|
||||
Op::RealtimeConversationAudio(_) => {
|
||||
let dispatch_span = match sub.op.as_protocol() {
|
||||
Some(Op::RealtimeConversationAudio(_)) => {
|
||||
debug_span!(
|
||||
"submission_dispatch",
|
||||
otel.name = span_name.as_str(),
|
||||
|
||||
@@ -8,7 +8,7 @@ use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
@@ -43,7 +43,7 @@ pub struct UserInputMetadata {
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub enum TurnInput {
|
||||
UserInput {
|
||||
content: Vec<UserInput>,
|
||||
content: Vec<ResolvedUserInput>,
|
||||
client_id: Option<String>,
|
||||
#[serde(flatten)]
|
||||
metadata: UserInputMetadata,
|
||||
@@ -569,7 +569,7 @@ mod tests {
|
||||
use codex_history::CodexHarnessMetadata;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
@@ -697,10 +697,7 @@ mod tests {
|
||||
&turn_state,
|
||||
vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "steer".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("steer".to_string())],
|
||||
client_id: None,
|
||||
}],
|
||||
)
|
||||
@@ -789,10 +786,7 @@ mod tests {
|
||||
&turn_state,
|
||||
vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "already pending".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("already pending".to_string())],
|
||||
client_id: None,
|
||||
}],
|
||||
)
|
||||
|
||||
@@ -263,9 +263,10 @@ mod guardian_checkpoint;
|
||||
mod handlers;
|
||||
pub(crate) mod inject;
|
||||
mod reasoning_effort;
|
||||
mod submission;
|
||||
pub(crate) mod submission;
|
||||
pub(crate) use reasoning_effort::RequestEffortUsage;
|
||||
pub(crate) use submission::Submission;
|
||||
use submission::SubmissionOp;
|
||||
mod cloud_plugins;
|
||||
mod input_queue;
|
||||
mod mcp;
|
||||
@@ -429,7 +430,7 @@ use codex_protocol::user_input::UserInput;
|
||||
use codex_skills_extension::HostSkillsService;
|
||||
use codex_tools::ToolName;
|
||||
use codex_user_input::LocalImagePreparation;
|
||||
use codex_user_input::from_user_input;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
#[cfg(test)]
|
||||
use codex_utils_stream_parser::ProposedPlanSegment;
|
||||
@@ -1203,7 +1204,7 @@ impl SessionIo {
|
||||
} = op.into();
|
||||
let sub = Submission {
|
||||
id: id.clone(),
|
||||
op,
|
||||
op: op.into(),
|
||||
turn_extension_init,
|
||||
trace,
|
||||
parent_turn_id,
|
||||
@@ -1233,7 +1234,7 @@ impl SessionIo {
|
||||
/// session loop exits before replying, the caller gets `InternalAgentDied`.
|
||||
pub(crate) async fn submit_turn_input(
|
||||
&self,
|
||||
request: impl Into<WithTurnExtensionData<TurnInputRequest>>,
|
||||
request: impl Into<WithTurnExtensionData<TurnInputRequest<ResolvedUserInput>>>,
|
||||
mode: TurnInputMode,
|
||||
) -> CodexResult<TurnInputSubmission> {
|
||||
let id = new_submission_id();
|
||||
@@ -1245,7 +1246,7 @@ impl SessionIo {
|
||||
let trace = request.trace.take();
|
||||
self.submit_with_id(Box::new(Submission {
|
||||
id,
|
||||
op: Op::TurnInput {
|
||||
op: SubmissionOp::TurnInput {
|
||||
request: Box::new(request),
|
||||
mode,
|
||||
reply: reply_tx,
|
||||
@@ -1276,7 +1277,8 @@ impl SessionIo {
|
||||
thread_settings,
|
||||
start_options,
|
||||
reply: reply_tx,
|
||||
},
|
||||
}
|
||||
.into(),
|
||||
trace,
|
||||
turn_extension_init,
|
||||
parent_turn_id: None,
|
||||
@@ -4007,20 +4009,21 @@ impl Session {
|
||||
assign_missing_item_id(item, &self.item_id_generator);
|
||||
}
|
||||
|
||||
pub(crate) fn response_item_from_user_input(&self, input: Vec<UserInput>) -> ResponseItem {
|
||||
self.response_item_from_user_input_with_image_positions(input, &mut HashMap::new())
|
||||
pub(crate) fn response_item_from_user_input(
|
||||
&self,
|
||||
input: &[ResolvedUserInput],
|
||||
) -> ResponseItem {
|
||||
self.response_item_from_user_input_with_image_positions(input)
|
||||
.0
|
||||
}
|
||||
|
||||
fn response_item_from_user_input_with_image_positions(
|
||||
&self,
|
||||
input: Vec<UserInput>,
|
||||
user_image_content_indices: &mut HashMap<usize, usize>,
|
||||
) -> ResponseItem {
|
||||
let mut item = ResponseItem::from(from_user_input(
|
||||
input,
|
||||
LocalImagePreparation::Defer,
|
||||
user_image_content_indices,
|
||||
));
|
||||
input: &[ResolvedUserInput],
|
||||
) -> (ResponseItem, HashMap<usize, usize>) {
|
||||
let (message, image_positions) =
|
||||
ResolvedUserInput::into_message(input, LocalImagePreparation::Defer);
|
||||
let mut item = ResponseItem::from(message);
|
||||
if let ResponseItem::Message {
|
||||
content,
|
||||
internal_chat_message_metadata_passthrough,
|
||||
@@ -4049,7 +4052,7 @@ impl Session {
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
item
|
||||
(item, image_positions)
|
||||
}
|
||||
|
||||
/// Appends to history, persists the prepared items, then notifies raw-item observers.
|
||||
@@ -5944,19 +5947,20 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
model_info: &ModelInfo,
|
||||
input: &[UserInput],
|
||||
input: &[ResolvedUserInput],
|
||||
client_id: Option<String>,
|
||||
metadata: UserInputMetadata,
|
||||
persist_context: PersistContext,
|
||||
) {
|
||||
// Persist the user message to history, but emit the turn item from `UserInput` so
|
||||
// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
|
||||
// those spans, and `record_response_item_and_emit_turn_item` would drop them.
|
||||
let mut user_image_content_indices = HashMap::new();
|
||||
let mut response_item = self.response_item_from_user_input_with_image_positions(
|
||||
input.to_vec(),
|
||||
&mut user_image_content_indices,
|
||||
);
|
||||
let display_input = input
|
||||
.iter()
|
||||
.map(ResolvedUserInput::to_user_input)
|
||||
.collect::<Vec<_>>();
|
||||
let mut user_message_item = UserMessageItem::new(&display_input);
|
||||
// Emit full prompt telemetry only for inputs selected for recording.
|
||||
turn_context.session_telemetry.user_prompt(&display_input);
|
||||
let (mut response_item, user_image_content_indices) =
|
||||
self.response_item_from_user_input_with_image_positions(input);
|
||||
if let Some(uuid) = metadata.response_item_uuid {
|
||||
response_item.set_id(Some(ResponseItemId::for_admitted_user_input(uuid)));
|
||||
}
|
||||
@@ -5985,7 +5989,6 @@ impl Session {
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let mut user_message_item = UserMessageItem::new(input);
|
||||
apply_prepared_image_file_ids(
|
||||
&mut user_message_item,
|
||||
&prepared_items,
|
||||
|
||||
@@ -34,10 +34,9 @@ async fn task_finish_preserves_notification_budget_and_queued_user_input() {
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
let user_input = vec![UserInput::Text {
|
||||
text: "keep this queued user input".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
let user_input = vec![ResolvedUserInput::text(
|
||||
"keep this queued user input".to_string(),
|
||||
)];
|
||||
assert!(matches!(
|
||||
submit_steer_only(&session, user_input.clone(), &turn.sub_id).await,
|
||||
TurnInputSubmission::Steered { .. }
|
||||
@@ -78,7 +77,7 @@ async fn task_finish_preserves_notification_budget_and_queued_user_input() {
|
||||
&text,
|
||||
codex_utils_output_truncation::TruncationPolicy::Tokens(120),
|
||||
)),
|
||||
session.response_item_from_user_input(user_input),
|
||||
session.response_item_from_user_input(&user_input),
|
||||
late[0].clone(),
|
||||
];
|
||||
let history = session.clone_history().await;
|
||||
|
||||
@@ -143,11 +143,7 @@ pub(super) async fn spawn_review_thread(
|
||||
// Seed the child task with the review prompt as the initial user message.
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: review_prompt,
|
||||
// Review prompt is synthesized; no UI element ranges to preserve.
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text(review_prompt)],
|
||||
client_id: None,
|
||||
}];
|
||||
let tc = Arc::new(review_turn_context);
|
||||
|
||||
@@ -518,7 +518,8 @@ fn settings_submission(
|
||||
turn_id: turn_id.to_string(),
|
||||
update,
|
||||
reply,
|
||||
},
|
||||
}
|
||||
.into(),
|
||||
trace: None,
|
||||
parent_turn_id: None,
|
||||
root_turn_id: None,
|
||||
|
||||
@@ -1,14 +1,64 @@
|
||||
//! Core-owned queue metadata; forwarding transfers the residency guard with the operation.
|
||||
|
||||
use codex_extension_api::ExtensionDataInit;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
use codex_protocol::turn_input::TurnInputMode;
|
||||
use codex_protocol::turn_input::TurnInputRequest;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use tokio::sync::OwnedRwLockReadGuard;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
/// Turn input crosses this queue only after the submitting host has resolved its attachments.
|
||||
pub(crate) enum SubmissionOp {
|
||||
Protocol(Box<Op>),
|
||||
TurnInput {
|
||||
request: Box<TurnInputRequest<ResolvedUserInput>>,
|
||||
mode: TurnInputMode,
|
||||
reply: oneshot::Sender<CodexResult<TurnInputSubmission>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<Op> for SubmissionOp {
|
||||
fn from(op: Op) -> Self {
|
||||
Self::Protocol(Box::new(op))
|
||||
}
|
||||
}
|
||||
|
||||
impl SubmissionOp {
|
||||
pub(crate) fn as_protocol(&self) -> Option<&Op> {
|
||||
match self {
|
||||
Self::Protocol(op) => Some(op),
|
||||
Self::TurnInput { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Protocol(op) => op.kind(),
|
||||
Self::TurnInput { .. } => "turn_input",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for SubmissionOp {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Protocol(op) => std::fmt::Debug::fmt(op, f),
|
||||
Self::TurnInput { request, .. } => f
|
||||
.debug_struct("TurnInput")
|
||||
.field("requested_settings", &request.thread_settings)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Submission {
|
||||
pub id: String,
|
||||
pub op: Op,
|
||||
pub op: SubmissionOp,
|
||||
/// Accepted with settings, never separately from the request that supplied them.
|
||||
pub turn_extension_init: Option<ExtensionDataInit>,
|
||||
/// Optional W3C trace carrier propagated across async submission handoffs.
|
||||
|
||||
@@ -6155,7 +6155,8 @@ async fn standalone_settings_invalidate_continuation_before_delivering_acceptanc
|
||||
op: Op::ThreadSettings {
|
||||
thread_settings: ThreadSettingsOverrides::default(),
|
||||
reply: Some(reply),
|
||||
},
|
||||
}
|
||||
.into(),
|
||||
trace: None,
|
||||
parent_turn_id: None,
|
||||
root_turn_id: None,
|
||||
@@ -8250,7 +8251,7 @@ fn submission_dispatch_span_prefers_submission_trace_context() {
|
||||
submission_dispatch_span(&Submission {
|
||||
turn_extension_init: None,
|
||||
id: "sub-1".into(),
|
||||
op: Op::Interrupt,
|
||||
op: Op::Interrupt.into(),
|
||||
parent_turn_id: None,
|
||||
root_turn_id: None,
|
||||
residency_guard: None,
|
||||
@@ -8281,7 +8282,8 @@ fn submission_dispatch_span_uses_debug_for_realtime_audio() {
|
||||
samples_per_channel: Some(160),
|
||||
item_id: None,
|
||||
},
|
||||
}),
|
||||
})
|
||||
.into(),
|
||||
parent_turn_id: None,
|
||||
root_turn_id: None,
|
||||
residency_guard: None,
|
||||
@@ -8644,7 +8646,7 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() {
|
||||
let dispatch_span = submission_dispatch_span(&Submission {
|
||||
turn_extension_init: None,
|
||||
id: "sub-1".into(),
|
||||
op: Op::Interrupt,
|
||||
op: Op::Interrupt.into(),
|
||||
parent_turn_id: None,
|
||||
root_turn_id: None,
|
||||
residency_guard: None,
|
||||
@@ -8660,10 +8662,7 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() {
|
||||
Arc::clone(&tc),
|
||||
vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}],
|
||||
TraceCaptureTask {
|
||||
@@ -9016,7 +9015,7 @@ async fn shutdown_and_wait_allows_multiple_waiters() {
|
||||
let (_tx_event, rx_event) = async_channel::unbounded();
|
||||
let session_loop_handle = tokio::spawn(async move {
|
||||
let shutdown = rx_sub.recv().await.expect("shutdown submission");
|
||||
assert!(matches!(shutdown.op, Op::Shutdown));
|
||||
assert!(matches!(shutdown.op.as_protocol(), Some(Op::Shutdown)));
|
||||
tokio::time::sleep(StdDuration::from_millis(50)).await;
|
||||
});
|
||||
let io = Arc::new(SessionIo {
|
||||
@@ -10419,10 +10418,7 @@ async fn spawn_task_does_not_update_previous_turn_settings_for_non_run_turn_task
|
||||
.await;
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
|
||||
@@ -12373,10 +12369,9 @@ async fn extension_interrupt_survives_the_calling_runtime() {
|
||||
let (sess, tc, rx) = make_session_and_context_with_rx().await;
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "keep turn active for extension interruption".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text(
|
||||
"keep turn active for extension interruption".to_string(),
|
||||
)],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(
|
||||
@@ -12438,10 +12433,7 @@ async fn turn_complete_flushes_terminal_event_after_delivery() {
|
||||
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "complete normally".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("complete normally".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(Arc::clone(&tc), input, CompletingTask)
|
||||
@@ -12466,10 +12458,7 @@ async fn turn_aborted_flushes_terminal_event_after_delivery() {
|
||||
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "interrupt me".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("interrupt me".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(
|
||||
@@ -12509,10 +12498,7 @@ async fn abort_regular_task_emits_marker_before_turn_aborted() {
|
||||
let (sess, tc, rx) = make_session_and_context_with_rx().await;
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(
|
||||
@@ -12551,10 +12537,7 @@ async fn abort_gracefully_emits_marker_before_turn_aborted() {
|
||||
let (sess, tc, rx) = make_session_and_context_with_rx().await;
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(
|
||||
@@ -12590,7 +12573,7 @@ async fn abort_gracefully_emits_marker_before_turn_aborted() {
|
||||
|
||||
async fn submit_steer_only(
|
||||
sess: &Arc<Session>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
expected_turn_id: &str,
|
||||
) -> TurnInputSubmission {
|
||||
super::turn_input::handle(
|
||||
@@ -12628,10 +12611,7 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
|
||||
);
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(
|
||||
@@ -12667,7 +12647,16 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
];
|
||||
let submission = submit_steer_only(&sess, pending_user_input.clone(), &tc.sub_id).await;
|
||||
let submission = submit_steer_only(
|
||||
&sess,
|
||||
pending_user_input
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect(),
|
||||
&tc.sub_id,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(submission, TurnInputSubmission::Steered { .. }));
|
||||
|
||||
let mut current = tc.initial_settings.as_ref().clone();
|
||||
@@ -13094,10 +13083,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() {
|
||||
.await;
|
||||
let submission = submit_steer_only(
|
||||
&sess,
|
||||
vec![UserInput::Text {
|
||||
text: "follow up".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("follow up".to_string())],
|
||||
&tc.sub_id,
|
||||
)
|
||||
.await;
|
||||
@@ -13111,10 +13097,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() {
|
||||
acceptance_order: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
content: vec![UserInput::Text {
|
||||
text: "follow up".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("follow up".to_string())],
|
||||
client_id: None,
|
||||
},
|
||||
TurnInput::InterAgentCommunication(communication.into()),
|
||||
@@ -13150,10 +13133,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() {
|
||||
.await;
|
||||
let submission = submit_steer_only(
|
||||
&sess,
|
||||
vec![UserInput::Text {
|
||||
text: "follow up".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("follow up".to_string())],
|
||||
&tc.sub_id,
|
||||
)
|
||||
.await;
|
||||
@@ -13171,10 +13151,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() {
|
||||
acceptance_order: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
content: vec![UserInput::Text {
|
||||
text: "follow up".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("follow up".to_string())],
|
||||
client_id: None,
|
||||
},
|
||||
TurnInput::InterAgentCommunication(communication.into()),
|
||||
@@ -13247,10 +13224,7 @@ async fn abort_review_task_emits_exited_then_aborted_and_records_history() {
|
||||
let (sess, tc, rx) = make_session_and_context_with_rx().await;
|
||||
let input = vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "start review".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("start review".to_string())],
|
||||
client_id: None,
|
||||
}];
|
||||
sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new())
|
||||
|
||||
@@ -102,7 +102,6 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::items::PlanItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::build_hook_prompt_message;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
@@ -136,6 +135,7 @@ use codex_tools::DiscoverableTool;
|
||||
use codex_tools::ToolName;
|
||||
use codex_tools::ToolSpec;
|
||||
use codex_tools::filter_request_plugin_install_discoverable_tools_for_client;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use codex_utils_stream_parser::AssistantTextChunk;
|
||||
use codex_utils_stream_parser::AssistantTextStreamParser;
|
||||
@@ -303,7 +303,7 @@ pub(crate) async fn run_turn(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let user_input = turn_user_input(&input);
|
||||
let user_input = input_for_extensions(&input);
|
||||
let allow_plugin_mentions =
|
||||
!crate::guardian::is_basic_session_source(&turn_context.session_source);
|
||||
let McpStartupRequirements {
|
||||
@@ -594,7 +594,7 @@ pub(crate) async fn run_turn(
|
||||
.await?
|
||||
}
|
||||
Some(_) | None => {
|
||||
let pending_user_input = turn_user_input(&pending_input);
|
||||
let pending_user_input = input_for_extensions(&pending_input);
|
||||
if allow_plugin_mentions {
|
||||
required_plugins.extend(crate::plugins::collect_explicit_plugin_ids(
|
||||
&pending_user_input,
|
||||
@@ -1169,7 +1169,7 @@ pub(crate) fn recordable_inputs<'a>(
|
||||
&& turn_input_has_uploaded_audio_file(&item)
|
||||
&& let TurnInput::UserInput { content, .. } = item.to_mut()
|
||||
{
|
||||
content.retain(|part| !is_uploaded_audio_file(part));
|
||||
content.retain(|part| !part.is_uploaded_audio_file());
|
||||
if content.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -1180,7 +1180,7 @@ pub(crate) fn recordable_inputs<'a>(
|
||||
}
|
||||
|
||||
fn turn_input_has_uploaded_audio_file(input: &TurnInput) -> bool {
|
||||
matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file))
|
||||
matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(ResolvedUserInput::is_uploaded_audio_file))
|
||||
}
|
||||
|
||||
fn unsupported_uploaded_audio_error() -> CodexErr {
|
||||
@@ -1189,16 +1189,8 @@ fn unsupported_uploaded_audio_error() -> CodexErr {
|
||||
)
|
||||
}
|
||||
|
||||
fn is_uploaded_audio_file(input: &UserInput) -> bool {
|
||||
matches!(
|
||||
input,
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File { .. }
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn turn_user_input(input: &[TurnInput]) -> Vec<UserInput> {
|
||||
/// Extensions and telemetry consume the protocol view, including the original mention and path labels.
|
||||
fn input_for_extensions(input: &[TurnInput]) -> Vec<UserInput> {
|
||||
input
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
@@ -1208,7 +1200,7 @@ pub(crate) fn turn_user_input(input: &[TurnInput]) -> Vec<UserInput> {
|
||||
| TurnInput::InterAgentCommunication(_) => None,
|
||||
})
|
||||
.flatten()
|
||||
.cloned()
|
||||
.map(ResolvedUserInput::to_user_input)
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -1568,9 +1560,7 @@ async fn track_turn_resolved_config_analytics(
|
||||
| TurnInput::InterAgentCommunication(_) => None,
|
||||
})
|
||||
.flatten()
|
||||
.filter(|item| {
|
||||
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
|
||||
})
|
||||
.filter(|item| item.is_image())
|
||||
.count(),
|
||||
submission_type: None,
|
||||
ephemeral: thread_config.ephemeral,
|
||||
|
||||
@@ -51,12 +51,12 @@ use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||
use codex_protocol::turn_input::AnnotatedResponseItem;
|
||||
use codex_protocol::turn_input::NotSubmittedReason;
|
||||
use codex_protocol::turn_input::ResponsesApiClientMetadataUpdate;
|
||||
use codex_protocol::turn_input::TurnInput as SubmittedTurnInput;
|
||||
type SubmittedTurnInput = codex_protocol::turn_input::TurnInput<ResolvedUserInput>;
|
||||
use codex_protocol::turn_input::TurnInputMode;
|
||||
use codex_protocol::turn_input::TurnInputRequest;
|
||||
type TurnInputRequest = codex_protocol::turn_input::TurnInputRequest<ResolvedUserInput>;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
@@ -926,14 +926,12 @@ impl Session {
|
||||
let submission_id = Uuid::now_v7().to_string();
|
||||
let submission = handle(
|
||||
self,
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text,
|
||||
text_elements: Vec::new(),
|
||||
}])
|
||||
.on_start(TurnStartOptions {
|
||||
turn_trigger: Some("realtime".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start(
|
||||
TurnStartOptions {
|
||||
turn_trigger: Some("realtime".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
TurnInputMode::StartOrSteer,
|
||||
submission_id.clone(),
|
||||
)
|
||||
|
||||
@@ -11,7 +11,8 @@ use crate::session::session::Session;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::turn_input::NotSubmittedReason;
|
||||
use codex_protocol::turn_input::TurnInput;
|
||||
use codex_protocol::turn_input::TurnInputRequest;
|
||||
type TurnInputRequest =
|
||||
codex_protocol::turn_input::TurnInputRequest<codex_user_input::ResolvedUserInput>;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use std::sync::Arc;
|
||||
|
||||
|
||||
@@ -22,8 +22,7 @@ use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||
use codex_protocol::protocol::TurnAbortReason;
|
||||
use codex_protocol::turn_input::TurnInput as SubmittedTurnInput;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use core_test_support::test_codex::local_selections;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -93,7 +92,7 @@ async fn submit_start_only(
|
||||
|
||||
async fn submit_steer_only(
|
||||
session: &Arc<Session>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
expected_turn_id: &str,
|
||||
) -> TurnInputSubmission {
|
||||
handle(
|
||||
@@ -148,10 +147,9 @@ async fn steering_does_not_wait_for_realtime_history() {
|
||||
std::time::Duration::from_secs(5),
|
||||
handle(
|
||||
&session,
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "steer without waiting for persistence".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}]),
|
||||
TurnInputRequest::user_input(vec![ResolvedUserInput::text(
|
||||
"steer without waiting for persistence".to_string(),
|
||||
)]),
|
||||
mode,
|
||||
"steer-submission".to_string(),
|
||||
),
|
||||
@@ -177,27 +175,24 @@ async fn accepted_input_applies_thread_settings() {
|
||||
let config = session.get_config().await;
|
||||
handle(
|
||||
&session,
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}])
|
||||
.with_thread_settings(ThreadSettingsOverrides {
|
||||
environments: Some(local_selections(config.cwd.clone()).into_requests()),
|
||||
approval_policy: Some(config.permissions.approval_policy.value()),
|
||||
approvals_reviewer: Some(codex_config::types::ApprovalsReviewer::AutoReview),
|
||||
permission_profile: Some(config.permissions.effective_permission_profile()),
|
||||
summary: config.model_reasoning_summary,
|
||||
personality: config.personality,
|
||||
collaboration_mode: Some(CollaborationMode {
|
||||
mode: ModeKind::Default,
|
||||
settings: Settings {
|
||||
model: turn_context.model_info().slug.clone(),
|
||||
reasoning_effort: config.model_reasoning_effort.clone(),
|
||||
developer_instructions: None,
|
||||
},
|
||||
TurnInputRequest::user_input(vec![ResolvedUserInput::text("hello".to_string())])
|
||||
.with_thread_settings(ThreadSettingsOverrides {
|
||||
environments: Some(local_selections(config.cwd.clone()).into_requests()),
|
||||
approval_policy: Some(config.permissions.approval_policy.value()),
|
||||
approvals_reviewer: Some(codex_config::types::ApprovalsReviewer::AutoReview),
|
||||
permission_profile: Some(config.permissions.effective_permission_profile()),
|
||||
summary: config.model_reasoning_summary,
|
||||
personality: config.personality,
|
||||
collaboration_mode: Some(CollaborationMode {
|
||||
mode: ModeKind::Default,
|
||||
settings: Settings {
|
||||
model: turn_context.model_info().slug.clone(),
|
||||
reasoning_effort: config.model_reasoning_effort.clone(),
|
||||
developer_instructions: None,
|
||||
},
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
TurnInputMode::StartOrSteer,
|
||||
"sub-1".to_string(),
|
||||
)
|
||||
@@ -787,10 +782,7 @@ async fn start_only_accepts_user_input_in_plan_mode() {
|
||||
let submission = submit_start_only(
|
||||
&session,
|
||||
SubmittedTurnInput::UserInput {
|
||||
content: vec![UserInput::Text {
|
||||
text: "queued user input".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("queued user input".to_string())],
|
||||
client_id: Some("queued-user-message".to_string()),
|
||||
},
|
||||
)
|
||||
@@ -928,10 +920,7 @@ async fn steer_only_requires_active_turn() {
|
||||
let (session, _turn_context, _rx) = make_session_and_context_with_rx().await;
|
||||
let submission = submit_steer_only(
|
||||
&session,
|
||||
vec![UserInput::Text {
|
||||
text: "steer".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("steer".to_string())],
|
||||
"missing-turn-id",
|
||||
)
|
||||
.await;
|
||||
@@ -955,10 +944,7 @@ async fn steer_only_enforces_expected_turn_id() {
|
||||
Arc::clone(&turn_context),
|
||||
vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}],
|
||||
NeverEndingTask {
|
||||
@@ -970,10 +956,7 @@ async fn steer_only_enforces_expected_turn_id() {
|
||||
|
||||
let submission = submit_steer_only(
|
||||
&session,
|
||||
vec![UserInput::Text {
|
||||
text: "steer".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("steer".to_string())],
|
||||
"different-turn-id",
|
||||
)
|
||||
.await;
|
||||
@@ -1059,10 +1042,7 @@ async fn rejects_non_regular_turns() {
|
||||
Arc::clone(&turn_context),
|
||||
vec![TurnInput::UserInput {
|
||||
metadata: Default::default(),
|
||||
content: vec![UserInput::Text {
|
||||
text: "hello".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("hello".to_string())],
|
||||
client_id: None,
|
||||
}],
|
||||
NeverEndingTask {
|
||||
@@ -1072,10 +1052,7 @@ async fn rejects_non_regular_turns() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let steer_input = vec![UserInput::Text {
|
||||
text: "steer".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
let steer_input = vec![ResolvedUserInput::text("steer".to_string())];
|
||||
let steer_submission = submit_steer_only(&session, steer_input.clone(), "turn").await;
|
||||
assert_eq!(
|
||||
TurnInputSubmission::NotSubmitted {
|
||||
@@ -1128,10 +1105,9 @@ async fn steer_preserves_request_origin(
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let content = vec![UserInput::Text {
|
||||
text: "Create the worktree now.".to_owned(),
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
let content = vec![ResolvedUserInput::text(
|
||||
"Create the worktree now.".to_owned(),
|
||||
)];
|
||||
handle(
|
||||
&session,
|
||||
TurnInputRequest::user_input(content.clone()).on_start(TurnStartOptions {
|
||||
@@ -1210,10 +1186,7 @@ async fn scored_prompt_keeps_its_submission_item_id() {
|
||||
// A legacy steer can present the same UI client ID as an admitted steer.
|
||||
for id in [Some(first), None, Some(second)] {
|
||||
let mut request = TurnInputRequest::new(SubmittedTurnInput::UserInput {
|
||||
content: vec![UserInput::Text {
|
||||
text: "identical input".into(),
|
||||
text_elements: vec![],
|
||||
}],
|
||||
content: vec![ResolvedUserInput::text("identical input".into())],
|
||||
client_id: Some("same-client-id".into()),
|
||||
});
|
||||
request.user_item_uuid = id;
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::state::TaskKind;
|
||||
use codex_features::Feature;
|
||||
use codex_model_provider::RemoteCompactionSupport;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
@@ -82,16 +82,13 @@ impl SessionTask for CompactTask {
|
||||
"local",
|
||||
/*manual*/ true,
|
||||
);
|
||||
let input = vec![UserInput::Text {
|
||||
text: ctx
|
||||
.config
|
||||
let input = vec![ResolvedUserInput::text(
|
||||
ctx.config
|
||||
.compact_prompt
|
||||
.as_deref()
|
||||
.unwrap_or(crate::compact::SUMMARIZATION_PROMPT)
|
||||
.to_string(),
|
||||
// Compaction prompt is synthesized; no UI element ranges to preserve.
|
||||
text_elements: Vec::new(),
|
||||
}];
|
||||
)];
|
||||
crate::compact::run_compact_task(session.clone(), step_context, world_state, input)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -95,7 +95,10 @@ async fn task_finish_rejects_uploaded_audio_without_losing_other_pending_input()
|
||||
file_id: "file_disabled_at_finish".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
]
|
||||
.into_iter()
|
||||
.map(codex_user_input::ResolvedUserInput::from_client)
|
||||
.collect(),
|
||||
client_id: None,
|
||||
metadata: Default::default(),
|
||||
}],
|
||||
|
||||
@@ -26,8 +26,8 @@ use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::state::TaskKind;
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
|
||||
use super::SessionTask;
|
||||
use super::SessionTaskResult;
|
||||
@@ -98,7 +98,7 @@ impl SessionTask for ReviewTask {
|
||||
async fn start_review_conversation(
|
||||
session: Arc<Session>,
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
input: Vec<ResolvedUserInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<async_channel::Receiver<Event>> {
|
||||
let config = ctx.config.clone();
|
||||
|
||||
@@ -29,6 +29,7 @@ use codex_protocol::turn_input::TurnInputRequest;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::ThreadMetadataPatch;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use core_test_support::PathExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::sync::Arc;
|
||||
@@ -97,7 +98,7 @@ async fn spawn_child(
|
||||
.control(parent.session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
config.clone(),
|
||||
text(),
|
||||
vec![ResolvedUserInput::text("Continue".into())],
|
||||
source,
|
||||
SpawnAgentOptions {
|
||||
parent_thread_id: Some(parent_id),
|
||||
|
||||
@@ -57,8 +57,8 @@ use codex_protocol::protocol::ThreadSource;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::InMemoryThreadStore;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use core_test_support::PathBufExt;
|
||||
use core_test_support::PathExt;
|
||||
@@ -794,7 +794,7 @@ async fn thread_id_generator_applies_to_roots_children_and_forks() {
|
||||
.control(root.thread.session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
config.clone(),
|
||||
vec![UserInput::Text {
|
||||
vec![ResolvedUserInput::Text {
|
||||
text: "child task".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::agent::control::render_input_preview;
|
||||
use crate::tools::handlers::multi_agents_spec::create_send_input_tool_v1;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_tools::ToolSpec;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
|
||||
pub(crate) struct Handler;
|
||||
|
||||
@@ -49,6 +50,10 @@ impl Handler {
|
||||
let receiver_thread_id = parse_agent_id_target(&args.target)?;
|
||||
let input_items = parse_collab_input(args.message, args.items)?;
|
||||
let prompt = render_input_preview(&input_items);
|
||||
let input_items = input_items
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
let local_agent_control = session
|
||||
.services
|
||||
.local_agent_runtime
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions;
|
||||
use crate::tools::handlers::multi_agents_spec::create_spawn_agent_tool_v1;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_tools::ToolSpec;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Handler {
|
||||
@@ -80,6 +81,10 @@ async fn handle_spawn_agent(
|
||||
"Agent depth limit reached. Solve the task yourself.".to_string(),
|
||||
));
|
||||
}
|
||||
let input_items = input_items
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
session
|
||||
.emit_turn_item_started(
|
||||
turn,
|
||||
|
||||
@@ -74,6 +74,7 @@ use codex_protocol::protocol::TurnAbortedEvent;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_state::DirectionalThreadSpawnEdgeStatus;
|
||||
use codex_user_input::ResolvedUserInput;
|
||||
use core_test_support::TempDirExt;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
@@ -1336,10 +1337,7 @@ async fn multi_agent_v2_send_message_accepts_root_target_from_child() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
(*turn.config).clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "inspect this repo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("inspect this repo".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
@@ -1417,10 +1415,7 @@ async fn multi_agent_v2_followup_task_rejects_root_target_from_child() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
(*turn.config).clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "inspect this repo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("inspect this repo".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
@@ -1584,10 +1579,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
config.clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "research".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("research".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
@@ -1605,10 +1597,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
config,
|
||||
vec![UserInput::Text {
|
||||
text: "build".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("build".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 2,
|
||||
@@ -4154,10 +4143,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_id() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
(*turn.config).clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "inspect this repo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("inspect this repo".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
@@ -4222,10 +4208,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_task_name() {
|
||||
.control(session.session_id())
|
||||
.spawn_agent_with_metadata(
|
||||
(*turn.config).clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "inspect this repo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
vec![ResolvedUserInput::text("inspect this repo".to_string())],
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
|
||||
@@ -124,19 +124,14 @@ async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Resu
|
||||
.update_thread_settings(ThreadSettingsOverrides::default())
|
||||
.await?;
|
||||
wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await;
|
||||
let (reply, resumed) = oneshot::channel();
|
||||
child
|
||||
.submit(Op::TurnInput {
|
||||
request: Box::new(codex_core::TurnInputRequest::user_input(vec![
|
||||
codex_protocol::user_input::UserInput::Text {
|
||||
text: "Resume.".into(),
|
||||
text_elements: vec![],
|
||||
},
|
||||
])),
|
||||
mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer,
|
||||
reply,
|
||||
})
|
||||
.await?;
|
||||
let resumed = child.start_or_steer_turn(codex_core::TurnInputRequest::user_input(vec![
|
||||
codex_protocol::user_input::UserInput::Text {
|
||||
text: "Resume.".into(),
|
||||
text_elements: vec![],
|
||||
},
|
||||
]));
|
||||
tokio::pin!(resumed);
|
||||
assert!(futures::poll!(resumed.as_mut()).is_pending());
|
||||
assert_eq!(child.agent_status().await, AgentStatus::Running);
|
||||
release_item.send(()).expect("release the sampled item");
|
||||
let next = |event: &EventMsg| match event {
|
||||
@@ -153,7 +148,7 @@ async fn interrupt_retains_one_sampled_item(mut done: serde_json::Value) -> Resu
|
||||
EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted
|
||||
));
|
||||
assert!(matches!(
|
||||
resumed.await??,
|
||||
resumed.await?,
|
||||
codex_core::TurnInputSubmission::Started { .. }
|
||||
));
|
||||
wait_for_event(&child, |event| matches!(event, EventMsg::TurnStarted(_))).await;
|
||||
|
||||
@@ -38,6 +38,7 @@ use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::config_types::Verbosity;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
@@ -56,6 +57,7 @@ use codex_protocol::models::WebSearchAction;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
@@ -88,6 +90,7 @@ use core_test_support::test_codex::TestCodex;
|
||||
use core_test_support::test_codex::local_requests;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_match;
|
||||
use futures::StreamExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
@@ -566,26 +569,63 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> {
|
||||
sse(vec![ev_response_created("resp1"), ev_completed("resp1")]),
|
||||
)
|
||||
.await;
|
||||
let codex = test_codex()
|
||||
let test = test_codex()
|
||||
.with_model_info_override("gpt-5.5", |model_info| {
|
||||
model_info.input_modalities.push(InputModality::Audio);
|
||||
})
|
||||
.build(&server)
|
||||
.await?
|
||||
.codex;
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
// Client-selected attachments live on the app-server host, even with a remote executor.
|
||||
let temp_dir = tempfile::tempdir()?;
|
||||
let audio_path = temp_dir.path().join("recording.wav");
|
||||
std::fs::write(&audio_path, b"audio")?;
|
||||
|
||||
codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::LocalAudio {
|
||||
let unsupported = temp_dir.path().join("clip.flac");
|
||||
std::fs::write(&unsupported, b"audio")?;
|
||||
let missing = temp_dir.path().join("missing.wav");
|
||||
let inline = "data:audio/wav;base64,YXVkaW8=";
|
||||
let input = vec![
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: inline.into(),
|
||||
},
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: unsupported.clone(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: audio_path.clone(),
|
||||
}]))
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: missing.clone(),
|
||||
},
|
||||
];
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(input.clone()))
|
||||
.await?;
|
||||
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
let displayed_input = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
item: TurnItem::UserMessage(message),
|
||||
..
|
||||
}) => Some(message.content.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(displayed_input, input);
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let request = response_mock.single_request();
|
||||
assert!(request.has_content_kinds(&["user.text", "user.audio", "user.text"]));
|
||||
assert!(request.has_content_kinds(&[
|
||||
"user.audio",
|
||||
"user.text",
|
||||
"user.text",
|
||||
"user.audio",
|
||||
"user.text",
|
||||
"user.text"
|
||||
]));
|
||||
let user_message = request
|
||||
.input()
|
||||
.into_iter()
|
||||
@@ -593,12 +633,20 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> {
|
||||
.find(|item| item.get("role").and_then(serde_json::Value::as_str) == Some("user"))
|
||||
.expect("request should include a user message");
|
||||
let audio_path = audio_path.display();
|
||||
let unsupported = unsupported.display();
|
||||
let missing_error = std::fs::File::open(&missing).unwrap_err();
|
||||
let missing = missing.display();
|
||||
assert_eq!(
|
||||
user_message["content"],
|
||||
json!([
|
||||
{"type": "input_audio", "audio_url": inline},
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": format!(r#"<audio name=[Audio #1] path="{audio_path}">"#),
|
||||
"text": format!("Codex cannot attach audio at `{unsupported}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg."),
|
||||
},
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": format!(r#"<audio name=[Audio #3] path="{audio_path}">"#),
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
@@ -608,6 +656,10 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> {
|
||||
"type": "input_text",
|
||||
"text": "</audio>",
|
||||
},
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": format!("Codex could not read the local audio at `{missing}`: {missing_error}"),
|
||||
},
|
||||
])
|
||||
);
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use codex_core::AgentSessionReader;
|
||||
use codex_core::AgentTurnAdmission;
|
||||
use codex_core::AgentTurnAdmissionRequest;
|
||||
use codex_core::CodexThread;
|
||||
use codex_core::SteerSubmission;
|
||||
use codex_core::SuspendTurnOutcome;
|
||||
use codex_core::TurnInputRequest;
|
||||
use codex_core::TurnInputSubmission;
|
||||
@@ -160,6 +161,58 @@ fn assert_denied(event: &EventMsg, turn_id: &str) {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn steered_audio_keeps_bytes_captured_while_admission_is_pending() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = responses::start_mock_server().await;
|
||||
let mock = responses::mount_sse_sequence(
|
||||
&server,
|
||||
["initial", "after-steer"]
|
||||
.map(|id| {
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created(id),
|
||||
responses::ev_completed(id),
|
||||
])
|
||||
})
|
||||
.to_vec(),
|
||||
)
|
||||
.await;
|
||||
let builder = test_codex().with_model_info_override("gpt-5.5", |model| {
|
||||
model
|
||||
.input_modalities
|
||||
.push(codex_protocol::openai_models::InputModality::Audio);
|
||||
});
|
||||
let (test, mut attempts) =
|
||||
root_with_admission(&server, builder, /*session_reader*/ None).await?;
|
||||
let (turn_id, decision) = start(&test, &mut attempts, "Listen to the attachment.").await?;
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("clip.wav");
|
||||
std::fs::write(&path, b"audio")?;
|
||||
let submission = test
|
||||
.codex
|
||||
.steer_turn(
|
||||
TurnInputRequest::user_input(vec![UserInput::LocalAudio { path: path.clone() }]),
|
||||
turn_id.clone(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(submission, SteerSubmission::Steered { turn_id });
|
||||
std::fs::remove_file(path)?;
|
||||
assert!(decision.send(Ok(AgentExecutionGuard::new(()))).is_ok());
|
||||
wait_for_successful_turn(&test.codex).await;
|
||||
let audio = mock.requests()[1]
|
||||
.input()
|
||||
.into_iter()
|
||||
.flat_map(|item| item["content"].as_array().cloned().unwrap_or_default())
|
||||
.filter(|part| part["type"] == "input_audio")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
audio,
|
||||
[json!({"type": "input_audio", "audio_url": "data:audio/wav;base64,YXVkaW8="})]
|
||||
);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn root_denial_and_interruption_retain_input_and_grants_span_execution() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
@@ -143,14 +143,10 @@ async fn interrupt_during_remote_compaction_preserves_summary_on_resume() -> Res
|
||||
.update_thread_settings(ThreadSettingsOverrides::default())
|
||||
.await?;
|
||||
assert_eq!(child.agent_status().await, AgentStatus::Running);
|
||||
let (reply, resumed) = oneshot::channel();
|
||||
child
|
||||
.submit(Op::TurnInput {
|
||||
request: Box::new(TurnInputRequest::user_input(vec![text("Resume.")])),
|
||||
mode: codex_protocol::turn_input::TurnInputMode::StartOrSteer,
|
||||
reply,
|
||||
})
|
||||
.await?;
|
||||
let resumed =
|
||||
child.start_or_steer_turn(TurnInputRequest::user_input(vec![text("Resume.")]));
|
||||
tokio::pin!(resumed);
|
||||
assert!(futures::poll!(resumed.as_mut()).is_pending());
|
||||
release_summary
|
||||
.send(())
|
||||
.expect("finish sampling the summary");
|
||||
@@ -169,7 +165,7 @@ async fn interrupt_during_remote_compaction_preserves_summary_on_resume() -> Res
|
||||
assert!(matches!(terminal,
|
||||
EventMsg::TurnAborted(event) if event.reason == TurnAbortReason::Interrupted));
|
||||
}
|
||||
let TurnInputSubmission::Started { turn_id, .. } = resumed.await?? else {
|
||||
let TurnInputSubmission::Started { turn_id, .. } = resumed.await? else {
|
||||
panic!("Resume must start a new turn");
|
||||
};
|
||||
assert_ne!(turn_id, previous_turn_id);
|
||||
|
||||
@@ -64,8 +64,6 @@ use crate::request_user_input::RequestUserInputResponse;
|
||||
use crate::response_item_id::OptionalWireItemId;
|
||||
use crate::turn_input::CyberAccessProgram;
|
||||
use crate::turn_input::SuspendTurnOutcome;
|
||||
use crate::turn_input::TurnInputMode;
|
||||
use crate::turn_input::TurnInputRequest;
|
||||
use crate::turn_input::TurnInputSubmission;
|
||||
use crate::turn_input::TurnStartOptions;
|
||||
use codex_extension_items::image_generation::GeneratedImageRef;
|
||||
@@ -786,14 +784,6 @@ pub enum Op {
|
||||
/// Request the list of voices supported by realtime conversation streams.
|
||||
RealtimeConversationListVoices,
|
||||
|
||||
/// Submit turn input using the requested routing behavior.
|
||||
#[debug("TurnInput {{ requested_settings: {:?} }}", request.thread_settings)]
|
||||
TurnInput {
|
||||
request: Box<TurnInputRequest>,
|
||||
mode: TurnInputMode,
|
||||
reply: oneshot::Sender<CodexResult<TurnInputSubmission>>,
|
||||
},
|
||||
|
||||
/// Resume an interrupted regular turn.
|
||||
#[debug("RecoverTurn {{ requested_settings: {thread_settings:?} }}")]
|
||||
RecoverTurn {
|
||||
@@ -1149,7 +1139,6 @@ impl Op {
|
||||
Self::RealtimeConversationClose => "realtime_conversation_close",
|
||||
Self::RealtimeConversationDetach { .. } => "realtime_conversation_detach",
|
||||
Self::RealtimeConversationListVoices => "realtime_conversation_list_voices",
|
||||
Self::TurnInput { .. } => "turn_input",
|
||||
Self::RecoverTurn { .. } => "recover_turn",
|
||||
Self::SuspendTurnAndShutdown { .. } => "suspend_turn_and_shutdown",
|
||||
Self::ThreadSettings { .. } => "thread_settings",
|
||||
|
||||
@@ -27,9 +27,9 @@ pub enum SuspendTurnOutcome {
|
||||
|
||||
/// Input consumed by a regular turn.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub enum TurnInput {
|
||||
pub enum TurnInput<I = UserInput> {
|
||||
UserInput {
|
||||
content: Vec<UserInput>,
|
||||
content: Vec<I>,
|
||||
client_id: Option<String>,
|
||||
},
|
||||
ResponseItem(ResponseItem),
|
||||
@@ -107,8 +107,8 @@ impl From<ResponseItem> for InjectedResponseItem {
|
||||
/// Callers choose start-or-steer, idle-start, or steer-only behavior through
|
||||
/// the corresponding `CodexThread` method.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TurnInputRequest {
|
||||
pub input: TurnInput,
|
||||
pub struct TurnInputRequest<I = UserInput> {
|
||||
pub input: TurnInput<I>,
|
||||
/// Optional host-assigned UUID for the Responses message created from [`TurnInput::UserInput`].
|
||||
/// Ignored for all other inputs, including raw user-role Responses items.
|
||||
pub user_item_uuid: Option<uuid::Uuid>,
|
||||
@@ -169,9 +169,9 @@ pub struct RecoverTurnRequest {
|
||||
pub cyber_access_program: Option<CyberAccessProgram>,
|
||||
}
|
||||
|
||||
impl TurnInputRequest {
|
||||
impl<I> TurnInputRequest<I> {
|
||||
/// Creates turn input that can be passed to one of the submission methods.
|
||||
pub fn new(input: TurnInput) -> Self {
|
||||
pub fn new(input: TurnInput<I>) -> Self {
|
||||
Self {
|
||||
input,
|
||||
user_item_uuid: None,
|
||||
@@ -185,7 +185,7 @@ impl TurnInputRequest {
|
||||
}
|
||||
|
||||
/// Creates ordinary user input without a client-provided message id.
|
||||
pub fn user_input(content: Vec<UserInput>) -> Self {
|
||||
pub fn user_input(content: Vec<I>) -> Self {
|
||||
Self::new(TurnInput::UserInput {
|
||||
content,
|
||||
client_id: None,
|
||||
|
||||
@@ -15,6 +15,8 @@ workspace = true
|
||||
[dependencies]
|
||||
codex-protocol = { workspace = true }
|
||||
codex-utils-image = { workspace = true }
|
||||
derive_more = { workspace = true, features = ["debug"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
//! Captures user attachments and renders resolved input for model requests.
|
||||
|
||||
mod local_media;
|
||||
mod message;
|
||||
mod resolved_user_input;
|
||||
|
||||
pub use local_media::audio_mime_for_path;
|
||||
pub use local_media::snapshot_local_user_input;
|
||||
pub use message::LocalImagePreparation;
|
||||
pub use message::from_user_input;
|
||||
pub use resolved_user_input::LocalImagePreparation;
|
||||
pub use resolved_user_input::ResolvedUserInput;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "message_tests.rs"]
|
||||
|
||||
@@ -26,7 +26,7 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
PromptImageMode::ResizeToFit
|
||||
}
|
||||
};
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
|
||||
let file_bytes = read_local_image_bytes(path)?;
|
||||
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
|
||||
*input = UserInput::Image {
|
||||
image: ImageReference::Inline {
|
||||
@@ -36,13 +36,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
};
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
let mime = audio_mime_for_path(path).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format")
|
||||
})?;
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
*input = UserInput::Audio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, &file_bytes),
|
||||
audio_url: read_local_audio_url(path)?,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -51,6 +47,18 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads a bounded client-local audio file into a data URL.
|
||||
pub(super) fn read_local_audio_url(path: &Path) -> io::Result<String> {
|
||||
let mime = audio_mime_for_path(path)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "unsupported audio format"))?;
|
||||
let bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
Ok(data_url_from_bytes(mime, &bytes))
|
||||
}
|
||||
|
||||
pub(super) fn read_local_image_bytes(path: &Path) -> io::Result<Vec<u8>> {
|
||||
read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")
|
||||
}
|
||||
|
||||
fn read_bounded_local_media(path: &Path, max_bytes: usize, kind: &str) -> io::Result<Vec<u8>> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
if file.metadata()?.len() > max_bytes as u64 {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
//! Converts user inputs to model content and records image positions for UI events.
|
||||
|
||||
use crate::local_media::audio_mime_for_path;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::audio_close_tag_text;
|
||||
use codex_protocol::models::local_audio_open_tag_text_with_path;
|
||||
use codex_protocol::models::local_image_content_items;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LocalMediaKind {
|
||||
Audio,
|
||||
Image,
|
||||
}
|
||||
|
||||
impl LocalMediaKind {
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Audio => "audio",
|
||||
Self::Image => "image",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn local_media_error_placeholder(
|
||||
path: &std::path::Path,
|
||||
error: impl std::fmt::Display,
|
||||
media_kind: LocalMediaKind,
|
||||
) -> ContentItem {
|
||||
let media_name = media_kind.name();
|
||||
let path = path.display();
|
||||
ContentItem::InputText {
|
||||
text: format!("Codex could not read the local {media_name} at `{path}`: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalImagePreparation {
|
||||
Process,
|
||||
Defer,
|
||||
}
|
||||
|
||||
fn unsupported_audio_error_placeholder(path: &std::path::Path) -> ContentItem {
|
||||
ContentItem::InputText {
|
||||
text: format!(
|
||||
"Codex cannot attach audio at `{}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_audio_content_items(
|
||||
path: &std::path::Path,
|
||||
file_bytes: &[u8],
|
||||
label_number: usize,
|
||||
) -> Vec<ContentItem> {
|
||||
let Some(mime) = audio_mime_for_path(path) else {
|
||||
return vec![unsupported_audio_error_placeholder(path)];
|
||||
};
|
||||
|
||||
vec![
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(label_number, path),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, file_bytes),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
pub fn from_user_input(
|
||||
items: Vec<UserInput>,
|
||||
local_image_preparation: LocalImagePreparation,
|
||||
user_image_content_indices: &mut HashMap<usize, usize>,
|
||||
) -> ResponseInputItem {
|
||||
user_image_content_indices.clear();
|
||||
let mut image_index = 0;
|
||||
let mut audio_index = 0;
|
||||
let mut content = Vec::new();
|
||||
for (input_index, input) in items.into_iter().enumerate() {
|
||||
let generated = match input {
|
||||
UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],
|
||||
UserInput::Image { image, detail, .. } => {
|
||||
image_index += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
vec![ContentItem::InputImage {
|
||||
image,
|
||||
detail: Some(detail),
|
||||
}]
|
||||
}
|
||||
UserInput::LocalImage { path, detail, .. } => {
|
||||
image_index += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
match std::fs::read(&path) {
|
||||
Ok(file_bytes) => match local_image_preparation {
|
||||
LocalImagePreparation::Process => {
|
||||
local_image_content_items_with_label_number(
|
||||
&path,
|
||||
file_bytes,
|
||||
Some(image_index),
|
||||
detail,
|
||||
)
|
||||
}
|
||||
LocalImagePreparation::Defer => local_image_content_items(
|
||||
&path,
|
||||
data_url_from_bytes("application/octet-stream", &file_bytes),
|
||||
Some(image_index),
|
||||
detail,
|
||||
),
|
||||
},
|
||||
Err(err) => vec![local_media_error_placeholder(
|
||||
&path,
|
||||
err,
|
||||
LocalMediaKind::Image,
|
||||
)],
|
||||
}
|
||||
}
|
||||
UserInput::Audio { audio } => {
|
||||
audio_index += 1;
|
||||
vec![ContentItem::InputAudio { audio }]
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
audio_index += 1;
|
||||
match std::fs::read(&path) {
|
||||
Ok(file_bytes) => local_audio_content_items(&path, &file_bytes, audio_index),
|
||||
Err(err) => vec![local_media_error_placeholder(
|
||||
&path,
|
||||
err,
|
||||
LocalMediaKind::Audio,
|
||||
)],
|
||||
}
|
||||
}
|
||||
_ => Vec::new(), // Skills and mentions are injected later in core.
|
||||
};
|
||||
// Local images add framing text, while mentions and skills add no content.
|
||||
// Capture the image's actual slot before preparation replaces its reference.
|
||||
for (offset, item) in generated.iter().enumerate() {
|
||||
if matches!(item, ContentItem::InputImage { .. }) {
|
||||
user_image_content_indices.insert(input_index, content.len() + offset);
|
||||
}
|
||||
}
|
||||
content.extend(generated);
|
||||
}
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content,
|
||||
phase: None,
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use crate::LocalImagePreparation;
|
||||
use crate::from_user_input;
|
||||
use crate::ResolvedUserInput;
|
||||
use anyhow::Result;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
@@ -29,7 +29,11 @@ const TINY_PNG_BYTES: &[u8] = &[
|
||||
];
|
||||
|
||||
fn user_message(items: Vec<UserInput>) -> ResponseInputItem {
|
||||
from_user_input(items, LocalImagePreparation::Process, &mut HashMap::new())
|
||||
let items: Vec<_> = items
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
ResolvedUserInput::into_message(&items, LocalImagePreparation::Process).0
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -294,9 +298,12 @@ fn user_input_image_indices_survive_content_expansion() -> Result<()> {
|
||||
inline_image.clone(),
|
||||
inline_image,
|
||||
];
|
||||
let inputs: Vec<_> = inputs
|
||||
.into_iter()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
for preparation in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
|
||||
let mut indices = HashMap::new();
|
||||
let item = from_user_input(inputs.clone(), preparation, &mut indices);
|
||||
let (item, indices) = ResolvedUserInput::into_message(&inputs, preparation);
|
||||
assert_eq!(indices, HashMap::from([(4, 3), (5, 5), (6, 6)]));
|
||||
let ResponseInputItem::Message { content, .. } = item else {
|
||||
panic!("expected a user message");
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Resolves one client input at a time. A local attachment carries its captured contents
|
||||
//! or its read error. Its path is retained for display, labels, and diagnostics.
|
||||
|
||||
use crate::local_media::audio_mime_for_path;
|
||||
use crate::local_media::read_local_audio_url;
|
||||
use crate::local_media::read_local_image_bytes;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::audio_close_tag_text;
|
||||
use codex_protocol::models::local_audio_open_tag_text_with_path;
|
||||
use codex_protocol::models::local_image_content_items;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::user_input::TextElement;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalImagePreparation {
|
||||
Process,
|
||||
Defer,
|
||||
}
|
||||
|
||||
/// One input admitted to the harness. Clones share captured attachment payloads.
|
||||
#[derive(Clone, PartialEq, derive_more::Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ResolvedUserInput {
|
||||
Text {
|
||||
#[debug("{:?} <{} bytes>", &text[..text.floor_char_boundary(/*index*/ 512)], text.len())]
|
||||
text: String,
|
||||
#[serde(default)]
|
||||
text_elements: Vec<TextElement>,
|
||||
},
|
||||
Image {
|
||||
#[debug(skip)]
|
||||
#[serde(flatten)]
|
||||
image: Arc<ImageReference>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<ImageDetail>,
|
||||
},
|
||||
LocalImage {
|
||||
path: PathBuf,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<ImageDetail>,
|
||||
#[debug(skip)]
|
||||
bytes: Result<Arc<[u8]>, String>,
|
||||
},
|
||||
Audio {
|
||||
#[debug(skip)]
|
||||
#[serde(flatten)]
|
||||
audio: Arc<AudioReference>,
|
||||
},
|
||||
LocalAudio {
|
||||
path: PathBuf,
|
||||
#[debug(skip)]
|
||||
audio_url: Result<Arc<str>, String>,
|
||||
},
|
||||
Skill {
|
||||
name: String,
|
||||
path: PathBuf,
|
||||
},
|
||||
Mention {
|
||||
name: String,
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ResolvedUserInput {
|
||||
/// Reads a client-selected attachment on this host. Executor-owned paths must be resolved there.
|
||||
pub fn from_client(input: UserInput) -> Self {
|
||||
match input {
|
||||
UserInput::Text {
|
||||
text,
|
||||
text_elements,
|
||||
} => Self::Text {
|
||||
text,
|
||||
text_elements,
|
||||
},
|
||||
UserInput::Image { image, detail } => Self::Image {
|
||||
image: Arc::new(image),
|
||||
detail,
|
||||
},
|
||||
UserInput::LocalImage { path, detail } => {
|
||||
let bytes = read_local_image_bytes(&path).map(Arc::from).map_err(|err| {
|
||||
format!(
|
||||
"Codex could not read the local image at `{}`: {err}",
|
||||
path.display()
|
||||
)
|
||||
});
|
||||
Self::LocalImage {
|
||||
path,
|
||||
detail,
|
||||
bytes,
|
||||
}
|
||||
}
|
||||
UserInput::Audio { audio } => Self::Audio {
|
||||
audio: Arc::new(audio),
|
||||
},
|
||||
UserInput::LocalAudio { path } => {
|
||||
let audio_url = read_local_audio_url(&path).map(Arc::from).map_err(|err| {
|
||||
let name = path.display();
|
||||
if audio_mime_for_path(&path).is_none() {
|
||||
format!("Codex cannot attach audio at `{name}`: unsupported audio format; use wav, mp3, m4a, webm, or ogg.")
|
||||
} else {
|
||||
format!("Codex could not read the local audio at `{name}`: {err}")
|
||||
}
|
||||
});
|
||||
Self::LocalAudio { path, audio_url }
|
||||
}
|
||||
UserInput::Skill { name, path } => Self::Skill { name, path },
|
||||
UserInput::Mention { name, path } => Self::Mention { name, path },
|
||||
_ => Self::text("Codex cannot prepare this input type.".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn text(text: String) -> Self {
|
||||
Self::Text {
|
||||
text,
|
||||
text_elements: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Projects the original client-facing input, including UI text spans and local display paths.
|
||||
pub fn to_user_input(&self) -> UserInput {
|
||||
match self {
|
||||
Self::Text {
|
||||
text,
|
||||
text_elements,
|
||||
} => UserInput::Text {
|
||||
text: text.clone(),
|
||||
text_elements: text_elements.clone(),
|
||||
},
|
||||
Self::Image { image, detail } => UserInput::Image {
|
||||
image: image.as_ref().clone(),
|
||||
detail: *detail,
|
||||
},
|
||||
Self::LocalImage { path, detail, .. } => UserInput::LocalImage {
|
||||
path: path.clone(),
|
||||
detail: *detail,
|
||||
},
|
||||
Self::Audio { audio } => UserInput::Audio {
|
||||
audio: audio.as_ref().clone(),
|
||||
},
|
||||
Self::LocalAudio { path, .. } => UserInput::LocalAudio { path: path.clone() },
|
||||
Self::Skill { name, path } => UserInput::Skill {
|
||||
name: name.clone(),
|
||||
path: path.clone(),
|
||||
},
|
||||
Self::Mention { name, path } => UserInput::Mention {
|
||||
name: name.clone(),
|
||||
path: path.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_text(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Text { text, .. } => Some(text),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_uploaded_audio_file(&self) -> bool {
|
||||
matches!(self, Self::Audio { audio } if matches!(audio.as_ref(), AudioReference::File { .. }))
|
||||
}
|
||||
|
||||
pub fn is_image(&self) -> bool {
|
||||
matches!(self, Self::Image { .. } | Self::LocalImage { .. })
|
||||
}
|
||||
|
||||
/// Renders the captured inputs and maps the supplied slice's image indices to message slots.
|
||||
pub fn into_message(
|
||||
items: &[Self],
|
||||
images: LocalImagePreparation,
|
||||
) -> (ResponseInputItem, HashMap<usize, usize>) {
|
||||
let mut image_positions = HashMap::new();
|
||||
let mut content = Vec::new();
|
||||
let mut image_number = 0;
|
||||
let mut audio_number = 0;
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
let start = content.len();
|
||||
match item {
|
||||
Self::Text { text, .. } => {
|
||||
content.push(ContentItem::InputText { text: text.clone() })
|
||||
}
|
||||
Self::Image { image, detail } => {
|
||||
image_number += 1;
|
||||
content.push(ContentItem::InputImage {
|
||||
image: image.as_ref().clone(),
|
||||
detail: Some(detail.unwrap_or(DEFAULT_IMAGE_DETAIL)),
|
||||
});
|
||||
}
|
||||
Self::LocalImage {
|
||||
path,
|
||||
bytes,
|
||||
detail,
|
||||
} => {
|
||||
image_number += 1;
|
||||
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
|
||||
match bytes {
|
||||
Ok(bytes) => content.extend(match images {
|
||||
LocalImagePreparation::Process => {
|
||||
local_image_content_items_with_label_number(
|
||||
path,
|
||||
bytes.to_vec(),
|
||||
Some(image_number),
|
||||
detail,
|
||||
)
|
||||
}
|
||||
LocalImagePreparation::Defer => local_image_content_items(
|
||||
path,
|
||||
data_url_from_bytes("application/octet-stream", bytes),
|
||||
Some(image_number),
|
||||
detail,
|
||||
),
|
||||
}),
|
||||
Err(message) => content.push(ContentItem::InputText {
|
||||
text: message.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
Self::Audio { audio } => {
|
||||
audio_number += 1;
|
||||
content.push(ContentItem::InputAudio {
|
||||
audio: audio.as_ref().clone(),
|
||||
});
|
||||
}
|
||||
Self::LocalAudio { path, audio_url } => {
|
||||
audio_number += 1;
|
||||
match audio_url {
|
||||
Ok(audio_url) => content.extend([
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(audio_number, path),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
},
|
||||
]),
|
||||
Err(message) => content.push(ContentItem::InputText {
|
||||
text: message.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
Self::Skill { .. } | Self::Mention { .. } => continue,
|
||||
}
|
||||
if let Some(offset) = content[start..]
|
||||
.iter()
|
||||
.position(|item| matches!(item, ContentItem::InputImage { .. }))
|
||||
{
|
||||
image_positions.insert(index, start + offset);
|
||||
}
|
||||
}
|
||||
(
|
||||
ResponseInputItem::Message {
|
||||
role: "user".into(),
|
||||
content,
|
||||
phase: None,
|
||||
},
|
||||
image_positions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "resolved_user_input_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,189 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
use codex_protocol::models::image_close_tag_text;
|
||||
use codex_protocol::models::local_image_open_tag_text_with_path;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
// A tiny valid PNG (1x1) so image conversion tests don't depend on cross-crate
|
||||
// file paths, which break under Bazel sandboxing.
|
||||
const TINY_PNG_BYTES: &[u8] = &[
|
||||
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
|
||||
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
|
||||
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn resolved_media_renders_after_source_files_are_removed() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let missing = dir.path().join("missing.png");
|
||||
let path = dir.path().join("image.png");
|
||||
let audio_path = dir.path().join("clip.wav");
|
||||
std::fs::write(&path, TINY_PNG_BYTES)?;
|
||||
std::fs::write(&audio_path, b"audio")?;
|
||||
|
||||
let image = ImageReference::File {
|
||||
file_id: "file_image".into(),
|
||||
};
|
||||
let audio = AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
};
|
||||
let inline = ImageReference::Inline {
|
||||
image_url: data_url_from_bytes("image/png", TINY_PNG_BYTES),
|
||||
};
|
||||
let inputs = vec![
|
||||
UserInput::Text {
|
||||
text: "compare".into(),
|
||||
text_elements: vec![TextElement::new((0..7).into(), Some("selection".into()))],
|
||||
},
|
||||
UserInput::Mention {
|
||||
name: "app".into(),
|
||||
path: "app://example".into(),
|
||||
},
|
||||
UserInput::Skill {
|
||||
name: "skill".into(),
|
||||
path: dir.path().join("SKILL.md"),
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: missing.clone(),
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Image {
|
||||
image: image.clone(),
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: audio.clone(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: audio_path.clone(),
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: path.clone(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Image {
|
||||
image: inline.clone(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
];
|
||||
let resolved: Vec<_> = inputs
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(ResolvedUserInput::from_client)
|
||||
.collect();
|
||||
for (original, item) in inputs.iter().zip(&resolved) {
|
||||
let wire = serde_json::to_value(original)?;
|
||||
if matches!(
|
||||
original,
|
||||
UserInput::LocalImage { .. } | UserInput::LocalAudio { .. }
|
||||
) {
|
||||
// A path alone cannot rehydrate an already-captured attachment.
|
||||
assert!(serde_json::from_value::<ResolvedUserInput>(wire).is_err());
|
||||
} else {
|
||||
assert_eq!(serde_json::to_value(item)?, wire);
|
||||
}
|
||||
}
|
||||
let serialized = serde_json::to_vec(&resolved)?;
|
||||
dir.close()?;
|
||||
let restored: Vec<ResolvedUserInput> = serde_json::from_slice(&serialized)?;
|
||||
assert_eq!(restored, resolved);
|
||||
assert_eq!(
|
||||
restored
|
||||
.iter()
|
||||
.map(ResolvedUserInput::to_user_input)
|
||||
.collect::<Vec<_>>(),
|
||||
inputs
|
||||
);
|
||||
|
||||
// Filtering a turn changes both label numbers and which UI positions own images.
|
||||
let (subset, positions) =
|
||||
ResolvedUserInput::into_message(&restored[6..8], LocalImagePreparation::Defer);
|
||||
let ResponseInputItem::Message { content, .. } = subset else {
|
||||
panic!("expected a user message");
|
||||
};
|
||||
assert_eq!(positions, HashMap::from([(1, 4)]));
|
||||
assert_eq!(
|
||||
content[0],
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(/*label_number*/ 1, &audio_path)
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
content[3],
|
||||
ContentItem::InputText {
|
||||
text: local_image_open_tag_text_with_path(/*label_number*/ 1, &path)
|
||||
}
|
||||
);
|
||||
|
||||
for mode in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {
|
||||
let (message, positions) = ResolvedUserInput::into_message(&restored, mode);
|
||||
let ResponseInputItem::Message { content, .. } = message else {
|
||||
panic!("expected a user message");
|
||||
};
|
||||
assert_eq!(positions, HashMap::from([(4, 2), (7, 8), (8, 10)]));
|
||||
assert_eq!(
|
||||
content[0],
|
||||
ContentItem::InputText {
|
||||
text: "compare".into()
|
||||
}
|
||||
);
|
||||
let ContentItem::InputText { text } = &content[1] else {
|
||||
panic!("expected a read error");
|
||||
};
|
||||
assert!(text.starts_with(&format!(
|
||||
"Codex could not read the local image at `{}`:",
|
||||
missing.display()
|
||||
)));
|
||||
let image_url = match mode {
|
||||
LocalImagePreparation::Defer => {
|
||||
data_url_from_bytes("application/octet-stream", TINY_PNG_BYTES)
|
||||
}
|
||||
LocalImagePreparation::Process => codex_utils_image::load_for_prompt_bytes(
|
||||
&path,
|
||||
TINY_PNG_BYTES.to_vec(),
|
||||
codex_utils_image::PromptImageMode::Original,
|
||||
)?
|
||||
.into_data_url(),
|
||||
};
|
||||
assert_eq!(
|
||||
&content[2..],
|
||||
&[
|
||||
ContentItem::InputImage {
|
||||
image: image.clone(),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL)
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: audio.clone()
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(/*label_number*/ 2, &audio_path)
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".into()
|
||||
}
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text()
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: local_image_open_tag_text_with_path(/*label_number*/ 3, &path)
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image: ImageReference::Inline { image_url },
|
||||
detail: Some(ImageDetail::Original)
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: image_close_tag_text()
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image: inline.clone(),
|
||||
detail: Some(ImageDetail::Original)
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user