Capture local attachments before queuing turn input (#61962)

## Why

Queued input can outlive its local attachment files. Reading those files only when rendering a model message can lose media submitted while turn admission is pending.

## What changed

- Introduce `ResolvedUserInput` to carry captured image bytes, audio data URLs, or read errors through session queues and agent input.
- Resolve client attachments on the submitting host before queuing, using a blocking task for local file reads.
- Render messages from captured payloads while preserving original paths and text spans for display, along with image position mappings.

## Testing

Add coverage for rendering and serialization after source files are removed, steered audio retained during pending admission, and local audio errors and display content.

GitOrigin-RevId: 4bbb6bda44167439a8cfdae15d3c476a7b1d336b
This commit is contained in:
Adam Perry @ OpenAI
2026-11-17 04:08:00 +00:00
committed by copyberry
parent 65a4450886
commit d8d840d32a
50 changed files with 1060 additions and 573 deletions
+2 -2
View File
@@ -24,8 +24,8 @@ use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::turn_input::TurnStartOptions;
use codex_protocol::user_input::UserInput;
use codex_rollout_trace::ThreadTraceContext;
use codex_user_input::ResolvedUserInput;
use futures::future::BoxFuture;
// Keep dynamic dispatch a compile-time property of the contract.
@@ -209,7 +209,7 @@ impl AgentInfo {
/// User input starts or steers a turn; agent messages retain their sender and wake mode.
pub enum AgentInput {
UserInput(Vec<UserInput>),
UserInput(Vec<ResolvedUserInput>),
Message {
message: AgentMessage,
mode: MessageDeliveryMode,
+5 -2
View File
@@ -47,6 +47,7 @@ use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadSource;
use codex_protocol::user_input::UserInput;
use codex_thread_store::ReadThreadParams;
use codex_user_input::ResolvedUserInput;
use futures::StreamExt;
use std::collections::VecDeque;
use std::sync::Arc;
@@ -137,13 +138,15 @@ impl LocalAgentControl {
pub(crate) async fn send_input(
&self,
agent_id: ThreadId,
input: Vec<UserInput>,
input: Vec<ResolvedUserInput>,
start_options: TurnStartOptions,
) -> CodexResult<String> {
let state = self.runtime.upgrade()?;
let thread = state.get_thread(agent_id).await?;
let result = match thread
.start_or_steer_turn(TurnInputRequest::user_input(input).on_start(start_options))
.start_or_steer_resolved_turn(
TurnInputRequest::<ResolvedUserInput>::user_input(input).on_start(start_options),
)
.await
{
Ok(TurnInputSubmission::Started { turn_id, .. }) => Ok(turn_id),
+2 -2
View File
@@ -38,7 +38,7 @@ const AGENT_NAMES: &str = include_str!("../../../assets/agent/agent_names.txt");
/// unrepresentable.
#[allow(clippy::large_enum_variant)]
pub(super) enum SpawnInitialInput {
UserInput(Vec<UserInput>),
UserInput(Vec<ResolvedUserInput>),
InterAgentCommunication(InterAgentCommunication, AgentCommunicationContext),
}
@@ -208,7 +208,7 @@ impl LocalAgentControl {
pub(crate) async fn spawn_agent(
&self,
config: Config,
initial_input: Vec<UserInput>,
initial_input: Vec<ResolvedUserInput>,
session_source: Option<SessionSource>,
) -> CodexResult<ThreadId> {
let (spawned_agent, _) = Box::pin(self.spawn_agent_internal(
+6 -23
View File
@@ -141,7 +141,7 @@ impl LocalAgentControl {
pub(crate) async fn spawn_agent_with_metadata(
&self,
config: Config,
initial_input: Vec<UserInput>,
initial_input: Vec<ResolvedUserInput>,
source: SessionSource,
options: SpawnAgentOptions,
) -> CodexResult<LiveAgent> {
@@ -181,8 +181,8 @@ async fn test_config() -> (TempDir, Config) {
test_config_with_cli_overrides(Vec::new()).await
}
fn text_input(text: &str) -> Vec<UserInput> {
vec![UserInput::Text {
fn text_input(text: &str) -> Vec<ResolvedUserInput> {
vec![ResolvedUserInput::Text {
text: text.to_string(),
text_elements: Vec::new(),
}]
@@ -584,14 +584,7 @@ async fn assert_thread_not_loaded(manager: &ThreadManager, thread_id: ThreadId)
async fn send_input_errors_when_manager_dropped() {
let control = LocalAgentControl::default();
let err = control
.send_input(
ThreadId::new(),
vec![UserInput::Text {
text: "hello".to_string(),
text_elements: Vec::new(),
}],
Default::default(),
)
.send_input(ThreadId::new(), text_input("hello"), Default::default())
.await
.expect_err("send_input should fail without a manager");
assert_eq!(
@@ -971,14 +964,7 @@ async fn send_input_errors_when_thread_missing() {
let thread_id = ThreadId::new();
let err = harness
.control
.send_input(
thread_id,
vec![UserInput::Text {
text: "hello".to_string(),
text_elements: Vec::new(),
}],
Default::default(),
)
.send_input(thread_id, text_input("hello"), Default::default())
.await
.expect_err("send_input should fail for missing thread");
assert_matches!(
@@ -1063,10 +1049,7 @@ async fn send_input_submits_user_message() {
caller: ThreadId::new(),
target: AgentTarget::Id(thread_id),
resume_config: harness.config.clone(),
input: AgentInput::UserInput(vec![UserInput::Text {
text: "hello from tests".to_string(),
text_elements: Vec::new(),
}]),
input: AgentInput::UserInput(text_input("hello from tests")),
start_options: Default::default(),
})
.await