Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
nhamidi-oai
2026-07-17 23:14:37 +00:00
committed by copyberry
parent e98765c212
commit 97dbd01a3d
49 changed files with 1330 additions and 33 deletions
+12
View File
@@ -13,6 +13,7 @@ use crate::history_cell::split_reasoning_summary_parts;
use crate::multi_agents::sub_agent_activity_summary;
use codex_app_server_protocol::Thread;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::UserInput;
use codex_protocol::ThreadId;
use codex_protocol::items::UserMessageItem;
use ratatui::style::Stylize as _;
@@ -54,6 +55,17 @@ pub(crate) fn thread_to_transcript_cells(
client_id,
content,
} => {
if content.iter().any(|input| {
matches!(
input,
UserInput::Audio { .. } | UserInput::LocalAudio { .. }
)
}) {
tracing::warn!(
user_message_id = id,
"audio user inputs are not supported by the TUI and will be omitted"
);
}
let item = UserMessageItem {
id: id.clone(),
client_id: client_id.clone(),