Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d

codex-rs/protocol/src/models.rs

@@ -718,6 +718,9 @@ pub enum ContentItem {
         #[ts(optional)]
         detail: Option<ImageDetail>,
     },
+    InputAudio {
+        audio_url: String,
+    },
     OutputText {
         text: String,
     },
@@ -1334,16 +1337,37 @@ fn should_serialize_reasoning_content(content: &Option<Vec<ReasoningItemContent>
     }
 }
 
-fn local_image_error_placeholder(
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum LocalMediaKind {
+    Audio,
+    Image,
+}
+
+impl LocalMediaKind {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Audio => "audio",
+            Self::Image => "image",
+        }
+    }
+}
+
+fn local_media_error_placeholder(
     path: &std::path::Path,
     error: impl std::fmt::Display,
+    media_kind: LocalMediaKind,
 ) -> ContentItem {
+    let media_name = media_kind.name();
+    let path = path.display();
     ContentItem::InputText {
-        text: format!(
-            "Codex could not read the local image at `{}`: {}",
-            path.display(),
-            error
-        ),
+        text: format!("Codex could not read the local {media_name} at `{path}`: {error}"),
+    }
+}
+
+fn local_media_kind_unsupported(media_kind: LocalMediaKind) -> ContentItem {
+    let media_name = media_kind.name();
+    ContentItem::InputText {
+        text: format!("Codex does not support local {media_name} input yet."),
     }
 }
 
@@ -1431,13 +1455,21 @@ pub fn local_image_content_items_with_label_number(
             | ImageProcessingError::Encode { .. }
             | ImageProcessingError::InvalidDataUrl { .. }
             | ImageProcessingError::ImageTooLarge { .. } => {
-                vec![local_image_error_placeholder(path, &err)]
+                vec![local_media_error_placeholder(
+                    path,
+                    &err,
+                    LocalMediaKind::Image,
+                )]
             }
             ImageProcessingError::Decode { .. } if err.is_invalid_image() => {
                 vec![invalid_image_error_placeholder(path, &err)]
             }
             ImageProcessingError::Decode { .. } => {
-                vec![local_image_error_placeholder(path, &err)]
+                vec![local_media_error_placeholder(
+                    path,
+                    &err,
+                    LocalMediaKind::Image,
+                )]
             }
             ImageProcessingError::UnsupportedImageFormat { mime } => {
                 vec![unsupported_image_error_placeholder(path, mime)]
@@ -1647,9 +1679,20 @@ impl ResponseInputItem {
                                     detail,
                                 ),
                             },
-                            Err(err) => vec![local_image_error_placeholder(&path, err)],
+                            Err(err) => vec![local_media_error_placeholder(
+                                &path,
+                                err,
+                                LocalMediaKind::Image,
+                            )],
                         }
                     }
+                    UserInput::Audio { .. } => vec![ContentItem::InputText {
+                        text: "Codex does not support audio input yet.".to_string(),
+                    }],
+                    // TODO: Load local audio inputs once they are supported.
+                    UserInput::LocalAudio { .. } => {
+                        vec![local_media_kind_unsupported(LocalMediaKind::Audio)]
+                    }
                     UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core
                 })
                 .collect::<Vec<ContentItem>>(),
@@ -1707,6 +1750,10 @@ pub enum FunctionCallOutputContentItem {
         #[ts(optional)]
         detail: Option<ImageDetail>,
     },
+    // Do not rename, these are serialized and used directly in the responses API.
+    InputAudio {
+        audio_url: String,
+    },
     EncryptedContent {
         encrypted_content: String,
     },
@@ -1734,6 +1781,7 @@ pub fn function_call_output_content_items_to_text(
             }
             FunctionCallOutputContentItem::InputText { .. }
             | FunctionCallOutputContentItem::InputImage { .. }
+            | FunctionCallOutputContentItem::InputAudio { .. }
             | FunctionCallOutputContentItem::EncryptedContent { .. } => None,
         })
         .collect::<Vec<_>>();
@@ -3192,6 +3240,44 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn replaces_unsupported_audio_user_input_with_placeholder() {
+        let item = ResponseInputItem::from(vec![UserInput::Audio {
+            audio_url: "data:audio/wav;base64,abc".to_string(),
+        }]);
+
+        assert_eq!(
+            item,
+            ResponseInputItem::Message {
+                role: "user".to_string(),
+                content: vec![ContentItem::InputText {
+                    text: "Codex does not support audio input yet.".to_string(),
+                }],
+                phase: None,
+            }
+        );
+    }
+
+    #[test]
+    fn replaces_unsupported_local_audio_user_input_with_placeholder() -> Result<()> {
+        let item = ResponseInputItem::from(vec![UserInput::LocalAudio {
+            path: "sample.mp3".into(),
+        }]);
+
+        assert_eq!(
+            item,
+            ResponseInputItem::Message {
+                role: "user".to_string(),
+                content: vec![ContentItem::InputText {
+                    text: "Codex does not support local audio input yet.".to_string(),
+                }],
+                phase: None,
+            }
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn image_user_input_preserves_requested_detail() -> Result<()> {
         let image_url = "data:image/png;base64,abc".to_string();

codex-rs/protocol/src/user_input.rs

@@ -29,7 +29,6 @@ pub enum UserInput {
         #[ts(optional)]
         detail: Option<ImageDetail>,
     },
-
     /// Local image path provided by the user.  This will be converted to an
     /// `Image` variant (base64 data URL) during request serialization.
     LocalImage {
@@ -38,6 +37,10 @@ pub enum UserInput {
         #[ts(optional)]
         detail: Option<ImageDetail>,
     },
+    /// Pre-encoded audio data URI. Audio input is not yet forwarded to the Responses API.
+    Audio { audio_url: String },
+    /// Local audio path provided by the user. Local audio is not converted in this layer.
+    LocalAudio { path: std::path::PathBuf },
 
     /// Skill selected by the user (name + path to SKILL.md).
     Skill {