mirror of
https://github.com/openai/codex.git
synced 2026-11-10 21:00:47 +00:00
Add audio variants to user input protocols (#33923)
## What changed - Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas. - Preserve audio fields when converting app-server inputs and response content into thread items. - Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input. ## Testing - Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation. GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
@@ -2545,6 +2545,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
path: PathBuf::from("local/image.png"),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
CoreUserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||
},
|
||||
CoreUserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
},
|
||||
CoreUserInput::Skill {
|
||||
name: "skill-creator".to_string(),
|
||||
path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"),
|
||||
@@ -2574,6 +2580,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
path: PathBuf::from("local/image.png"),
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "data:audio/wav;base64,AAA".to_string(),
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
},
|
||||
UserInput::Skill {
|
||||
name: "skill-creator".to_string(),
|
||||
path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"),
|
||||
@@ -3022,7 +3034,7 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_input_into_core_preserves_image_detail() {
|
||||
fn user_input_into_core_preserves_media_fields() {
|
||||
assert_eq!(
|
||||
UserInput::Image {
|
||||
url: "https://example.com/image.png".to_string(),
|
||||
@@ -3046,6 +3058,26 @@ fn user_input_into_core_preserves_image_detail() {
|
||||
detail: Some(ImageDetail::Original),
|
||||
}
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
UserInput::Audio {
|
||||
url: "data:audio/wav;base64,AAA".to_string(),
|
||||
}
|
||||
.into_core(),
|
||||
CoreUserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||
}
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
UserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
}
|
||||
.into_core(),
|
||||
CoreUserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -308,6 +308,12 @@ pub enum UserInput {
|
||||
detail: Option<ImageDetail>,
|
||||
path: PathBuf,
|
||||
},
|
||||
Audio {
|
||||
url: String,
|
||||
},
|
||||
LocalAudio {
|
||||
path: PathBuf,
|
||||
},
|
||||
Skill {
|
||||
name: String,
|
||||
path: PathBuf,
|
||||
@@ -333,6 +339,8 @@ impl UserInput {
|
||||
detail,
|
||||
},
|
||||
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
|
||||
UserInput::Audio { url } => CoreUserInput::Audio { audio_url: url },
|
||||
UserInput::LocalAudio { path } => CoreUserInput::LocalAudio { path },
|
||||
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
|
||||
UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
|
||||
}
|
||||
@@ -354,6 +362,8 @@ impl From<CoreUserInput> for UserInput {
|
||||
detail,
|
||||
},
|
||||
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
|
||||
CoreUserInput::Audio { audio_url } => UserInput::Audio { url: audio_url },
|
||||
CoreUserInput::LocalAudio { path } => UserInput::LocalAudio { path },
|
||||
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
|
||||
CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
|
||||
_ => unreachable!("unsupported user input variant"),
|
||||
@@ -367,6 +377,8 @@ impl UserInput {
|
||||
UserInput::Text { text, .. } => text.chars().count(),
|
||||
UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Audio { .. }
|
||||
| UserInput::LocalAudio { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => 0,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user