Support audio input

This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-05-15 22:11:50 -07:00
parent 326e31ab65
commit b42b2ff9ec
37 changed files with 2112 additions and 110 deletions
+121
View File
@@ -12,6 +12,7 @@ use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
@@ -175,6 +176,54 @@ async fn run_code_mode_turn(
Ok((test, second_mock))
}
async fn run_code_mode_turn_with_audio_model(
server: &MockServer,
prompt: &str,
code: &str,
) -> Result<(TestCodex, ResponseMock)> {
let mut builder = test_codex()
.with_model("gpt-5.4")
.with_config(move |config| {
let _ = config.features.enable(Feature::CodeMode);
let mut model_catalog = bundled_models_response()
.unwrap_or_else(|err| panic!("bundled models.json should parse: {err}"));
let model = model_catalog
.models
.iter_mut()
.find(|model| model.slug == "gpt-5.4")
.expect("gpt-5.4 exists in bundled models.json");
model.input_modalities = vec![
InputModality::Text,
InputModality::Image,
InputModality::Audio,
];
config.model_catalog = Some(model_catalog);
});
let test = builder.build(server).await?;
responses::mount_sse_once(
server,
sse(vec![
ev_response_created("resp-1"),
ev_custom_tool_call("call-1", "exec", code),
ev_completed("resp-1"),
]),
)
.await;
let second_mock = responses::mount_sse_once(
server,
sse(vec![
ev_assistant_message("msg-1", "done"),
ev_completed("resp-2"),
]),
)
.await;
test.submit_turn(prompt).await?;
Ok((test, second_mock))
}
async fn run_code_mode_turn_with_rmcp(
server: &MockServer,
prompt: &str,
@@ -1974,6 +2023,77 @@ image("data:image/png;base64,AAA");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_output_audio_via_global_helper_for_audio_model() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_audio_model(
&server,
"use exec to return audio",
r#"
audio({ data: "BASE64", format: "wav" });
audio({ data: "data:audio/mpeg;base64,MP3BASE64" });
"#,
)
.await?;
let req = second_mock.single_request();
let items = custom_tool_output_items(&req, "call-1");
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
assert_ne!(
success,
Some(false),
"code_mode audio output failed unexpectedly"
);
assert_eq!(items.len(), 3);
assert_regex_match(
concat!(
r"(?s)\A",
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
),
text_item(&items, /*index*/ 0),
);
assert_eq!(
items[1],
serde_json::json!({
"type": "input_audio",
"input_audio": { "data": "BASE64", "format": "wav" }
}),
);
assert_eq!(
items[2],
serde_json::json!({
"type": "input_audio",
"input_audio": { "data": "MP3BASE64", "format": "mp3" }
}),
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_audio_output_fails_for_non_audio_model() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn(
&server,
"use exec to return audio",
r#"audio({ data: "BASE64", format: "wav" });"#,
)
.await?;
let req = second_mock.single_request();
let (output, _success) = custom_tool_output_body_and_success(&req, "call-1");
assert_eq!(
output,
"audio content emitted by code mode but the selected model does not support audio input"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -2404,6 +2524,7 @@ text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort()));
"WeakSet",
"__codexContentItems",
"add_content",
"audio",
"decodeURI",
"decodeURIComponent",
"encodeURI",
+296 -1
View File
@@ -93,6 +93,50 @@ fn assert_wall_time_header(output: &str) {
assert_eq!(marker, "Output:");
}
fn test_model_info_with_modalities(
slug: &str,
description: &str,
input_modalities: Vec<InputModality>,
) -> ModelInfo {
ModelInfo {
slug: slug.to_string(),
display_name: slug.to_string(),
description: Some(description.to_string()),
default_reasoning_level: None,
supported_reasoning_levels: vec![ReasoningEffortPreset {
effort: codex_protocol::openai_models::ReasoningEffort::Medium,
description: "Medium".to_string(),
}],
shell_type: ConfigShellToolType::Default,
visibility: ModelVisibility::List,
supported_in_api: true,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
upgrade: None,
base_instructions: "base instructions".to_string(),
model_messages: None,
supports_reasoning_summaries: false,
default_reasoning_summary: ReasoningSummary::Auto,
support_verbosity: false,
default_verbosity: None,
availability_nux: None,
apply_patch_tool_type: None,
web_search_tool_type: Default::default(),
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
context_window: Some(272_000),
max_context_window: None,
auto_compact_token_limit: None,
effective_context_window_percent: 95,
experimental_supported_tools: Vec::new(),
input_modalities,
used_fallback_model_metadata: false,
supports_search_tool: false,
}
}
fn read_only_user_turn(fixture: &TestCodex, text: impl Into<String>) -> Op {
read_only_user_turn_with_model(fixture, text, fixture.session_configured.model.clone())
}
@@ -154,7 +198,7 @@ fn remote_aware_stdio_server_bin() -> anyhow::Result<String> {
return Ok(bin);
};
// Keep the Docker path rewrite scoped to tests that use `build_remote_aware`.
// Keep the Docker path rewrite scoped to tests that use `build_with_remote_env`.
// Other MCP tests still start their stdio server from the orchestrator test
// process, even when the full-ci remote env is present.
//
@@ -1386,6 +1430,257 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial(mcp_test_value)]
async fn stdio_audio_responses_are_forwarded_for_audio_model() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let call_id = "audio-supported-1";
let server_name = "rmcp";
let namespace = format!("mcp__{server_name}__");
let audio_model_slug = "rmcp-audio-model";
let models_mock = mount_models_once(
&server,
ModelsResponse {
models: vec![test_model_info_with_modalities(
audio_model_slug,
"Test model with audio input support",
vec![
InputModality::Text,
InputModality::Image,
InputModality::Audio,
],
)],
},
)
.await;
mount_sse_once(
&server,
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call_with_namespace(call_id, &namespace, "audio", "{}"),
responses::ev_completed("resp-1"),
]),
)
.await;
let final_mock = mount_sse_once(
&server,
responses::sse(vec![
responses::ev_assistant_message("msg-1", "rmcp audio tool completed successfully."),
responses::ev_completed("resp-2"),
]),
)
.await;
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
let fixture = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
insert_mcp_server(
config,
server_name,
stdio_transport(
rmcp_test_server_bin,
Some(HashMap::from([
("MCP_TEST_AUDIO_DATA".to_string(), "UklGRg==".to_string()),
(
"MCP_TEST_AUDIO_MIME_TYPE".to_string(),
"audio/mpeg".to_string(),
),
])),
Vec::new(),
),
TestMcpServerOptions {
experimental_environment: remote_aware_experimental_environment(),
..Default::default()
},
);
})
.build_with_remote_env(&server)
.await?;
fixture
.thread_manager
.get_models_manager()
.list_models(RefreshStrategy::Online)
.await;
assert_eq!(models_mock.requests().len(), 1);
fixture
.codex
.submit(read_only_user_turn_with_model(
&fixture,
"call the rmcp audio tool",
audio_model_slug.to_string(),
))
.await?;
wait_for_event(&fixture.codex, |ev| {
matches!(ev, EventMsg::McpToolCallBegin(_))
})
.await;
wait_for_event(&fixture.codex, |ev| {
matches!(ev, EventMsg::McpToolCallEnd(_))
})
.await;
wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let output_item = final_mock.single_request().function_call_output(call_id);
let output = output_item["output"]
.as_array()
.expect("audio MCP output should be content items");
assert_eq!(output.len(), 2);
assert_wall_time_header(
output[0]["text"]
.as_str()
.expect("first MCP audio output item should be wall-time text"),
);
assert_eq!(
output[1],
json!({
"type": "input_audio",
"input_audio": {
"data": "UklGRg==",
"format": "mp3",
},
})
);
server.verify().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial(mcp_test_value)]
async fn stdio_audio_responses_fail_for_text_only_model() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let call_id = "audio-text-only-1";
let server_name = "rmcp";
let namespace = format!("mcp__{server_name}__");
let text_only_model_slug = "rmcp-audio-text-only-model";
let models_mock = mount_models_once(
&server,
ModelsResponse {
models: vec![test_model_info_with_modalities(
text_only_model_slug,
"Test model without audio input support",
vec![InputModality::Text, InputModality::Image],
)],
},
)
.await;
mount_sse_once(
&server,
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call_with_namespace(call_id, &namespace, "audio", "{}"),
responses::ev_completed("resp-1"),
]),
)
.await;
let final_mock = mount_sse_once(
&server,
responses::sse(vec![
responses::ev_assistant_message("msg-1", "rmcp audio tool failed."),
responses::ev_completed("resp-2"),
]),
)
.await;
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
let fixture = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
insert_mcp_server(
config,
server_name,
stdio_transport(
rmcp_test_server_bin,
Some(HashMap::from([
("MCP_TEST_AUDIO_DATA".to_string(), "UklGRg==".to_string()),
(
"MCP_TEST_AUDIO_MIME_TYPE".to_string(),
"audio/wav".to_string(),
),
])),
Vec::new(),
),
TestMcpServerOptions {
experimental_environment: remote_aware_experimental_environment(),
..Default::default()
},
);
})
.build_with_remote_env(&server)
.await?;
fixture
.thread_manager
.get_models_manager()
.list_models(RefreshStrategy::Online)
.await;
assert_eq!(models_mock.requests().len(), 1);
fixture
.codex
.submit(read_only_user_turn_with_model(
&fixture,
"call the rmcp audio tool",
text_only_model_slug.to_string(),
))
.await?;
wait_for_event(&fixture.codex, |ev| {
matches!(ev, EventMsg::McpToolCallBegin(_))
})
.await;
let end_event = wait_for_event(&fixture.codex, |ev| {
matches!(ev, EventMsg::McpToolCallEnd(_))
})
.await;
let EventMsg::McpToolCallEnd(end) = end_event else {
unreachable!("event guard guarantees McpToolCallEnd");
};
assert_eq!(
end.result,
Err(
"audio content returned by MCP tool but the selected model does not support audio input"
.to_string()
)
);
wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let output_item = final_mock.single_request().function_call_output(call_id);
let output_text = output_item
.get("output")
.and_then(Value::as_str)
.expect("function_call_output output should be a JSON string");
let wrapped_payload = split_wall_time_wrapped_output(output_text);
let output_json: Value = serde_json::from_str(wrapped_payload)
.expect("function_call_output output should be valid JSON");
assert_eq!(
output_json,
json!([{
"type": "text",
"text": "audio content returned by MCP tool but the selected model does not support audio input"
}])
);
server.verify().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial(mcp_test_value)]
async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> {