mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Support audio input
This commit is contained in:
@@ -12,6 +12,7 @@ use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
|
||||
use codex_protocol::dynamic_tools::DynamicToolResponse;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
@@ -175,6 +176,54 @@ async fn run_code_mode_turn(
|
||||
Ok((test, second_mock))
|
||||
}
|
||||
|
||||
async fn run_code_mode_turn_with_audio_model(
|
||||
server: &MockServer,
|
||||
prompt: &str,
|
||||
code: &str,
|
||||
) -> Result<(TestCodex, ResponseMock)> {
|
||||
let mut builder = test_codex()
|
||||
.with_model("gpt-5.4")
|
||||
.with_config(move |config| {
|
||||
let _ = config.features.enable(Feature::CodeMode);
|
||||
let mut model_catalog = bundled_models_response()
|
||||
.unwrap_or_else(|err| panic!("bundled models.json should parse: {err}"));
|
||||
let model = model_catalog
|
||||
.models
|
||||
.iter_mut()
|
||||
.find(|model| model.slug == "gpt-5.4")
|
||||
.expect("gpt-5.4 exists in bundled models.json");
|
||||
model.input_modalities = vec![
|
||||
InputModality::Text,
|
||||
InputModality::Image,
|
||||
InputModality::Audio,
|
||||
];
|
||||
config.model_catalog = Some(model_catalog);
|
||||
});
|
||||
let test = builder.build(server).await?;
|
||||
|
||||
responses::mount_sse_once(
|
||||
server,
|
||||
sse(vec![
|
||||
ev_response_created("resp-1"),
|
||||
ev_custom_tool_call("call-1", "exec", code),
|
||||
ev_completed("resp-1"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let second_mock = responses::mount_sse_once(
|
||||
server,
|
||||
sse(vec![
|
||||
ev_assistant_message("msg-1", "done"),
|
||||
ev_completed("resp-2"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
test.submit_turn(prompt).await?;
|
||||
Ok((test, second_mock))
|
||||
}
|
||||
|
||||
async fn run_code_mode_turn_with_rmcp(
|
||||
server: &MockServer,
|
||||
prompt: &str,
|
||||
@@ -1974,6 +2023,77 @@ image("data:image/png;base64,AAA");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn code_mode_can_output_audio_via_global_helper_for_audio_model() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let (_test, second_mock) = run_code_mode_turn_with_audio_model(
|
||||
&server,
|
||||
"use exec to return audio",
|
||||
r#"
|
||||
audio({ data: "BASE64", format: "wav" });
|
||||
audio({ data: "data:audio/mpeg;base64,MP3BASE64" });
|
||||
"#,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let req = second_mock.single_request();
|
||||
let items = custom_tool_output_items(&req, "call-1");
|
||||
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
||||
assert_ne!(
|
||||
success,
|
||||
Some(false),
|
||||
"code_mode audio output failed unexpectedly"
|
||||
);
|
||||
assert_eq!(items.len(), 3);
|
||||
assert_regex_match(
|
||||
concat!(
|
||||
r"(?s)\A",
|
||||
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
||||
),
|
||||
text_item(&items, /*index*/ 0),
|
||||
);
|
||||
assert_eq!(
|
||||
items[1],
|
||||
serde_json::json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": { "data": "BASE64", "format": "wav" }
|
||||
}),
|
||||
);
|
||||
assert_eq!(
|
||||
items[2],
|
||||
serde_json::json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": { "data": "MP3BASE64", "format": "mp3" }
|
||||
}),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn code_mode_audio_output_fails_for_non_audio_model() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let (_test, second_mock) = run_code_mode_turn(
|
||||
&server,
|
||||
"use exec to return audio",
|
||||
r#"audio({ data: "BASE64", format: "wav" });"#,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let req = second_mock.single_request();
|
||||
let (output, _success) = custom_tool_output_body_and_success(&req, "call-1");
|
||||
assert_eq!(
|
||||
output,
|
||||
"audio content emitted by code mode but the selected model does not support audio input"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
@@ -2404,6 +2524,7 @@ text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort()));
|
||||
"WeakSet",
|
||||
"__codexContentItems",
|
||||
"add_content",
|
||||
"audio",
|
||||
"decodeURI",
|
||||
"decodeURIComponent",
|
||||
"encodeURI",
|
||||
|
||||
@@ -93,6 +93,50 @@ fn assert_wall_time_header(output: &str) {
|
||||
assert_eq!(marker, "Output:");
|
||||
}
|
||||
|
||||
fn test_model_info_with_modalities(
|
||||
slug: &str,
|
||||
description: &str,
|
||||
input_modalities: Vec<InputModality>,
|
||||
) -> ModelInfo {
|
||||
ModelInfo {
|
||||
slug: slug.to_string(),
|
||||
display_name: slug.to_string(),
|
||||
description: Some(description.to_string()),
|
||||
default_reasoning_level: None,
|
||||
supported_reasoning_levels: vec![ReasoningEffortPreset {
|
||||
effort: codex_protocol::openai_models::ReasoningEffort::Medium,
|
||||
description: "Medium".to_string(),
|
||||
}],
|
||||
shell_type: ConfigShellToolType::Default,
|
||||
visibility: ModelVisibility::List,
|
||||
supported_in_api: true,
|
||||
priority: 1,
|
||||
additional_speed_tiers: Vec::new(),
|
||||
service_tiers: Vec::new(),
|
||||
upgrade: None,
|
||||
base_instructions: "base instructions".to_string(),
|
||||
model_messages: None,
|
||||
supports_reasoning_summaries: false,
|
||||
default_reasoning_summary: ReasoningSummary::Auto,
|
||||
support_verbosity: false,
|
||||
default_verbosity: None,
|
||||
availability_nux: None,
|
||||
apply_patch_tool_type: None,
|
||||
web_search_tool_type: Default::default(),
|
||||
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
|
||||
supports_parallel_tool_calls: false,
|
||||
supports_image_detail_original: false,
|
||||
context_window: Some(272_000),
|
||||
max_context_window: None,
|
||||
auto_compact_token_limit: None,
|
||||
effective_context_window_percent: 95,
|
||||
experimental_supported_tools: Vec::new(),
|
||||
input_modalities,
|
||||
used_fallback_model_metadata: false,
|
||||
supports_search_tool: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_only_user_turn(fixture: &TestCodex, text: impl Into<String>) -> Op {
|
||||
read_only_user_turn_with_model(fixture, text, fixture.session_configured.model.clone())
|
||||
}
|
||||
@@ -154,7 +198,7 @@ fn remote_aware_stdio_server_bin() -> anyhow::Result<String> {
|
||||
return Ok(bin);
|
||||
};
|
||||
|
||||
// Keep the Docker path rewrite scoped to tests that use `build_remote_aware`.
|
||||
// Keep the Docker path rewrite scoped to tests that use `build_with_remote_env`.
|
||||
// Other MCP tests still start their stdio server from the orchestrator test
|
||||
// process, even when the full-ci remote env is present.
|
||||
//
|
||||
@@ -1386,6 +1430,257 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial(mcp_test_value)]
|
||||
async fn stdio_audio_responses_are_forwarded_for_audio_model() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
|
||||
let call_id = "audio-supported-1";
|
||||
let server_name = "rmcp";
|
||||
let namespace = format!("mcp__{server_name}__");
|
||||
let audio_model_slug = "rmcp-audio-model";
|
||||
|
||||
let models_mock = mount_models_once(
|
||||
&server,
|
||||
ModelsResponse {
|
||||
models: vec![test_model_info_with_modalities(
|
||||
audio_model_slug,
|
||||
"Test model with audio input support",
|
||||
vec![
|
||||
InputModality::Text,
|
||||
InputModality::Image,
|
||||
InputModality::Audio,
|
||||
],
|
||||
)],
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(call_id, &namespace, "audio", "{}"),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let final_mock = mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message("msg-1", "rmcp audio tool completed successfully."),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
||||
|
||||
let fixture = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(move |config| {
|
||||
insert_mcp_server(
|
||||
config,
|
||||
server_name,
|
||||
stdio_transport(
|
||||
rmcp_test_server_bin,
|
||||
Some(HashMap::from([
|
||||
("MCP_TEST_AUDIO_DATA".to_string(), "UklGRg==".to_string()),
|
||||
(
|
||||
"MCP_TEST_AUDIO_MIME_TYPE".to_string(),
|
||||
"audio/mpeg".to_string(),
|
||||
),
|
||||
])),
|
||||
Vec::new(),
|
||||
),
|
||||
TestMcpServerOptions {
|
||||
experimental_environment: remote_aware_experimental_environment(),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
})
|
||||
.build_with_remote_env(&server)
|
||||
.await?;
|
||||
|
||||
fixture
|
||||
.thread_manager
|
||||
.get_models_manager()
|
||||
.list_models(RefreshStrategy::Online)
|
||||
.await;
|
||||
assert_eq!(models_mock.requests().len(), 1);
|
||||
|
||||
fixture
|
||||
.codex
|
||||
.submit(read_only_user_turn_with_model(
|
||||
&fixture,
|
||||
"call the rmcp audio tool",
|
||||
audio_model_slug.to_string(),
|
||||
))
|
||||
.await?;
|
||||
|
||||
wait_for_event(&fixture.codex, |ev| {
|
||||
matches!(ev, EventMsg::McpToolCallBegin(_))
|
||||
})
|
||||
.await;
|
||||
wait_for_event(&fixture.codex, |ev| {
|
||||
matches!(ev, EventMsg::McpToolCallEnd(_))
|
||||
})
|
||||
.await;
|
||||
wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let output_item = final_mock.single_request().function_call_output(call_id);
|
||||
let output = output_item["output"]
|
||||
.as_array()
|
||||
.expect("audio MCP output should be content items");
|
||||
assert_eq!(output.len(), 2);
|
||||
assert_wall_time_header(
|
||||
output[0]["text"]
|
||||
.as_str()
|
||||
.expect("first MCP audio output item should be wall-time text"),
|
||||
);
|
||||
assert_eq!(
|
||||
output[1],
|
||||
json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": "UklGRg==",
|
||||
"format": "mp3",
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
server.verify().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial(mcp_test_value)]
|
||||
async fn stdio_audio_responses_fail_for_text_only_model() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
|
||||
let call_id = "audio-text-only-1";
|
||||
let server_name = "rmcp";
|
||||
let namespace = format!("mcp__{server_name}__");
|
||||
let text_only_model_slug = "rmcp-audio-text-only-model";
|
||||
|
||||
let models_mock = mount_models_once(
|
||||
&server,
|
||||
ModelsResponse {
|
||||
models: vec![test_model_info_with_modalities(
|
||||
text_only_model_slug,
|
||||
"Test model without audio input support",
|
||||
vec![InputModality::Text, InputModality::Image],
|
||||
)],
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(call_id, &namespace, "audio", "{}"),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let final_mock = mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message("msg-1", "rmcp audio tool failed."),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
|
||||
|
||||
let fixture = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(move |config| {
|
||||
insert_mcp_server(
|
||||
config,
|
||||
server_name,
|
||||
stdio_transport(
|
||||
rmcp_test_server_bin,
|
||||
Some(HashMap::from([
|
||||
("MCP_TEST_AUDIO_DATA".to_string(), "UklGRg==".to_string()),
|
||||
(
|
||||
"MCP_TEST_AUDIO_MIME_TYPE".to_string(),
|
||||
"audio/wav".to_string(),
|
||||
),
|
||||
])),
|
||||
Vec::new(),
|
||||
),
|
||||
TestMcpServerOptions {
|
||||
experimental_environment: remote_aware_experimental_environment(),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
})
|
||||
.build_with_remote_env(&server)
|
||||
.await?;
|
||||
|
||||
fixture
|
||||
.thread_manager
|
||||
.get_models_manager()
|
||||
.list_models(RefreshStrategy::Online)
|
||||
.await;
|
||||
assert_eq!(models_mock.requests().len(), 1);
|
||||
|
||||
fixture
|
||||
.codex
|
||||
.submit(read_only_user_turn_with_model(
|
||||
&fixture,
|
||||
"call the rmcp audio tool",
|
||||
text_only_model_slug.to_string(),
|
||||
))
|
||||
.await?;
|
||||
|
||||
wait_for_event(&fixture.codex, |ev| {
|
||||
matches!(ev, EventMsg::McpToolCallBegin(_))
|
||||
})
|
||||
.await;
|
||||
let end_event = wait_for_event(&fixture.codex, |ev| {
|
||||
matches!(ev, EventMsg::McpToolCallEnd(_))
|
||||
})
|
||||
.await;
|
||||
let EventMsg::McpToolCallEnd(end) = end_event else {
|
||||
unreachable!("event guard guarantees McpToolCallEnd");
|
||||
};
|
||||
assert_eq!(
|
||||
end.result,
|
||||
Err(
|
||||
"audio content returned by MCP tool but the selected model does not support audio input"
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let output_item = final_mock.single_request().function_call_output(call_id);
|
||||
let output_text = output_item
|
||||
.get("output")
|
||||
.and_then(Value::as_str)
|
||||
.expect("function_call_output output should be a JSON string");
|
||||
let wrapped_payload = split_wall_time_wrapped_output(output_text);
|
||||
let output_json: Value = serde_json::from_str(wrapped_payload)
|
||||
.expect("function_call_output output should be valid JSON");
|
||||
assert_eq!(
|
||||
output_json,
|
||||
json!([{
|
||||
"type": "text",
|
||||
"text": "audio content returned by MCP tool but the selected model does not support audio input"
|
||||
}])
|
||||
);
|
||||
|
||||
server.verify().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial(mcp_test_value)]
|
||||
async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> {
|
||||
|
||||
Reference in New Issue
Block a user