From 6412cfd024fcd7f140804cfeb0b69287fb475ae5 Mon Sep 17 00:00:00 2001 From: Aaron Levine Date: Mon, 2 Feb 2026 10:38:19 -0800 Subject: [PATCH] dynamic tool call response can include content_items --- .../app-server-protocol/src/protocol/v2.rs | 11 +- .../app-server/src/bespoke_event_handling.rs | 1 + codex-rs/app-server/src/dynamic_tools.rs | 30 +++- .../tests/suite/v2/dynamic_tools.rs | 145 +++++++++++++++++- codex-rs/core/src/tools/handlers/dynamic.rs | 2 +- codex-rs/protocol/src/dynamic_tools.rs | 5 + 6 files changed, 183 insertions(+), 11 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index f99659cd4d..76231e1ed2 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -14,6 +14,7 @@ use codex_protocol::config_types::Verbosity; use codex_protocol::config_types::WebSearchMode; use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; use codex_protocol::items::TurnItem as CoreTurnItem; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::parse_command::ParsedCommand as CoreParsedCommand; @@ -2670,8 +2671,16 @@ pub struct DynamicToolCallParams { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct DynamicToolCallResponse { - pub output: String, + /// Optional plain-text output. When omitted, the app server will derive a + /// string representation from `contentItems` for legacy consumers. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub output: Option, pub success: bool, + /// Optional structured content for the tool output (for example, images). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub content_items: Option>, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index bf8fad3a9c..5e4b9f6209 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -354,6 +354,7 @@ pub(crate) async fn apply_bespoke_event_handling( call_id, output: "dynamic tool calls require api v2".to_string(), success: false, + content_items: None, }, }) .await; diff --git a/codex-rs/app-server/src/dynamic_tools.rs b/codex-rs/app-server/src/dynamic_tools.rs index a1b424d0ee..29df399f14 100644 --- a/codex-rs/app-server/src/dynamic_tools.rs +++ b/codex-rs/app-server/src/dynamic_tools.rs @@ -1,6 +1,7 @@ use codex_app_server_protocol::DynamicToolCallResponse; use codex_core::CodexThread; use codex_protocol::dynamic_tools::DynamicToolResponse as CoreDynamicToolResponse; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::protocol::Op; use std::sync::Arc; use tokio::sync::oneshot; @@ -20,6 +21,7 @@ pub(crate) async fn on_call_response( call_id: call_id.clone(), output: "dynamic tool request failed".to_string(), success: false, + content_items: None, }; if let Err(err) = conversation .submit(Op::DynamicToolResponse { @@ -37,14 +39,17 @@ pub(crate) async fn on_call_response( let response = serde_json::from_value::(value).unwrap_or_else(|err| { error!("failed to deserialize DynamicToolCallResponse: {err}"); DynamicToolCallResponse { - output: "dynamic tool response was invalid".to_string(), + output: Some("dynamic tool response was invalid".to_string()), success: false, + content_items: None, } }); + let output = normalize_output(response.output, response.content_items.as_deref()); let response = CoreDynamicToolResponse { call_id: call_id.clone(), - output: response.output, + output, success: response.success, + content_items: response.content_items, }; if let Err(err) = conversation .submit(Op::DynamicToolResponse { @@ -56,3 +61,24 @@ pub(crate) async fn on_call_response( error!("failed to submit DynamicToolResponse: {err}"); } } + +fn normalize_output( + output: Option, + content_items: Option<&[FunctionCallOutputContentItem]>, +) -> String { + if let Some(output) = output { + return output; + } + + if let Some(items) = content_items { + return match serde_json::to_string(items) { + Ok(json) => json, + Err(err) => { + error!("failed to serialize dynamic tool content_items: {err}"); + String::new() + } + }; + } + + String::new() +} diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index dc985ac49f..ec7a7ef825 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -15,6 +15,8 @@ use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::UserInput as V2UserInput; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; use core_test_support::responses; use pretty_assertions::assert_eq; use serde_json::Value; @@ -200,8 +202,9 @@ async fn dynamic_tool_call_round_trip_sends_output_to_model() -> Result<()> { // Respond to the tool call so the model receives a function_call_output. let response = DynamicToolCallResponse { - output: "dynamic-ok".to_string(), + output: Some("dynamic-ok".to_string()), success: true, + content_items: None, }; mcp.send_response(request_id, serde_json::to_value(response)?) .await?; @@ -213,11 +216,139 @@ async fn dynamic_tool_call_round_trip_sends_output_to_model() -> Result<()> { .await??; let bodies = responses_bodies(&server).await?; - let output = bodies + let payload = bodies .iter() - .find_map(|body| function_call_output_text(body, call_id)) + .find_map(|body| function_call_output_payload(body, call_id)) .context("expected function_call_output in follow-up request")?; - assert_eq!(output, "dynamic-ok"); + let expected_payload = FunctionCallOutputPayload { + content: "dynamic-ok".to_string(), + content_items: None, + success: None, + }; + assert_eq!(payload, expected_payload); + + Ok(()) +} + +/// Ensures dynamic tool call responses can include structured content items. +#[tokio::test] +async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<()> { + let call_id = "dyn-call-items-1"; + let tool_name = "demo_tool"; + let tool_args = json!({ "city": "Paris" }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let dynamic_tool = DynamicToolSpec { + name: tool_name.to_string(), + description: "Demo dynamic tool".to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"], + "additionalProperties": false, + }), + }; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + dynamic_tools: Some(vec![dynamic_tool]), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "Run the tool".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + let request = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let (request_id, params) = match request { + ServerRequest::DynamicToolCall { request_id, params } => (request_id, params), + other => panic!("expected DynamicToolCall request, got {other:?}"), + }; + + let expected = DynamicToolCallParams { + thread_id: thread.id, + turn_id: turn.id, + call_id: call_id.to_string(), + tool: tool_name.to_string(), + arguments: tool_args, + }; + assert_eq!(params, expected); + + let content_items = vec![ + FunctionCallOutputContentItem::InputText { + text: "dynamic-ok".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + ]; + let response = DynamicToolCallResponse { + output: None, + success: true, + content_items: Some(content_items.clone()), + }; + mcp.send_response(request_id, serde_json::to_value(response)?) + .await?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let payload = bodies + .iter() + .find_map(|body| function_call_output_payload(body, call_id)) + .context("expected function_call_output in follow-up request")?; + let expected_payload = FunctionCallOutputPayload { + content: serde_json::to_string(&content_items)?, + content_items: Some(content_items), + success: None, + }; + assert_eq!(payload, expected_payload); Ok(()) } @@ -248,7 +379,7 @@ fn find_tool<'a>(body: &'a Value, name: &str) -> Option<&'a Value> { }) } -fn function_call_output_text(body: &Value, call_id: &str) -> Option { +fn function_call_output_payload(body: &Value, call_id: &str) -> Option { body.get("input") .and_then(Value::as_array) .and_then(|items| { @@ -258,8 +389,8 @@ fn function_call_output_text(body: &Value, call_id: &str) -> Option { }) }) .and_then(|item| item.get("output")) - .and_then(Value::as_str) - .map(str::to_string) + .cloned() + .and_then(|output| serde_json::from_value(output).ok()) } fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs index a68c70b98d..f2cb10e870 100644 --- a/codex-rs/core/src/tools/handlers/dynamic.rs +++ b/codex-rs/core/src/tools/handlers/dynamic.rs @@ -57,7 +57,7 @@ impl ToolHandler for DynamicToolHandler { Ok(ToolOutput::Function { content: response.output, - content_items: None, + content_items: response.content_items, success: Some(response.success), }) } diff --git a/codex-rs/protocol/src/dynamic_tools.rs b/codex-rs/protocol/src/dynamic_tools.rs index e55d372d8e..5a21cb6926 100644 --- a/codex-rs/protocol/src/dynamic_tools.rs +++ b/codex-rs/protocol/src/dynamic_tools.rs @@ -4,6 +4,8 @@ use serde::Serialize; use serde_json::Value as JsonValue; use ts_rs::TS; +use crate::models::FunctionCallOutputContentItem; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] pub struct DynamicToolSpec { @@ -27,4 +29,7 @@ pub struct DynamicToolResponse { pub call_id: String, pub output: String, pub success: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub content_items: Option>, }