mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
Add a content helper for forwarding MCP blocks in Code Mode (#60665)
## Why Code Mode needs a way to forward mixed MCP tool results while preserving their content types and order. ## What changed Add `content(block)` and `content(iterable)` so callers can forward tool output with `content(result.content)`. Text, image, and audio blocks become their corresponding output items; resource links and embedded resources are serialized as text. Invalid blocks and encrypted text are rejected, and iterators are closed when a block is rejected. Share MCP block conversion and image detail types with the protocol layer, while preserving existing `image()` and `audio()` input handling. ## Testing Add coverage for mixed content ordering, iterator behavior and cleanup, invalid blocks, image metadata, short audio clips, and output suppression after exit. Add an integration scenario verifying that MCP text and image blocks reach the model in order. GitOrigin-RevId: 83832ec6c530dd35ea1e3373680627997c6e13c1
This commit is contained in:
committed by
copyberry
parent
f7aad26f1d
commit
733a1cca06
+19
-111
@@ -37,6 +37,9 @@ mod configuration_update;
|
||||
mod executed_tool_calls;
|
||||
|
||||
mod item_metadata;
|
||||
mod mcp_content;
|
||||
|
||||
use mcp_content::convert_mcp_content_to_items;
|
||||
|
||||
pub use crate::local_media::MAX_PROMPT_AUDIO_INPUT_BYTES;
|
||||
pub use crate::local_media::snapshot_local_user_input;
|
||||
@@ -2576,108 +2579,6 @@ impl CallToolResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_mcp_content_to_items(
|
||||
contents: &[serde_json::Value],
|
||||
) -> Vec<FunctionCallOutputContentItem> {
|
||||
const CODEX_ENCRYPTED_CONTENT_META_KEY: &str = "codex/encryptedContent";
|
||||
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
enum McpContent {
|
||||
#[serde(rename = "text")]
|
||||
Text {
|
||||
text: String,
|
||||
#[serde(rename = "_meta", default)]
|
||||
meta: Option<serde_json::Value>,
|
||||
},
|
||||
#[serde(rename = "image")]
|
||||
Image {
|
||||
data: String,
|
||||
#[serde(rename = "mimeType", alias = "mime_type")]
|
||||
mime_type: Option<String>,
|
||||
#[serde(rename = "_meta", default)]
|
||||
meta: Option<serde_json::Value>,
|
||||
},
|
||||
#[serde(rename = "audio")]
|
||||
Audio {
|
||||
data: String,
|
||||
#[serde(rename = "mimeType", alias = "mime_type")]
|
||||
mime_type: Option<String>,
|
||||
#[serde(rename = "_meta", default)]
|
||||
_meta: Option<serde_json::Value>,
|
||||
},
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
let mut items = Vec::with_capacity(contents.len());
|
||||
|
||||
for content in contents {
|
||||
let item = match serde_json::from_value::<McpContent>(content.clone()) {
|
||||
Ok(McpContent::Text { text, meta }) => {
|
||||
if meta
|
||||
.as_ref()
|
||||
.and_then(|meta| meta.get(CODEX_ENCRYPTED_CONTENT_META_KEY))
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
== Some(true)
|
||||
{
|
||||
FunctionCallOutputContentItem::EncryptedContent {
|
||||
encrypted_content: text,
|
||||
}
|
||||
} else {
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
}
|
||||
}
|
||||
Ok(McpContent::Image {
|
||||
data,
|
||||
mime_type,
|
||||
meta,
|
||||
}) => {
|
||||
let image_url = if data.starts_with("data:") {
|
||||
data
|
||||
} else {
|
||||
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
|
||||
format!("data:{mime_type};base64,{data}")
|
||||
};
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image: ImageReference::Inline { image_url },
|
||||
detail: meta
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|detail| match detail {
|
||||
"auto" => Some(ImageDetail::Auto),
|
||||
"low" => Some(ImageDetail::Low),
|
||||
"high" => Some(ImageDetail::High),
|
||||
"original" => Some(ImageDetail::Original),
|
||||
_ => None,
|
||||
})
|
||||
.or(Some(DEFAULT_IMAGE_DETAIL)),
|
||||
}
|
||||
}
|
||||
Ok(McpContent::Audio {
|
||||
data, mime_type, ..
|
||||
}) => {
|
||||
let audio_url = if data.starts_with("data:") {
|
||||
data
|
||||
} else {
|
||||
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
|
||||
format!("data:{mime_type};base64,{data}")
|
||||
};
|
||||
FunctionCallOutputContentItem::InputAudio { audio_url }
|
||||
}
|
||||
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
|
||||
},
|
||||
};
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
// Implement Display so callers can treat the payload like a plain string when logging or doing
|
||||
// trivial substring checks in tests (existing tests call `.contains()` on the output). For
|
||||
// `ContentItems`, Display emits a JSON representation.
|
||||
@@ -3841,12 +3742,20 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn preserves_existing_image_data_urls() -> Result<()> {
|
||||
let urls = [
|
||||
"data:image/png;base64,BASE64",
|
||||
"DaTa:image/png;base64,BASE64",
|
||||
];
|
||||
let call_tool_result = CallToolResult {
|
||||
content: vec![serde_json::json!({
|
||||
"type": "image",
|
||||
"data": "data:image/png;base64,BASE64",
|
||||
"mimeType": "image/png"
|
||||
})],
|
||||
content: urls
|
||||
.map(|url| {
|
||||
serde_json::json!({
|
||||
"type": "image",
|
||||
"data": url,
|
||||
"mimeType": "image/png"
|
||||
})
|
||||
})
|
||||
.into(),
|
||||
structured_content: None,
|
||||
is_error: Some(false),
|
||||
meta: None,
|
||||
@@ -3856,15 +3765,14 @@ mod tests {
|
||||
let Some(items) = payload.content_items() else {
|
||||
panic!("expected content items");
|
||||
};
|
||||
let items = items.to_vec();
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
&urls.map(|url| FunctionCallOutputContentItem::InputImage {
|
||||
image: ImageReference::Inline {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
image_url: url.into(),
|
||||
},
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
}]
|
||||
})
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! Converts individual MCP content blocks into model-visible tool output.
|
||||
|
||||
use super::DEFAULT_IMAGE_DETAIL;
|
||||
use super::FunctionCallOutputContentItem;
|
||||
use super::ImageDetail;
|
||||
use super::ImageReference;
|
||||
use crate::mcp::Resource;
|
||||
use crate::mcp::ResourceContent;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum McpContent<'a> {
|
||||
Text {
|
||||
text: String,
|
||||
},
|
||||
Image(#[serde(borrow)] McpMedia<'a>),
|
||||
Audio(#[serde(borrow)] McpMedia<'a>),
|
||||
ResourceLink {
|
||||
#[serde(flatten)]
|
||||
_link: Resource,
|
||||
},
|
||||
Resource {
|
||||
#[serde(rename = "resource")]
|
||||
_resource: ResourceContent,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct McpMedia<'a> {
|
||||
data: &'a str,
|
||||
#[serde(rename = "mimeType", alias = "mime_type")]
|
||||
mime_type: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl McpMedia<'_> {
|
||||
fn data_url(&self) -> String {
|
||||
if self
|
||||
.data
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
{
|
||||
self.data.to_owned()
|
||||
} else {
|
||||
let mime_type = self.mime_type.unwrap_or("application/octet-stream");
|
||||
format!("data:{mime_type};base64,{}", self.data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FunctionCallOutputContentItem {
|
||||
/// Projects one MCP ContentBlock using its wire discriminator and Codex metadata.
|
||||
pub fn from_mcp_content_block(block: &Value) -> serde_json::Result<Self> {
|
||||
let meta = block.get("_meta");
|
||||
Ok(match McpContent::deserialize(block)? {
|
||||
McpContent::Text { text } => {
|
||||
if meta
|
||||
.and_then(|meta| meta.get("codex/encryptedContent"))
|
||||
.and_then(Value::as_bool)
|
||||
== Some(true)
|
||||
{
|
||||
Self::EncryptedContent {
|
||||
encrypted_content: text,
|
||||
}
|
||||
} else {
|
||||
Self::InputText { text }
|
||||
}
|
||||
}
|
||||
McpContent::Image(media) => Self::InputImage {
|
||||
image: ImageReference::Inline {
|
||||
image_url: media.data_url(),
|
||||
},
|
||||
detail: Some(
|
||||
meta.and_then(|meta| meta.get("codex/imageDetail"))
|
||||
.filter(|detail| detail.is_string())
|
||||
.and_then(|detail| ImageDetail::deserialize(detail).ok())
|
||||
.unwrap_or(DEFAULT_IMAGE_DETAIL),
|
||||
),
|
||||
},
|
||||
McpContent::Audio(media) => Self::InputAudio {
|
||||
audio_url: media.data_url(),
|
||||
},
|
||||
McpContent::ResourceLink { .. } | McpContent::Resource { .. } => Self::InputText {
|
||||
text: serde_json::to_string(block)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn convert_mcp_content_to_items(
|
||||
contents: &[Value],
|
||||
) -> Vec<FunctionCallOutputContentItem> {
|
||||
contents
|
||||
.iter()
|
||||
.map(|block| {
|
||||
FunctionCallOutputContentItem::from_mcp_content_block(block).unwrap_or_else(|_| {
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(block).unwrap_or_else(|_| "<content>".to_string()),
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
Reference in New Issue
Block a user