Materialize MCP tool images into local refs

This commit is contained in:
Aismit Das
2026-03-16 17:24:15 -04:00
parent b33edebd6a
commit 4d46aea4dc
2 changed files with 283 additions and 33 deletions
+91 -7
View File
@@ -1307,6 +1307,8 @@ fn convert_mcp_content_to_items(
#[serde(rename = "mimeType", alias = "mime_type")]
mime_type: Option<String>,
},
#[serde(rename = "local_image", alias = "localImage")]
LocalImage { path: String },
#[serde(other)]
Unknown,
}
@@ -1315,8 +1317,10 @@ fn convert_mcp_content_to_items(
let mut items = Vec::with_capacity(contents.len());
for content in contents {
let item = match serde_json::from_value::<McpContent>(content.clone()) {
Ok(McpContent::Text { text }) => FunctionCallOutputContentItem::InputText { text },
let content_items = match serde_json::from_value::<McpContent>(content.clone()) {
Ok(McpContent::Text { text }) => {
vec![FunctionCallOutputContentItem::InputText { text }]
}
Ok(McpContent::Image { data, mime_type }) => {
saw_image = true;
let image_url = if data.starts_with("data:") {
@@ -1325,16 +1329,37 @@ fn convert_mcp_content_to_items(
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
format!("data:{mime_type};base64,{data}")
};
FunctionCallOutputContentItem::InputImage {
vec![FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
}
}]
}
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
Ok(McpContent::LocalImage { path }) => {
saw_image = true;
local_image_content_items_with_label_number(
Path::new(&path),
None,
PromptImageMode::ResizeToFit,
)
.into_iter()
.map(|item| match item {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
ContentItem::InputImage { image_url } => {
FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
}
}
})
.collect()
}
Ok(McpContent::Unknown) | Err(_) => vec![FunctionCallOutputContentItem::InputText {
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
},
}],
};
items.push(item);
items.extend(content_items);
}
if saw_image { Some(items) } else { None }
@@ -1965,6 +1990,65 @@ mod tests {
Ok(())
}
#[test]
fn serializes_local_image_outputs_as_array() -> Result<()> {
let dir = tempdir()?;
let local_path = dir.path().join("local.png");
const TINY_PNG_BYTES: &[u8] = &[
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1,
8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2,
0, 0, 5, 0, 1, 122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
];
std::fs::write(&local_path, TINY_PNG_BYTES)?;
let call_tool_result = CallToolResult {
content: vec![serde_json::json!({
"type": "local_image",
"path": local_path,
})],
structured_content: None,
is_error: Some(false),
meta: None,
};
let payload = McpToolOutput::from(&call_tool_result).into_function_call_output_payload();
let Some(items) = payload.content_items() else {
panic!("expected content items");
};
assert!(matches!(
items,
[FunctionCallOutputContentItem::InputImage { image_url, detail: None }]
if image_url.starts_with("data:image/png;base64,")
));
Ok(())
}
#[test]
fn local_image_output_missing_file_becomes_text_placeholder() -> Result<()> {
let call_tool_result = CallToolResult {
content: vec![serde_json::json!({
"type": "local_image",
"path": "/tmp/does-not-exist.png",
})],
structured_content: None,
is_error: Some(false),
meta: None,
};
let payload = McpToolOutput::from(&call_tool_result).into_function_call_output_payload();
let Some(items) = payload.content_items() else {
panic!("expected content items");
};
assert!(matches!(
items,
[FunctionCallOutputContentItem::InputText { text }]
if text.contains("could not read the local image")
));
Ok(())
}
#[test]
fn deserializes_array_payload_into_items() -> Result<()> {
let json = r#"[