mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add a content helper for forwarding MCP blocks in Code Mode (#60665)
## Why Code Mode needs a way to forward mixed MCP tool results while preserving their content types and order. ## What changed Add `content(block)` and `content(iterable)` so callers can forward tool output with `content(result.content)`. Text, image, and audio blocks become their corresponding output items; resource links and embedded resources are serialized as text. Invalid blocks and encrypted text are rejected, and iterators are closed when a block is rejected. Share MCP block conversion and image detail types with the protocol layer, while preserving existing `image()` and `audio()` input handling. ## Testing Add coverage for mixed content ordering, iterator behavior and cleanup, invalid blocks, image metadata, short audio clips, and output suppression after exit. Add an integration scenario verifying that MCP text and image blocks reach the model in order. GitOrigin-RevId: 83832ec6c530dd35ea1e3373680627997c6e13c1
This commit is contained in:
committed by
copyberry
parent
f7aad26f1d
commit
733a1cca06
@@ -1450,6 +1450,7 @@ async fn global_scope_contains_only_allowed_items() {
|
||||
"add_content",
|
||||
"as_settled",
|
||||
"audio",
|
||||
"content",
|
||||
"decodeURI",
|
||||
"decodeURIComponent",
|
||||
"encodeURI",
|
||||
@@ -1644,13 +1645,17 @@ text(JSON.stringify(returnsUndefined));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn text_helper_serializes_objects() {
|
||||
async fn text_serializes_objects_and_content_forwards_mcp_text() {
|
||||
let service = InProcessCodeModeSession::new();
|
||||
|
||||
let response = execute(
|
||||
&service,
|
||||
ExecuteRequest {
|
||||
source: "text({ json: true });".to_string(),
|
||||
source: r#"const block = {type: "text", text: "caption", json: true};
|
||||
text(block);
|
||||
text(block.text);
|
||||
content(block);"#
|
||||
.to_string(),
|
||||
yield_time_ms: None,
|
||||
..execute_request("")
|
||||
},
|
||||
@@ -1662,9 +1667,13 @@ async fn text_helper_serializes_objects() {
|
||||
RuntimeResponse::Result {
|
||||
code_mode_host_duration: None,
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![FunctionCallOutputContentItem::InputText {
|
||||
text: r#"{"json":true}"#.to_string(),
|
||||
}],
|
||||
content_items: [
|
||||
r#"{"type":"text","text":"caption","json":true}"#,
|
||||
"caption",
|
||||
"caption"
|
||||
]
|
||||
.map(|text| FunctionCallOutputContentItem::InputText { text: text.into() })
|
||||
.into(),
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
@@ -1672,7 +1681,7 @@ async fn text_helper_serializes_objects() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn output_helpers_surface_serialization_errors() {
|
||||
async fn output_helpers_surface_input_errors() {
|
||||
for (source, expected_error) in [
|
||||
(
|
||||
"const circular = {}; circular.self = circular; text(circular);",
|
||||
@@ -1686,6 +1695,14 @@ async fn output_helpers_surface_serialization_errors() {
|
||||
"image({type: 'image', get data() { throw new Error('MCP getter failed'); }});",
|
||||
"MCP getter failed",
|
||||
),
|
||||
(
|
||||
"image({type: 'image', data: ''});",
|
||||
"image expected MCP image data",
|
||||
),
|
||||
(
|
||||
"image({type: 'audio'});",
|
||||
r#"image only accepts MCP image blocks, got "audio""#,
|
||||
),
|
||||
] {
|
||||
let service = InProcessCodeModeSession::new();
|
||||
let response = execute(
|
||||
@@ -1803,17 +1820,19 @@ async fn audio_helper_rejects_non_data_urls() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() {
|
||||
async fn image_and_content_forward_the_same_raw_mcp_image() {
|
||||
let service = InProcessCodeModeSession::new();
|
||||
|
||||
let response = execute(&service, ExecuteRequest {
|
||||
source: r#"
|
||||
image({
|
||||
const block = {
|
||||
type: "image",
|
||||
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
|
||||
mimeType: "image/png",
|
||||
mime_type: "image/png",
|
||||
_meta: { "codex/imageDetail": "original" },
|
||||
});
|
||||
};
|
||||
image(block);
|
||||
content(block);
|
||||
"#
|
||||
.to_string(),
|
||||
yield_time_ms: None,
|
||||
@@ -1829,13 +1848,221 @@ image({
|
||||
content_items: vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
|
||||
detail: Some(crate::ImageDetail::Original),
|
||||
}],
|
||||
}; 2],
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn media_helpers_preserve_legacy_mime_inputs_and_detail_overrides() {
|
||||
let response = execute(
|
||||
&InProcessCodeModeSession::new(),
|
||||
ExecuteRequest {
|
||||
yield_time_ms: None,
|
||||
..execute_request(
|
||||
r#"
|
||||
image({type: "image", data: "YQ==", mimeType: "image/png", mime_type: "image/jpeg"});
|
||||
audio({type: "audio", data: "YQ==", mimeType: 1, mime_type: "audio/wav"});
|
||||
image({type: "image", data: "DaTa:image/png;base64,YQ==", mimeType: ""}, "LOW");
|
||||
"#,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
code_mode_host_duration: None,
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,YQ==".into(),
|
||||
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:application/octet-stream;base64,YQ==".into(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "DaTa:image/png;base64,YQ==".into(),
|
||||
detail: Some(crate::ImageDetail::Low),
|
||||
},
|
||||
],
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn content_helper_forwards_mixed_mcp_blocks_in_order() {
|
||||
let link = serde_json::json!({
|
||||
"type": "resource_link", "name": "notes", "uri": "file:///notes",
|
||||
});
|
||||
let resource = serde_json::json!({
|
||||
"type": "resource", "resource": {"uri": "file:///notes", "text": "notes"},
|
||||
});
|
||||
let blob = serde_json::json!({
|
||||
"type": "resource", "resource": {"uri": "file:///sample", "blob": "YWJj"},
|
||||
});
|
||||
let blocks = serde_json::json!([
|
||||
{"type": "text", "text": "caption", "annotations": {"priority": 0.5}},
|
||||
{
|
||||
"type": "image",
|
||||
"data": "aW1hZ2U=",
|
||||
"mimeType": "image/png",
|
||||
"_meta": {"codex/imageDetail": "original"},
|
||||
},
|
||||
{"type": "audio", "data": "YXVkaW8=", "mimeType": "audio/mpeg"},
|
||||
link, resource, blob,
|
||||
]);
|
||||
let response = execute(
|
||||
&InProcessCodeModeSession::new(),
|
||||
ExecuteRequest {
|
||||
yield_time_ms: None,
|
||||
..execute_request(&format!("content({blocks})"))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
code_mode_host_duration: None,
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "caption".into()
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,aW1hZ2U=".into(),
|
||||
detail: Some(crate::ImageDetail::Original),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".into(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: link.to_string()
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: resource.to_string()
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: blob.to_string()
|
||||
},
|
||||
],
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn content_helper_consumes_iterables_in_order_and_closes_on_error() {
|
||||
let response = execute(
|
||||
&InProcessCodeModeSession::new(),
|
||||
ExecuteRequest {
|
||||
yield_time_ms: None,
|
||||
..execute_request(
|
||||
r#"
|
||||
const block = (text) => ({type: "text", text});
|
||||
content([]);
|
||||
[block("single")].forEach(content);
|
||||
content(new Set([block("set")]));
|
||||
|
||||
const failure = new Error("iterator failed");
|
||||
function* generator() { yield block("generator"); throw failure; }
|
||||
try { content(generator()); } catch (error) {
|
||||
if (error !== failure) throw error;
|
||||
text("caught iterator error");
|
||||
}
|
||||
|
||||
let reads = 0;
|
||||
const iterable = {
|
||||
get [Symbol.iterator]() {
|
||||
if (++reads !== 1) throw new Error("iterator getter read twice");
|
||||
return function () {
|
||||
if (this !== iterable) throw new Error("wrong iterator receiver");
|
||||
const items = [block("first"), {type: "image"}, block("past error")].values();
|
||||
return {
|
||||
next() { text("next"); return items.next(); },
|
||||
return() { text("closed"); return {}; },
|
||||
};
|
||||
};
|
||||
},
|
||||
};
|
||||
try { content(iterable); } catch (error) { text(error); }
|
||||
"#,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let expected = [
|
||||
"single",
|
||||
"set",
|
||||
"generator",
|
||||
"caught iterator error",
|
||||
"next",
|
||||
"first",
|
||||
"next",
|
||||
"closed",
|
||||
"content expects one MCP ContentBlock",
|
||||
];
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
code_mode_host_duration: None,
|
||||
cell_id: cell_id("1"),
|
||||
content_items: expected
|
||||
.into_iter()
|
||||
.map(|text| FunctionCallOutputContentItem::InputText { text: text.into() })
|
||||
.collect(),
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn content_helper_rejects_invalid_blocks_and_encrypted_text_without_emitting_them() {
|
||||
let response = execute(
|
||||
&InProcessCodeModeSession::new(),
|
||||
ExecuteRequest {
|
||||
yield_time_ms: None,
|
||||
..execute_request(
|
||||
r#"
|
||||
for (const block of [
|
||||
undefined, "", {content: []}, [[{type: "text", text: "nested"}]], ["text", "nested"],
|
||||
{type: "unknown"}, {type: "text", text: 1}, {type: "image"},
|
||||
{type: "resource", resource: {uri: "file:///notes"}}, {type: "resource_link"},
|
||||
{type: "text", text: "opaque", _meta: {"codex/encryptedContent": true}},
|
||||
]) {
|
||||
try { content(block); } catch (error) {
|
||||
const expected = block?._meta?.["codex/encryptedContent"]
|
||||
? "content cannot emit encrypted MCP text"
|
||||
: "content expects one MCP ContentBlock";
|
||||
if (error === expected) continue;
|
||||
throw error;
|
||||
}
|
||||
throw new Error("expected content to reject the block");
|
||||
}
|
||||
"#,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
code_mode_host_duration: None,
|
||||
cell_id: cell_id("1"),
|
||||
content_items: Vec::new(),
|
||||
result_buffered_for: Duration::ZERO,
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_image_helper_appends_image_and_output_hint() {
|
||||
let service = InProcessCodeModeSession::new();
|
||||
|
||||
Reference in New Issue
Block a user