Record attempted tool calls in response metadata (#36181)

## What changed

- Add the opt-in `executed_tool_call_metadata` feature to record model-attempted direct and code-mode tool calls.
- Attach recorded names and arguments to the matching tool output in the next Responses request, preserving the metadata across sampling retries.
- Bound pending calls and argument sizes, and emit truncation metadata when limits are exceeded.

## Testing

- Cover disabled-by-default behavior, namespaced and nested calls, blocked or failed attempts, retry handling, and recorder limits.

GitOrigin-RevId: 5ce917b73e797b0f6904d46477b9d2cf8ea71bf2
This commit is contained in:
ningyi-oai
2026-07-30 16:22:43 +00:00
committed by copyberry
parent 355d2a802a
commit ceb4bc72c4
16 changed files with 570 additions and 0 deletions
+15
View File
@@ -195,6 +195,7 @@ async fn run_code_mode_turn_with_model_and_config(
) -> Result<(TestCodex, ResponseMock)> {
let builder = test_codex().with_model(model).with_config(move |config| {
let _ = config.features.enable(Feature::CodeMode);
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
configure(config);
});
run_code_mode_turn_with_builder(server, prompt, code, builder).await
@@ -985,6 +986,13 @@ text(output.output);
"code_mode_only nested tool call failed unexpectedly: {output}"
);
assert_eq!(output, "code_mode_only_nested_tool_marker");
assert!(
request
.custom_tool_call_output("call-1")
.pointer("/internal_chat_message_metadata_passthrough/executed_tool_calls")
.is_none(),
"nested tool calls must not be recorded while executed-tool-call metadata is disabled",
);
Ok(())
}
@@ -1581,6 +1589,13 @@ try {
.await?;
let request = second_mock.single_request();
assert_eq!(
request.custom_tool_call_output("call-1")["internal_chat_message_metadata_passthrough"]["executed_tool_calls"],
serde_json::json!([
{"name": "exec_command", "arguments": {}},
]),
"failed nested tool attempts remain private request metadata",
);
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
assert_ne!(
success,
+9
View File
@@ -277,6 +277,7 @@ async fn pre_tool_use_blocks_mcp_tool_before_execution(
.expect("failed to write MCP pre tool use hook fixture");
})
.with_config(move |config| {
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
enable_hooks_and_rmcp_server(
config,
rmcp_test_server_bin,
@@ -294,6 +295,14 @@ async fn pre_tool_use_blocks_mcp_tool_before_execution(
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let output_item = requests[1].function_call_output(call_id);
assert_eq!(
output_item["internal_chat_message_metadata_passthrough"]["executed_tool_calls"],
json!([{
"name": format!("{mcp_namespace}__echo"),
"arguments": { "message": RMCP_ECHO_MESSAGE },
}]),
"a blocked MCP request must still retain the original model-attempted call",
);
let output = output_item
.get("output")
.and_then(Value::as_str)
+102
View File
@@ -175,6 +175,11 @@ async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> {
.unwrap_or_default();
let expected = format!("unsupported custom tool call: {tool_name}");
assert_eq!(output, expected);
assert!(
item.pointer("/internal_chat_message_metadata_passthrough/executed_tool_calls")
.is_none(),
"attempted-tool metadata must be disabled by default",
);
Ok(())
}
@@ -186,6 +191,9 @@ async fn namespaced_custom_tool_call_preserves_namespace_through_dispatch_and_re
let server = start_mock_server().await;
let mut builder = test_codex();
builder = builder.with_config(|config| {
let _ = config.features.enable(Feature::ExecutedToolCallMetadata);
});
let test = builder.build(&server).await?;
let call_id = "custom-namespaced";
@@ -248,10 +256,104 @@ async fn namespaced_custom_tool_call_preserves_namespace_through_dispatch_and_re
"output": format!("unsupported custom tool call: {namespace}{tool_name}"),
"internal_chat_message_metadata_passthrough": {
"turn_id": turn_id,
"executed_tool_calls": [{
"name": format!("{namespace}__{tool_name}"),
"arguments": input,
}],
},
}),
)
);
let escaped_call_id = "custom-namespaced-escaped";
let escaped_input = "\\".repeat(4_096);
mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-3"),
ev_custom_tool_call_with_namespace(
escaped_call_id,
namespace,
tool_name,
&escaped_input,
),
ev_completed("resp-3"),
]),
)
.await;
let escaped_mock = mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-2", "done"),
ev_completed("resp-4"),
]),
)
.await;
test.submit_turn_with_approval_and_permission_profile(
"invoke namespaced custom tool with escaped arguments",
AskForApproval::Never,
PermissionProfile::Disabled,
)
.await?;
assert_eq!(
escaped_mock
.single_request()
.custom_tool_call_output(escaped_call_id)["internal_chat_message_metadata_passthrough"]
["executed_tool_calls"],
json!([{
"name": format!("{namespace}__{tool_name}"),
"arguments": {
"_codex_executed_tool_call_truncated": {
"original_bytes": serde_json::to_vec(&escaped_input)?.len(),
"max_bytes": 8 * 1024,
},
},
}]),
);
let direct_exec_call_id = "custom-direct-exec";
mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-5"),
ev_custom_tool_call(
direct_exec_call_id,
codex_code_mode::PUBLIC_TOOL_NAME,
input,
),
ev_completed("resp-5"),
]),
)
.await;
let direct_exec_mock = mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-3", "done"),
ev_completed("resp-6"),
]),
)
.await;
test.submit_turn_with_approval_and_permission_profile(
"invoke direct custom exec outside code mode",
AskForApproval::Never,
PermissionProfile::Disabled,
)
.await?;
let direct_exec_output = direct_exec_mock
.single_request()
.custom_tool_call_output(direct_exec_call_id);
assert_eq!(
direct_exec_output["output"],
json!("unsupported custom tool call: exec"),
);
assert_eq!(
direct_exec_output["internal_chat_message_metadata_passthrough"]["executed_tool_calls"],
json!([{
"name": codex_code_mode::PUBLIC_TOOL_NAME,
"arguments": input,
}]),
);
Ok(())
}