openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-05 20:10:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable app tool result metadata with analytics controls (#46010)

Browse Source 
## What changed

Capture result metadata for host-owned app calls when analytics and executed tool call metadata recording are enabled. Respect both `analytics.enabled = false` and host-disabled analytics, using the prepared call's ownership to determine eligibility.

Disable incremental WebSocket request reuse when raw result metadata or its call binding changes, so late results attached to an already-sent output are included in the next request. Continue allowing reuse when only other metadata changes or result metadata is filtered out for the endpoint.

## Testing

Extend coverage for direct and code-mode calls, analytics settings and host overrides, runtime recording changes, and prepared call ownership. Add metadata comparison and WebSocket reuse tests, and verify that raw response item notifications continue to omit result metadata.

GitOrigin-RevId: 411843fbff3749ee96fd6f0ebc591362942a4555
This commit is contained in:
ningyi-oai
2026-09-16 19:32:16 +00:00
committed by copyberry
parent 7f0ab95827
commit 3a589370a4
7 changed files with 276 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server/tests/suite/v2/turn_start.rs
+15 -7
View File
@@ -358,8 +358,13 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
Ok(())
}
#[test_case(None; "analytics_unset")]
#[test_case(Some(true); "analytics_enabled")]
#[test_case(Some(false); "analytics_disabled")]
#[tokio::test]
async fn tool_call_metadata_stays_out_of_raw_response_item_notifications() -> Result<()> {
async fn tool_call_metadata_stays_out_of_raw_response_item_notifications(
analytics_enabled: Option<bool>,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let arguments = json!({"query": "redaction"});
@@ -392,7 +397,8 @@ async fn tool_call_metadata_stays_out_of_raw_response_item_notifications() -> Re
"params": {"name": "calendar_list_events", "arguments": arguments},
})))
.respond_with(move |request: &Request| {
let request: Value = serde_json::from_slice(&request.body).unwrap();
let request: Value =
serde_json::from_slice(&request.body).expect("valid MCP tool call JSON");
ResponseTemplate::new(200).set_body_json(json!({
"jsonrpc": "2.0",
"id": request["id"],
@@ -404,7 +410,7 @@ async fn tool_call_metadata_stays_out_of_raw_response_item_notifications() -> Re
.mount(&server)
.await;
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri())
let mut config = MockResponsesConfig::new(&server.uri())
.with_provider_name("OpenAI")
.with_provider_config("supports_websockets = false")
.with_root_config(&format!(
@@ -412,8 +418,11 @@ async fn tool_call_metadata_stays_out_of_raw_response_item_notifications() -> Re
apps.chatgpt_base_url
))
.enable_feature(Feature::Apps)
.enable_feature(Feature::ExecutedToolCallMetadata)
.write(codex_home.path())?;
.enable_feature(Feature::ExecutedToolCallMetadata);
if let Some(enabled) = analytics_enabled {
config = config.with_extra_config(&format!("[analytics]\nenabled = {enabled}"));
}
config.write(codex_home.path())?;
write_chatgpt_auth(
codex_home.path(),
ChatGptAuthFixture::new("chatgpt-test-token")
@@ -530,11 +539,10 @@ async fn tool_call_metadata_stays_out_of_raw_response_item_notifications() -> Re
.context("persisted MCP output")?;
assert_eq!(captured["output"], raw_output["output"]);
// The custom inference endpoint omits raw metadata, so verify capture in the rollout.
let expected_metadata: Option<&Value> = None;
assert_eq!(
captured["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]
.get("tool_result_metadata"),
expected_metadata,
(analytics_enabled != Some(false)).then_some(&result_metadata),
);
Ok(())
}
codex-rs/core/src/client.rs
+6 -1
View File
@@ -314,7 +314,7 @@ struct WebsocketSession {
}
// This is intentionally not a `PartialEq` implementation: request equality includes `input` and
// `client_metadata`, while websocket reuse compares the input separately and ignores metadata.
// `client_metadata`, while websocket reuse compares input and late tool-result metadata separately.
// Access programs are authorized per response, including continuations, without replaying input.
// Keep the destructuring exhaustive so new request fields require an explicit reuse decision.
fn responses_request_properties_match(
@@ -382,6 +382,11 @@ fn response_items_equal_ignoring_internal_metadata(
return true;
}
// Late results update an already-sent output. A delta cannot carry that update.
if !previous.has_same_tool_result_metadata(current) {
return false;
}
let mut previous = previous.clone();
previous.clear_internal_chat_message_metadata_passthrough();
let mut current = current.clone();
codex-rs/core/src/client_tests.rs
+115
View File
@@ -541,6 +541,121 @@ fn responses_request_limits_raw_tool_metadata_to_resolved_first_party_https_endp
Ok(())
}
#[test]
fn websocket_incremental_reuse_tracks_raw_result_metadata() -> anyhow::Result<()> {
let provider =
ModelProviderInfo::create_openai_provider(Some("https://api.openai.com/v1".to_string()));
let mut api_provider = provider.to_api_provider(/*auth_mode*/ None)?;
let mut client = test_model_client(SessionSource::Cli);
Arc::get_mut(&mut client.state)
.expect("test client should have unique session state")
.provider = create_model_provider(provider, /*auth_manager*/ None);
let responses_metadata = test_responses_metadata_for_client(
&client,
/*turn_id*/ None,
format!("{}:0", client.state.thread_id),
/*parent_thread_id*/ None,
TestCodexResponsesRequestKind::Turn,
);
for (scenario, base_url, previous_metadata, current_metadata, expect_incremental) in [
(
"late_result",
"https://api.openai.com/v1",
None,
Some("first"),
false,
),
(
"unchanged_result",
"https://api.openai.com/v1",
Some("first"),
Some("first"),
true,
),
(
"changed_result",
"https://api.openai.com/v1",
Some("first"),
Some("second"),
false,
),
(
"ordinary_metadata_only",
"https://api.openai.com/v1",
None,
None,
true,
),
(
"filtered_result",
"https://proxy.example.com/v1",
None,
Some("first"),
true,
),
] {
let [mut previous_output, mut current_output] =
[previous_metadata, current_metadata].map(|metadata| {
let mut call =
ExecutedToolCall::new("apps_tool".to_string(), json!({ "query": "same" }));
if let Some(id) = metadata {
call.set_tool_result_metadata(ToolResultMetadata::new(&json!({ "id": id })));
}
let mut output = ResponseItem::from(ResponseInputItem::CustomToolCallOutput {
call_id: "exec-call".to_string(),
name: None,
output: FunctionCallOutputPayload::from_text(
"Script running with cell ID cell".to_string(),
),
});
output.append_executed_tool_calls(vec![call]);
output.set_tool_call_cell_id("exec-call");
output
});
previous_output.set_turn_id_if_missing("previous-turn");
current_output.set_turn_id_if_missing("current-turn");
let mut previous = client.build_responses_request(
&Prompt {
input: vec![previous_output],
..Default::default()
},
&test_model_info(),
/*effort*/ None,
codex_protocol::config_types::ReasoningSummary::None,
/*service_tier*/ None,
&responses_metadata,
)?;
let follow_up = ResponseItem::from(ResponseInputItem::FunctionCallOutput {
call_id: "wait-call".to_string(),
output: FunctionCallOutputPayload::from_text("done".to_string()),
});
let mut current = previous.clone();
current.input = vec![current_output, follow_up.clone()];
api_provider.base_url = base_url.to_string();
ModelClient::filter_tool_result_metadata(&mut previous.input, &api_provider);
ModelClient::filter_tool_result_metadata(&mut current.input, &api_provider);
let mut session = client.new_session();
session.websocket_session.last_request = Some(previous);
let (sender, receiver) = tokio::sync::oneshot::channel();
sender
.send(super::LastResponse {
response_id: "previous-response".to_string(),
items_added: Vec::new(),
})
.unwrap();
session.websocket_session.last_response_rx = Some(receiver);
let (incremental, from_warmup) = session.prepare_websocket_request(&current);
assert_eq!(
incremental,
expect_incremental.then_some(("previous-response".to_string(), vec![follow_up])),
"{scenario}",
);
assert!(!from_warmup);
}
Ok(())
}
#[tokio::test]
async fn responses_http_omits_raw_tool_metadata_for_openai_named_custom_endpoint()
-> anyhow::Result<()> {
codex-rs/core/src/tools/handlers/mcp.rs
+8 -2
View File
@@ -185,8 +185,14 @@ impl McpHandler {
)
.await;
// Use the executed call's binding; a later catalog refresh must not change eligibility.
// Only the new metadata is internal; tool execution and call accounting are not.
let result_metadata_capture_allowed = false;
let result_metadata_capture_allowed = invocation
.session
.services
.analytics_events_client
.is_enabled()
&& prepared_mcp_call
.as_ref()
.is_some_and(codex_mcp::PreparedMcpCall::is_host_owned_apps);
let mcp_tool = prepared_mcp_call.as_ref().map(|call| {
McpToolContext::from_prepared_call(
call,
codex-rs/core/tests/suite/code_mode.rs
+46 -18
View File
@@ -3,6 +3,7 @@
use anyhow::Result;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_analytics::AnalyticsEventsClient;
use codex_config::types::McpServerConfig;
use codex_config::types::McpServerTransportConfig;
use codex_core::StartThreadOptions;
@@ -1751,17 +1752,31 @@ fn assert_result_metadata_call(
assert!(metadata.get("tool_result_metadata").is_none());
}
enum ResultMetadataAnalytics {
Config(Option<bool>),
HostDisabled(Option<bool>),
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(true, true, true, false, "employee@openai.com", ToolMode::CodeModeOnly; "copies_full_metadata_without_rules")]
#[test_case(true, true, true, true, "employee@openai.com", ToolMode::CodeModeOnly; "accepted_error_keeps_metadata")]
#[test_case(true, true, false, false, "employee@openai.com", ToolMode::CodeModeOnly; "missing_metadata_stays_absent")]
#[test_case(false, true, true, false, "employee@openai.com", ToolMode::CodeModeOnly; "feature_off_does_not_record")]
#[test_case(true, false, true, false, "employee@openai.com", ToolMode::CodeModeOnly; "extension_owned_apps_do_not_record")]
#[test_case(true, true, true, false, "employee@example.com", ToolMode::CodeModeOnly; "external_user_keeps_calls_without_metadata")]
#[test_case(true, true, true, false, "employee@openai.com.attacker.invalid", ToolMode::CodeModeOnly; "lookalike_domain_does_not_record")]
#[test_case(true, true, true, false, "employee@openai.com", ToolMode::Direct; "direct_keeps_metadata")]
#[test_case(true, true, true, true, "employee@openai.com", ToolMode::Direct; "direct_accepted_error_keeps_metadata")]
#[test_case(false, true, true, false, "employee@openai.com", ToolMode::Direct; "direct_feature_off_does_not_record")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "copies_full_metadata_without_rules")]
#[test_case(true, true, true, true, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "accepted_error_keeps_metadata")]
#[test_case(true, true, false, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "missing_metadata_stays_absent")]
#[test_case(false, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "feature_off_does_not_record")]
#[test_case(true, false, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "extension_owned_apps_do_not_record")]
#[test_case(true, true, true, false, "employee@openai.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(None); "employee_keeps_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(None); "direct_keeps_metadata")]
#[test_case(true, true, true, true, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(None); "direct_accepted_error_keeps_metadata")]
#[test_case(true, true, false, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(None); "direct_missing_metadata_stays_absent")]
#[test_case(false, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(None); "direct_feature_off_does_not_record")]
#[test_case(true, false, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(None); "direct_extension_owned_apps_do_not_record")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(Some(true)); "analytics_enabled_keeps_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::Config(Some(false)); "analytics_disabled_omits_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(Some(true)); "direct_analytics_enabled_keeps_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::Config(Some(false)); "direct_analytics_disabled_omits_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::HostDisabled(None); "host_analytics_disabled_with_config_unset_omits_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::CodeModeOnly, ResultMetadataAnalytics::HostDisabled(Some(true)); "host_analytics_disabled_with_config_enabled_omits_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::HostDisabled(None); "direct_host_analytics_disabled_with_config_unset_omits_metadata")]
#[test_case(true, true, true, false, "user@example.com", ToolMode::Direct, ResultMetadataAnalytics::HostDisabled(Some(true)); "direct_host_analytics_disabled_with_config_enabled_omits_metadata")]
async fn result_metadata_follows_call_binding(
metadata_enabled: bool,
host_owned: bool,
@@ -1769,8 +1784,14 @@ async fn result_metadata_follows_call_binding(
is_error: bool,
account_email: &str,
tool_mode: ToolMode,
analytics: ResultMetadataAnalytics,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let (analytics_enabled, host_disables_analytics) = match analytics {
ResultMetadataAnalytics::Config(enabled) => (enabled, false),
ResultMetadataAnalytics::HostDisabled(enabled) => (enabled, true),
};
let effective_analytics_enabled = analytics_enabled != Some(false) && !host_disables_analytics;
let direct = matches!(tool_mode, ToolMode::Direct);
let server = responses::start_mock_server().await;
let result_metadata = has_metadata.then(|| {
@@ -1791,6 +1812,7 @@ async fn result_metadata_follows_call_binding(
let mut builder =
result_metadata_apps_builder(apps_server.chatgpt_base_url.clone(), account_email)
.with_config(move |config| {
config.analytics_enabled = analytics_enabled;
if direct {
config.features.disable(Feature::CodeMode).unwrap();
config.features.disable(Feature::CodeModeOnly).unwrap();
@@ -1802,6 +1824,9 @@ async fn result_metadata_follows_call_binding(
.unwrap();
}
});
if host_disables_analytics {
builder = builder.with_analytics_events_client(AnalyticsEventsClient::disabled());
}
if !host_owned {
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
extensions.mcp_server_contributor(Arc::new(ResultMetadataTestControl {
@@ -1844,6 +1869,7 @@ async fn result_metadata_follows_call_binding(
);
run_code_mode_turn_with_builder(&server, "Search a connected app", &code, builder).await?
};
assert_eq!(test.codex.analytics_enabled(), effective_analytics_enabled);
let request = follow_up.single_request();
assert_eq!(recorded_apps_tool_calls(&server).await.len(), 1);
let output = if direct {
@@ -1899,8 +1925,9 @@ async fn result_metadata_follows_call_binding(
.iter()
.find(|item| item["type"] == output["type"] && item["call_id"] == "call-1")
.expect("captured tool output");
// The public build never captures result metadata, even with employee test credentials.
let expected_metadata = None;
let expected_metadata = (host_owned && effective_analytics_enabled)
.then_some(result_metadata)
.flatten();
assert_result_metadata_call(captured_output, &arguments, expected_metadata);
assert_eq!(
output["internal_chat_message_metadata_passthrough"]["tool_calls_complete"],
@@ -1928,13 +1955,14 @@ async fn code_mode_result_metadata_follows_runtime_recording_enablement() -> Res
)
.await?;
let mut builder =
result_metadata_apps_builder(apps_server.chatgpt_base_url, "employee@openai.com")
.with_config(|config| {
result_metadata_apps_builder(apps_server.chatgpt_base_url, "user@example.com").with_config(
|config| {
config
.features
.disable(Feature::ExecutedToolCallMetadata)
.unwrap();
});
},
);
let test = builder.build_with_auto_env(&server).await?;
let arguments = serde_json::json!({ "search": "launch plan" });
let code = format!(
@@ -2008,7 +2036,7 @@ async fn code_mode_result_metadata_follows_runtime_recording_enablement() -> Res
.iter()
.find(|item| item["type"] == "custom_tool_call_output" && item["call_id"] == "call-on")
.expect("captured exec output after runtime enablement");
let expected_metadata = None;
let expected_metadata = Some(result_metadata);
assert_result_metadata_call(captured_output, &arguments, expected_metadata);
Ok(())
}
@@ -2048,7 +2076,7 @@ async fn code_mode_result_metadata_keeps_prepared_call_binding_across_runtime_re
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
extensions.mcp_server_contributor(control.clone());
extensions.tool_lifecycle_contributor(control.clone());
let builder = result_metadata_apps_builder(apps_server.chatgpt_base_url, "employee@openai.com")
let builder = result_metadata_apps_builder(apps_server.chatgpt_base_url, "user@example.com")
.with_extensions(Arc::new(extensions.build()));
let arguments = serde_json::json!({
"query": "launch plan",
@@ -2142,7 +2170,7 @@ async fn code_mode_result_metadata_keeps_prepared_call_binding_across_runtime_re
let captured = codex_core::test_support::history_with_tool_call_metadata(&test.codex).await;
let captured = serde_json::to_value(captured)?;
// A's accepted result must update the output that first reported it, not the final wait.
let expected_metadata = None;
let expected_metadata = Some(original_metadata);
for (call_id, call_type, expected_metadata) in [
(
original_output["call_id"].as_str().unwrap(),
codex-rs/protocol/src/models/executed_tool_calls.rs
+25
View File
@@ -485,6 +485,31 @@ impl ResponseItem {
self.internal_chat_message_metadata_passthrough()
}
/// Compares raw result metadata and its call bindings, ignoring other internal metadata.
pub fn has_same_tool_result_metadata(&self, other: &Self) -> bool {
fn result_metadata(
item: &ResponseItem,
) -> impl Iterator<Item = (usize, &str, &ExecutedToolCallArguments, &ToolResultMetadata)>
{
item.executed_tool_call_metadata()
.and_then(|metadata| metadata.executed_tool_calls.as_ref())
.into_iter()
.flatten()
.enumerate()
.filter(|(_, call)| call.tool_result_metadata.is_some())
.map(|(index, call)| {
(
index,
call.name.as_str(),
call.arguments(),
&call.tool_result_metadata,
)
})
}
result_metadata(self).eq(result_metadata(other))
}
/// Omits raw tool results without changing existing call, source or completion metadata.
pub fn clear_tool_result_metadata(&mut self) {
if let Some(metadata) = self
codex-rs/protocol/src/models/executed_tool_calls_tests.rs
+61
View File
@@ -26,6 +26,67 @@ fn first_executed_tool_call(item: &mut ResponseItem) -> Option<&mut ExecutedTool
.and_then(|calls| calls.first_mut())
}
#[test]
fn result_metadata_comparison_tracks_raw_values_and_call_bindings() {
let empty = output("output");
let mut without_metadata = empty.clone();
without_metadata.append_executed_tool_calls(vec![ExecutedToolCall::new(
"apps_tool".to_string(),
serde_json::json!({ "query": "same" }),
)]);
assert!(empty.has_same_tool_result_metadata(&without_metadata));
let mut with_metadata = without_metadata.clone();
first_executed_tool_call(&mut with_metadata)
.unwrap()
.set_tool_result_metadata(ToolResultMetadata::new(
&serde_json::json!({ "id": "first" }),
));
assert!(!without_metadata.has_same_tool_result_metadata(&with_metadata));
assert!(!with_metadata.has_same_tool_result_metadata(&without_metadata));
assert!(with_metadata.has_same_tool_result_metadata(&with_metadata.clone()));
let mut changed = with_metadata.clone();
first_executed_tool_call(&mut changed)
.unwrap()
.set_tool_result_metadata(ToolResultMetadata::new(
&serde_json::json!({ "id": "second" }),
));
assert!(!with_metadata.has_same_tool_result_metadata(&changed));
changed.clear_tool_result_metadata();
assert!(without_metadata.has_same_tool_result_metadata(&changed));
let mut ordinary_metadata = with_metadata.clone();
ordinary_metadata.set_turn_id_if_missing("other-turn");
ordinary_metadata.set_tool_call_cell_id("other-cell");
ordinary_metadata.mark_tool_calls_complete();
first_executed_tool_call(&mut ordinary_metadata)
.unwrap()
.set_tool_result_sources(ToolResultSources::new(vec![ToolResultSource {
r#type: "resource".to_string(),
id: "source".to_string(),
}]));
assert!(with_metadata.has_same_tool_result_metadata(&ordinary_metadata));
let mut different_call = with_metadata.clone();
first_executed_tool_call(&mut different_call).unwrap().name = "other_tool".to_string();
assert!(!with_metadata.has_same_tool_result_metadata(&different_call));
first_executed_tool_call(&mut different_call).unwrap().name = "apps_tool".to_string();
different_call
.internal_chat_message_metadata_passthrough_mut()
.unwrap()
.as_mut()
.unwrap()
.executed_tool_calls
.as_mut()
.unwrap()
.insert(
0,
ExecutedToolCall::new("other_tool".to_string(), serde_json::json!({})),
);
assert!(!with_metadata.has_same_tool_result_metadata(&different_call));
}
#[test]
fn executed_tool_call_prompt_budget_includes_metadata_fields() -> Result<()> {
let metadata_bytes = |items: &[ResponseItem]| -> Result<usize> {