mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
Classify MCP auth and approval outcomes in analytics (#45716)
## What changed - Record `auth_or_link` for Codex app calls with trusted connector authentication failure metadata, capturing it before result callbacks or model-facing rewrites. Propagate the classification to MCP tool-call and app-usage analytics. - Record `approval` when a dispatched approval request is denied, times out, or aborts, covering both MCP elicitation and legacy user-input approvals. ## Testing Add unit and turn-level coverage for authentication failures, metadata removal by callbacks, successful and rejected approvals, and closed approval response channels. Verify event counts, classification, and exclusion of sensitive test values and authentication URLs from analytics payloads. GitOrigin-RevId: 493df2a2923f346a0f644f1f26ab8b6b9b8888e0
This commit is contained in:
committed by
copyberry
parent
2fdcdeaf0e
commit
7784318b5f
@@ -23,7 +23,9 @@ use crate::tools::sandboxing::ApprovalAction;
|
||||
use crate::tools::sandboxing::ToolError;
|
||||
use crate::turn_metadata::ExecutionMetadata;
|
||||
use codex_analytics::AppInvocation;
|
||||
use codex_analytics::ElicitationType;
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_analytics::McpToolCallElicitation;
|
||||
use codex_analytics::build_track_events_context;
|
||||
use codex_api::HostedFileUploadContext;
|
||||
use codex_config::ConfigLayerSource;
|
||||
@@ -41,6 +43,7 @@ use codex_mcp::SandboxState;
|
||||
use codex_mcp::ToolInfo;
|
||||
use codex_mcp::auth_elicitation_completed_result;
|
||||
use codex_mcp::build_auth_elicitation_plan;
|
||||
use codex_mcp::is_connector_auth_failure_from_tool_result;
|
||||
use codex_mcp::mcp_permission_prompt_is_auto_approved;
|
||||
use codex_protocol::approvals::ElicitationRequest;
|
||||
use codex_protocol::items::McpToolCallError;
|
||||
@@ -450,6 +453,7 @@ async fn handle_approved_mcp_tool_call(
|
||||
let mut tool_input = arguments_value
|
||||
.clone()
|
||||
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new()));
|
||||
let mut elicitation_type = None;
|
||||
let result = async {
|
||||
let result = async {
|
||||
let mut result = prepared_call
|
||||
@@ -532,6 +536,8 @@ async fn handle_approved_mcp_tool_call(
|
||||
})
|
||||
.await
|
||||
.map_err(|error| format!("tool call error: {error:?}"))?;
|
||||
// Capture trusted server metadata before result callbacks or model-facing rewrites.
|
||||
elicitation_type = mcp_tool_call_auth_elicitation_type(&server, connector_id, &result);
|
||||
let mcp_tool = McpToolContext::from_prepared_call(
|
||||
&prepared_call,
|
||||
turn_context.config.mcp_servers.get().get(&server),
|
||||
@@ -581,6 +587,9 @@ async fn handle_approved_mcp_tool_call(
|
||||
tracing::warn!("MCP tool call error: {error:?}");
|
||||
}
|
||||
let duration = start.elapsed();
|
||||
if let Some(elicitation_type) = elicitation_type {
|
||||
track_mcp_tool_call_elicitation(sess, turn_context, call_id, elicitation_type);
|
||||
}
|
||||
notify_mcp_tool_call_completed(
|
||||
sess,
|
||||
turn_context,
|
||||
@@ -591,7 +600,7 @@ async fn handle_approved_mcp_tool_call(
|
||||
truncate_mcp_tool_result_for_event(&result),
|
||||
)
|
||||
.await;
|
||||
maybe_track_codex_app_used(sess, step_context, &server, &metadata).await;
|
||||
maybe_track_codex_app_used(sess, step_context, &server, &metadata, elicitation_type).await;
|
||||
|
||||
let outcome = mcp_call_metric_outcome(&result);
|
||||
emit_mcp_call_metrics(
|
||||
@@ -1059,6 +1068,7 @@ async fn maybe_track_codex_app_used(
|
||||
step_context: &StepContext,
|
||||
server: &str,
|
||||
metadata: &McpToolApprovalMetadata,
|
||||
elicitation_type: Option<ElicitationType>,
|
||||
) {
|
||||
if server != CODEX_APPS_MCP_SERVER_NAME {
|
||||
return;
|
||||
@@ -1090,10 +1100,36 @@ async fn maybe_track_codex_app_used(
|
||||
app_name,
|
||||
invocation_type: Some(invocation_type),
|
||||
},
|
||||
/*elicitation_type*/ None,
|
||||
elicitation_type,
|
||||
);
|
||||
}
|
||||
|
||||
fn track_mcp_tool_call_elicitation(
|
||||
sess: &Session,
|
||||
turn_context: &TurnContext,
|
||||
call_id: &str,
|
||||
elicitation_type: ElicitationType,
|
||||
) {
|
||||
sess.services
|
||||
.analytics_events_client
|
||||
.track_mcp_tool_call_elicitation(McpToolCallElicitation {
|
||||
thread_id: sess.thread_id.to_string(),
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
item_id: call_id.to_string(),
|
||||
elicitation_type,
|
||||
});
|
||||
}
|
||||
|
||||
fn mcp_tool_call_auth_elicitation_type(
|
||||
server: &str,
|
||||
connector_id: Option<&str>,
|
||||
result: &CallToolResult,
|
||||
) -> Option<ElicitationType> {
|
||||
(server == CODEX_APPS_MCP_SERVER_NAME
|
||||
&& is_connector_auth_failure_from_tool_result(result, connector_id))
|
||||
.then_some(ElicitationType::AuthOrLink)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct McpToolApprovalPolicy {
|
||||
mode: AppToolApproval,
|
||||
@@ -1583,7 +1619,7 @@ pub(crate) async fn request_mcp_tool_user_approval(
|
||||
.as_ref()
|
||||
.map(|rendered_template| rendered_template.question.as_str()),
|
||||
);
|
||||
if tool_call_mcp_elicitation_enabled {
|
||||
let (request_dispatched, decision) = if tool_call_mcp_elicitation_enabled {
|
||||
let link_id = sess
|
||||
.mcp_tool_approval_metadata(server, id)
|
||||
.and_then(|(_, metadata)| metadata.link_id);
|
||||
@@ -1617,27 +1653,40 @@ pub(crate) async fn request_mcp_tool_user_approval(
|
||||
.map(|rendered_template| rendered_template.elicitation_message.as_str()),
|
||||
prompt_options,
|
||||
});
|
||||
let decision = parse_mcp_tool_approval_elicitation_response(
|
||||
sess.request_mcp_server_elicitation(turn_context, server.clone(), request_id, request)
|
||||
.await
|
||||
.response,
|
||||
&question_id,
|
||||
);
|
||||
return normalize_approval_decision_for_mode(decision, *approval_mode);
|
||||
}
|
||||
|
||||
let args = RequestUserInputArgs {
|
||||
questions: vec![question],
|
||||
is_blocking: true,
|
||||
auto_resolution_ms: None,
|
||||
let outcome = sess
|
||||
.request_mcp_server_elicitation(turn_context, server.clone(), request_id, request)
|
||||
.await;
|
||||
(
|
||||
outcome.sent,
|
||||
parse_mcp_tool_approval_elicitation_response(outcome.response, &question_id),
|
||||
)
|
||||
} else {
|
||||
let args = RequestUserInputArgs {
|
||||
questions: vec![question],
|
||||
is_blocking: true,
|
||||
auto_resolution_ms: None,
|
||||
};
|
||||
let response = sess
|
||||
.request_user_input(turn_context, call_id.to_string(), args)
|
||||
.await;
|
||||
(
|
||||
true,
|
||||
parse_mcp_tool_approval_response(
|
||||
response.map(|accepted| accepted.response),
|
||||
&question_id,
|
||||
),
|
||||
)
|
||||
};
|
||||
let response = sess
|
||||
.request_user_input(turn_context, call_id.to_string(), args)
|
||||
.await;
|
||||
normalize_approval_decision_for_mode(
|
||||
parse_mcp_tool_approval_response(response.map(|accepted| accepted.response), &question_id),
|
||||
*approval_mode,
|
||||
)
|
||||
let decision = normalize_approval_decision_for_mode(decision, *approval_mode);
|
||||
if request_dispatched
|
||||
&& matches!(
|
||||
decision,
|
||||
ReviewDecision::Denied { .. } | ReviewDecision::TimedOut | ReviewDecision::Abort
|
||||
)
|
||||
{
|
||||
track_mcp_tool_call_elicitation(sess, turn_context, call_id, ElicitationType::Approval);
|
||||
}
|
||||
decision
|
||||
}
|
||||
|
||||
fn session_mcp_tool_approval_key(
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::state::ActiveTurn;
|
||||
use crate::test_support::models_manager_with_provider;
|
||||
use crate::tools::hook_names::HookToolName;
|
||||
use crate::turn_metadata::ExecutionMetadata;
|
||||
use codex_app_server_protocol as app_server_protocol;
|
||||
use codex_config::CONFIG_TOML_FILE;
|
||||
use codex_config::config_toml::ConfigToml;
|
||||
use codex_config::types::AppConfig;
|
||||
@@ -1615,6 +1616,28 @@ fn codex_apps_auth_failure_metadata() -> McpToolApprovalMetadata {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_apps_auth_classification_requires_the_trusted_server_and_connector() {
|
||||
let result = codex_apps_auth_failure_result();
|
||||
|
||||
assert_eq!(
|
||||
mcp_tool_call_auth_elicitation_type(
|
||||
"untrusted_mcp_server",
|
||||
Some("connector_calendar"),
|
||||
&result,
|
||||
),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
mcp_tool_call_auth_elicitation_type(
|
||||
CODEX_APPS_MCP_SERVER_NAME,
|
||||
Some("connector_drive"),
|
||||
&result,
|
||||
),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_apps_auth_elicitation_feature_disabled_returns_original_result() {
|
||||
let (session, mut turn_context, rx_event) = make_session_and_context_with_rx().await;
|
||||
@@ -2153,6 +2176,147 @@ fn accepted_elicitation_without_content_defaults_to_accept() {
|
||||
assert_eq!(response, ReviewDecision::Approved);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dispatched_mcp_approval_with_closed_response_is_classified_as_approval() {
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
let server = wiremock::MockServer::start().await;
|
||||
wiremock::Mock::given(method("POST"))
|
||||
.and(path("/codex/analytics-events/events"))
|
||||
.respond_with(wiremock::ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let (mut session, turn_context, rx_event) = make_session_and_context_with_rx().await;
|
||||
let client = codex_analytics::AnalyticsEventsClient::new(
|
||||
crate::test_support::auth_manager_from_auth(
|
||||
codex_login::CodexAuth::create_dummy_chatgpt_auth_for_testing(),
|
||||
),
|
||||
server.uri(),
|
||||
/*analytics_enabled*/ Some(true),
|
||||
);
|
||||
Arc::get_mut(&mut session)
|
||||
.expect("session should be uniquely owned")
|
||||
.services
|
||||
.analytics_events_client = client.clone();
|
||||
*session.active_turn.lock().await = Some(ActiveTurn::default());
|
||||
|
||||
let call_id = "modern-missing-response";
|
||||
let thread_id = session.thread_id.to_string();
|
||||
let turn_id = turn_context.sub_id.clone();
|
||||
client.track_initialize(
|
||||
/*connection_id*/ 1,
|
||||
app_server_protocol::InitializeParams::default(),
|
||||
"test-client".to_string(),
|
||||
codex_analytics::AppServerRpcTransport::Stdio,
|
||||
);
|
||||
let response = serde_json::from_value(serde_json::json!({
|
||||
"thread": {
|
||||
"id": thread_id, "sessionId": thread_id, "preview": "", "ephemeral": false,
|
||||
"modelProvider": "openai", "createdAt": 1, "updatedAt": 1,
|
||||
"status": {"type": "idle"}, "cwd": &turn_context.config.cwd,
|
||||
"cliVersion": "0.0.0", "source": "exec", "turns": [],
|
||||
},
|
||||
"model": "test-model", "modelProvider": "openai", "cwd": &turn_context.config.cwd,
|
||||
"approvalPolicy": "on-request", "approvalsReviewer": "user",
|
||||
"sandbox": {"type": "dangerFullAccess"},
|
||||
}))
|
||||
.expect("thread-start response should deserialize");
|
||||
client.track_response(
|
||||
/*connection_id*/ 1,
|
||||
app_server_protocol::RequestId::Integer(1),
|
||||
&app_server_protocol::ClientResponsePayload::ThreadStart(response),
|
||||
);
|
||||
client.track_notification(&app_server_protocol::ServerNotification::TurnStarted(
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"threadId": thread_id,
|
||||
"turn": {"id": turn_id, "items": [], "status": "inProgress"},
|
||||
}))
|
||||
.expect("turn-started notification should deserialize"),
|
||||
));
|
||||
let mut item = serde_json::json!({
|
||||
"type": "mcpToolCall", "id": call_id, "server": "calendar", "tool": "send",
|
||||
"status": "inProgress", "arguments": {},
|
||||
"appContext": {"connectorId": "calendar"},
|
||||
});
|
||||
client.track_notification(&app_server_protocol::ServerNotification::ItemStarted(
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"threadId": thread_id, "turnId": turn_id, "startedAtMs": 1, "item": item,
|
||||
}))
|
||||
.expect("item-started notification should deserialize"),
|
||||
));
|
||||
|
||||
let action = ApprovalAction::McpToolCall {
|
||||
id: call_id.to_string(),
|
||||
server: "calendar".to_string(),
|
||||
tool_name: "send".to_string(),
|
||||
arguments: None,
|
||||
connector_id: Some("calendar".to_string()),
|
||||
connector_name: None,
|
||||
connector_description: None,
|
||||
connected_account_email: None,
|
||||
tool_title: None,
|
||||
tool_description: None,
|
||||
annotations: None,
|
||||
hook_tool_name: HookToolName::new("mcp__calendar__send"),
|
||||
approval_policy: AskForApproval::OnRequest,
|
||||
reviewer: ApprovalsReviewer::User,
|
||||
approval_mode: AppToolApproval::Auto,
|
||||
allow_session_remember: true,
|
||||
allow_persistent_approval: true,
|
||||
};
|
||||
let approval = tokio::spawn({
|
||||
let session = Arc::clone(&session);
|
||||
let turn_context = Arc::clone(&turn_context);
|
||||
async move { request_mcp_tool_user_approval(&session, &turn_context, call_id, &action).await }
|
||||
});
|
||||
let event = tokio::time::timeout(std::time::Duration::from_secs(1), rx_event.recv())
|
||||
.await
|
||||
.expect("approval request should be dispatched")
|
||||
.expect("approval request should be received");
|
||||
assert!(matches!(event.msg, EventMsg::ElicitationRequest(_)));
|
||||
*session.active_turn.lock().await = None;
|
||||
assert_eq!(
|
||||
approval.await.expect("approval task should complete"),
|
||||
ReviewDecision::Abort
|
||||
);
|
||||
|
||||
item["status"] = serde_json::json!("failed");
|
||||
client.track_notification(&app_server_protocol::ServerNotification::ItemCompleted(
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"threadId": thread_id, "turnId": turn_id, "completedAtMs": 2, "item": item,
|
||||
}))
|
||||
.expect("item-completed notification should deserialize"),
|
||||
));
|
||||
client.flush().await;
|
||||
let events = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("analytics requests should be recorded")
|
||||
.into_iter()
|
||||
.flat_map(|request| {
|
||||
serde_json::from_slice::<serde_json::Value>(&request.body)
|
||||
.expect("analytics request should deserialize")["events"]
|
||||
.as_array()
|
||||
.expect("analytics events should be an array")
|
||||
.clone()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mcp_events = events
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
event["event_type"] == "codex_mcp_tool_call_event"
|
||||
&& event["event_params"]["item_id"] == call_id
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(mcp_events.len(), 1);
|
||||
assert_eq!(
|
||||
mcp_events[0]["event_params"]["elicitation_type"],
|
||||
"approval"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persist_codex_app_tool_approval_writes_tool_override() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
|
||||
Reference in New Issue
Block a user