From ca21fe65d18a9b6326759fb4263c65dfbb645aa5 Mon Sep 17 00:00:00 2001 From: cazhang-oai Date: Wed, 18 Nov 2026 08:05:56 +0000 Subject: [PATCH] Tag Guardian review counts with turn origin (#62225) ## Why Guardian review counts need a way to distinguish automation-triggered turns from other turns, including when a subagent requests the review. ## What changed Add a `turn_origin` tag to the `codex.guardian.review` counter based on the trigger recorded when the turn started. Recognized automation triggers map to `automation`, missing or empty triggers map to `unknown`, and all other values map to `other`. This keeps the set of metric labels bounded. ## Testing Cover trigger classification for both allow and deny verdicts and verify that a later user turn on the same thread is counted separately from an automation turn in both main and subagent threads. GitOrigin-RevId: 6a5808798df7d184203cea18e60a94a82d71f036 --- codex-rs/core/src/guardian/review_request.rs | 1 + .../guardian_automation_metrics_tests.rs | 126 ++++++++++++++++++ codex-rs/core/tests/suite/guardian_review.rs | 3 + codex-rs/ext/guardian-reviewer/src/metrics.rs | 4 + .../ext/guardian-reviewer/src/reporting.rs | 19 +++ .../guardian-reviewer/src/reporting_tests.rs | 112 ++++++++++++++++ .../ext/guardian-reviewer/src/review_tests.rs | 1 + 7 files changed, 266 insertions(+) create mode 100644 codex-rs/core/tests/suite/guardian_automation_metrics_tests.rs diff --git a/codex-rs/core/src/guardian/review_request.rs b/codex-rs/core/src/guardian/review_request.rs index 3e03eb042f..eeff43b739 100644 --- a/codex-rs/core/src/guardian/review_request.rs +++ b/codex-rs/core/src/guardian/review_request.rs @@ -178,6 +178,7 @@ impl ReviewHost for super::super::runtime::ReviewRuntime { codex_guardian_reviewer::ReviewReport::new(codex_guardian_reviewer::ReviewMetadata { thread_id: session.thread_id.to_string(), turn_id: assessment_turn_id, + turn_trigger: turn.turn_metadata_state.current_turn_trigger(), review_id: review_id.to_owned(), target_item_id, plugin_id, diff --git a/codex-rs/core/tests/suite/guardian_automation_metrics_tests.rs b/codex-rs/core/tests/suite/guardian_automation_metrics_tests.rs new file mode 100644 index 0000000000..9e1e01d96c --- /dev/null +++ b/codex-rs/core/tests/suite/guardian_automation_metrics_tests.rs @@ -0,0 +1,126 @@ +#![cfg(test)] +//! Checks review counter attribution from admitted main and subagent turns. + +use super::*; +use codex_core::TurnStartOptions; +use codex_otel::GUARDIAN_REVIEW_COUNT_METRIC; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use opentelemetry_sdk::metrics::data::ScopeMetrics; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_metrics_distinguish_automation_main_and_subagent_turns() -> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let test = test_codex() + .with_config(|config| { + config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); + config.approvals_reviewer = ApprovalsReviewer::User; + }) + .build_with_auto_env(&server) + .await?; + let metrics = MetricsClient::new( + MetricsConfig::in_memory("test", "test", "1", Default::default()).with_runtime_reader(), + )?; + let mut expected = BTreeMap::new(); + for source in [ + SessionSource::Custom("codex-cloud-agent".into()), + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: test.session_configured.thread_id, + depth: 1, + agent_path: None, + agent_nickname: None, + agent_role: None, + }), + ] { + let mut options = test.start_thread_options().await; + options.session_source = Some(source.clone()); + options.thread_extension_init.insert(metrics.clone()); + let thread = test.thread_manager.start_thread(options).await?.thread; + // A later user turn on the same session must not retain automation attribution. + for (trigger, origin) in [ + ("automation_cron_scheduled", "automation"), + ("composer", "other"), + ] { + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("parent-tool"), + ev_function_call( + "reviewed-command", + "exec_command", + r#"{"cmd":"true","sandbox_permissions":"require_escalated","justification":"Exercise review attribution."}"#, + ), + ev_completed("parent-tool"), + ]), + sse(vec![ + ev_response_created("guardian-verdict"), + ev_assistant_message( + "guardian", + r#"{"risk_level":"high","user_authorization":"low","outcome":"deny","rationale":"Test denial."}"#, + ), + ev_completed("guardian-verdict"), + ]), + sse(vec![ev_completed("parent-complete")]), + ], + ) + .await; + thread + .start_or_steer_turn( + TurnInputRequest::user_input(vec![UserInput::Text { + text: "Run the reviewed command.".into(), + text_elements: Vec::new(), + }]) + .on_start(TurnStartOptions { + turn_trigger: Some(trigger.into()), + ..Default::default() + }) + .with_thread_settings(ThreadSettingsOverrides { + approvals_reviewer: Some(ApprovalsReviewer::AutoReview), + ..Default::default() + }), + ) + .await?; + wait_for_event(&thread, |event| matches!(event, EventMsg::TurnComplete(_))).await; + assert_eq!(responses.requests().len(), 3); + expected.insert( + (source.to_string(), origin.to_string(), "deny".to_string()), + 1, + ); + } + thread.shutdown_and_wait().await?; + } + let snapshot = metrics.snapshot()?; + let metric = snapshot + .scope_metrics() + .flat_map(ScopeMetrics::metrics) + .find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC) + .context("Guardian review counter")?; + let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else { + panic!("expected review counter"); + }; + let observed = counter + .data_points() + .map(|point| { + let tags = point + .attributes() + .map(|tag| (tag.key.as_str(), tag.value.as_str().to_string())) + .collect::>(); + ( + ( + tags["session_source"].clone(), + tags["turn_origin"].clone(), + tags["outcome"].clone(), + ), + point.value(), + ) + }) + .collect::>(); + assert_eq!(observed, expected); + Ok(()) +} diff --git a/codex-rs/core/tests/suite/guardian_review.rs b/codex-rs/core/tests/suite/guardian_review.rs index 5714c93a02..8dd60ae531 100644 --- a/codex-rs/core/tests/suite/guardian_review.rs +++ b/codex-rs/core/tests/suite/guardian_review.rs @@ -1,5 +1,8 @@ #![cfg(not(target_os = "windows"))] +#[path = "guardian_automation_metrics_tests.rs"] +mod automation_metrics; + use anyhow::Context; use anyhow::Result; use chrono::DateTime; diff --git a/codex-rs/ext/guardian-reviewer/src/metrics.rs b/codex-rs/ext/guardian-reviewer/src/metrics.rs index 9bed7e42df..4893313eab 100644 --- a/codex-rs/ext/guardian-reviewer/src/metrics.rs +++ b/codex-rs/ext/guardian-reviewer/src/metrics.rs @@ -41,6 +41,7 @@ pub(crate) fn emit_guardian_review_metrics( result: &GuardianReviewAnalyticsResult, approval_request_source: GuardianApprovalRequestSource, reviewed_action: &GuardianReviewedAction, + turn_origin: &'static str, elapsed: Duration, ) { let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action); @@ -62,6 +63,7 @@ pub(crate) fn emit_guardian_review_metrics( let mut counter_tag_refs = tag_refs.clone(); counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str()))); counter_tag_refs.push(("context_changed", bool_tag(result.context_changed))); + counter_tag_refs.push(("turn_origin", turn_origin)); session_telemetry.counter( GUARDIAN_REVIEW_COUNT_METRIC, /*inc*/ 1, @@ -499,6 +501,7 @@ mod tests { &result, GuardianApprovalRequestSource::DelegatedSubagent, &reviewed_action, + "unknown", Duration::from_millis(456), ); @@ -529,6 +532,7 @@ mod tests { let mut expected_counter_tags = expected_base_tags.clone(); expected_counter_tags.extend(expected_mcp_tags); expected_counter_tags.insert("context_changed".to_string(), "false".to_string()); + expected_counter_tags.insert("turn_origin".to_string(), "unknown".to_string()); assert_eq!((attrs, value), (expected_counter_tags, 1)); let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = diff --git a/codex-rs/ext/guardian-reviewer/src/reporting.rs b/codex-rs/ext/guardian-reviewer/src/reporting.rs index ecca70f7c7..4629afe594 100644 --- a/codex-rs/ext/guardian-reviewer/src/reporting.rs +++ b/codex-rs/ext/guardian-reviewer/src/reporting.rs @@ -34,6 +34,8 @@ use crate::ReviewCompletion; pub struct ReviewMetadata { pub thread_id: String, pub turn_id: String, + /// Origin of the issuing turn, inherited by delegated turns at admission. + pub turn_trigger: Option, pub review_id: String, pub target_item_id: Option, pub plugin_id: Option, @@ -54,11 +56,26 @@ pub struct ReviewReport { tracking: GuardianReviewTrackContext, approval_request_source: GuardianApprovalRequestSource, reviewed_action: GuardianReviewedAction, + turn_origin: &'static str, } impl ReviewReport { pub fn new(metadata: ReviewMetadata) -> Self { let started_instant = Instant::now(); + // Keep client-supplied trigger strings out of metric labels. This describes + // the turn's origin, not later input steered into an already-running turn. + let turn_origin = match metadata.turn_trigger.as_deref() { + Some( + "automation" + | "mcp_event" + | "automation_cron_scheduled" + | "automation_cron_run_now" + | "automation_heartbeat_scheduled" + | "automation_heartbeat_run_now", + ) => "automation", + None | Some("") => "unknown", + Some(_) => "other", + }; let tracking = GuardianReviewTrackContext::new( metadata.thread_id, metadata.turn_id.clone(), @@ -93,6 +110,7 @@ impl ReviewReport { tracking, approval_request_source: metadata.approval_request_source, reviewed_action: metadata.reviewed_action, + turn_origin, } } @@ -134,6 +152,7 @@ impl ReviewReport { &result, self.approval_request_source, &self.reviewed_action, + self.turn_origin, elapsed, ); analytics.track_guardian_review(&self.tracking, result, completed_at_ms, elapsed); diff --git a/codex-rs/ext/guardian-reviewer/src/reporting_tests.rs b/codex-rs/ext/guardian-reviewer/src/reporting_tests.rs index f4c88ff694..1cfbb6812e 100644 --- a/codex-rs/ext/guardian-reviewer/src/reporting_tests.rs +++ b/codex-rs/ext/guardian-reviewer/src/reporting_tests.rs @@ -17,8 +17,119 @@ use opentelemetry_sdk::metrics::data::MetricData; use opentelemetry_sdk::metrics::data::ScopeMetrics; use pretty_assertions::assert_eq; use serde_json::json; +use std::collections::BTreeMap; use std::time::Duration; +#[test_case::test_case(Some("automation"), "automation"; "automation")] +#[test_case::test_case(Some("mcp_event"), "automation"; "webhook")] +#[test_case::test_case(Some("automation_cron_scheduled"), "automation"; "scheduled")] +#[test_case::test_case(Some("automation_cron_run_now"), "automation"; "run_now")] +#[test_case::test_case(Some("automation_heartbeat_scheduled"), "automation"; "heartbeat_scheduled")] +#[test_case::test_case(Some("automation_heartbeat_run_now"), "automation"; "heartbeat_run_now")] +#[test_case::test_case(Some("composer"), "other"; "user")] +#[test_case::test_case(Some("arbitrary client supplied value"), "other"; "bounded_labels")] +#[test_case::test_case(Some(""), "unknown"; "empty")] +#[test_case::test_case(None, "unknown"; "missing")] +fn review_report_bounds_turn_origin_on_both_verdicts( + turn_trigger: Option<&str>, + expected_origin: &str, +) { + use codex_analytics::GuardianReviewDecision; + use codex_analytics::GuardianReviewTerminalStatus; + use codex_protocol::protocol::GuardianAssessmentOutcome; + + let metrics = MetricsClient::new( + MetricsConfig::in_memory("test", "test", "1", InMemoryMetricExporter::default()) + .with_runtime_reader(), + ) + .unwrap(); + let telemetry = SessionTelemetry::new( + ThreadId::new(), + "test-model", + "test-model", + /*account_id*/ None, + /*account_email*/ None, + /*auth_mode*/ None, + "test".to_string(), + /*log_user_prompts*/ false, + "tty".to_string(), + SessionSource::Cli, + ) + .with_metrics_without_metadata_tags(metrics.clone()); + let report = ReviewReport::new(ReviewMetadata { + thread_id: "thread".into(), + turn_id: "turn".into(), + turn_trigger: turn_trigger.map(str::to_owned), + review_id: "review".into(), + target_item_id: None, + plugin_id: None, + script_path: None, + approval_request_source: GuardianApprovalRequestSource::MainTurn, + reviewed_action: GuardianReviewedAction::NativeToolCall {}, + action: None, + review_reason: GuardianReviewReason::Policy, + model_context: ModelInvocationContext { + model_slug: "test-model".into(), + reasoning_effort: None, + }, + timeout: None, + }); + for (decision, terminal_status, outcome) in [ + ( + GuardianReviewDecision::Approved, + GuardianReviewTerminalStatus::Approved, + GuardianAssessmentOutcome::Allow, + ), + ( + GuardianReviewDecision::Denied, + GuardianReviewTerminalStatus::Denied, + GuardianAssessmentOutcome::Deny, + ), + ] { + report.track( + &telemetry, + &AnalyticsEventsClient::disabled(), + GuardianReviewAnalyticsResult { + decision, + terminal_status, + outcome: Some(outcome), + ..GuardianReviewAnalyticsResult::without_session() + }, + /*completed_at_ms*/ 1_792_141_011_000, + Duration::from_millis(10), + ); + } + let snapshot = metrics.snapshot().unwrap(); + let metric = snapshot + .scope_metrics() + .flat_map(ScopeMetrics::metrics) + .find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC) + .unwrap(); + let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else { + panic!("expected review counter"); + }; + let observed = counter + .data_points() + .map(|point| { + let tags = point + .attributes() + .map(|tag| (tag.key.as_str(), tag.value.as_str().to_string())) + .collect::>(); + ( + (tags["turn_origin"].clone(), tags["outcome"].clone()), + point.value(), + ) + }) + .collect::>(); + assert_eq!( + observed, + BTreeMap::from([ + ((expected_origin.to_string(), "allow".to_string()), 1), + ((expected_origin.to_string(), "deny".to_string()), 1), + ]) + ); +} + #[test_case::test_case( Some(GuardianAssessmentAction::NetworkAccess { target: "example.com".to_string(), @@ -57,6 +168,7 @@ fn review_completion_uses_one_monotonic_duration( let mut report = ReviewReport::new(ReviewMetadata { thread_id: "thread".to_string(), turn_id: "turn".to_string(), + turn_trigger: None, review_id: "review".to_string(), target_item_id: None, plugin_id: None, diff --git a/codex-rs/ext/guardian-reviewer/src/review_tests.rs b/codex-rs/ext/guardian-reviewer/src/review_tests.rs index 78c2911e8f..ec6d79eadb 100644 --- a/codex-rs/ext/guardian-reviewer/src/review_tests.rs +++ b/codex-rs/ext/guardian-reviewer/src/review_tests.rs @@ -58,6 +58,7 @@ impl ReviewHost for GatedHost { ReviewReport::new(ReviewMetadata { thread_id: "thread".into(), turn_id: "turn".into(), + turn_trigger: None, review_id: approval_id.into(), target_item_id: Some("call".into()), plugin_id: None,