Tag Guardian review counts with turn origin (#62225)

## Why

Guardian review counts need a way to distinguish automation-triggered
turns from other turns, including when a subagent requests the review.

## What changed

Add a `turn_origin` tag to the `codex.guardian.review` counter based on
the trigger recorded when the turn started. Recognized automation
triggers map to `automation`, missing or empty triggers map to
`unknown`, and all other values map to `other`. This keeps the set of
metric labels bounded.

## Testing

Cover trigger classification for both allow and deny verdicts and
verify that a later user turn on the same thread is counted separately
from an automation turn in both main and subagent threads.

GitOrigin-RevId: 6a5808798df7d184203cea18e60a94a82d71f036
This commit is contained in:
cazhang-oai
2026-11-18 08:20:26 +00:00
committed by copyberry
parent 24e236514b
commit ca21fe65d1
7 changed files with 266 additions and 0 deletions
@@ -41,6 +41,7 @@ pub(crate) fn emit_guardian_review_metrics(
result: &GuardianReviewAnalyticsResult,
approval_request_source: GuardianApprovalRequestSource,
reviewed_action: &GuardianReviewedAction,
turn_origin: &'static str,
elapsed: Duration,
) {
let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action);
@@ -62,6 +63,7 @@ pub(crate) fn emit_guardian_review_metrics(
let mut counter_tag_refs = tag_refs.clone();
counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str())));
counter_tag_refs.push(("context_changed", bool_tag(result.context_changed)));
counter_tag_refs.push(("turn_origin", turn_origin));
session_telemetry.counter(
GUARDIAN_REVIEW_COUNT_METRIC,
/*inc*/ 1,
@@ -499,6 +501,7 @@ mod tests {
&result,
GuardianApprovalRequestSource::DelegatedSubagent,
&reviewed_action,
"unknown",
Duration::from_millis(456),
);
@@ -529,6 +532,7 @@ mod tests {
let mut expected_counter_tags = expected_base_tags.clone();
expected_counter_tags.extend(expected_mcp_tags);
expected_counter_tags.insert("context_changed".to_string(), "false".to_string());
expected_counter_tags.insert("turn_origin".to_string(), "unknown".to_string());
assert_eq!((attrs, value), (expected_counter_tags, 1));
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) =
@@ -34,6 +34,8 @@ use crate::ReviewCompletion;
pub struct ReviewMetadata {
pub thread_id: String,
pub turn_id: String,
/// Origin of the issuing turn, inherited by delegated turns at admission.
pub turn_trigger: Option<String>,
pub review_id: String,
pub target_item_id: Option<String>,
pub plugin_id: Option<String>,
@@ -54,11 +56,26 @@ pub struct ReviewReport {
tracking: GuardianReviewTrackContext,
approval_request_source: GuardianApprovalRequestSource,
reviewed_action: GuardianReviewedAction,
turn_origin: &'static str,
}
impl ReviewReport {
pub fn new(metadata: ReviewMetadata) -> Self {
let started_instant = Instant::now();
// Keep client-supplied trigger strings out of metric labels. This describes
// the turn's origin, not later input steered into an already-running turn.
let turn_origin = match metadata.turn_trigger.as_deref() {
Some(
"automation"
| "mcp_event"
| "automation_cron_scheduled"
| "automation_cron_run_now"
| "automation_heartbeat_scheduled"
| "automation_heartbeat_run_now",
) => "automation",
None | Some("") => "unknown",
Some(_) => "other",
};
let tracking = GuardianReviewTrackContext::new(
metadata.thread_id,
metadata.turn_id.clone(),
@@ -93,6 +110,7 @@ impl ReviewReport {
tracking,
approval_request_source: metadata.approval_request_source,
reviewed_action: metadata.reviewed_action,
turn_origin,
}
}
@@ -134,6 +152,7 @@ impl ReviewReport {
&result,
self.approval_request_source,
&self.reviewed_action,
self.turn_origin,
elapsed,
);
analytics.track_guardian_review(&self.tracking, result, completed_at_ms, elapsed);
@@ -17,8 +17,119 @@ use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::BTreeMap;
use std::time::Duration;
#[test_case::test_case(Some("automation"), "automation"; "automation")]
#[test_case::test_case(Some("mcp_event"), "automation"; "webhook")]
#[test_case::test_case(Some("automation_cron_scheduled"), "automation"; "scheduled")]
#[test_case::test_case(Some("automation_cron_run_now"), "automation"; "run_now")]
#[test_case::test_case(Some("automation_heartbeat_scheduled"), "automation"; "heartbeat_scheduled")]
#[test_case::test_case(Some("automation_heartbeat_run_now"), "automation"; "heartbeat_run_now")]
#[test_case::test_case(Some("composer"), "other"; "user")]
#[test_case::test_case(Some("arbitrary client supplied value"), "other"; "bounded_labels")]
#[test_case::test_case(Some(""), "unknown"; "empty")]
#[test_case::test_case(None, "unknown"; "missing")]
fn review_report_bounds_turn_origin_on_both_verdicts(
turn_trigger: Option<&str>,
expected_origin: &str,
) {
use codex_analytics::GuardianReviewDecision;
use codex_analytics::GuardianReviewTerminalStatus;
use codex_protocol::protocol::GuardianAssessmentOutcome;
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "test", "1", InMemoryMetricExporter::default())
.with_runtime_reader(),
)
.unwrap();
let telemetry = SessionTelemetry::new(
ThreadId::new(),
"test-model",
"test-model",
/*account_id*/ None,
/*account_email*/ None,
/*auth_mode*/ None,
"test".to_string(),
/*log_user_prompts*/ false,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics_without_metadata_tags(metrics.clone());
let report = ReviewReport::new(ReviewMetadata {
thread_id: "thread".into(),
turn_id: "turn".into(),
turn_trigger: turn_trigger.map(str::to_owned),
review_id: "review".into(),
target_item_id: None,
plugin_id: None,
script_path: None,
approval_request_source: GuardianApprovalRequestSource::MainTurn,
reviewed_action: GuardianReviewedAction::NativeToolCall {},
action: None,
review_reason: GuardianReviewReason::Policy,
model_context: ModelInvocationContext {
model_slug: "test-model".into(),
reasoning_effort: None,
},
timeout: None,
});
for (decision, terminal_status, outcome) in [
(
GuardianReviewDecision::Approved,
GuardianReviewTerminalStatus::Approved,
GuardianAssessmentOutcome::Allow,
),
(
GuardianReviewDecision::Denied,
GuardianReviewTerminalStatus::Denied,
GuardianAssessmentOutcome::Deny,
),
] {
report.track(
&telemetry,
&AnalyticsEventsClient::disabled(),
GuardianReviewAnalyticsResult {
decision,
terminal_status,
outcome: Some(outcome),
..GuardianReviewAnalyticsResult::without_session()
},
/*completed_at_ms*/ 1_792_141_011_000,
Duration::from_millis(10),
);
}
let snapshot = metrics.snapshot().unwrap();
let metric = snapshot
.scope_metrics()
.flat_map(ScopeMetrics::metrics)
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
.unwrap();
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
panic!("expected review counter");
};
let observed = counter
.data_points()
.map(|point| {
let tags = point
.attributes()
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
.collect::<BTreeMap<_, _>>();
(
(tags["turn_origin"].clone(), tags["outcome"].clone()),
point.value(),
)
})
.collect::<BTreeMap<_, _>>();
assert_eq!(
observed,
BTreeMap::from([
((expected_origin.to_string(), "allow".to_string()), 1),
((expected_origin.to_string(), "deny".to_string()), 1),
])
);
}
#[test_case::test_case(
Some(GuardianAssessmentAction::NetworkAccess {
target: "example.com".to_string(),
@@ -57,6 +168,7 @@ fn review_completion_uses_one_monotonic_duration(
let mut report = ReviewReport::new(ReviewMetadata {
thread_id: "thread".to_string(),
turn_id: "turn".to_string(),
turn_trigger: None,
review_id: "review".to_string(),
target_item_id: None,
plugin_id: None,
@@ -58,6 +58,7 @@ impl ReviewHost for GatedHost {
ReviewReport::new(ReviewMetadata {
thread_id: "thread".into(),
turn_id: "turn".into(),
turn_trigger: None,
review_id: approval_id.into(),
target_item_id: Some("call".into()),
plugin_id: None,