Tag Guardian review counts with turn origin (#62225)

## Why

Guardian review counts need a way to distinguish automation-triggered
turns from other turns, including when a subagent requests the review.

## What changed

Add a `turn_origin` tag to the `codex.guardian.review` counter based on
the trigger recorded when the turn started. Recognized automation
triggers map to `automation`, missing or empty triggers map to
`unknown`, and all other values map to `other`. This keeps the set of
metric labels bounded.

## Testing

Cover trigger classification for both allow and deny verdicts and
verify that a later user turn on the same thread is counted separately
from an automation turn in both main and subagent threads.

GitOrigin-RevId: 6a5808798df7d184203cea18e60a94a82d71f036
This commit is contained in:
cazhang-oai
2026-11-18 08:20:26 +00:00
committed by copyberry
parent 24e236514b
commit ca21fe65d1
7 changed files with 266 additions and 0 deletions
@@ -178,6 +178,7 @@ impl ReviewHost for super::super::runtime::ReviewRuntime {
codex_guardian_reviewer::ReviewReport::new(codex_guardian_reviewer::ReviewMetadata {
thread_id: session.thread_id.to_string(),
turn_id: assessment_turn_id,
turn_trigger: turn.turn_metadata_state.current_turn_trigger(),
review_id: review_id.to_owned(),
target_item_id,
plugin_id,
@@ -0,0 +1,126 @@
#![cfg(test)]
//! Checks review counter attribution from admitted main and subagent turns.
use super::*;
use codex_core::TurnStartOptions;
use codex_otel::GUARDIAN_REVIEW_COUNT_METRIC;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use std::collections::BTreeMap;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn guardian_metrics_distinguish_automation_main_and_subagent_turns() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let test = test_codex()
.with_config(|config| {
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
config.approvals_reviewer = ApprovalsReviewer::User;
})
.build_with_auto_env(&server)
.await?;
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "test", "1", Default::default()).with_runtime_reader(),
)?;
let mut expected = BTreeMap::new();
for source in [
SessionSource::Custom("codex-cloud-agent".into()),
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
parent_thread_id: test.session_configured.thread_id,
depth: 1,
agent_path: None,
agent_nickname: None,
agent_role: None,
}),
] {
let mut options = test.start_thread_options().await;
options.session_source = Some(source.clone());
options.thread_extension_init.insert(metrics.clone());
let thread = test.thread_manager.start_thread(options).await?.thread;
// A later user turn on the same session must not retain automation attribution.
for (trigger, origin) in [
("automation_cron_scheduled", "automation"),
("composer", "other"),
] {
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("parent-tool"),
ev_function_call(
"reviewed-command",
"exec_command",
r#"{"cmd":"true","sandbox_permissions":"require_escalated","justification":"Exercise review attribution."}"#,
),
ev_completed("parent-tool"),
]),
sse(vec![
ev_response_created("guardian-verdict"),
ev_assistant_message(
"guardian",
r#"{"risk_level":"high","user_authorization":"low","outcome":"deny","rationale":"Test denial."}"#,
),
ev_completed("guardian-verdict"),
]),
sse(vec![ev_completed("parent-complete")]),
],
)
.await;
thread
.start_or_steer_turn(
TurnInputRequest::user_input(vec![UserInput::Text {
text: "Run the reviewed command.".into(),
text_elements: Vec::new(),
}])
.on_start(TurnStartOptions {
turn_trigger: Some(trigger.into()),
..Default::default()
})
.with_thread_settings(ThreadSettingsOverrides {
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
..Default::default()
}),
)
.await?;
wait_for_event(&thread, |event| matches!(event, EventMsg::TurnComplete(_))).await;
assert_eq!(responses.requests().len(), 3);
expected.insert(
(source.to_string(), origin.to_string(), "deny".to_string()),
1,
);
}
thread.shutdown_and_wait().await?;
}
let snapshot = metrics.snapshot()?;
let metric = snapshot
.scope_metrics()
.flat_map(ScopeMetrics::metrics)
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
.context("Guardian review counter")?;
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
panic!("expected review counter");
};
let observed = counter
.data_points()
.map(|point| {
let tags = point
.attributes()
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
.collect::<BTreeMap<_, _>>();
(
(
tags["session_source"].clone(),
tags["turn_origin"].clone(),
tags["outcome"].clone(),
),
point.value(),
)
})
.collect::<BTreeMap<_, _>>();
assert_eq!(observed, expected);
Ok(())
}
@@ -1,5 +1,8 @@
#![cfg(not(target_os = "windows"))]
#[path = "guardian_automation_metrics_tests.rs"]
mod automation_metrics;
use anyhow::Context;
use anyhow::Result;
use chrono::DateTime;
@@ -41,6 +41,7 @@ pub(crate) fn emit_guardian_review_metrics(
result: &GuardianReviewAnalyticsResult,
approval_request_source: GuardianApprovalRequestSource,
reviewed_action: &GuardianReviewedAction,
turn_origin: &'static str,
elapsed: Duration,
) {
let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action);
@@ -62,6 +63,7 @@ pub(crate) fn emit_guardian_review_metrics(
let mut counter_tag_refs = tag_refs.clone();
counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str())));
counter_tag_refs.push(("context_changed", bool_tag(result.context_changed)));
counter_tag_refs.push(("turn_origin", turn_origin));
session_telemetry.counter(
GUARDIAN_REVIEW_COUNT_METRIC,
/*inc*/ 1,
@@ -499,6 +501,7 @@ mod tests {
&result,
GuardianApprovalRequestSource::DelegatedSubagent,
&reviewed_action,
"unknown",
Duration::from_millis(456),
);
@@ -529,6 +532,7 @@ mod tests {
let mut expected_counter_tags = expected_base_tags.clone();
expected_counter_tags.extend(expected_mcp_tags);
expected_counter_tags.insert("context_changed".to_string(), "false".to_string());
expected_counter_tags.insert("turn_origin".to_string(), "unknown".to_string());
assert_eq!((attrs, value), (expected_counter_tags, 1));
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) =
@@ -34,6 +34,8 @@ use crate::ReviewCompletion;
pub struct ReviewMetadata {
pub thread_id: String,
pub turn_id: String,
/// Origin of the issuing turn, inherited by delegated turns at admission.
pub turn_trigger: Option<String>,
pub review_id: String,
pub target_item_id: Option<String>,
pub plugin_id: Option<String>,
@@ -54,11 +56,26 @@ pub struct ReviewReport {
tracking: GuardianReviewTrackContext,
approval_request_source: GuardianApprovalRequestSource,
reviewed_action: GuardianReviewedAction,
turn_origin: &'static str,
}
impl ReviewReport {
pub fn new(metadata: ReviewMetadata) -> Self {
let started_instant = Instant::now();
// Keep client-supplied trigger strings out of metric labels. This describes
// the turn's origin, not later input steered into an already-running turn.
let turn_origin = match metadata.turn_trigger.as_deref() {
Some(
"automation"
| "mcp_event"
| "automation_cron_scheduled"
| "automation_cron_run_now"
| "automation_heartbeat_scheduled"
| "automation_heartbeat_run_now",
) => "automation",
None | Some("") => "unknown",
Some(_) => "other",
};
let tracking = GuardianReviewTrackContext::new(
metadata.thread_id,
metadata.turn_id.clone(),
@@ -93,6 +110,7 @@ impl ReviewReport {
tracking,
approval_request_source: metadata.approval_request_source,
reviewed_action: metadata.reviewed_action,
turn_origin,
}
}
@@ -134,6 +152,7 @@ impl ReviewReport {
&result,
self.approval_request_source,
&self.reviewed_action,
self.turn_origin,
elapsed,
);
analytics.track_guardian_review(&self.tracking, result, completed_at_ms, elapsed);
@@ -17,8 +17,119 @@ use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::BTreeMap;
use std::time::Duration;
#[test_case::test_case(Some("automation"), "automation"; "automation")]
#[test_case::test_case(Some("mcp_event"), "automation"; "webhook")]
#[test_case::test_case(Some("automation_cron_scheduled"), "automation"; "scheduled")]
#[test_case::test_case(Some("automation_cron_run_now"), "automation"; "run_now")]
#[test_case::test_case(Some("automation_heartbeat_scheduled"), "automation"; "heartbeat_scheduled")]
#[test_case::test_case(Some("automation_heartbeat_run_now"), "automation"; "heartbeat_run_now")]
#[test_case::test_case(Some("composer"), "other"; "user")]
#[test_case::test_case(Some("arbitrary client supplied value"), "other"; "bounded_labels")]
#[test_case::test_case(Some(""), "unknown"; "empty")]
#[test_case::test_case(None, "unknown"; "missing")]
fn review_report_bounds_turn_origin_on_both_verdicts(
turn_trigger: Option<&str>,
expected_origin: &str,
) {
use codex_analytics::GuardianReviewDecision;
use codex_analytics::GuardianReviewTerminalStatus;
use codex_protocol::protocol::GuardianAssessmentOutcome;
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "test", "1", InMemoryMetricExporter::default())
.with_runtime_reader(),
)
.unwrap();
let telemetry = SessionTelemetry::new(
ThreadId::new(),
"test-model",
"test-model",
/*account_id*/ None,
/*account_email*/ None,
/*auth_mode*/ None,
"test".to_string(),
/*log_user_prompts*/ false,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics_without_metadata_tags(metrics.clone());
let report = ReviewReport::new(ReviewMetadata {
thread_id: "thread".into(),
turn_id: "turn".into(),
turn_trigger: turn_trigger.map(str::to_owned),
review_id: "review".into(),
target_item_id: None,
plugin_id: None,
script_path: None,
approval_request_source: GuardianApprovalRequestSource::MainTurn,
reviewed_action: GuardianReviewedAction::NativeToolCall {},
action: None,
review_reason: GuardianReviewReason::Policy,
model_context: ModelInvocationContext {
model_slug: "test-model".into(),
reasoning_effort: None,
},
timeout: None,
});
for (decision, terminal_status, outcome) in [
(
GuardianReviewDecision::Approved,
GuardianReviewTerminalStatus::Approved,
GuardianAssessmentOutcome::Allow,
),
(
GuardianReviewDecision::Denied,
GuardianReviewTerminalStatus::Denied,
GuardianAssessmentOutcome::Deny,
),
] {
report.track(
&telemetry,
&AnalyticsEventsClient::disabled(),
GuardianReviewAnalyticsResult {
decision,
terminal_status,
outcome: Some(outcome),
..GuardianReviewAnalyticsResult::without_session()
},
/*completed_at_ms*/ 1_792_141_011_000,
Duration::from_millis(10),
);
}
let snapshot = metrics.snapshot().unwrap();
let metric = snapshot
.scope_metrics()
.flat_map(ScopeMetrics::metrics)
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
.unwrap();
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
panic!("expected review counter");
};
let observed = counter
.data_points()
.map(|point| {
let tags = point
.attributes()
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
.collect::<BTreeMap<_, _>>();
(
(tags["turn_origin"].clone(), tags["outcome"].clone()),
point.value(),
)
})
.collect::<BTreeMap<_, _>>();
assert_eq!(
observed,
BTreeMap::from([
((expected_origin.to_string(), "allow".to_string()), 1),
((expected_origin.to_string(), "deny".to_string()), 1),
])
);
}
#[test_case::test_case(
Some(GuardianAssessmentAction::NetworkAccess {
target: "example.com".to_string(),
@@ -57,6 +168,7 @@ fn review_completion_uses_one_monotonic_duration(
let mut report = ReviewReport::new(ReviewMetadata {
thread_id: "thread".to_string(),
turn_id: "turn".to_string(),
turn_trigger: None,
review_id: "review".to_string(),
target_item_id: None,
plugin_id: None,
@@ -58,6 +58,7 @@ impl ReviewHost for GatedHost {
ReviewReport::new(ReviewMetadata {
thread_id: "thread".into(),
turn_id: "turn".into(),
turn_trigger: None,
review_id: approval_id.into(),
target_item_id: Some("call".into()),
plugin_id: None,