mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Tag Guardian review counts with turn origin (#62225)
## Why Guardian review counts need a way to distinguish automation-triggered turns from other turns, including when a subagent requests the review. ## What changed Add a `turn_origin` tag to the `codex.guardian.review` counter based on the trigger recorded when the turn started. Recognized automation triggers map to `automation`, missing or empty triggers map to `unknown`, and all other values map to `other`. This keeps the set of metric labels bounded. ## Testing Cover trigger classification for both allow and deny verdicts and verify that a later user turn on the same thread is counted separately from an automation turn in both main and subagent threads. GitOrigin-RevId: 6a5808798df7d184203cea18e60a94a82d71f036
This commit is contained in:
@@ -178,6 +178,7 @@ impl ReviewHost for super::super::runtime::ReviewRuntime {
|
||||
codex_guardian_reviewer::ReviewReport::new(codex_guardian_reviewer::ReviewMetadata {
|
||||
thread_id: session.thread_id.to_string(),
|
||||
turn_id: assessment_turn_id,
|
||||
turn_trigger: turn.turn_metadata_state.current_turn_trigger(),
|
||||
review_id: review_id.to_owned(),
|
||||
target_item_id,
|
||||
plugin_id,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#![cfg(test)]
|
||||
//! Checks review counter attribution from admitted main and subagent turns.
|
||||
|
||||
use super::*;
|
||||
use codex_core::TurnStartOptions;
|
||||
use codex_otel::GUARDIAN_REVIEW_COUNT_METRIC;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use opentelemetry_sdk::metrics::data::ScopeMetrics;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_metrics_distinguish_automation_main_and_subagent_turns() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
|
||||
config.approvals_reviewer = ApprovalsReviewer::User;
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "test", "1", Default::default()).with_runtime_reader(),
|
||||
)?;
|
||||
let mut expected = BTreeMap::new();
|
||||
for source in [
|
||||
SessionSource::Custom("codex-cloud-agent".into()),
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: test.session_configured.thread_id,
|
||||
depth: 1,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
}),
|
||||
] {
|
||||
let mut options = test.start_thread_options().await;
|
||||
options.session_source = Some(source.clone());
|
||||
options.thread_extension_init.insert(metrics.clone());
|
||||
let thread = test.thread_manager.start_thread(options).await?.thread;
|
||||
// A later user turn on the same session must not retain automation attribution.
|
||||
for (trigger, origin) in [
|
||||
("automation_cron_scheduled", "automation"),
|
||||
("composer", "other"),
|
||||
] {
|
||||
let responses = mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_response_created("parent-tool"),
|
||||
ev_function_call(
|
||||
"reviewed-command",
|
||||
"exec_command",
|
||||
r#"{"cmd":"true","sandbox_permissions":"require_escalated","justification":"Exercise review attribution."}"#,
|
||||
),
|
||||
ev_completed("parent-tool"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("guardian-verdict"),
|
||||
ev_assistant_message(
|
||||
"guardian",
|
||||
r#"{"risk_level":"high","user_authorization":"low","outcome":"deny","rationale":"Test denial."}"#,
|
||||
),
|
||||
ev_completed("guardian-verdict"),
|
||||
]),
|
||||
sse(vec![ev_completed("parent-complete")]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
thread
|
||||
.start_or_steer_turn(
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "Run the reviewed command.".into(),
|
||||
text_elements: Vec::new(),
|
||||
}])
|
||||
.on_start(TurnStartOptions {
|
||||
turn_trigger: Some(trigger.into()),
|
||||
..Default::default()
|
||||
})
|
||||
.with_thread_settings(ThreadSettingsOverrides {
|
||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
wait_for_event(&thread, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
assert_eq!(responses.requests().len(), 3);
|
||||
expected.insert(
|
||||
(source.to_string(), origin.to_string(), "deny".to_string()),
|
||||
1,
|
||||
);
|
||||
}
|
||||
thread.shutdown_and_wait().await?;
|
||||
}
|
||||
let snapshot = metrics.snapshot()?;
|
||||
let metric = snapshot
|
||||
.scope_metrics()
|
||||
.flat_map(ScopeMetrics::metrics)
|
||||
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
|
||||
.context("Guardian review counter")?;
|
||||
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
|
||||
panic!("expected review counter");
|
||||
};
|
||||
let observed = counter
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
let tags = point
|
||||
.attributes()
|
||||
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
(
|
||||
(
|
||||
tags["session_source"].clone(),
|
||||
tags["turn_origin"].clone(),
|
||||
tags["outcome"].clone(),
|
||||
),
|
||||
point.value(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(observed, expected);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
#![cfg(not(target_os = "windows"))]
|
||||
|
||||
#[path = "guardian_automation_metrics_tests.rs"]
|
||||
mod automation_metrics;
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use chrono::DateTime;
|
||||
|
||||
@@ -41,6 +41,7 @@ pub(crate) fn emit_guardian_review_metrics(
|
||||
result: &GuardianReviewAnalyticsResult,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: &GuardianReviewedAction,
|
||||
turn_origin: &'static str,
|
||||
elapsed: Duration,
|
||||
) {
|
||||
let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action);
|
||||
@@ -62,6 +63,7 @@ pub(crate) fn emit_guardian_review_metrics(
|
||||
let mut counter_tag_refs = tag_refs.clone();
|
||||
counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str())));
|
||||
counter_tag_refs.push(("context_changed", bool_tag(result.context_changed)));
|
||||
counter_tag_refs.push(("turn_origin", turn_origin));
|
||||
session_telemetry.counter(
|
||||
GUARDIAN_REVIEW_COUNT_METRIC,
|
||||
/*inc*/ 1,
|
||||
@@ -499,6 +501,7 @@ mod tests {
|
||||
&result,
|
||||
GuardianApprovalRequestSource::DelegatedSubagent,
|
||||
&reviewed_action,
|
||||
"unknown",
|
||||
Duration::from_millis(456),
|
||||
);
|
||||
|
||||
@@ -529,6 +532,7 @@ mod tests {
|
||||
let mut expected_counter_tags = expected_base_tags.clone();
|
||||
expected_counter_tags.extend(expected_mcp_tags);
|
||||
expected_counter_tags.insert("context_changed".to_string(), "false".to_string());
|
||||
expected_counter_tags.insert("turn_origin".to_string(), "unknown".to_string());
|
||||
assert_eq!((attrs, value), (expected_counter_tags, 1));
|
||||
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) =
|
||||
|
||||
@@ -34,6 +34,8 @@ use crate::ReviewCompletion;
|
||||
pub struct ReviewMetadata {
|
||||
pub thread_id: String,
|
||||
pub turn_id: String,
|
||||
/// Origin of the issuing turn, inherited by delegated turns at admission.
|
||||
pub turn_trigger: Option<String>,
|
||||
pub review_id: String,
|
||||
pub target_item_id: Option<String>,
|
||||
pub plugin_id: Option<String>,
|
||||
@@ -54,11 +56,26 @@ pub struct ReviewReport {
|
||||
tracking: GuardianReviewTrackContext,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: GuardianReviewedAction,
|
||||
turn_origin: &'static str,
|
||||
}
|
||||
|
||||
impl ReviewReport {
|
||||
pub fn new(metadata: ReviewMetadata) -> Self {
|
||||
let started_instant = Instant::now();
|
||||
// Keep client-supplied trigger strings out of metric labels. This describes
|
||||
// the turn's origin, not later input steered into an already-running turn.
|
||||
let turn_origin = match metadata.turn_trigger.as_deref() {
|
||||
Some(
|
||||
"automation"
|
||||
| "mcp_event"
|
||||
| "automation_cron_scheduled"
|
||||
| "automation_cron_run_now"
|
||||
| "automation_heartbeat_scheduled"
|
||||
| "automation_heartbeat_run_now",
|
||||
) => "automation",
|
||||
None | Some("") => "unknown",
|
||||
Some(_) => "other",
|
||||
};
|
||||
let tracking = GuardianReviewTrackContext::new(
|
||||
metadata.thread_id,
|
||||
metadata.turn_id.clone(),
|
||||
@@ -93,6 +110,7 @@ impl ReviewReport {
|
||||
tracking,
|
||||
approval_request_source: metadata.approval_request_source,
|
||||
reviewed_action: metadata.reviewed_action,
|
||||
turn_origin,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,6 +152,7 @@ impl ReviewReport {
|
||||
&result,
|
||||
self.approval_request_source,
|
||||
&self.reviewed_action,
|
||||
self.turn_origin,
|
||||
elapsed,
|
||||
);
|
||||
analytics.track_guardian_review(&self.tracking, result, completed_at_ms, elapsed);
|
||||
|
||||
@@ -17,8 +17,119 @@ use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use opentelemetry_sdk::metrics::data::ScopeMetrics;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test_case::test_case(Some("automation"), "automation"; "automation")]
|
||||
#[test_case::test_case(Some("mcp_event"), "automation"; "webhook")]
|
||||
#[test_case::test_case(Some("automation_cron_scheduled"), "automation"; "scheduled")]
|
||||
#[test_case::test_case(Some("automation_cron_run_now"), "automation"; "run_now")]
|
||||
#[test_case::test_case(Some("automation_heartbeat_scheduled"), "automation"; "heartbeat_scheduled")]
|
||||
#[test_case::test_case(Some("automation_heartbeat_run_now"), "automation"; "heartbeat_run_now")]
|
||||
#[test_case::test_case(Some("composer"), "other"; "user")]
|
||||
#[test_case::test_case(Some("arbitrary client supplied value"), "other"; "bounded_labels")]
|
||||
#[test_case::test_case(Some(""), "unknown"; "empty")]
|
||||
#[test_case::test_case(None, "unknown"; "missing")]
|
||||
fn review_report_bounds_turn_origin_on_both_verdicts(
|
||||
turn_trigger: Option<&str>,
|
||||
expected_origin: &str,
|
||||
) {
|
||||
use codex_analytics::GuardianReviewDecision;
|
||||
use codex_analytics::GuardianReviewTerminalStatus;
|
||||
use codex_protocol::protocol::GuardianAssessmentOutcome;
|
||||
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "test", "1", InMemoryMetricExporter::default())
|
||||
.with_runtime_reader(),
|
||||
)
|
||||
.unwrap();
|
||||
let telemetry = SessionTelemetry::new(
|
||||
ThreadId::new(),
|
||||
"test-model",
|
||||
"test-model",
|
||||
/*account_id*/ None,
|
||||
/*account_email*/ None,
|
||||
/*auth_mode*/ None,
|
||||
"test".to_string(),
|
||||
/*log_user_prompts*/ false,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics_without_metadata_tags(metrics.clone());
|
||||
let report = ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".into(),
|
||||
turn_id: "turn".into(),
|
||||
turn_trigger: turn_trigger.map(str::to_owned),
|
||||
review_id: "review".into(),
|
||||
target_item_id: None,
|
||||
plugin_id: None,
|
||||
script_path: None,
|
||||
approval_request_source: GuardianApprovalRequestSource::MainTurn,
|
||||
reviewed_action: GuardianReviewedAction::NativeToolCall {},
|
||||
action: None,
|
||||
review_reason: GuardianReviewReason::Policy,
|
||||
model_context: ModelInvocationContext {
|
||||
model_slug: "test-model".into(),
|
||||
reasoning_effort: None,
|
||||
},
|
||||
timeout: None,
|
||||
});
|
||||
for (decision, terminal_status, outcome) in [
|
||||
(
|
||||
GuardianReviewDecision::Approved,
|
||||
GuardianReviewTerminalStatus::Approved,
|
||||
GuardianAssessmentOutcome::Allow,
|
||||
),
|
||||
(
|
||||
GuardianReviewDecision::Denied,
|
||||
GuardianReviewTerminalStatus::Denied,
|
||||
GuardianAssessmentOutcome::Deny,
|
||||
),
|
||||
] {
|
||||
report.track(
|
||||
&telemetry,
|
||||
&AnalyticsEventsClient::disabled(),
|
||||
GuardianReviewAnalyticsResult {
|
||||
decision,
|
||||
terminal_status,
|
||||
outcome: Some(outcome),
|
||||
..GuardianReviewAnalyticsResult::without_session()
|
||||
},
|
||||
/*completed_at_ms*/ 1_792_141_011_000,
|
||||
Duration::from_millis(10),
|
||||
);
|
||||
}
|
||||
let snapshot = metrics.snapshot().unwrap();
|
||||
let metric = snapshot
|
||||
.scope_metrics()
|
||||
.flat_map(ScopeMetrics::metrics)
|
||||
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
|
||||
.unwrap();
|
||||
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
|
||||
panic!("expected review counter");
|
||||
};
|
||||
let observed = counter
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
let tags = point
|
||||
.attributes()
|
||||
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
(
|
||||
(tags["turn_origin"].clone(), tags["outcome"].clone()),
|
||||
point.value(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(
|
||||
observed,
|
||||
BTreeMap::from([
|
||||
((expected_origin.to_string(), "allow".to_string()), 1),
|
||||
((expected_origin.to_string(), "deny".to_string()), 1),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case::test_case(
|
||||
Some(GuardianAssessmentAction::NetworkAccess {
|
||||
target: "example.com".to_string(),
|
||||
@@ -57,6 +168,7 @@ fn review_completion_uses_one_monotonic_duration(
|
||||
let mut report = ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".to_string(),
|
||||
turn_id: "turn".to_string(),
|
||||
turn_trigger: None,
|
||||
review_id: "review".to_string(),
|
||||
target_item_id: None,
|
||||
plugin_id: None,
|
||||
|
||||
@@ -58,6 +58,7 @@ impl ReviewHost for GatedHost {
|
||||
ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".into(),
|
||||
turn_id: "turn".into(),
|
||||
turn_trigger: None,
|
||||
review_id: approval_id.into(),
|
||||
target_item_id: Some("call".into()),
|
||||
plugin_id: None,
|
||||
|
||||
Reference in New Issue
Block a user