mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Tag Guardian review counts with turn origin (#62225)
## Why Guardian review counts need a way to distinguish automation-triggered turns from other turns, including when a subagent requests the review. ## What changed Add a `turn_origin` tag to the `codex.guardian.review` counter based on the trigger recorded when the turn started. Recognized automation triggers map to `automation`, missing or empty triggers map to `unknown`, and all other values map to `other`. This keeps the set of metric labels bounded. ## Testing Cover trigger classification for both allow and deny verdicts and verify that a later user turn on the same thread is counted separately from an automation turn in both main and subagent threads. GitOrigin-RevId: 6a5808798df7d184203cea18e60a94a82d71f036
This commit is contained in:
@@ -41,6 +41,7 @@ pub(crate) fn emit_guardian_review_metrics(
|
||||
result: &GuardianReviewAnalyticsResult,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: &GuardianReviewedAction,
|
||||
turn_origin: &'static str,
|
||||
elapsed: Duration,
|
||||
) {
|
||||
let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action);
|
||||
@@ -62,6 +63,7 @@ pub(crate) fn emit_guardian_review_metrics(
|
||||
let mut counter_tag_refs = tag_refs.clone();
|
||||
counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str())));
|
||||
counter_tag_refs.push(("context_changed", bool_tag(result.context_changed)));
|
||||
counter_tag_refs.push(("turn_origin", turn_origin));
|
||||
session_telemetry.counter(
|
||||
GUARDIAN_REVIEW_COUNT_METRIC,
|
||||
/*inc*/ 1,
|
||||
@@ -499,6 +501,7 @@ mod tests {
|
||||
&result,
|
||||
GuardianApprovalRequestSource::DelegatedSubagent,
|
||||
&reviewed_action,
|
||||
"unknown",
|
||||
Duration::from_millis(456),
|
||||
);
|
||||
|
||||
@@ -529,6 +532,7 @@ mod tests {
|
||||
let mut expected_counter_tags = expected_base_tags.clone();
|
||||
expected_counter_tags.extend(expected_mcp_tags);
|
||||
expected_counter_tags.insert("context_changed".to_string(), "false".to_string());
|
||||
expected_counter_tags.insert("turn_origin".to_string(), "unknown".to_string());
|
||||
assert_eq!((attrs, value), (expected_counter_tags, 1));
|
||||
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) =
|
||||
|
||||
@@ -34,6 +34,8 @@ use crate::ReviewCompletion;
|
||||
pub struct ReviewMetadata {
|
||||
pub thread_id: String,
|
||||
pub turn_id: String,
|
||||
/// Origin of the issuing turn, inherited by delegated turns at admission.
|
||||
pub turn_trigger: Option<String>,
|
||||
pub review_id: String,
|
||||
pub target_item_id: Option<String>,
|
||||
pub plugin_id: Option<String>,
|
||||
@@ -54,11 +56,26 @@ pub struct ReviewReport {
|
||||
tracking: GuardianReviewTrackContext,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: GuardianReviewedAction,
|
||||
turn_origin: &'static str,
|
||||
}
|
||||
|
||||
impl ReviewReport {
|
||||
pub fn new(metadata: ReviewMetadata) -> Self {
|
||||
let started_instant = Instant::now();
|
||||
// Keep client-supplied trigger strings out of metric labels. This describes
|
||||
// the turn's origin, not later input steered into an already-running turn.
|
||||
let turn_origin = match metadata.turn_trigger.as_deref() {
|
||||
Some(
|
||||
"automation"
|
||||
| "mcp_event"
|
||||
| "automation_cron_scheduled"
|
||||
| "automation_cron_run_now"
|
||||
| "automation_heartbeat_scheduled"
|
||||
| "automation_heartbeat_run_now",
|
||||
) => "automation",
|
||||
None | Some("") => "unknown",
|
||||
Some(_) => "other",
|
||||
};
|
||||
let tracking = GuardianReviewTrackContext::new(
|
||||
metadata.thread_id,
|
||||
metadata.turn_id.clone(),
|
||||
@@ -93,6 +110,7 @@ impl ReviewReport {
|
||||
tracking,
|
||||
approval_request_source: metadata.approval_request_source,
|
||||
reviewed_action: metadata.reviewed_action,
|
||||
turn_origin,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,6 +152,7 @@ impl ReviewReport {
|
||||
&result,
|
||||
self.approval_request_source,
|
||||
&self.reviewed_action,
|
||||
self.turn_origin,
|
||||
elapsed,
|
||||
);
|
||||
analytics.track_guardian_review(&self.tracking, result, completed_at_ms, elapsed);
|
||||
|
||||
@@ -17,8 +17,119 @@ use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use opentelemetry_sdk::metrics::data::ScopeMetrics;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test_case::test_case(Some("automation"), "automation"; "automation")]
|
||||
#[test_case::test_case(Some("mcp_event"), "automation"; "webhook")]
|
||||
#[test_case::test_case(Some("automation_cron_scheduled"), "automation"; "scheduled")]
|
||||
#[test_case::test_case(Some("automation_cron_run_now"), "automation"; "run_now")]
|
||||
#[test_case::test_case(Some("automation_heartbeat_scheduled"), "automation"; "heartbeat_scheduled")]
|
||||
#[test_case::test_case(Some("automation_heartbeat_run_now"), "automation"; "heartbeat_run_now")]
|
||||
#[test_case::test_case(Some("composer"), "other"; "user")]
|
||||
#[test_case::test_case(Some("arbitrary client supplied value"), "other"; "bounded_labels")]
|
||||
#[test_case::test_case(Some(""), "unknown"; "empty")]
|
||||
#[test_case::test_case(None, "unknown"; "missing")]
|
||||
fn review_report_bounds_turn_origin_on_both_verdicts(
|
||||
turn_trigger: Option<&str>,
|
||||
expected_origin: &str,
|
||||
) {
|
||||
use codex_analytics::GuardianReviewDecision;
|
||||
use codex_analytics::GuardianReviewTerminalStatus;
|
||||
use codex_protocol::protocol::GuardianAssessmentOutcome;
|
||||
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "test", "1", InMemoryMetricExporter::default())
|
||||
.with_runtime_reader(),
|
||||
)
|
||||
.unwrap();
|
||||
let telemetry = SessionTelemetry::new(
|
||||
ThreadId::new(),
|
||||
"test-model",
|
||||
"test-model",
|
||||
/*account_id*/ None,
|
||||
/*account_email*/ None,
|
||||
/*auth_mode*/ None,
|
||||
"test".to_string(),
|
||||
/*log_user_prompts*/ false,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics_without_metadata_tags(metrics.clone());
|
||||
let report = ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".into(),
|
||||
turn_id: "turn".into(),
|
||||
turn_trigger: turn_trigger.map(str::to_owned),
|
||||
review_id: "review".into(),
|
||||
target_item_id: None,
|
||||
plugin_id: None,
|
||||
script_path: None,
|
||||
approval_request_source: GuardianApprovalRequestSource::MainTurn,
|
||||
reviewed_action: GuardianReviewedAction::NativeToolCall {},
|
||||
action: None,
|
||||
review_reason: GuardianReviewReason::Policy,
|
||||
model_context: ModelInvocationContext {
|
||||
model_slug: "test-model".into(),
|
||||
reasoning_effort: None,
|
||||
},
|
||||
timeout: None,
|
||||
});
|
||||
for (decision, terminal_status, outcome) in [
|
||||
(
|
||||
GuardianReviewDecision::Approved,
|
||||
GuardianReviewTerminalStatus::Approved,
|
||||
GuardianAssessmentOutcome::Allow,
|
||||
),
|
||||
(
|
||||
GuardianReviewDecision::Denied,
|
||||
GuardianReviewTerminalStatus::Denied,
|
||||
GuardianAssessmentOutcome::Deny,
|
||||
),
|
||||
] {
|
||||
report.track(
|
||||
&telemetry,
|
||||
&AnalyticsEventsClient::disabled(),
|
||||
GuardianReviewAnalyticsResult {
|
||||
decision,
|
||||
terminal_status,
|
||||
outcome: Some(outcome),
|
||||
..GuardianReviewAnalyticsResult::without_session()
|
||||
},
|
||||
/*completed_at_ms*/ 1_792_141_011_000,
|
||||
Duration::from_millis(10),
|
||||
);
|
||||
}
|
||||
let snapshot = metrics.snapshot().unwrap();
|
||||
let metric = snapshot
|
||||
.scope_metrics()
|
||||
.flat_map(ScopeMetrics::metrics)
|
||||
.find(|metric| metric.name() == GUARDIAN_REVIEW_COUNT_METRIC)
|
||||
.unwrap();
|
||||
let AggregatedMetrics::U64(MetricData::Sum(counter)) = metric.data() else {
|
||||
panic!("expected review counter");
|
||||
};
|
||||
let observed = counter
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
let tags = point
|
||||
.attributes()
|
||||
.map(|tag| (tag.key.as_str(), tag.value.as_str().to_string()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
(
|
||||
(tags["turn_origin"].clone(), tags["outcome"].clone()),
|
||||
point.value(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(
|
||||
observed,
|
||||
BTreeMap::from([
|
||||
((expected_origin.to_string(), "allow".to_string()), 1),
|
||||
((expected_origin.to_string(), "deny".to_string()), 1),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case::test_case(
|
||||
Some(GuardianAssessmentAction::NetworkAccess {
|
||||
target: "example.com".to_string(),
|
||||
@@ -57,6 +168,7 @@ fn review_completion_uses_one_monotonic_duration(
|
||||
let mut report = ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".to_string(),
|
||||
turn_id: "turn".to_string(),
|
||||
turn_trigger: None,
|
||||
review_id: "review".to_string(),
|
||||
target_item_id: None,
|
||||
plugin_id: None,
|
||||
|
||||
@@ -58,6 +58,7 @@ impl ReviewHost for GatedHost {
|
||||
ReviewReport::new(ReviewMetadata {
|
||||
thread_id: "thread".into(),
|
||||
turn_id: "turn".into(),
|
||||
turn_trigger: None,
|
||||
review_id: approval_id.into(),
|
||||
target_item_id: Some("call".into()),
|
||||
plugin_id: None,
|
||||
|
||||
Reference in New Issue
Block a user