mirror of
https://github.com/openai/codex.git
synced 2026-10-29 19:08:43 +00:00
Move Guardian reporting and denial accounting into the extension (#44544)
## What changed - Move assessment event construction, metrics, and analytics tracking into `codex-guardian-reviewer` through `ReviewReport`. - Store denial accounting in thread extension data through `ReviewDenials`, with core retaining turn interruption and lifecycle cleanup. - Move failed-review record selection and bounded serialization into the extension, with core supplying captured review context. ## Testing Add coverage that denial accounting clears on turn completion and interruption. Move the oversized-record test into the extension and exercise the new API. GitOrigin-RevId: 6e5b3d9d4128b356f99b1d0e74a68111e40261ab
This commit is contained in:
jif
@@ -17,12 +17,19 @@ anyhow = { workspace = true }
|
||||
codex-analytics = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
codex-features = { workspace = true }
|
||||
codex-feedback = { workspace = true }
|
||||
codex-mcp = { workspace = true }
|
||||
codex-otel = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["experimental_metrics_custom_reader", "metrics"] }
|
||||
test-case = "3.3.1"
|
||||
pretty_assertions = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Selects failed reviews for opt-in feedback and bounds each serialized record.
|
||||
//! The host supplies captured context; records outlive short-lived reviewer sessions.
|
||||
|
||||
use crate::GuardianReviewSessionOutcome;
|
||||
use crate::parse_guardian_assessment;
|
||||
use codex_feedback::record_guardian_review_failure;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::GuardianAssessmentOutcome;
|
||||
use serde::Serialize;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
|
||||
const MAX_RECORD_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ReviewFeedbackRecord<'a> {
|
||||
reviewed_thread_id: ThreadId,
|
||||
reviewed_turn_id: &'a str,
|
||||
target_item_id: Option<&'a str>,
|
||||
reviewer_thread_id: ThreadId,
|
||||
model: &'a str,
|
||||
status: &'a str,
|
||||
decision: Option<&'a str>,
|
||||
action: &'a str,
|
||||
action_truncated: bool,
|
||||
instructions: Option<&'a str>,
|
||||
history: Vec<&'a ResponseItem>,
|
||||
context_omitted: bool,
|
||||
}
|
||||
|
||||
/// Context captured by the host only after feedback policy selects a failed review.
|
||||
pub struct ReviewFeedbackContext<'a> {
|
||||
pub reviewed_thread_id: ThreadId,
|
||||
pub reviewed_turn_id: &'a str,
|
||||
pub target_item_id: Option<&'a str>,
|
||||
pub reviewer_thread_id: ThreadId,
|
||||
pub model: &'a str,
|
||||
pub action: &'a str,
|
||||
pub action_truncated: bool,
|
||||
pub instructions: Option<&'a str>,
|
||||
pub history: Vec<&'a ResponseItem>,
|
||||
}
|
||||
|
||||
/// Host-supplied opt-in and persistence facts for the reviewed session.
|
||||
pub struct ReviewFeedbackSettings {
|
||||
pub enabled: bool,
|
||||
pub ephemeral: bool,
|
||||
}
|
||||
|
||||
pub struct FailedReviewFeedback<'a> {
|
||||
status: &'static str,
|
||||
decision: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl<'a> FailedReviewFeedback<'a> {
|
||||
pub fn for_outcome(
|
||||
outcome: &'a GuardianReviewSessionOutcome,
|
||||
settings: ReviewFeedbackSettings,
|
||||
) -> Option<Self> {
|
||||
if !settings.enabled || settings.ephemeral {
|
||||
return None;
|
||||
}
|
||||
let (status, decision) = match outcome {
|
||||
GuardianReviewSessionOutcome::Completed(Ok(Some(decision))) => {
|
||||
match parse_guardian_assessment(Some(decision)) {
|
||||
Ok(assessment) if assessment.outcome == GuardianAssessmentOutcome::Allow => {
|
||||
return None;
|
||||
}
|
||||
Ok(_) => ("denied", Some(decision.as_str())),
|
||||
Err(_) => ("invalid_decision", Some(decision.as_str())),
|
||||
}
|
||||
}
|
||||
GuardianReviewSessionOutcome::Completed(Ok(None)) => ("missing_decision", None),
|
||||
GuardianReviewSessionOutcome::Completed(Err(_))
|
||||
| GuardianReviewSessionOutcome::PromptBuildFailed(_)
|
||||
| GuardianReviewSessionOutcome::SessionFailed { .. } => ("failed", None),
|
||||
GuardianReviewSessionOutcome::TimedOut => ("timed_out", None),
|
||||
GuardianReviewSessionOutcome::Aborted => ("aborted", None),
|
||||
};
|
||||
Some(Self { status, decision })
|
||||
}
|
||||
|
||||
pub fn store(self, context: ReviewFeedbackContext<'_>) {
|
||||
ReviewFeedbackRecord {
|
||||
reviewed_thread_id: context.reviewed_thread_id,
|
||||
reviewed_turn_id: context.reviewed_turn_id,
|
||||
target_item_id: context.target_item_id,
|
||||
reviewer_thread_id: context.reviewer_thread_id,
|
||||
model: context.model,
|
||||
action: context.action,
|
||||
action_truncated: context.action_truncated,
|
||||
instructions: context.instructions,
|
||||
history: context.history,
|
||||
status: self.status,
|
||||
decision: self.decision,
|
||||
context_omitted: false,
|
||||
}
|
||||
.store();
|
||||
}
|
||||
}
|
||||
|
||||
impl ReviewFeedbackRecord<'_> {
|
||||
fn store(mut self) {
|
||||
let mut buffer = BoundedBuffer(Vec::new());
|
||||
if serde_json::to_writer(&mut buffer, &self).is_err() {
|
||||
// Preserve the action and decision even when optional reviewer history is too large.
|
||||
self.instructions = None;
|
||||
self.history.clear();
|
||||
self.context_omitted = true;
|
||||
buffer = BoundedBuffer(Vec::new());
|
||||
if serde_json::to_writer(&mut buffer, &self).is_err() {
|
||||
tracing::warn!("Guardian feedback action and decision exceed the record limit");
|
||||
return;
|
||||
}
|
||||
}
|
||||
record_guardian_review_failure(self.reviewed_thread_id, buffer.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct BoundedBuffer(Vec<u8>);
|
||||
|
||||
impl Write for BoundedBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.len() > MAX_RECORD_BYTES.saturating_sub(self.0.len()) {
|
||||
return Err(io::Error::other(
|
||||
"Guardian feedback record exceeds its size limit",
|
||||
));
|
||||
}
|
||||
self.0.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "feedback_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,53 @@
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn oversized_context_keeps_the_action_and_decision() -> anyhow::Result<()> {
|
||||
let thread_id = ThreadId::new();
|
||||
let reviewer_thread_id = ThreadId::new();
|
||||
let decision = r#"{"outcome":"deny","rationale":"Missing approval."}"#;
|
||||
let action = r#"{"command":"git push"}"#;
|
||||
let outcome = GuardianReviewSessionOutcome::Completed(Ok(Some(decision.to_owned())));
|
||||
let feedback = FailedReviewFeedback::for_outcome(
|
||||
&outcome,
|
||||
ReviewFeedbackSettings {
|
||||
enabled: true,
|
||||
ephemeral: false,
|
||||
},
|
||||
)
|
||||
.expect("denials retain feedback");
|
||||
feedback.store(ReviewFeedbackContext {
|
||||
reviewed_thread_id: thread_id,
|
||||
reviewed_turn_id: "parent-turn",
|
||||
target_item_id: Some("push-call"),
|
||||
reviewer_thread_id,
|
||||
model: "review-model",
|
||||
action,
|
||||
action_truncated: false,
|
||||
instructions: Some(&"x".repeat(MAX_RECORD_BYTES)),
|
||||
history: Vec::new(),
|
||||
});
|
||||
let contents = codex_feedback::guardian_review_failures(&[thread_id])
|
||||
.attachment
|
||||
.expect("failed-review record")
|
||||
.buffer;
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&contents)?,
|
||||
json!({
|
||||
"reviewed_thread_id": thread_id,
|
||||
"reviewed_turn_id": "parent-turn",
|
||||
"target_item_id": "push-call",
|
||||
"reviewer_thread_id": reviewer_thread_id,
|
||||
"model": "review-model",
|
||||
"status": "denied",
|
||||
"decision": decision,
|
||||
"action": action,
|
||||
"action_truncated": false,
|
||||
"instructions": null,
|
||||
"history": [],
|
||||
"context_omitted": true,
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -6,9 +6,12 @@ mod circuit_breaker;
|
||||
mod completion;
|
||||
mod deadline;
|
||||
mod execution;
|
||||
mod feedback;
|
||||
mod metrics;
|
||||
mod model;
|
||||
mod outcome;
|
||||
mod pool;
|
||||
mod reporting;
|
||||
mod retry;
|
||||
mod review;
|
||||
mod settings;
|
||||
@@ -53,3 +56,10 @@ pub use execution::wait_for_guardian_review;
|
||||
pub use settings::ReviewerConfigOverrides;
|
||||
pub use settings::ReviewerTurn;
|
||||
pub use settings::reviewer_config_overrides;
|
||||
|
||||
pub use feedback::FailedReviewFeedback;
|
||||
pub use feedback::ReviewFeedbackContext;
|
||||
pub use feedback::ReviewFeedbackSettings;
|
||||
pub use reporting::ReviewDenials;
|
||||
pub use reporting::ReviewMetadata;
|
||||
pub use reporting::ReviewReport;
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
//! Emits Guardian metrics with bounded tags and the existing telemetry schema.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_analytics::GuardianApprovalRequestSource;
|
||||
use codex_analytics::GuardianReviewAnalyticsResult;
|
||||
use codex_analytics::GuardianReviewDecision;
|
||||
use codex_analytics::GuardianReviewFailureReason;
|
||||
use codex_analytics::GuardianReviewSessionKind;
|
||||
use codex_analytics::GuardianReviewTerminalStatus;
|
||||
use codex_analytics::GuardianReviewedAction;
|
||||
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
||||
use codex_otel::GUARDIAN_REVIEW_COUNT_METRIC;
|
||||
use codex_otel::GUARDIAN_REVIEW_DURATION_METRIC;
|
||||
use codex_otel::GUARDIAN_REVIEW_TOKEN_USAGE_METRIC;
|
||||
use codex_otel::GUARDIAN_REVIEW_TTFT_DURATION_METRIC;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_otel::sanitize_metric_tag_value;
|
||||
use codex_protocol::protocol::GuardianAssessmentOutcome;
|
||||
use codex_protocol::protocol::GuardianRiskLevel;
|
||||
use codex_protocol::protocol::GuardianUserAuthorization;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
pub(crate) fn emit_guardian_review_metrics(
|
||||
session_telemetry: &SessionTelemetry,
|
||||
result: &GuardianReviewAnalyticsResult,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: &GuardianReviewedAction,
|
||||
completion_latency_ms: u64,
|
||||
) {
|
||||
let tags = guardian_review_metric_tags(result, approval_request_source, reviewed_action);
|
||||
let tag_refs: Vec<(&str, &str)> = tags
|
||||
.iter()
|
||||
.map(|(key, value)| (*key, value.as_str()))
|
||||
.collect();
|
||||
|
||||
// Custom MCP tool names may contain user data; only attribute tools from the OpenAI Apps server.
|
||||
let mcp_tags = if let GuardianReviewedAction::McpToolCall {
|
||||
server, tool_name, ..
|
||||
} = reviewed_action
|
||||
&& server == CODEX_APPS_MCP_SERVER_NAME
|
||||
{
|
||||
vec![("tool", sanitize_metric_tag_value(tool_name))]
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut counter_tag_refs = tag_refs.clone();
|
||||
counter_tag_refs.extend(mcp_tags.iter().map(|(key, value)| (*key, value.as_str())));
|
||||
session_telemetry.counter(
|
||||
GUARDIAN_REVIEW_COUNT_METRIC,
|
||||
/*inc*/ 1,
|
||||
&counter_tag_refs,
|
||||
);
|
||||
session_telemetry.record_duration(
|
||||
GUARDIAN_REVIEW_DURATION_METRIC,
|
||||
Duration::from_millis(completion_latency_ms),
|
||||
&tag_refs,
|
||||
);
|
||||
|
||||
if let Some(time_to_first_token_ms) = result.time_to_first_token_ms {
|
||||
session_telemetry.record_duration(
|
||||
GUARDIAN_REVIEW_TTFT_DURATION_METRIC,
|
||||
Duration::from_millis(time_to_first_token_ms),
|
||||
&tag_refs,
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(token_usage) = result.token_usage.as_ref() {
|
||||
emit_guardian_token_usage_histograms(session_telemetry, token_usage, tags);
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_guardian_token_usage_histograms(
|
||||
session_telemetry: &SessionTelemetry,
|
||||
token_usage: &TokenUsage,
|
||||
base_tags: Vec<(&'static str, String)>,
|
||||
) {
|
||||
for (token_type, value) in [
|
||||
("total", token_usage.total_tokens.max(0)),
|
||||
("input", token_usage.input_tokens.max(0)),
|
||||
("cached_input", token_usage.cached_input()),
|
||||
(
|
||||
"cache_write_input",
|
||||
token_usage.cache_write_input_tokens.max(0),
|
||||
),
|
||||
("non_cached_input", token_usage.non_cached_input()),
|
||||
("output", token_usage.output_tokens.max(0)),
|
||||
(
|
||||
"reasoning_output",
|
||||
token_usage.reasoning_output_tokens.max(0),
|
||||
),
|
||||
] {
|
||||
let mut tags = base_tags.clone();
|
||||
tags.push(("token_type", token_type.to_string()));
|
||||
let tag_refs: Vec<(&str, &str)> = tags
|
||||
.iter()
|
||||
.map(|(key, value)| (*key, value.as_str()))
|
||||
.collect();
|
||||
session_telemetry.histogram(GUARDIAN_REVIEW_TOKEN_USAGE_METRIC, value, &tag_refs);
|
||||
}
|
||||
}
|
||||
|
||||
fn guardian_review_metric_tags(
|
||||
result: &GuardianReviewAnalyticsResult,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: &GuardianReviewedAction,
|
||||
) -> Vec<(&'static str, String)> {
|
||||
vec![
|
||||
("decision", decision_tag(result.decision).to_string()),
|
||||
(
|
||||
"terminal_status",
|
||||
terminal_status_tag(result.terminal_status).to_string(),
|
||||
),
|
||||
(
|
||||
"failure_reason",
|
||||
failure_reason_tag(result.failure_reason).to_string(),
|
||||
),
|
||||
(
|
||||
"approval_request_source",
|
||||
approval_request_source_tag(approval_request_source).to_string(),
|
||||
),
|
||||
("action", reviewed_action_tag(reviewed_action).to_string()),
|
||||
(
|
||||
"session_kind",
|
||||
session_kind_tag(result.guardian_session_kind).to_string(),
|
||||
),
|
||||
(
|
||||
"had_prior_review_context",
|
||||
optional_bool_tag(result.had_prior_review_context).to_string(),
|
||||
),
|
||||
(
|
||||
"reviewed_action_truncated",
|
||||
bool_tag(result.reviewed_action_truncated).to_string(),
|
||||
),
|
||||
("risk_level", risk_level_tag(result.risk_level).to_string()),
|
||||
(
|
||||
"user_authorization",
|
||||
user_authorization_tag(result.user_authorization).to_string(),
|
||||
),
|
||||
("outcome", outcome_tag(result.outcome).to_string()),
|
||||
(
|
||||
"guardian_model",
|
||||
result
|
||||
.guardian_model
|
||||
.as_deref()
|
||||
.map(sanitize_metric_tag_value)
|
||||
.unwrap_or_else(|| "none".to_string()),
|
||||
),
|
||||
(
|
||||
"guardian_reasoning_effort",
|
||||
result
|
||||
.guardian_reasoning_effort
|
||||
.as_deref()
|
||||
.map(sanitize_metric_tag_value)
|
||||
.unwrap_or_else(|| "none".to_string()),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn decision_tag(decision: GuardianReviewDecision) -> &'static str {
|
||||
match decision {
|
||||
GuardianReviewDecision::Approved => "approved",
|
||||
GuardianReviewDecision::Denied => "denied",
|
||||
GuardianReviewDecision::Aborted => "aborted",
|
||||
}
|
||||
}
|
||||
|
||||
fn terminal_status_tag(status: GuardianReviewTerminalStatus) -> &'static str {
|
||||
match status {
|
||||
GuardianReviewTerminalStatus::Approved => "approved",
|
||||
GuardianReviewTerminalStatus::Denied => "denied",
|
||||
GuardianReviewTerminalStatus::Aborted => "aborted",
|
||||
GuardianReviewTerminalStatus::TimedOut => "timed_out",
|
||||
GuardianReviewTerminalStatus::FailedClosed => "failed_closed",
|
||||
}
|
||||
}
|
||||
|
||||
fn failure_reason_tag(reason: Option<GuardianReviewFailureReason>) -> &'static str {
|
||||
match reason {
|
||||
Some(GuardianReviewFailureReason::Timeout) => "timeout",
|
||||
Some(GuardianReviewFailureReason::Cancelled) => "cancelled",
|
||||
Some(GuardianReviewFailureReason::PromptBuildError) => "prompt_build_error",
|
||||
Some(GuardianReviewFailureReason::SessionError) => "session_error",
|
||||
Some(GuardianReviewFailureReason::ParseError) => "parse_error",
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
fn approval_request_source_tag(source: GuardianApprovalRequestSource) -> &'static str {
|
||||
match source {
|
||||
GuardianApprovalRequestSource::MainTurn => "main_turn",
|
||||
GuardianApprovalRequestSource::DelegatedSubagent => "delegated_subagent",
|
||||
}
|
||||
}
|
||||
|
||||
fn reviewed_action_tag(action: &GuardianReviewedAction) -> &'static str {
|
||||
match action {
|
||||
GuardianReviewedAction::Shell { .. } => "shell",
|
||||
GuardianReviewedAction::UnifiedExec { .. } => "unified_exec",
|
||||
GuardianReviewedAction::WriteStdin { .. } => "write_stdin",
|
||||
GuardianReviewedAction::Execve { .. } => "execve",
|
||||
GuardianReviewedAction::ApplyPatch {} => "apply_patch",
|
||||
GuardianReviewedAction::NetworkAccess { .. } => "network_access",
|
||||
GuardianReviewedAction::McpToolCall { .. } => "mcp_tool_call",
|
||||
GuardianReviewedAction::RequestPermissions {} => "request_permissions",
|
||||
}
|
||||
}
|
||||
|
||||
fn session_kind_tag(kind: Option<GuardianReviewSessionKind>) -> &'static str {
|
||||
match kind {
|
||||
Some(GuardianReviewSessionKind::TrunkNew) => "trunk_new",
|
||||
Some(GuardianReviewSessionKind::TrunkReused) => "trunk_reused",
|
||||
Some(GuardianReviewSessionKind::EphemeralForked) => "ephemeral_forked",
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_bool_tag(value: Option<bool>) -> &'static str {
|
||||
match value {
|
||||
Some(true) => "true",
|
||||
Some(false) => "false",
|
||||
None => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn bool_tag(value: bool) -> &'static str {
|
||||
if value { "true" } else { "false" }
|
||||
}
|
||||
|
||||
fn risk_level_tag(risk_level: Option<GuardianRiskLevel>) -> &'static str {
|
||||
match risk_level {
|
||||
Some(GuardianRiskLevel::Low) => "low",
|
||||
Some(GuardianRiskLevel::Medium) => "medium",
|
||||
Some(GuardianRiskLevel::High) => "high",
|
||||
Some(GuardianRiskLevel::Critical) => "critical",
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
fn user_authorization_tag(user_authorization: Option<GuardianUserAuthorization>) -> &'static str {
|
||||
match user_authorization {
|
||||
Some(GuardianUserAuthorization::Unknown) => "unknown",
|
||||
Some(GuardianUserAuthorization::Low) => "low",
|
||||
Some(GuardianUserAuthorization::Medium) => "medium",
|
||||
Some(GuardianUserAuthorization::High) => "high",
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
fn outcome_tag(outcome: Option<GuardianAssessmentOutcome>) -> &'static str {
|
||||
match outcome {
|
||||
Some(GuardianAssessmentOutcome::Allow) => "allow",
|
||||
Some(GuardianAssessmentOutcome::Deny) => "deny",
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use codex_otel::MetricsClient;
|
||||
use codex_otel::MetricsConfig;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
use opentelemetry_sdk::metrics::data::Metric;
|
||||
use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use opentelemetry_sdk::metrics::data::ResourceMetrics;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::collections::BTreeMap;
|
||||
use test_case::test_case;
|
||||
|
||||
fn test_session_telemetry() -> SessionTelemetry {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "codex-core", env!("CARGO_PKG_VERSION"), exporter)
|
||||
.with_runtime_reader(),
|
||||
)
|
||||
.expect("in-memory metrics client");
|
||||
SessionTelemetry::new(
|
||||
ThreadId::new(),
|
||||
"gpt-5.4",
|
||||
"gpt-5.4",
|
||||
/*account_id*/ None,
|
||||
/*account_email*/ None,
|
||||
/*auth_mode*/ None,
|
||||
"test_originator".to_string(),
|
||||
/*log_user_prompts*/ false,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics_without_metadata_tags(metrics)
|
||||
}
|
||||
|
||||
fn find_metric<'a>(resource_metrics: &'a ResourceMetrics, name: &str) -> &'a Metric {
|
||||
for scope_metrics in resource_metrics.scope_metrics() {
|
||||
for metric in scope_metrics.metrics() {
|
||||
if metric.name() == name {
|
||||
return metric;
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("metric {name} missing");
|
||||
}
|
||||
|
||||
fn attributes_to_map<'a>(
|
||||
attributes: impl Iterator<Item = &'a KeyValue>,
|
||||
) -> BTreeMap<String, String> {
|
||||
attributes
|
||||
.map(|kv| (kv.key.as_str().to_string(), kv.value.as_str().to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn counter_point(
|
||||
resource_metrics: &ResourceMetrics,
|
||||
name: &str,
|
||||
) -> (BTreeMap<String, String>, u64) {
|
||||
let metric = find_metric(resource_metrics, name);
|
||||
match metric.data() {
|
||||
AggregatedMetrics::U64(data) => match data {
|
||||
MetricData::Sum(sum) => {
|
||||
let points: Vec<_> = sum.data_points().collect();
|
||||
assert_eq!(points.len(), 1);
|
||||
let point = points[0];
|
||||
(attributes_to_map(point.attributes()), point.value())
|
||||
}
|
||||
_ => panic!("unexpected counter aggregation"),
|
||||
},
|
||||
_ => panic!("unexpected counter data type"),
|
||||
}
|
||||
}
|
||||
|
||||
fn histogram_sums(
|
||||
resource_metrics: &ResourceMetrics,
|
||||
name: &str,
|
||||
expected_base_tags: &BTreeMap<String, String>,
|
||||
) -> BTreeMap<String, u64> {
|
||||
let metric = find_metric(resource_metrics, name);
|
||||
match metric.data() {
|
||||
AggregatedMetrics::F64(data) => match data {
|
||||
MetricData::Histogram(histogram) => histogram
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
let mut attrs = attributes_to_map(point.attributes());
|
||||
let token_type = attrs
|
||||
.remove("token_type")
|
||||
.unwrap_or_else(|| "sample".to_string());
|
||||
assert_eq!(&attrs, expected_base_tags);
|
||||
(token_type, point.sum() as u64)
|
||||
})
|
||||
.collect(),
|
||||
_ => panic!("unexpected histogram aggregation"),
|
||||
},
|
||||
_ => panic!("unexpected histogram data type"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case(
|
||||
GuardianReviewedAction::NetworkAccess {
|
||||
protocol: codex_protocol::approvals::NetworkApprovalProtocol::Https,
|
||||
port: 443,
|
||||
},
|
||||
"network_access",
|
||||
BTreeMap::new();
|
||||
"network access"
|
||||
)]
|
||||
#[test_case(
|
||||
GuardianReviewedAction::McpToolCall {
|
||||
server: CODEX_APPS_MCP_SERVER_NAME.to_string(),
|
||||
tool_name: "search docs".to_string(),
|
||||
connector_id: None,
|
||||
connector_name: None,
|
||||
tool_title: None,
|
||||
},
|
||||
"mcp_tool_call",
|
||||
BTreeMap::from([("tool".to_string(), "search_docs".to_string())]);
|
||||
"openai apps tool call"
|
||||
)]
|
||||
#[test_case(
|
||||
GuardianReviewedAction::McpToolCall {
|
||||
server: "codex_apps_custom".to_string(),
|
||||
tool_name: "search_private_docs".to_string(),
|
||||
connector_id: None,
|
||||
connector_name: None,
|
||||
tool_title: None,
|
||||
},
|
||||
"mcp_tool_call",
|
||||
BTreeMap::new();
|
||||
"custom mcp tool call"
|
||||
)]
|
||||
fn guardian_review_metrics_record_counts_durations_and_token_usage(
|
||||
reviewed_action: GuardianReviewedAction,
|
||||
expected_action: &str,
|
||||
expected_mcp_tags: BTreeMap<String, String>,
|
||||
) {
|
||||
let session_telemetry = test_session_telemetry();
|
||||
let result = GuardianReviewAnalyticsResult {
|
||||
decision: GuardianReviewDecision::Approved,
|
||||
terminal_status: GuardianReviewTerminalStatus::Approved,
|
||||
risk_level: Some(GuardianRiskLevel::Low),
|
||||
user_authorization: Some(GuardianUserAuthorization::High),
|
||||
outcome: Some(GuardianAssessmentOutcome::Allow),
|
||||
guardian_session_kind: Some(GuardianReviewSessionKind::TrunkReused),
|
||||
guardian_model: Some("gpt-5.4 guardian".to_string()),
|
||||
guardian_reasoning_effort: Some("low".to_string()),
|
||||
had_prior_review_context: Some(true),
|
||||
reviewed_action_truncated: true,
|
||||
token_usage: Some(TokenUsage {
|
||||
input_tokens: 10,
|
||||
cached_input_tokens: 4,
|
||||
cache_write_input_tokens: 2,
|
||||
output_tokens: 3,
|
||||
reasoning_output_tokens: 2,
|
||||
total_tokens: 15,
|
||||
codex_rollout_budget_units: None,
|
||||
}),
|
||||
time_to_first_token_ms: Some(123),
|
||||
..GuardianReviewAnalyticsResult::without_session()
|
||||
};
|
||||
|
||||
emit_guardian_review_metrics(
|
||||
&session_telemetry,
|
||||
&result,
|
||||
GuardianApprovalRequestSource::DelegatedSubagent,
|
||||
&reviewed_action,
|
||||
/*completion_latency_ms*/ 456,
|
||||
);
|
||||
|
||||
let snapshot = session_telemetry
|
||||
.snapshot_metrics()
|
||||
.expect("runtime metrics snapshot");
|
||||
let (attrs, value) = counter_point(&snapshot, GUARDIAN_REVIEW_COUNT_METRIC);
|
||||
|
||||
let expected_base_tags = BTreeMap::from([
|
||||
("action".to_string(), expected_action.to_string()),
|
||||
(
|
||||
"approval_request_source".to_string(),
|
||||
"delegated_subagent".to_string(),
|
||||
),
|
||||
("decision".to_string(), "approved".to_string()),
|
||||
("failure_reason".to_string(), "none".to_string()),
|
||||
("guardian_model".to_string(), "gpt-5.4_guardian".to_string()),
|
||||
("guardian_reasoning_effort".to_string(), "low".to_string()),
|
||||
("had_prior_review_context".to_string(), "true".to_string()),
|
||||
("outcome".to_string(), "allow".to_string()),
|
||||
("reviewed_action_truncated".to_string(), "true".to_string()),
|
||||
("risk_level".to_string(), "low".to_string()),
|
||||
("session_kind".to_string(), "trunk_reused".to_string()),
|
||||
("terminal_status".to_string(), "approved".to_string()),
|
||||
("user_authorization".to_string(), "high".to_string()),
|
||||
]);
|
||||
let mut expected_counter_tags = expected_base_tags.clone();
|
||||
expected_counter_tags.extend(expected_mcp_tags);
|
||||
assert_eq!((attrs, value), (expected_counter_tags, 1));
|
||||
|
||||
assert_eq!(
|
||||
histogram_sums(
|
||||
&snapshot,
|
||||
GUARDIAN_REVIEW_TOKEN_USAGE_METRIC,
|
||||
&expected_base_tags
|
||||
),
|
||||
BTreeMap::from([
|
||||
("cached_input".to_string(), 4),
|
||||
("cache_write_input".to_string(), 2),
|
||||
("input".to_string(), 10),
|
||||
("non_cached_input".to_string(), 6),
|
||||
("output".to_string(), 3),
|
||||
("reasoning_output".to_string(), 2),
|
||||
("total".to_string(), 15),
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_sums(
|
||||
&snapshot,
|
||||
GUARDIAN_REVIEW_DURATION_METRIC,
|
||||
&expected_base_tags
|
||||
),
|
||||
BTreeMap::from([("sample".to_string(), 456)])
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_sums(
|
||||
&snapshot,
|
||||
GUARDIAN_REVIEW_TTFT_DURATION_METRIC,
|
||||
&expected_base_tags
|
||||
),
|
||||
BTreeMap::from([("sample".to_string(), 123)])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Owns review reporting and the per-thread denial window. The host publishes
|
||||
//! events and enforces interruption only on the still-active owning turn.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_analytics::GuardianApprovalRequestSource;
|
||||
use codex_analytics::GuardianReviewAnalyticsResult;
|
||||
use codex_analytics::GuardianReviewTrackContext;
|
||||
use codex_analytics::GuardianReviewedAction;
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_protocol::approvals::GuardianAssessmentAction;
|
||||
use codex_protocol::approvals::GuardianReviewReason;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
use codex_protocol::protocol::GuardianAssessmentDecisionSource;
|
||||
use codex_protocol::protocol::GuardianAssessmentEvent;
|
||||
use codex_protocol::protocol::GuardianAssessmentStatus;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::GuardianRejectionCircuitBreaker;
|
||||
use crate::GuardianRejectionCircuitBreakerAction;
|
||||
use crate::GuardianRejectionCircuitBreakerPolicy;
|
||||
use crate::GuardianReviewOutcome;
|
||||
use crate::ReviewCompletion;
|
||||
|
||||
/// Captured action attribution used for both the started and terminal report.
|
||||
pub struct ReviewMetadata {
|
||||
pub thread_id: String,
|
||||
pub turn_id: String,
|
||||
pub review_id: String,
|
||||
pub target_item_id: Option<String>,
|
||||
pub plugin_id: Option<String>,
|
||||
pub script_path: Option<String>,
|
||||
pub approval_request_source: GuardianApprovalRequestSource,
|
||||
pub reviewed_action: GuardianReviewedAction,
|
||||
pub action: GuardianAssessmentAction,
|
||||
pub review_reason: GuardianReviewReason,
|
||||
}
|
||||
|
||||
pub struct ReviewReport {
|
||||
started: GuardianAssessmentEvent,
|
||||
tracking: GuardianReviewTrackContext,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
reviewed_action: GuardianReviewedAction,
|
||||
}
|
||||
|
||||
impl ReviewReport {
|
||||
pub fn new(metadata: ReviewMetadata) -> Self {
|
||||
let tracking = GuardianReviewTrackContext::new(
|
||||
metadata.thread_id,
|
||||
metadata.turn_id.clone(),
|
||||
metadata.review_id.clone(),
|
||||
metadata.target_item_id.clone(),
|
||||
metadata.approval_request_source,
|
||||
metadata.reviewed_action.clone(),
|
||||
crate::REVIEW_TIMEOUT.as_millis() as u64,
|
||||
);
|
||||
let started = GuardianAssessmentEvent {
|
||||
review_reason: Some(metadata.review_reason),
|
||||
id: metadata.review_id,
|
||||
target_item_id: metadata.target_item_id,
|
||||
plugin_id: metadata.plugin_id,
|
||||
script_path: metadata.script_path,
|
||||
turn_id: metadata.turn_id,
|
||||
started_at_ms: tracking.started_at_ms.try_into().unwrap_or_default(),
|
||||
completed_at_ms: None,
|
||||
status: GuardianAssessmentStatus::InProgress,
|
||||
risk_level: None,
|
||||
user_authorization: None,
|
||||
rationale: None,
|
||||
decision_source: None,
|
||||
action: metadata.action,
|
||||
};
|
||||
Self {
|
||||
started,
|
||||
tracking,
|
||||
approval_request_source: metadata.approval_request_source,
|
||||
reviewed_action: metadata.reviewed_action,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn turn_id(&self) -> &str {
|
||||
&self.started.turn_id
|
||||
}
|
||||
|
||||
pub fn started_event(&self) -> GuardianAssessmentEvent {
|
||||
self.started.clone()
|
||||
}
|
||||
|
||||
pub fn complete(
|
||||
&self,
|
||||
outcome: GuardianReviewOutcome,
|
||||
model: &ModelInfo,
|
||||
analytics: GuardianReviewAnalyticsResult,
|
||||
completed_at_ms: i64,
|
||||
) -> ReviewCompletion {
|
||||
let mut event = self.started.clone();
|
||||
event.completed_at_ms = Some(completed_at_ms);
|
||||
event.decision_source = Some(GuardianAssessmentDecisionSource::Agent);
|
||||
crate::complete_review(outcome, model, event, analytics)
|
||||
}
|
||||
|
||||
pub fn track(
|
||||
&self,
|
||||
telemetry: &SessionTelemetry,
|
||||
analytics: &AnalyticsEventsClient,
|
||||
result: GuardianReviewAnalyticsResult,
|
||||
completed_at_ms: u64,
|
||||
) {
|
||||
crate::metrics::emit_guardian_review_metrics(
|
||||
telemetry,
|
||||
&result,
|
||||
self.approval_request_source,
|
||||
&self.reviewed_action,
|
||||
completed_at_ms.saturating_sub(self.tracking.started_at_ms),
|
||||
);
|
||||
analytics.track_guardian_review(&self.tracking, result, completed_at_ms);
|
||||
}
|
||||
}
|
||||
|
||||
/// Thread-scoped accounting shared by synchronous and cached approvals.
|
||||
#[derive(Default)]
|
||||
pub struct ReviewDenials(Mutex<GuardianRejectionCircuitBreaker>);
|
||||
|
||||
impl ReviewDenials {
|
||||
pub fn for_thread(store: &ExtensionData) -> Arc<Self> {
|
||||
store.get_or_init(Self::default)
|
||||
}
|
||||
|
||||
pub async fn clear_turn(store: &ExtensionData, turn_id: &str) {
|
||||
if let Some(state) = store.get::<Self>() {
|
||||
state.0.lock().await.clear_turn(turn_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn record_non_denial(&self, turn_id: &str) {
|
||||
self.0.lock().await.record_non_denial(turn_id);
|
||||
}
|
||||
|
||||
/// Returns a warning exactly once when this turn reaches its denial limit.
|
||||
pub async fn record_denial(&self, turn_id: &str, model: &ModelInfo) -> Option<String> {
|
||||
let action = self
|
||||
.0
|
||||
.lock()
|
||||
.await
|
||||
.record_denial(turn_id, GuardianRejectionCircuitBreakerPolicy::from(model));
|
||||
match action {
|
||||
GuardianRejectionCircuitBreakerAction::Continue => None,
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials,
|
||||
recent_denials,
|
||||
} => {
|
||||
let window = crate::AUTO_REVIEW_DENIAL_WINDOW_SIZE;
|
||||
Some(format!(
|
||||
"Automatic approval review rejected too many approval requests for this turn ({consecutive_denials} consecutive, {recent_denials} in the last {window} reviews); interrupting the turn."
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user