openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-02 19:43:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Guardian V2 analytics events (#42144)

Browse Source 
## What changed

- Emit `codex_guardian_v2_classification` events with the outcome, risk
  level, duration, model, and thread attribution.
- Emit `codex_guardian_v2_fast_decision` events when Guardian V2 approves an
  action without a full review.
- Add `guardian_v2_enabled` to turn analytics, accounting for review scope
  and model requirements.

## Testing

- Extend the Guardian V2 app-server coverage to verify event payloads,
  attribution, enablement state, and scoped event emission.

GitOrigin-RevId: 0805975f8e8f9d954aa1058ea8c60545a090ae74
This commit is contained in:
jif
2026-09-01 22:02:42 +00:00
committed by copyberry
parent a876a9e494
commit 86b7d12742
17 changed files with 272 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/Cargo.lock
Generated
+1
View File
@@ -3382,6 +3382,7 @@ name = "codex-guardian-v2"
version = "0.0.0"
dependencies = [
"anyhow",
"codex-analytics",
"codex-api",
"codex-core",
"codex-extension-api",
codex-rs/analytics/src/analytics_client_tests.rs
+4
View File
@@ -480,6 +480,7 @@ fn sample_turn_resolved_config(thread_id: &str, turn_id: &str) -> TurnResolvedCo
service_tier: None,
approval_policy: AskForApproval::OnRequest,
approvals_reviewer: ApprovalsReviewer::AutoReview,
guardian_v2_enabled: false,
sandbox_network_access: true,
collaboration_mode: ModeKind::Plan,
personality: None,
@@ -4674,6 +4675,7 @@ fn turn_event_serializes_expected_shape() {
service_tier: "flex".to_string(),
approval_policy: "on-request".to_string(),
approvals_reviewer: "auto_review".to_string(),
guardian_v2_enabled: true,
sandbox_network_access: true,
collaboration_mode: Some("plan"),
personality: Some("pragmatic".to_string()),
@@ -4761,6 +4763,7 @@ fn turn_event_serializes_expected_shape() {
"service_tier": "flex",
"approval_policy": "on-request",
"approvals_reviewer": "auto_review",
"guardian_v2_enabled": true,
"sandbox_network_access": true,
"collaboration_mode": "plan",
"personality": "pragmatic",
@@ -5114,6 +5117,7 @@ async fn turn_lifecycle_emits_turn_event() {
})
);
assert!(payload["event_params"].get("product_client_id").is_none());
assert_eq!(payload["event_params"]["guardian_v2_enabled"], json!(false));
assert_eq!(payload["event_params"]["ephemeral"], json!(false));
assert_eq!(payload["event_params"]["workspace_kind"], json!(null));
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
codex-rs/analytics/src/client.rs
+7
View File
@@ -33,6 +33,7 @@ use crate::facts::TurnCodexErrorFact;
use crate::facts::TurnProfileFact;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnTokenUsageFact;
use crate::guardian_v2::GuardianV2Event;
use crate::now_unix_millis;
use crate::reducer::AnalyticsReducer;
use crate::reducer::MAX_PLUGIN_MEASUREMENTS_PER_BATCH;
@@ -492,6 +493,12 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_guardian_v2_event(&self, event: GuardianV2Event) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianV2(
Box::new(event),
)));
}
pub fn track_goal_event(&self, event: CodexGoalEvent) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(
event,
codex-rs/analytics/src/events.rs
+3
View File
@@ -25,6 +25,7 @@ use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerResult;
use crate::facts::TurnSubmissionType;
use crate::guardian_v2::GuardianV2EventRequest;
use crate::now_unix_millis;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::CommandExecutionSource;
@@ -69,6 +70,7 @@ pub(crate) enum TrackEventRequest {
ThreadInitialized(ThreadInitializedEvent),
ThreadArchive(ThreadArchiveEvent),
GuardianReview(Box<GuardianReviewEventRequest>),
GuardianV2(Box<GuardianV2EventRequest>),
AppMentioned(CodexAppMentionedEventRequest),
AppUsed(CodexAppUsedEventRequest),
HookRun(CodexHookRunEventRequest),
@@ -1032,6 +1034,7 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) service_tier: String,
pub(crate) approval_policy: String,
pub(crate) approvals_reviewer: String,
pub(crate) guardian_v2_enabled: bool,
pub(crate) sandbox_network_access: bool,
pub(crate) collaboration_mode: Option<&'static str>,
pub(crate) personality: Option<String>,
codex-rs/analytics/src/facts.rs
+3
View File
@@ -1,6 +1,7 @@
use crate::events::AppServerRpcTransport;
use crate::events::CodexRuntimeMetadata;
use crate::events::GuardianReviewEventParams;
use crate::guardian_v2::GuardianV2Event;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponsePayload;
use codex_app_server_protocol::InitializeParams;
@@ -199,6 +200,7 @@ pub struct TurnResolvedConfigFact {
pub service_tier: Option<ServiceTier>,
pub approval_policy: AskForApproval,
pub approvals_reviewer: ApprovalsReviewer,
pub guardian_v2_enabled: bool,
pub sandbox_network_access: bool,
pub collaboration_mode: ModeKind,
pub personality: Option<Personality>,
@@ -565,6 +567,7 @@ pub(crate) enum CustomAnalyticsFact {
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
GuardianReview(Box<GuardianReviewEventParams>),
GuardianV2(Box<GuardianV2Event>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),
TurnProfile(Box<TurnProfileFact>),
codex-rs/analytics/src/guardian_v2.rs
+49
View File
@@ -0,0 +1,49 @@
//! Guardian V2 classifier and fast-decision facts, enriched by the existing reducer.
//! Payloads contain attribution and bounded outcomes, never prompts or tool arguments.
use crate::events::CodexAppServerClientMetadata;
use crate::events::CodexRuntimeMetadata;
use codex_protocol::protocol::ThreadSource;
use serde::Serialize;
#[derive(Serialize)]
pub struct GuardianV2Event {
pub thread_id: String,
pub turn_id: String,
pub item_id: Option<String>,
pub model: Option<String>,
pub occurred_at_ms: u64,
#[serde(flatten)]
pub kind: GuardianV2EventKind,
}
#[derive(Serialize)]
#[serde(untagged)]
pub enum GuardianV2EventKind {
Classification {
outcome: &'static str,
risk_level: Option<&'static str>,
duration_ms: u64,
},
FastDecision {
decision: &'static str,
},
}
#[derive(Serialize)]
pub(crate) struct GuardianV2EventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: GuardianV2EventParams,
}
#[derive(Serialize)]
pub(crate) struct GuardianV2EventParams {
pub(crate) session_id: String,
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
pub(crate) thread_source: Option<ThreadSource>,
pub(crate) subagent_source: Option<String>,
pub(crate) parent_thread_id: Option<String>,
#[serde(flatten)]
pub(crate) guardian_v2: GuardianV2Event,
}
codex-rs/analytics/src/lib.rs
+3
View File
@@ -4,6 +4,7 @@ mod analytics_capture;
mod client;
mod events;
mod facts;
mod guardian_v2;
mod reducer;
use std::time::SystemTime;
@@ -71,6 +72,8 @@ pub use facts::TurnSteerRequestError;
pub use facts::TurnSteerResult;
pub use facts::TurnTokenUsageFact;
pub use facts::build_track_events_context;
pub use guardian_v2::GuardianV2Event;
pub use guardian_v2::GuardianV2EventKind;
#[cfg(test)]
mod analytics_client_tests;
codex-rs/analytics/src/reducer.rs
+39
View File
@@ -113,6 +113,9 @@ use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerResult;
use crate::facts::TurnTokenUsageFact;
use crate::guardian_v2::GuardianV2EventKind;
use crate::guardian_v2::GuardianV2EventParams;
use crate::guardian_v2::GuardianV2EventRequest;
use crate::now_unix_millis;
use crate::now_unix_seconds;
use crate::option_i64_to_u64;
@@ -618,6 +621,40 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Goal(input) => {
self.ingest_goal(*input, out);
}
CustomAnalyticsFact::GuardianV2(input) => {
let event_type = match &input.kind {
GuardianV2EventKind::Classification { .. } => {
"codex_guardian_v2_classification"
}
GuardianV2EventKind::FastDecision { .. } => {
"codex_guardian_v2_fast_decision"
}
};
if let Some((connection, thread, metadata)) =
self.thread_context_or_warn(AnalyticsDropSite {
event_name: event_type,
thread_id: &input.thread_id,
turn_id: Some(&input.turn_id),
review_id: None,
item_id: input.item_id.as_deref(),
})
{
out.push(TrackEventRequest::GuardianV2(Box::new(
GuardianV2EventRequest {
event_type,
event_params: GuardianV2EventParams {
session_id: metadata.session_id.clone(),
app_server_client: thread.app_server_client(connection),
runtime: connection.runtime.clone(),
thread_source: metadata.thread_source.clone(),
subagent_source: metadata.subagent_source.clone(),
parent_thread_id: metadata.parent_thread_id.clone(),
guardian_v2: *input,
},
},
)));
}
}
CustomAnalyticsFact::GuardianReview(input) => {
self.ingest_guardian_review(*input, out);
}
@@ -3403,6 +3440,7 @@ fn codex_turn_event_params(
service_tier,
approval_policy,
approvals_reviewer,
guardian_v2_enabled,
sandbox_network_access,
collaboration_mode,
personality,
@@ -3449,6 +3487,7 @@ fn codex_turn_event_params(
.unwrap_or_else(|| "default".to_string()),
approval_policy: approval_policy.to_string(),
approvals_reviewer: approvals_reviewer.to_string(),
guardian_v2_enabled,
sandbox_network_access,
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
personality: personality_mode(personality),
codex-rs/app-server/tests/suite/v2/analytics.rs
+1
View File
@@ -198,6 +198,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
.expect("turn completion time");
assert!(started_at > 0 && completed_at >= started_at);
assert!(params["duration_ms"].is_u64());
assert_eq!(params["guardian_v2_enabled"], false);
json!([
params["turn_id"],
params["status"],
codex-rs/app-server/tests/suite/v2/guardian_v2.rs
+80 -1
View File
@@ -56,6 +56,9 @@ use tokio::net::TcpListener;
use tokio::sync::Notify;
use tokio::time::timeout;
use super::analytics::captured_analytics_events;
use super::analytics::mount_analytics_capture;
use super::analytics::wait_for_matching_analytics_event;
use super::mcp_tool::TEST_SERVER_NAME;
use super::mcp_tool::TEST_TOOL_NAME;
use super::mcp_tool::start_mcp_server;
@@ -633,6 +636,8 @@ async fn guardian_v2_routes_scoped_tool_approvals(
let (mcp_server_url, mcp_server_handle) = start_mcp_server(sensitive_action).await?;
let codex_home = TempDir::new()?;
let analytics_server = responses::start_mock_server().await;
mount_analytics_capture(&analytics_server, codex_home.path()).await?;
let root_skill = if matches!(lifecycle, ThreadLifecycle::RootTrustedSkill) {
let path = codex_home.path().join("skills/root-trusted/SKILL.md");
std::fs::create_dir_all(path.parent().expect("root skill parent"))?;
@@ -700,7 +705,10 @@ async fn guardian_v2_routes_scoped_tool_approvals(
.with_model(MODEL)
.with_provider_config("supports_websockets = false")
.with_approval_policy("on-request")
.with_root_config(reviewer_config)
.with_root_config(&format!(
"{reviewer_config}\nchatgpt_base_url = \"{}\"",
analytics_server.uri(),
))
.with_extra_config(&format!(
"[mcp_servers.{server_name}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"{tool_approval_mode}\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}{guardian_scope_config}"
))
@@ -1358,6 +1366,77 @@ async fn guardian_v2_routes_scoped_tool_approvals(
timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?;
}
if matches!(lifecycle, ThreadLifecycle::New)
&& matches!(scope, GuardianToolScope::AllTools)
&& sensitive_action.is_none()
{
if classifier_in_scope {
wait_for_matching_analytics_event(&analytics_server, TIMEOUT, |event| {
event["event_type"] == "codex_guardian_v2_classification"
&& event["event_params"]["item_id"] == "guardian-action-0"
})
.await?;
}
timeout(TIMEOUT, app_server.shutdown_gracefully()).await??;
let events = captured_analytics_events(&analytics_server).await;
let turn = &events
.iter()
.find(|event| {
event["event_type"] == "codex_turn_event"
&& event["event_params"]["thread_id"] == reviewed_thread_id
})
.expect("parent turn analytics")["event_params"];
assert_eq!(turn["guardian_v2_enabled"], classifier_in_scope);
let classification = events.iter().find(|event| {
event["event_type"] == "codex_guardian_v2_classification"
&& event["event_params"]["item_id"] == "guardian-action-0"
});
assert_eq!(classification.is_some(), classifier_in_scope);
if let Some(event) = classification {
assert_eq!(
json!([
event["event_params"]["outcome"],
event["event_params"]["risk_level"]
]),
match risk {
GuardianRisk::Low => json!(["success", "low"]),
GuardianRisk::Threshold | GuardianRisk::High => json!(["success", "high"]),
GuardianRisk::InvalidResponse => json!(["failure", null]),
}
);
}
let approvals = events
.iter()
.filter(|event| event["event_type"] == "codex_guardian_v2_fast_decision")
.collect::<Vec<_>>();
assert_eq!(
approvals.len(),
usize::from(classifier_in_scope && matches!(risk, GuardianRisk::Low))
);
for event in classification.into_iter().chain(approvals) {
let params = &event["event_params"];
for key in [
"session_id",
"thread_id",
"turn_id",
"model",
"app_server_client",
"runtime",
"thread_source",
"subagent_source",
"parent_thread_id",
] {
assert_eq!(params[key], turn[key], "{key}");
}
if event["event_type"] == "codex_guardian_v2_fast_decision" {
assert_eq!(
json!([params["item_id"], params["decision"]]),
json!(["guardian-action-1", "approved"])
);
}
}
}
mcp_server_handle.abort();
responses_server.abort();
Ok(())
codex-rs/core/src/guardian/review.rs
+23
View File
@@ -5,8 +5,11 @@ use codex_analytics::GuardianReviewFailureReason;
use codex_analytics::GuardianReviewTerminalStatus;
use codex_analytics::GuardianReviewTrackContext;
use codex_analytics::GuardianReviewedAction;
use codex_analytics::GuardianV2Event;
use codex_analytics::GuardianV2EventKind;
use codex_async_utils::THREAD_STACK_SIZE_BYTES;
use codex_core_plugins::PluginCommandAttribution;
use codex_extension_api::GuardianV2Enabled;
use codex_extension_api::ThreadIdleCause;
use codex_features::Feature;
use codex_protocol::config_types::ApprovalsReviewer;
@@ -362,6 +365,26 @@ async fn run_guardian_review(
.await
{
if decision == ReviewDecision::Approved {
if session
.services
.thread_extension_data
.get::<GuardianV2Enabled>()
.is_some()
{
session
.services
.analytics_events_client
.track_guardian_v2_event(GuardianV2Event {
thread_id: session.thread_id.to_string(),
turn_id: guardian_request_turn_id(&request, &turn.sub_id).to_owned(),
item_id: guardian_request_target_item_id(&request).map(str::to_owned),
model: Some(turn.model_info().slug.clone()),
occurred_at_ms: codex_analytics::now_unix_millis(),
kind: GuardianV2EventKind::FastDecision {
decision: "approved",
},
});
}
record_guardian_non_denial(&session, guardian_request_turn_id(&request, &turn.sub_id))
.await;
}
codex-rs/core/src/session/turn.rs
+12
View File
@@ -1029,6 +1029,18 @@ async fn track_turn_resolved_config_analytics(
.and_then(ServiceTier::from_request_value),
approval_policy: turn_context.approval_policy(),
approvals_reviewer: turn_context.config.approvals_reviewer,
guardian_v2_enabled: sess
.services
.thread_extension_data
.get::<codex_extension_api::GuardianV2Enabled>()
.is_some_and(|state| {
state.computer_use_only
|| !turn_context
.config
.config_layer_stack
.requirements()
.auto_review_required_for_model(&turn_context.model_info().slug)
}),
sandbox_network_access: turn_context.network_sandbox_policy().is_enabled(),
collaboration_mode: turn_context.mode(),
personality: turn_context.personality(),
codex-rs/ext/extension-api/src/contributors.rs
+1
View File
@@ -26,6 +26,7 @@ mod world_state;
pub use approval_review::ApprovalAssessment;
pub use approval_review::ApprovalReviewError;
pub use approval_review::ApprovalReviewInput;
pub use approval_review::GuardianV2Enabled;
pub use context::TurnContextContributionInput;
pub use mcp::McpServerContribution;
pub use mcp::McpServerContributionContext;
codex-rs/ext/extension-api/src/contributors/approval_review.rs
+6
View File
@@ -10,6 +10,12 @@ use serde_json::Value;
use crate::ConversationHistorySnapshot;
use crate::ExtensionData;
/// Thread-local state installed only after Guardian V2's async classifier initializes.
pub struct GuardianV2Enabled {
/// Computer-use classification remains active for models requiring synchronous review.
pub computer_use_only: bool,
}
/// Classification returned by an approval reviewer.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ApprovalAssessment {
codex-rs/ext/extension-api/src/lib.rs
+1
View File
@@ -47,6 +47,7 @@ pub use contributors::ApprovalReviewInput;
pub use contributors::ConfigContributor;
pub use contributors::ContextContributor;
pub use contributors::ExtensionFuture;
pub use contributors::GuardianV2Enabled;
pub use contributors::McpServerContribution;
pub use contributors::McpServerContributionContext;
pub use contributors::McpServerContributor;
codex-rs/ext/guardian-v2/Cargo.toml
+1
View File
@@ -13,6 +13,7 @@ doctest = false
workspace = true
[dependencies]
codex-analytics = { workspace = true }
codex-api = { workspace = true }
codex-core = { workspace = true }
codex-extension-api = { workspace = true }
codex-rs/ext/guardian-v2/src/async_scorer/extension.rs
+38 -19
View File
@@ -7,6 +7,9 @@ use std::sync::atomic::Ordering;
use std::time::Instant;
use std::time::SystemTime;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::GuardianV2Event;
use codex_analytics::GuardianV2EventKind;
use codex_core::ThreadManager;
use codex_core::config::Config;
use codex_core::context::GuardianReviewEvidence;
@@ -18,6 +21,7 @@ use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ExtensionWarning;
use codex_extension_api::GuardianV2Enabled;
use codex_extension_api::ResponseItem;
use codex_extension_api::SkillInvocationContributor;
use codex_extension_api::SkillInvocationInput;
@@ -103,8 +107,6 @@ pub enum StrictReviewReason {
StaleScore,
}
struct GuardianV2Enabled;
enum ClassificationOutcome {
Scored,
Superseded,
@@ -196,6 +198,10 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
let sampler = input
.thread_store
.get_or_init(|| LunaSampler::new(sampler_config));
let guardian_v2_enabled = GuardianV2Enabled {
computer_use_only: guardian_config.review_scope
== GuardianV2ReviewScope::ComputerUseOnly,
};
input.thread_store.insert(guardian_config);
input.thread_store.insert(GuardianV2ScoreProgress {
metrics: input.extension_metrics.clone(),
@@ -205,7 +211,7 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
input
.thread_store
.insert(TrustedSkillRoots::from_config(input.config));
input.thread_store.insert(GuardianV2Enabled);
input.thread_store.insert(guardian_v2_enabled);
tokio::spawn(async move {
sampler.prewarm().await;
@@ -431,6 +437,7 @@ impl GuardianV2Extension {
.fetch_add(/*val*/ 1, Ordering::Relaxed);
}
let metrics = score_progress.metrics.clone();
let analytics = input.session_store.get::<AnalyticsEventsClient>();
let sampled_at = SystemTime::now();
let tool_call_index = score_progress
.latest_tool_call
@@ -673,6 +680,7 @@ impl GuardianV2Extension {
format!("{planned_action}\n"),
">>> APPROVAL REQUEST END\n".to_owned(),
]);
let mut classification_risk = None;
let mut classification_finished_at = None;
let result: Result<ClassificationOutcome, String> = async {
let review_model_messages = if config.guardian_policy_config.is_none() {
@@ -722,11 +730,12 @@ impl GuardianV2Extension {
}
Err(error) => return Err(error.to_string()),
};
let action_risk = match output.as_str() {
"high" => 1.0,
"low" => 0.0,
let (action_risk, risk_level) = match output.as_str() {
"high" => (1.0, "high"),
"low" => (0.0, "low"),
_ => return Err("invalid Guardian V2 classification".to_owned()),
};
classification_risk = Some(risk_level);
let score = SecurityRiskScore {
scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]),
call_id: Some(call_id.clone()),
@@ -794,19 +803,29 @@ impl GuardianV2Extension {
if result.is_err() {
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
}
record_classification(
metrics.as_deref(),
classification_finished_at
.map(|finished_at: Instant| {
finished_at.duration_since(classification_started_at)
})
.unwrap_or_else(|| classification_started_at.elapsed()),
match &result {
Ok(ClassificationOutcome::Scored) => "success",
Ok(ClassificationOutcome::Superseded) => "superseded",
Err(_) => "failure",
},
);
let duration = classification_finished_at
.map(|finished_at: Instant| finished_at.duration_since(classification_started_at))
.unwrap_or_else(|| classification_started_at.elapsed());
let outcome = match &result {
Ok(ClassificationOutcome::Scored) => "success",
Ok(ClassificationOutcome::Superseded) => "superseded",
Err(_) => "failure",
};
record_classification(metrics.as_deref(), duration, outcome);
if let Some(analytics) = analytics {
analytics.track_guardian_v2_event(GuardianV2Event {
thread_id: thread_id.clone(),
turn_id: turn_id.clone(),
item_id: Some(call_id),
model: parent_model.as_ref().map(|model| model.slug.clone()),
occurred_at_ms: codex_analytics::now_unix_millis(),
kind: GuardianV2EventKind::Classification {
outcome,
risk_level: classification_risk,
duration_ms: u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
},
});
}
if matches!(result, Ok(ClassificationOutcome::Scored)) {
truncations.emit(metrics.as_deref());
}