From 3869f396aa6d748015b3636e8f97ed9d353371b9 Mon Sep 17 00:00:00 2001 From: jif Date: Thu, 19 Nov 2026 18:24:20 +0000 Subject: [PATCH] Keep async Guardian scoring active for serving probes (#62571) ## Why A serving probe that reports no review needed can bypass async scoring. Keeping the async path active lets Guardian compare the two signals while using the existing approval policy. ## What changed Treat streamed review flags as observations. Match each flag to its response item, carry the enclosing item's flag into Code Mode tool calls, and discard the flag when a pre-tool hook rewrites the payload. Emit correlated `codex.guardian_probe_comparison` logs for tool calls, async classification outcomes, and synchronous review results. ## Testing Update integration tests to expect async scoring even when a probe reports no review needed, and verify that Code Mode retains the probe across yield and resume. GitOrigin-RevId: 715f56ba4a82d544ce59f0291bcd1a84dd2daa51 --- codex-rs/core/src/guardian/decision.rs | 2 + codex-rs/core/src/session/mod.rs | 1 + codex-rs/core/src/session/step_context.rs | 2 + codex-rs/core/src/session/tests.rs | 1 + codex-rs/core/src/session/turn.rs | 29 +++------ codex-rs/core/src/tools/lifecycle.rs | 2 + codex-rs/core/src/tools/registry.rs | 27 ++++++++ codex-rs/core/tests/suite/code_mode.rs | 23 +++---- .../suite/guardian_cached_score_tests.rs | 22 +++---- codex-rs/core/tests/suite/remote_env.rs | 26 ++++++-- codex-rs/core/tests/suite/step_settings.rs | 34 ++++++++-- .../src/contributors/tool_lifecycle.rs | 5 ++ codex-rs/ext/guardian-reviewer/src/review.rs | 20 ++++++ .../ext/guardian-reviewer/src/review_tests.rs | 2 + codex-rs/ext/guardian-reviewer/src/routing.rs | 3 + .../src/async_scorer/budget_tests.rs | 2 + .../src/async_scorer/classification.rs | 8 +++ .../src/async_scorer/conversation_tests.rs | 2 + .../src/async_scorer/extension_tests.rs | 41 ++++++++++-- .../guardian-v2/src/async_scorer/metrics.rs | 13 ++++ .../ext/guardian-v2/src/async_scorer/mod.rs | 1 + .../src/async_scorer/observation.rs | 16 +++++ .../src/async_scorer/probe_shadow.rs | 65 +++++++++++++++++++ 23 files changed, 290 insertions(+), 57 deletions(-) create mode 100644 codex-rs/ext/guardian-v2/src/async_scorer/probe_shadow.rs diff --git a/codex-rs/core/src/guardian/decision.rs b/codex-rs/core/src/guardian/decision.rs index 70b52216af..927cc3a84e 100644 --- a/codex-rs/core/src/guardian/decision.rs +++ b/codex-rs/core/src/guardian/decision.rs @@ -190,6 +190,8 @@ pub(crate) fn decide_approval( } let retried = reasons.retry.is_some(); let decision = codex_guardian_reviewer::ReviewRequest { + sampling_model: &context.model_info.slug, + turn_id: &turn.sub_id, serving_assessment: context.serving_assessment.as_ref().map(|value| value.requires_review), host: ReviewRuntime { attached_reviewer, diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 832679d31c..851bdcfc66 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -4768,6 +4768,7 @@ impl Session { Ok(Arc::new(StepContext { sampled_through_sequence: None, serving_assessment: None, + guardian_probe: None, turn_cancellation: cancellation_token.clone(), preempt: turn_context .config diff --git a/codex-rs/core/src/session/step_context.rs b/codex-rs/core/src/session/step_context.rs index 6041bf0f46..6b3e47abae 100644 --- a/codex-rs/core/src/session/step_context.rs +++ b/codex-rs/core/src/session/step_context.rs @@ -30,6 +30,8 @@ pub(crate) struct StepContext { pub(crate) sampled_through_sequence: Option, /// Final serving value for this invocation; retained by its code cell across waits. pub(crate) serving_assessment: Option>, + /// Existing serving review flag, retained only for shadow logging. + pub(crate) guardian_probe: Option, pub(crate) turn: Arc, /// Cancels work owned by this turn, independently of steers or session shutdown. pub(crate) turn_cancellation: CancellationToken, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 4b5e4c93a9..27f26c552b 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -327,6 +327,7 @@ impl StepContext { environments, executor_hook_sources: Vec::new(), serving_assessment: None, + guardian_probe: None, extension_data: Arc::new(codex_extension_api::ExtensionData::new(turn.sub_id.clone())), mcp: Arc::new(codex_mcp::McpBinding::empty(mcp_config_for_test( &turn.config, diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index d175129c94..6ae0b1e65e 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -3076,17 +3076,6 @@ async fn try_run_sampling_request( let mut pending_assessment: Option = None; let mut response_completed = false; let mut includes_reasoning = step_context.settings.model_info.use_responses_lite; - // Catalog policies can enable adaptive review independently of the legacy scorer flag. - // Capture evidence whenever Guardian is available; the reviewer resolves the policy. - let assessment_authorization = if turn_context - .config - .features - .enabled(Feature::GuardianApproval) - { - Some(crate::guardian::ScoreAuthorization::for_session(&sess, &Default::default()).await) - } else { - None - }; let mut resampling = crate::guardian::ResponseResampling::new(&sess); let prompt = Arc::new(prompt); let interrupt_at_boundary = SamplingInterrupt::enabled(&turn_context); @@ -3318,6 +3307,7 @@ async fn try_run_sampling_request( match event { ResponseEvent::Created { response_id } => { answer_moderation = None; + pending_assessment = None; response_id_for_assessment.clone_from(&response_id); if let Some(response_id) = response_id { turn_context @@ -3335,16 +3325,10 @@ async fn try_run_sampling_request( between_items = true; // Serving emits one final value immediately before this item's completion. // After this handoff, the invocation/cell owns it independently of the stream. - let assessment = pending_assessment + let guardian_probe = pending_assessment .take() .filter(|value| item.id().is_some_and(|id| id.as_str() == value.item_id)) - .zip(assessment_authorization.as_ref()) - .map(|(value, authorization)| { - Arc::new(crate::guardian::ServingAssessment { - requires_review: value.requires_review, - authorization: authorization.clone(), - }) - }); + .map(|value| value.requires_review); if item.id().is_none_or(|id| id.is_empty()) && let Some(active_item) = &active_item { @@ -3411,8 +3395,11 @@ async fn try_run_sampling_request( } let mut tool_runtime = tool_runtime.clone(); - if assessment.is_some() { - Arc::make_mut(&mut tool_runtime.step_context).serving_assessment = assessment; + if guardian_probe.is_some() { + let tool_step = Arc::make_mut(&mut tool_runtime.step_context); + // Probe results are observations; async Guardian remains authoritative. + tool_step.serving_assessment = None; + tool_step.guardian_probe = guardian_probe; } let mut ctx = HandleOutputCtx { sess: sess.clone(), diff --git a/codex-rs/core/src/tools/lifecycle.rs b/codex-rs/core/src/tools/lifecycle.rs index 1b6bec2b0d..da9924a265 100644 --- a/codex-rs/core/src/tools/lifecycle.rs +++ b/codex-rs/core/src/tools/lifecycle.rs @@ -45,11 +45,13 @@ pub(crate) async fn notify_tool_start( for contributor in contributors { contributor .on_tool_start(ToolStartInput { + sampling_model: &invocation.step_context.settings.model_info.slug, serving_assessment: invocation .step_context .serving_assessment .as_ref() .map(|value| value.requires_review), + guardian_probe: invocation.step_context.guardian_probe, session_store: &invocation.session.services.session_extension_data, thread_store, turn_store: invocation.turn.extension_data.as_ref(), diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 62446c1e67..803a9ee8dc 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -15,6 +15,7 @@ use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::tools::context::AbortedToolOutput; use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolCallSource; use crate::tools::context::ToolCallState; use crate::tools::context::ToolEnvironments; use crate::tools::context::ToolInvocation; @@ -956,6 +957,31 @@ impl ToolRegistry { Some(Arc::new(assessment)); } + { + let conversation_history = invocation.session.conversation_history_snapshot().await; + tracing::event!( + target: "codex_otel.log_only", + tracing::Level::INFO, + event.name = "codex.guardian_probe_comparison", + stage = "tool", + thread_id = %invocation.session.thread_id, + turn_id = %invocation.turn.sub_id, + call_id = %invocation.call_id, + model = %invocation.step_context.settings.model_info.slug, + guardian_context_mode = crate::context::GuardianContextMode::from_history(conversation_history.as_ref()).as_str(), + scope = match invocation.source { + ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => "tool_item", + ToolCallSource::CodeMode { .. } => "enclosing_item", + }, + probe_review = match invocation.step_context.guardian_probe { + Some(true) => "required", + Some(false) => "not_required", + None => "not_reported", + }, + "Guardian probe comparison" + ); + } + if registered_tool.native_review == NativeReview::Required && let Err(err) = super::review_native_tool_call(&invocation).await { @@ -1232,6 +1258,7 @@ pub(crate) async fn apply_pre_tool_use( .map_err(PreToolUseError::Failed)?; if updated.payload != invocation.payload { Arc::make_mut(&mut updated.step_context).serving_assessment = None; + Arc::make_mut(&mut updated.step_context).guardian_probe = None; } *invocation = updated; Ok(()) diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index fec8839d68..6a0aa3685b 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -5165,7 +5165,7 @@ try { #[derive(Default)] struct ResponseIdObserver { response_ids: Mutex)>>, - serving_assessments: Mutex>>, + guardian_probes: Mutex>>, originating_items: Mutex)>>, wait_started: tokio::sync::Notify, } @@ -5180,10 +5180,10 @@ impl ToolLifecycleContributor for ResponseIdObserver { .as_str() .is_some_and(|cmd| cmd.starts_with("printf 'phase ")) { - self.serving_assessments + self.guardian_probes .lock() .unwrap() - .push(input.serving_assessment); + .push(input.guardian_probe); } if matches!(input.tool_name.name.as_str(), "exec" | "exec_command") { self.originating_items.lock().unwrap().push(( @@ -5208,9 +5208,9 @@ impl ToolLifecycleContributor for ResponseIdObserver { } #[cfg_attr(windows, ignore = "no exec_command on Windows")] -#[test_case::test_case(None; "without_assessment")] -#[test_case::test_case(Some(false); "low_risk_assessment")] -#[test_case::test_case(Some(true); "review_required_assessment")] +#[test_case::test_case(None; "without_probe")] +#[test_case::test_case(Some(false); "low_risk_probe")] +#[test_case::test_case(Some(true); "review_required_probe")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_yield_and_resume_with_wait(requires_review: Option) -> Result<()> { skip_if_no_network!(Ok(())); @@ -5220,10 +5220,11 @@ async fn code_mode_can_yield_and_resume_with_wait(requires_review: Option) .and(wiremock::matchers::body_partial_json( serde_json::json!({"model": "gpt-5.6-luna"}), )) - .respond_with(responses::sse_response(sse(vec![ev_completed( - "unexpected-score", - )]))) - .expect(0) + .respond_with(responses::sse_response(sse(vec![ + ev_assistant_message("score", "low"), + ev_completed("score"), + ]))) + .with_priority(/*p*/ 1) .mount(&server) .await; let observer = Arc::new(ResponseIdObserver::default()); @@ -5406,7 +5407,7 @@ text((await tools.exec_command({{cmd: "printf 'phase 3'"}})).output); ); assert_eq!( - *observer.serving_assessments.lock().unwrap(), + *observer.guardian_probes.lock().unwrap(), vec![requires_review; 3] ); let observed = observer.response_ids.lock().unwrap(); diff --git a/codex-rs/core/tests/suite/guardian_cached_score_tests.rs b/codex-rs/core/tests/suite/guardian_cached_score_tests.rs index 00e341cc37..89813010fd 100644 --- a/codex-rs/core/tests/suite/guardian_cached_score_tests.rs +++ b/codex-rs/core/tests/suite/guardian_cached_score_tests.rs @@ -1,4 +1,4 @@ -//! Exercises async and serving assessments through real tool calls, hooks, and approval results. +//! Exercises async Guardian and observational probe flags through real tool calls and approvals. use std::collections::BTreeMap; use std::sync::Arc; @@ -374,9 +374,9 @@ enum ProbeCase { IncompleteAuthorization, } -#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Standard, 0; "direct_low")] +#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Standard, 1; "direct_low")] #[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Standard, 1; "direct_review")] -#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Standard, 0; "code_cell_low")] +#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Standard, 2; "code_cell_low")] #[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Standard, 2; "code_cell_review")] #[test_case::test_case(None, false, "probe-item", ProbeCase::Standard, 1; "missing_uses_async")] #[test_case::test_case(Some(false), false, "other-item", ProbeCase::Standard, 1; "mismatched_uses_async")] @@ -385,11 +385,11 @@ enum ProbeCase { #[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Rewrite, 2; "nested_hook_rewrite")] #[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 0; "interrupted_direct_review")] #[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 0; "interrupted_cell_review")] -#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 0; "catalog_low_without_legacy_flag")] +#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_low_without_legacy_flag")] #[test_case::test_case(Some(true), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_review_without_legacy_flag")] #[test_case::test_case(Some(false), false, "probe-item", ProbeCase::IncompleteAuthorization, 1; "incomplete_authorization_still_reviews")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serving_assessment_replaces_async_scoring( +async fn probe_flags_keep_async_scoring( requires_review: Option, code_mode: bool, assessed_item: &str, @@ -459,13 +459,11 @@ async fn serving_assessment_replaces_async_scoring( ], ) .await; - let uses_async = requires_review.is_none() - || assessed_item != "probe-item" - || scenario == ProbeCase::Rewrite; - let expected_scores = if uses_async { - 1..=if code_mode { 2 } else { 1 } - } else { + let expected_scores = if scenario == ProbeCase::Interrupted { 0..=0 + } else { + // The enclosing code wrapper also remains on the async path. + 1..=if code_mode { 3 } else { 1 } }; Mock::given(method("POST")) .and(body_partial_json(json!({"model": "gpt-5.6-luna"}))) @@ -600,7 +598,7 @@ async fn serving_assessment_replaces_async_scoring( matches!(event, EventMsg::TurnComplete(_)) }) .await; - if uses_async { + if scenario != ProbeCase::Interrupted { timeout(Duration::from_secs(30), async { loop { if server diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index 936fde9087..6c51704c30 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -52,7 +52,6 @@ use codex_http_client::HttpClientFactory; use codex_http_client::OutboundProxyPolicy; use codex_network_proxy::NetworkProxyConfig; use codex_protocol::approvals::GuardianAssessmentStatus; -use codex_protocol::approvals::GuardianReviewReason; use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::config_types::CollaborationMode; @@ -3258,6 +3257,24 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment( let server = start_mock_server().await; let completed_response = |id: &str, item| sse(vec![ev_response_created(id), item, ev_completed(id)]); + if tool == "mcp_elicitation" { + // Probe flags are observational, so the async scorer still runs. Keep its + // responses separate from the parent and synchronous Guardian review. + wiremock::Mock::given(wiremock::matchers::method("POST")) + .and(wiremock::matchers::body_partial_json(json!({ + "model": "gpt-5.6-luna", + "tool_choice": "none", + }))) + .respond_with(core_test_support::responses::sse_response( + completed_response( + "resp-guardian-score", + ev_assistant_message("msg-guardian-score", "high"), + ), + )) + .with_priority(/*p*/ 1) + .mount(&server) + .await; + } let responses = mount_sse_sequence( &server, vec![ @@ -3409,17 +3426,18 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment( .await?; wait_for_response_request_count(&responses, /*expected_count*/ 1).await; attach_tx.send(()).expect("attach remote environment"); - let mut denial_reason = None; + let mut was_denied = false; wait_for_event(&test.codex, |event| { if let EventMsg::GuardianAssessment(assessment) = event && assessment.status == GuardianAssessmentStatus::Denied && assessment.target_item_id.as_deref() == Some(EXEC_CALL_ID) { - denial_reason = assessment.review_reason; + was_denied = true; } matches!(event, EventMsg::TurnComplete(_)) }) .await; + assert!(was_denied, "Guardian should deny the action"); let requests = responses.requests(); assert!( @@ -3437,8 +3455,6 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment( let guardian_context = guardian_request.message_input_texts("user").join("\n"); // Both actions start in the remote-primary step, but the MCP server runs locally. let (owner_denied_path, other_denied_path) = if tool == "mcp_elicitation" { - // The low score is labeled local, but the model saw the remote primary's rules. - assert_eq!(denial_reason, Some(GuardianReviewReason::FreshRequired)); (&local_denied_path, &remote_denied_path) } else { (&remote_denied_path, &local_denied_path) diff --git a/codex-rs/core/tests/suite/step_settings.rs b/codex-rs/core/tests/suite/step_settings.rs index 15394819dc..3abced94da 100644 --- a/codex-rs/core/tests/suite/step_settings.rs +++ b/codex-rs/core/tests/suite/step_settings.rs @@ -16,6 +16,9 @@ use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; use codex_extension_api::ToolContributor; +use codex_extension_api::ToolLifecycleContributor; +use codex_extension_api::ToolLifecycleFuture; +use codex_extension_api::ToolStartInput; use codex_extension_api::TurnContextContributionInput; use codex_features::Feature; use codex_history::RolloutItem; @@ -2958,7 +2961,23 @@ async fn captured_step_controls_mcp_output_limit(supports_images: bool) -> Resul } #[derive(Clone)] -struct SettingsEcho(Arc>>); +struct SettingsEcho { + history: Arc>>, + sampling_model: Arc>>, +} + +impl ToolLifecycleContributor for SettingsEcho { + fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> { + Box::pin(async move { + if input.tool_name.is_default_namespace() + && input.tool_name.name == "extension_settings" + { + *self.sampling_model.lock().expect("sampling model lock") = + Some(input.sampling_model.to_owned()); + } + }) + } +} impl ToolContributor for SettingsEcho { fn tools( @@ -2991,7 +3010,7 @@ impl<'call> ToolExecutor> for SettingsEcho { 'call: 'a, { Box::pin(async move { - *self.0.lock().expect("history lock") = Some(call.conversation_history); + *self.history.lock().expect("history lock") = Some(call.conversation_history); let metadata: Value = serde_json::from_str( call.codex_turn_metadata .as_deref() @@ -3000,6 +3019,7 @@ impl<'call> ToolExecutor> for SettingsEcho { .expect("metadata JSON"); Ok(Box::new(JsonToolOutput::new(json!({ "model": call.model, + "sampling_model": self.sampling_model.lock().expect("sampling model lock").clone(), "metadata_model": metadata["model"], "effort": metadata["reasoning_effort"], "output_bytes": call.truncation_policy.byte_budget(), @@ -3011,7 +3031,7 @@ impl<'call> ToolExecutor> for SettingsEcho { #[test_case(ThreadHistoryMode::Legacy; "legacy")] #[test_case(ThreadHistoryMode::Paginated; "paginated")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn captured_step_settings_and_history_reach_extension_executor( +async fn captured_step_settings_and_history_reach_extensions( history_mode: ThreadHistoryMode, ) -> Result<()> { skip_if_no_network!(Ok(())); @@ -3032,7 +3052,12 @@ async fn captured_step_settings_and_history_reach_extension_executor( .await; let history = Arc::default(); let mut extensions = ExtensionRegistryBuilder::::new(); - extensions.tool_contributor(Arc::new(SettingsEcho(Arc::clone(&history)))); + let extension = Arc::new(SettingsEcho { + history: Arc::clone(&history), + sampling_model: Arc::default(), + }); + extensions.tool_contributor(extension.clone()); + extensions.tool_lifecycle_contributor(extension); let test = direct_tool_settings_test() .with_history_mode(history_mode) .with_extensions(Arc::new(extensions.build())) @@ -3067,6 +3092,7 @@ async fn captured_step_settings_and_history_reach_extension_executor( serde_json::from_str::(&output)?, json!({ "model": MODEL_B, + "sampling_model": MODEL_B, "metadata_model": MODEL_B, "effort": "high", "output_bytes": 512, diff --git a/codex-rs/ext/extension-api/src/contributors/tool_lifecycle.rs b/codex-rs/ext/extension-api/src/contributors/tool_lifecycle.rs index cbca227ec6..2d7ba82333 100644 --- a/codex-rs/ext/extension-api/src/contributors/tool_lifecycle.rs +++ b/codex-rs/ext/extension-api/src/contributors/tool_lifecycle.rs @@ -122,9 +122,14 @@ impl McpToolContext { /// Input supplied when the host starts executing one tool call. pub struct ToolStartInput<'a> { + /// Model that issued this call or its enclosing code cell, preserved across yields and waits. + pub sampling_model: &'a str, /// Final serving assessment for the unchanged action, including its code cell's value. /// When present, replaces asynchronous scoring; it does not grant execution permission. pub serving_assessment: Option, + /// Existing serving review flag for this action or its enclosing code cell, in shadow mode. + /// Absence means no flag was reported, not that the action was scored as low risk. + pub guardian_probe: Option, /// Store scoped to the host session runtime. pub session_store: &'a ExtensionData, /// Store scoped to this thread runtime. diff --git a/codex-rs/ext/guardian-reviewer/src/review.rs b/codex-rs/ext/guardian-reviewer/src/review.rs index 2f18f505a5..d1fe558de6 100644 --- a/codex-rs/ext/guardian-reviewer/src/review.rs +++ b/codex-rs/ext/guardian-reviewer/src/review.rs @@ -187,6 +187,26 @@ impl SynchronousApprovalReviewer for ReviewRequest<'_, H> { analytics, completed_at_ms.try_into().unwrap_or_default(), ); + tracing::event!( + target: "codex_otel.log_only", + tracing::Level::INFO, + event.name = "codex.guardian_probe_comparison", + stage = "sync_guardian", + thread_id = %self.thread_id, + turn_id = self.turn_id, + call_id = self.tool_call_id, + review_id = self.approval_id, + model = self.sampling_model, + guardian_context_mode = completed.analytics.guardian_context_mode.unwrap_or("not_reported"), + sync_outcome = if cached_approval { "async_approval" } else if completed.assessment_outcome.is_some() { "completed" } else { "failure" }, + sync_decision = match completed.assessment_outcome { + Some(GuardianAssessmentOutcome::Allow) => "allow", + Some(GuardianAssessmentOutcome::Deny) => "deny", + None => "not_reported", + }, + failure_reason = ?completed.analytics.failure_reason, + "Guardian probe comparison" + ); if self.log_assessments && !cached_approval && let Some(event) = &completed.event diff --git a/codex-rs/ext/guardian-reviewer/src/review_tests.rs b/codex-rs/ext/guardian-reviewer/src/review_tests.rs index 9b3b68b0b1..2b04a43546 100644 --- a/codex-rs/ext/guardian-reviewer/src/review_tests.rs +++ b/codex-rs/ext/guardian-reviewer/src/review_tests.rs @@ -164,6 +164,8 @@ async fn async_approval_revalidates_without_waiting_for_tracked_cleanup(state: A let analytics = AnalyticsEventsClient::disabled(); let action = json!({}); let request = ReviewRequest { + sampling_model: "test", + turn_id: "turn", serving_assessment: None, host: host.clone(), approval_id: "approval", diff --git a/codex-rs/ext/guardian-reviewer/src/routing.rs b/codex-rs/ext/guardian-reviewer/src/routing.rs index c173bd376a..f1a5b3f646 100644 --- a/codex-rs/ext/guardian-reviewer/src/routing.rs +++ b/codex-rs/ext/guardian-reviewer/src/routing.rs @@ -25,6 +25,9 @@ use tokio_util::sync::CancellationToken; /// One captured action and its host-enforced constraints. Guardian derives the /// contributor input and owns cancellation for the complete review operation. pub struct ReviewRequest<'a, H> { + /// Issuing action's model and turn, retained across yielded code cells. + pub sampling_model: &'a str, + pub turn_id: &'a str, /// Final serving value retained by the host for this action. pub serving_assessment: Option, pub host: H, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs index 6d07b9cdbc..57a4affaaa 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs @@ -236,7 +236,9 @@ async fn assert_catalog_budget(evidence: BudgetEvidence) -> Result<()> { }; fixture.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &fixture.session_store, thread_store, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs b/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs index 5b1e467389..3a116a6560 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/classification.rs @@ -58,6 +58,7 @@ use super::trusted_tools::trusted_tool_context; pub(super) struct Classification { pub(super) reservation: Reservation, + pub(super) probe_comparison: Option, pub(super) classification_started_at: Instant, pub(super) decisions_sampler: Option>, pub(super) guardian_config: GuardianV2Config, @@ -113,6 +114,7 @@ impl Classification { pub(super) async fn run(self) { let Self { reservation, + probe_comparison, classification_started_at, decisions_sampler, guardian_config, @@ -177,9 +179,11 @@ impl Classification { else { record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "superseded", + /*classification_risk*/ None, /*failure_reason*/ None, ); return; @@ -271,9 +275,11 @@ impl Classification { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("context_build_error"), ); event_sink.emit_warning(ExtensionWarning { @@ -453,9 +459,11 @@ impl Classification { }; record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, duration, outcome, + classification_risk, result.is_err().then_some(failure_reason), ); if let Some(analytics) = analytics { diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/conversation_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/conversation_tests.rs index d94d7f03a6..d90ac4b194 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/conversation_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/conversation_tests.rs @@ -712,7 +712,9 @@ async fn delayed_classification_skips_after_newer_score_and_history_complete() - let history = fixture.test.codex.conversation_history_snapshot().await; let observe = |call_id, permissions| { fixture.registry.tool_lifecycle_contributors()[0].on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: Some(true), permissions, session_store: &fixture.session_store, thread_store: store, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs index 4281b9642c..6c0c3083f7 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs @@ -362,7 +362,9 @@ async fn installed_extension_keeps_parent_program_out_of_websocket_and_http_revi } registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &session_store, thread_store, @@ -645,7 +647,9 @@ async fn sandboxed_shell_classification_respects_review_scope() -> Result<()> { ] { fixture.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { panic!("unexpected permission resolution") }), session_store: &fixture.session_store, thread_store, @@ -749,7 +753,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> { }; fixture.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &fixture.session_store, thread_store, @@ -974,7 +980,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> { for _ in 0..ignored_calls { fixture.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &fixture.session_store, thread_store, @@ -1257,7 +1265,9 @@ async fn sample_configured_conversation_history_with_delivery( registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &session_store, thread_store, @@ -1408,7 +1418,9 @@ impl GuardianFailureFixture { }; self.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &self.session_store, thread_store, @@ -1568,7 +1580,7 @@ async fn contributor_fails_closed_when_luna_classification_fails() -> Result<()> #[tokio::test(flavor = "multi_thread")] async fn late_turn_actions_invalidate_the_current_policy_score() -> Result<()> { skip_if_no_network!(Ok(())); - let fixture = GuardianFailureFixture::new().await?; + let fixture = GuardianFailureFixture::with_config("").await?; let server = responses::start_mock_server().await; let responses = ["low", "high", "invalid"].map(|risk| { responses::sse(vec![ @@ -1612,21 +1624,30 @@ async fn late_turn_actions_invalidate_the_current_policy_score() -> Result<()> { executions + 1 ); let mut expected: Option = None; - for (turn, call_id, risk, expected_approval) in [ - (&new_turn, "new-score", 0.0, Some(ReviewDecision::Approved)), + for (turn, call_id, guardian_probe, risk, expected_approval) in [ + ( + &new_turn, + "new-score", + Some(true), + 0.0, + Some(ReviewDecision::Approved), + ), ( &old_turn, "late-old-score", + Some(false), 1.0, Some(ReviewDecision::Approved), ), - (&old_turn, "late-old-failure", 1.0, None), + (&old_turn, "late-old-failure", None, 1.0, None), ] { let backend = ConversationBackend::for_turn(store, turn).unwrap(); let mut updates = backend.progress.updates.subscribe(); fixture.registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe, permissions: Box::pin(async { Some(Default::default()) }), session_store: &fixture.session_store, thread_store: store, @@ -2885,7 +2906,9 @@ async fn contributor_skips_required_models_in_standard_scope() -> Result<()> { let previous_score = cached_score(thread_store); registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { panic!("unexpected permission resolution") }), session_store: &session_store, thread_store, @@ -3241,7 +3264,9 @@ async fn contributor_counts_failed_thread_lookups_toward_score_lag() -> Result<( }; registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &session_store, thread_store, @@ -3777,7 +3802,9 @@ async fn assert_parent_compaction_reuse(parent_context_for_review: bool) -> Resu registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &session_store, thread_store, @@ -3873,7 +3900,9 @@ async fn assert_parent_compaction_reuse(parent_context_for_review: bool) -> Resu let call_id = format!("call-{}", index + 2); registry.tool_lifecycle_contributors()[0] .on_tool_start(ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &session_store, thread_store, @@ -3985,7 +4014,9 @@ async fn cached_approval_respects_action_order_and_wrapper_lag() -> Result<()> { }, }; progress.observe(&ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: &fixture.session_store, thread_store: store, @@ -4194,7 +4225,9 @@ fn seed_cached_score( fn observe_unscored_call(progress: &GuardianV2ScoreProgress, store: &ExtensionData) -> usize { progress.observe(&ToolStartInput { + sampling_model: "gpt-5.5", serving_assessment: None, + guardian_probe: None, permissions: Box::pin(async { Some(Default::default()) }), session_store: store, thread_store: store, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs index 25a08f346c..ab4b20962f 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs @@ -8,6 +8,7 @@ use codex_core::context::GuardianContextMode; use codex_extension_api::ExtensionMetrics; use super::decisions::DecisionsError; +use super::probe_shadow::ProbeComparison; use super::sampler::LunaSamplerError; pub(super) const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification"; @@ -73,11 +74,23 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str { pub(super) fn record_classification( metrics: Option<&dyn ExtensionMetrics>, + probe_comparison: Option<&ProbeComparison>, context_mode: GuardianContextMode, duration: Duration, outcome: &str, + classification_risk: Option<&str>, failure_reason: Option<&str>, ) { + // Shadow comparisons cover every reported classification outcome, including early exits. + if let Some(comparison) = probe_comparison { + comparison.record( + context_mode, + outcome, + classification_risk, + duration, + failure_reason, + ); + } if outcome == "failure" { tracing::warn!( failure_reason, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs index 275cc126ff..62ed1d6d60 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs @@ -10,6 +10,7 @@ mod extension; mod metrics; mod observation; mod parent_compaction; +mod probe_shadow; mod request; mod sampler; mod score; diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs index 3c0329761d..a93d494b15 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs @@ -97,6 +97,7 @@ impl GuardianV2Extension { return; } let parent_model = input.thread_store.get::(); + let probe_comparison = Some(super::probe_shadow::ProbeComparison::capture(&input)); let policy = guardian_config.policy_for_model( parent_model.as_deref(), config.config_layer_stack.requirements(), @@ -176,9 +177,11 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some(super::metrics::sampler_failure_reason(&error)), ); return; @@ -216,9 +219,11 @@ impl GuardianV2Extension { score_progress.invalidate(tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("thread_context_error"), ); event_sink.emit_warning(ExtensionWarning { @@ -265,9 +270,11 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("configuration_error"), ); self.event_sink.emit_warning(ExtensionWarning { @@ -317,9 +324,11 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), outcome, + /*classification_risk*/ None, failure_reason, ); return; @@ -339,9 +348,11 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("input_too_large"), ); return; @@ -350,9 +361,11 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("action_serialization_error"), ); event_sink.emit_warning(ExtensionWarning { @@ -395,15 +408,18 @@ impl GuardianV2Extension { score_progress.fail_closed(sampled_at, tool_call_index); record_classification( metrics.as_deref(), + probe_comparison.as_ref(), context_mode, classification_started_at.elapsed(), "failure", + /*classification_risk*/ None, Some("permission_resolution_error"), ); return; }; let score_authorization = ScoreAuthorization::current(&thread, &permissions).await; let classification = Classification { + probe_comparison, reservation, classification_started_at, decisions_sampler: input diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/probe_shadow.rs b/codex-rs/ext/guardian-v2/src/async_scorer/probe_shadow.rs new file mode 100644 index 0000000000..a2dd10bcdb --- /dev/null +++ b/codex-rs/ext/guardian-v2/src/async_scorer/probe_shadow.rs @@ -0,0 +1,65 @@ +//! Logs existing probe flags alongside async results without waiting for other review stages. +//! Action IDs join these records to tool observations and synchronous review outcomes. + +use std::time::Duration; + +use codex_core::context::GuardianContextMode; +use codex_extension_api::ToolCallSource; +use codex_extension_api::ToolStartInput; + +pub(super) struct ProbeComparison { + probe: Option, + scope: &'static str, + model: String, + thread_id: String, + turn_id: String, + call_id: String, +} + +impl ProbeComparison { + pub(super) fn capture(input: &ToolStartInput<'_>) -> Self { + Self { + probe: input.guardian_probe, + scope: match input.source { + ToolCallSource::Direct => "tool_item", + ToolCallSource::CodeMode { .. } => "enclosing_item", + }, + model: input.sampling_model.to_owned(), + thread_id: input.thread_store.level_id().to_owned(), + turn_id: input.turn_id.to_owned(), + call_id: input.call_id.to_owned(), + } + } + + pub(super) fn record( + &self, + context_mode: GuardianContextMode, + outcome: &str, + risk: Option<&str>, + duration: Duration, + failure_reason: Option<&str>, + ) { + tracing::event!( + target: "codex_otel.log_only", + tracing::Level::INFO, + event.name = "codex.guardian_probe_comparison", + stage = "async_scorer", + thread_id = %self.thread_id, + turn_id = %self.turn_id, + call_id = %self.call_id, + model = %self.model, + guardian_context_mode = context_mode.as_str(), + scope = self.scope, + probe_review = match self.probe { + Some(true) => "required", + Some(false) => "not_required", + None => "not_reported", + }, + async_outcome = outcome, + async_risk = risk.unwrap_or("not_reported"), + failure_reason, + duration_ms = u64::try_from(duration.as_millis()).unwrap_or(u64::MAX), + "Guardian probe comparison" + ); + } +}