Keep async Guardian scoring active for serving probes (#62571)

## Why

A serving probe that reports no review needed can bypass async scoring.
Keeping the async path active lets Guardian compare the two signals
while using the existing approval policy.

## What changed

Treat streamed review flags as observations. Match each flag to its
response item, carry the enclosing item's flag into Code Mode tool
calls, and discard the flag when a pre-tool hook rewrites the payload.

Emit correlated `codex.guardian_probe_comparison` logs for tool calls,
async classification outcomes, and synchronous review results.

## Testing

Update integration tests to expect async scoring even when a probe
reports no review needed, and verify that Code Mode retains the probe
across yield and resume.

GitOrigin-RevId: 715f56ba4a82d544ce59f0291bcd1a84dd2daa51
This commit is contained in:
jif
2026-11-19 18:54:22 +00:00
committed by copyberry
parent 99870c4f49
commit 3869f396aa
23 changed files with 290 additions and 57 deletions
+2
View File
@@ -190,6 +190,8 @@ pub(crate) fn decide_approval(
}
let retried = reasons.retry.is_some();
let decision = codex_guardian_reviewer::ReviewRequest {
sampling_model: &context.model_info.slug,
turn_id: &turn.sub_id,
serving_assessment: context.serving_assessment.as_ref().map(|value| value.requires_review),
host: ReviewRuntime {
attached_reviewer,
+1
View File
@@ -4768,6 +4768,7 @@ impl Session {
Ok(Arc::new(StepContext {
sampled_through_sequence: None,
serving_assessment: None,
guardian_probe: None,
turn_cancellation: cancellation_token.clone(),
preempt: turn_context
.config
@@ -30,6 +30,8 @@ pub(crate) struct StepContext {
pub(crate) sampled_through_sequence: Option<u64>,
/// Final serving value for this invocation; retained by its code cell across waits.
pub(crate) serving_assessment: Option<Arc<crate::guardian::ServingAssessment>>,
/// Existing serving review flag, retained only for shadow logging.
pub(crate) guardian_probe: Option<bool>,
pub(crate) turn: Arc<TurnContext>,
/// Cancels work owned by this turn, independently of steers or session shutdown.
pub(crate) turn_cancellation: CancellationToken,
+1
View File
@@ -327,6 +327,7 @@ impl StepContext {
environments,
executor_hook_sources: Vec::new(),
serving_assessment: None,
guardian_probe: None,
extension_data: Arc::new(codex_extension_api::ExtensionData::new(turn.sub_id.clone())),
mcp: Arc::new(codex_mcp::McpBinding::empty(mcp_config_for_test(
&turn.config,
+8 -21
View File
@@ -3076,17 +3076,6 @@ async fn try_run_sampling_request(
let mut pending_assessment: Option<codex_api::GuardianAssessment> = None;
let mut response_completed = false;
let mut includes_reasoning = step_context.settings.model_info.use_responses_lite;
// Catalog policies can enable adaptive review independently of the legacy scorer flag.
// Capture evidence whenever Guardian is available; the reviewer resolves the policy.
let assessment_authorization = if turn_context
.config
.features
.enabled(Feature::GuardianApproval)
{
Some(crate::guardian::ScoreAuthorization::for_session(&sess, &Default::default()).await)
} else {
None
};
let mut resampling = crate::guardian::ResponseResampling::new(&sess);
let prompt = Arc::new(prompt);
let interrupt_at_boundary = SamplingInterrupt::enabled(&turn_context);
@@ -3318,6 +3307,7 @@ async fn try_run_sampling_request(
match event {
ResponseEvent::Created { response_id } => {
answer_moderation = None;
pending_assessment = None;
response_id_for_assessment.clone_from(&response_id);
if let Some(response_id) = response_id {
turn_context
@@ -3335,16 +3325,10 @@ async fn try_run_sampling_request(
between_items = true;
// Serving emits one final value immediately before this item's completion.
// After this handoff, the invocation/cell owns it independently of the stream.
let assessment = pending_assessment
let guardian_probe = pending_assessment
.take()
.filter(|value| item.id().is_some_and(|id| id.as_str() == value.item_id))
.zip(assessment_authorization.as_ref())
.map(|(value, authorization)| {
Arc::new(crate::guardian::ServingAssessment {
requires_review: value.requires_review,
authorization: authorization.clone(),
})
});
.map(|value| value.requires_review);
if item.id().is_none_or(|id| id.is_empty())
&& let Some(active_item) = &active_item
{
@@ -3411,8 +3395,11 @@ async fn try_run_sampling_request(
}
let mut tool_runtime = tool_runtime.clone();
if assessment.is_some() {
Arc::make_mut(&mut tool_runtime.step_context).serving_assessment = assessment;
if guardian_probe.is_some() {
let tool_step = Arc::make_mut(&mut tool_runtime.step_context);
// Probe results are observations; async Guardian remains authoritative.
tool_step.serving_assessment = None;
tool_step.guardian_probe = guardian_probe;
}
let mut ctx = HandleOutputCtx {
sess: sess.clone(),
+2
View File
@@ -45,11 +45,13 @@ pub(crate) async fn notify_tool_start(
for contributor in contributors {
contributor
.on_tool_start(ToolStartInput {
sampling_model: &invocation.step_context.settings.model_info.slug,
serving_assessment: invocation
.step_context
.serving_assessment
.as_ref()
.map(|value| value.requires_review),
guardian_probe: invocation.step_context.guardian_probe,
session_store: &invocation.session.services.session_extension_data,
thread_store,
turn_store: invocation.turn.extension_data.as_ref(),
+27
View File
@@ -15,6 +15,7 @@ use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
use crate::tools::context::AbortedToolOutput;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolCallState;
use crate::tools::context::ToolEnvironments;
use crate::tools::context::ToolInvocation;
@@ -956,6 +957,31 @@ impl ToolRegistry {
Some(Arc::new(assessment));
}
{
let conversation_history = invocation.session.conversation_history_snapshot().await;
tracing::event!(
target: "codex_otel.log_only",
tracing::Level::INFO,
event.name = "codex.guardian_probe_comparison",
stage = "tool",
thread_id = %invocation.session.thread_id,
turn_id = %invocation.turn.sub_id,
call_id = %invocation.call_id,
model = %invocation.step_context.settings.model_info.slug,
guardian_context_mode = crate::context::GuardianContextMode::from_history(conversation_history.as_ref()).as_str(),
scope = match invocation.source {
ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => "tool_item",
ToolCallSource::CodeMode { .. } => "enclosing_item",
},
probe_review = match invocation.step_context.guardian_probe {
Some(true) => "required",
Some(false) => "not_required",
None => "not_reported",
},
"Guardian probe comparison"
);
}
if registered_tool.native_review == NativeReview::Required
&& let Err(err) = super::review_native_tool_call(&invocation).await
{
@@ -1232,6 +1258,7 @@ pub(crate) async fn apply_pre_tool_use(
.map_err(PreToolUseError::Failed)?;
if updated.payload != invocation.payload {
Arc::make_mut(&mut updated.step_context).serving_assessment = None;
Arc::make_mut(&mut updated.step_context).guardian_probe = None;
}
*invocation = updated;
Ok(())
+12 -11
View File
@@ -5165,7 +5165,7 @@ try {
#[derive(Default)]
struct ResponseIdObserver {
response_ids: Mutex<Vec<(String, Option<String>)>>,
serving_assessments: Mutex<Vec<Option<bool>>>,
guardian_probes: Mutex<Vec<Option<bool>>>,
originating_items: Mutex<Vec<(String, Option<codex_protocol::ResponseItemId>)>>,
wait_started: tokio::sync::Notify,
}
@@ -5180,10 +5180,10 @@ impl ToolLifecycleContributor for ResponseIdObserver {
.as_str()
.is_some_and(|cmd| cmd.starts_with("printf 'phase "))
{
self.serving_assessments
self.guardian_probes
.lock()
.unwrap()
.push(input.serving_assessment);
.push(input.guardian_probe);
}
if matches!(input.tool_name.name.as_str(), "exec" | "exec_command") {
self.originating_items.lock().unwrap().push((
@@ -5208,9 +5208,9 @@ impl ToolLifecycleContributor for ResponseIdObserver {
}
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[test_case::test_case(None; "without_assessment")]
#[test_case::test_case(Some(false); "low_risk_assessment")]
#[test_case::test_case(Some(true); "review_required_assessment")]
#[test_case::test_case(None; "without_probe")]
#[test_case::test_case(Some(false); "low_risk_probe")]
#[test_case::test_case(Some(true); "review_required_probe")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_yield_and_resume_with_wait(requires_review: Option<bool>) -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -5220,10 +5220,11 @@ async fn code_mode_can_yield_and_resume_with_wait(requires_review: Option<bool>)
.and(wiremock::matchers::body_partial_json(
serde_json::json!({"model": "gpt-5.6-luna"}),
))
.respond_with(responses::sse_response(sse(vec![ev_completed(
"unexpected-score",
)])))
.expect(0)
.respond_with(responses::sse_response(sse(vec![
ev_assistant_message("score", "low"),
ev_completed("score"),
])))
.with_priority(/*p*/ 1)
.mount(&server)
.await;
let observer = Arc::new(ResponseIdObserver::default());
@@ -5406,7 +5407,7 @@ text((await tools.exec_command({{cmd: "printf 'phase 3'"}})).output);
);
assert_eq!(
*observer.serving_assessments.lock().unwrap(),
*observer.guardian_probes.lock().unwrap(),
vec![requires_review; 3]
);
let observed = observer.response_ids.lock().unwrap();
@@ -1,4 +1,4 @@
//! Exercises async and serving assessments through real tool calls, hooks, and approval results.
//! Exercises async Guardian and observational probe flags through real tool calls and approvals.
use std::collections::BTreeMap;
use std::sync::Arc;
@@ -374,9 +374,9 @@ enum ProbeCase {
IncompleteAuthorization,
}
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Standard, 0; "direct_low")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Standard, 1; "direct_low")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Standard, 1; "direct_review")]
#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Standard, 0; "code_cell_low")]
#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Standard, 2; "code_cell_low")]
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Standard, 2; "code_cell_review")]
#[test_case::test_case(None, false, "probe-item", ProbeCase::Standard, 1; "missing_uses_async")]
#[test_case::test_case(Some(false), false, "other-item", ProbeCase::Standard, 1; "mismatched_uses_async")]
@@ -385,11 +385,11 @@ enum ProbeCase {
#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Rewrite, 2; "nested_hook_rewrite")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 0; "interrupted_direct_review")]
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 0; "interrupted_cell_review")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 0; "catalog_low_without_legacy_flag")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_low_without_legacy_flag")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_review_without_legacy_flag")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::IncompleteAuthorization, 1; "incomplete_authorization_still_reviews")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn serving_assessment_replaces_async_scoring(
async fn probe_flags_keep_async_scoring(
requires_review: Option<bool>,
code_mode: bool,
assessed_item: &str,
@@ -459,13 +459,11 @@ async fn serving_assessment_replaces_async_scoring(
],
)
.await;
let uses_async = requires_review.is_none()
|| assessed_item != "probe-item"
|| scenario == ProbeCase::Rewrite;
let expected_scores = if uses_async {
1..=if code_mode { 2 } else { 1 }
} else {
let expected_scores = if scenario == ProbeCase::Interrupted {
0..=0
} else {
// The enclosing code wrapper also remains on the async path.
1..=if code_mode { 3 } else { 1 }
};
Mock::given(method("POST"))
.and(body_partial_json(json!({"model": "gpt-5.6-luna"})))
@@ -600,7 +598,7 @@ async fn serving_assessment_replaces_async_scoring(
matches!(event, EventMsg::TurnComplete(_))
})
.await;
if uses_async {
if scenario != ProbeCase::Interrupted {
timeout(Duration::from_secs(30), async {
loop {
if server
+21 -5
View File
@@ -52,7 +52,6 @@ use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_network_proxy::NetworkProxyConfig;
use codex_protocol::approvals::GuardianAssessmentStatus;
use codex_protocol::approvals::GuardianReviewReason;
use codex_protocol::capabilities::CapabilityRootLocation;
use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::config_types::CollaborationMode;
@@ -3258,6 +3257,24 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment(
let server = start_mock_server().await;
let completed_response =
|id: &str, item| sse(vec![ev_response_created(id), item, ev_completed(id)]);
if tool == "mcp_elicitation" {
// Probe flags are observational, so the async scorer still runs. Keep its
// responses separate from the parent and synchronous Guardian review.
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::body_partial_json(json!({
"model": "gpt-5.6-luna",
"tool_choice": "none",
})))
.respond_with(core_test_support::responses::sse_response(
completed_response(
"resp-guardian-score",
ev_assistant_message("msg-guardian-score", "high"),
),
))
.with_priority(/*p*/ 1)
.mount(&server)
.await;
}
let responses = mount_sse_sequence(
&server,
vec![
@@ -3409,17 +3426,18 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment(
.await?;
wait_for_response_request_count(&responses, /*expected_count*/ 1).await;
attach_tx.send(()).expect("attach remote environment");
let mut denial_reason = None;
let mut was_denied = false;
wait_for_event(&test.codex, |event| {
if let EventMsg::GuardianAssessment(assessment) = event
&& assessment.status == GuardianAssessmentStatus::Denied
&& assessment.target_item_id.as_deref() == Some(EXEC_CALL_ID)
{
denial_reason = assessment.review_reason;
was_denied = true;
}
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert!(was_denied, "Guardian should deny the action");
let requests = responses.requests();
assert!(
@@ -3437,8 +3455,6 @@ async fn deferred_executor_guardian_uses_newly_ready_step_environment(
let guardian_context = guardian_request.message_input_texts("user").join("\n");
// Both actions start in the remote-primary step, but the MCP server runs locally.
let (owner_denied_path, other_denied_path) = if tool == "mcp_elicitation" {
// The low score is labeled local, but the model saw the remote primary's rules.
assert_eq!(denial_reason, Some(GuardianReviewReason::FreshRequired));
(&local_denied_path, &remote_denied_path)
} else {
(&remote_denied_path, &local_denied_path)
+30 -4
View File
@@ -16,6 +16,9 @@ use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::PromptFragment;
use codex_extension_api::ToolContributor;
use codex_extension_api::ToolLifecycleContributor;
use codex_extension_api::ToolLifecycleFuture;
use codex_extension_api::ToolStartInput;
use codex_extension_api::TurnContextContributionInput;
use codex_features::Feature;
use codex_history::RolloutItem;
@@ -2958,7 +2961,23 @@ async fn captured_step_controls_mcp_output_limit(supports_images: bool) -> Resul
}
#[derive(Clone)]
struct SettingsEcho(Arc<Mutex<Option<ConversationHistory>>>);
struct SettingsEcho {
history: Arc<Mutex<Option<ConversationHistory>>>,
sampling_model: Arc<Mutex<Option<String>>>,
}
impl ToolLifecycleContributor for SettingsEcho {
fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> {
Box::pin(async move {
if input.tool_name.is_default_namespace()
&& input.tool_name.name == "extension_settings"
{
*self.sampling_model.lock().expect("sampling model lock") =
Some(input.sampling_model.to_owned());
}
})
}
}
impl ToolContributor for SettingsEcho {
fn tools(
@@ -2991,7 +3010,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for SettingsEcho {
'call: 'a,
{
Box::pin(async move {
*self.0.lock().expect("history lock") = Some(call.conversation_history);
*self.history.lock().expect("history lock") = Some(call.conversation_history);
let metadata: Value = serde_json::from_str(
call.codex_turn_metadata
.as_deref()
@@ -3000,6 +3019,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for SettingsEcho {
.expect("metadata JSON");
Ok(Box::new(JsonToolOutput::new(json!({
"model": call.model,
"sampling_model": self.sampling_model.lock().expect("sampling model lock").clone(),
"metadata_model": metadata["model"],
"effort": metadata["reasoning_effort"],
"output_bytes": call.truncation_policy.byte_budget(),
@@ -3011,7 +3031,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for SettingsEcho {
#[test_case(ThreadHistoryMode::Legacy; "legacy")]
#[test_case(ThreadHistoryMode::Paginated; "paginated")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn captured_step_settings_and_history_reach_extension_executor(
async fn captured_step_settings_and_history_reach_extensions(
history_mode: ThreadHistoryMode,
) -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -3032,7 +3052,12 @@ async fn captured_step_settings_and_history_reach_extension_executor(
.await;
let history = Arc::default();
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
extensions.tool_contributor(Arc::new(SettingsEcho(Arc::clone(&history))));
let extension = Arc::new(SettingsEcho {
history: Arc::clone(&history),
sampling_model: Arc::default(),
});
extensions.tool_contributor(extension.clone());
extensions.tool_lifecycle_contributor(extension);
let test = direct_tool_settings_test()
.with_history_mode(history_mode)
.with_extensions(Arc::new(extensions.build()))
@@ -3067,6 +3092,7 @@ async fn captured_step_settings_and_history_reach_extension_executor(
serde_json::from_str::<Value>(&output)?,
json!({
"model": MODEL_B,
"sampling_model": MODEL_B,
"metadata_model": MODEL_B,
"effort": "high",
"output_bytes": 512,
@@ -122,9 +122,14 @@ impl McpToolContext {
/// Input supplied when the host starts executing one tool call.
pub struct ToolStartInput<'a> {
/// Model that issued this call or its enclosing code cell, preserved across yields and waits.
pub sampling_model: &'a str,
/// Final serving assessment for the unchanged action, including its code cell's value.
/// When present, replaces asynchronous scoring; it does not grant execution permission.
pub serving_assessment: Option<bool>,
/// Existing serving review flag for this action or its enclosing code cell, in shadow mode.
/// Absence means no flag was reported, not that the action was scored as low risk.
pub guardian_probe: Option<bool>,
/// Store scoped to the host session runtime.
pub session_store: &'a ExtensionData,
/// Store scoped to this thread runtime.
@@ -187,6 +187,26 @@ impl<H: ReviewHost> SynchronousApprovalReviewer for ReviewRequest<'_, H> {
analytics,
completed_at_ms.try_into().unwrap_or_default(),
);
tracing::event!(
target: "codex_otel.log_only",
tracing::Level::INFO,
event.name = "codex.guardian_probe_comparison",
stage = "sync_guardian",
thread_id = %self.thread_id,
turn_id = self.turn_id,
call_id = self.tool_call_id,
review_id = self.approval_id,
model = self.sampling_model,
guardian_context_mode = completed.analytics.guardian_context_mode.unwrap_or("not_reported"),
sync_outcome = if cached_approval { "async_approval" } else if completed.assessment_outcome.is_some() { "completed" } else { "failure" },
sync_decision = match completed.assessment_outcome {
Some(GuardianAssessmentOutcome::Allow) => "allow",
Some(GuardianAssessmentOutcome::Deny) => "deny",
None => "not_reported",
},
failure_reason = ?completed.analytics.failure_reason,
"Guardian probe comparison"
);
if self.log_assessments
&& !cached_approval
&& let Some(event) = &completed.event
@@ -164,6 +164,8 @@ async fn async_approval_revalidates_without_waiting_for_tracked_cleanup(state: A
let analytics = AnalyticsEventsClient::disabled();
let action = json!({});
let request = ReviewRequest {
sampling_model: "test",
turn_id: "turn",
serving_assessment: None,
host: host.clone(),
approval_id: "approval",
@@ -25,6 +25,9 @@ use tokio_util::sync::CancellationToken;
/// One captured action and its host-enforced constraints. Guardian derives the
/// contributor input and owns cancellation for the complete review operation.
pub struct ReviewRequest<'a, H> {
/// Issuing action's model and turn, retained across yielded code cells.
pub sampling_model: &'a str,
pub turn_id: &'a str,
/// Final serving value retained by the host for this action.
pub serving_assessment: Option<bool>,
pub host: H,
@@ -236,7 +236,9 @@ async fn assert_catalog_budget(evidence: BudgetEvidence) -> Result<()> {
};
fixture.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &fixture.session_store,
thread_store,
@@ -58,6 +58,7 @@ use super::trusted_tools::trusted_tool_context;
pub(super) struct Classification {
pub(super) reservation: Reservation,
pub(super) probe_comparison: Option<super::probe_shadow::ProbeComparison>,
pub(super) classification_started_at: Instant,
pub(super) decisions_sampler: Option<Arc<super::decisions::DecisionsSampler>>,
pub(super) guardian_config: GuardianV2Config,
@@ -113,6 +114,7 @@ impl Classification {
pub(super) async fn run(self) {
let Self {
reservation,
probe_comparison,
classification_started_at,
decisions_sampler,
guardian_config,
@@ -177,9 +179,11 @@ impl Classification {
else {
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"superseded",
/*classification_risk*/ None,
/*failure_reason*/ None,
);
return;
@@ -271,9 +275,11 @@ impl Classification {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("context_build_error"),
);
event_sink.emit_warning(ExtensionWarning {
@@ -453,9 +459,11 @@ impl Classification {
};
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
duration,
outcome,
classification_risk,
result.is_err().then_some(failure_reason),
);
if let Some(analytics) = analytics {
@@ -712,7 +712,9 @@ async fn delayed_classification_skips_after_newer_score_and_history_complete() -
let history = fixture.test.codex.conversation_history_snapshot().await;
let observe = |call_id, permissions| {
fixture.registry.tool_lifecycle_contributors()[0].on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: Some(true),
permissions,
session_store: &fixture.session_store,
thread_store: store,
@@ -362,7 +362,9 @@ async fn installed_extension_keeps_parent_program_out_of_websocket_and_http_revi
}
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &session_store,
thread_store,
@@ -645,7 +647,9 @@ async fn sandboxed_shell_classification_respects_review_scope() -> Result<()> {
] {
fixture.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { panic!("unexpected permission resolution") }),
session_store: &fixture.session_store,
thread_store,
@@ -749,7 +753,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {
};
fixture.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &fixture.session_store,
thread_store,
@@ -974,7 +980,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {
for _ in 0..ignored_calls {
fixture.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &fixture.session_store,
thread_store,
@@ -1257,7 +1265,9 @@ async fn sample_configured_conversation_history_with_delivery(
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &session_store,
thread_store,
@@ -1408,7 +1418,9 @@ impl GuardianFailureFixture {
};
self.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &self.session_store,
thread_store,
@@ -1568,7 +1580,7 @@ async fn contributor_fails_closed_when_luna_classification_fails() -> Result<()>
#[tokio::test(flavor = "multi_thread")]
async fn late_turn_actions_invalidate_the_current_policy_score() -> Result<()> {
skip_if_no_network!(Ok(()));
let fixture = GuardianFailureFixture::new().await?;
let fixture = GuardianFailureFixture::with_config("").await?;
let server = responses::start_mock_server().await;
let responses = ["low", "high", "invalid"].map(|risk| {
responses::sse(vec![
@@ -1612,21 +1624,30 @@ async fn late_turn_actions_invalidate_the_current_policy_score() -> Result<()> {
executions + 1
);
let mut expected: Option<crate::async_scorer::score::CachedScore> = None;
for (turn, call_id, risk, expected_approval) in [
(&new_turn, "new-score", 0.0, Some(ReviewDecision::Approved)),
for (turn, call_id, guardian_probe, risk, expected_approval) in [
(
&new_turn,
"new-score",
Some(true),
0.0,
Some(ReviewDecision::Approved),
),
(
&old_turn,
"late-old-score",
Some(false),
1.0,
Some(ReviewDecision::Approved),
),
(&old_turn, "late-old-failure", 1.0, None),
(&old_turn, "late-old-failure", None, 1.0, None),
] {
let backend = ConversationBackend::for_turn(store, turn).unwrap();
let mut updates = backend.progress.updates.subscribe();
fixture.registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &fixture.session_store,
thread_store: store,
@@ -2885,7 +2906,9 @@ async fn contributor_skips_required_models_in_standard_scope() -> Result<()> {
let previous_score = cached_score(thread_store);
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { panic!("unexpected permission resolution") }),
session_store: &session_store,
thread_store,
@@ -3241,7 +3264,9 @@ async fn contributor_counts_failed_thread_lookups_toward_score_lag() -> Result<(
};
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &session_store,
thread_store,
@@ -3777,7 +3802,9 @@ async fn assert_parent_compaction_reuse(parent_context_for_review: bool) -> Resu
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &session_store,
thread_store,
@@ -3873,7 +3900,9 @@ async fn assert_parent_compaction_reuse(parent_context_for_review: bool) -> Resu
let call_id = format!("call-{}", index + 2);
registry.tool_lifecycle_contributors()[0]
.on_tool_start(ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &session_store,
thread_store,
@@ -3985,7 +4014,9 @@ async fn cached_approval_respects_action_order_and_wrapper_lag() -> Result<()> {
},
};
progress.observe(&ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: &fixture.session_store,
thread_store: store,
@@ -4194,7 +4225,9 @@ fn seed_cached_score(
fn observe_unscored_call(progress: &GuardianV2ScoreProgress, store: &ExtensionData) -> usize {
progress.observe(&ToolStartInput {
sampling_model: "gpt-5.5",
serving_assessment: None,
guardian_probe: None,
permissions: Box::pin(async { Some(Default::default()) }),
session_store: store,
thread_store: store,
@@ -8,6 +8,7 @@ use codex_core::context::GuardianContextMode;
use codex_extension_api::ExtensionMetrics;
use super::decisions::DecisionsError;
use super::probe_shadow::ProbeComparison;
use super::sampler::LunaSamplerError;
pub(super) const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification";
@@ -73,11 +74,23 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str {
pub(super) fn record_classification(
metrics: Option<&dyn ExtensionMetrics>,
probe_comparison: Option<&ProbeComparison>,
context_mode: GuardianContextMode,
duration: Duration,
outcome: &str,
classification_risk: Option<&str>,
failure_reason: Option<&str>,
) {
// Shadow comparisons cover every reported classification outcome, including early exits.
if let Some(comparison) = probe_comparison {
comparison.record(
context_mode,
outcome,
classification_risk,
duration,
failure_reason,
);
}
if outcome == "failure" {
tracing::warn!(
failure_reason,
@@ -10,6 +10,7 @@ mod extension;
mod metrics;
mod observation;
mod parent_compaction;
mod probe_shadow;
mod request;
mod sampler;
mod score;
@@ -97,6 +97,7 @@ impl GuardianV2Extension {
return;
}
let parent_model = input.thread_store.get::<ModelInfo>();
let probe_comparison = Some(super::probe_shadow::ProbeComparison::capture(&input));
let policy = guardian_config.policy_for_model(
parent_model.as_deref(),
config.config_layer_stack.requirements(),
@@ -176,9 +177,11 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some(super::metrics::sampler_failure_reason(&error)),
);
return;
@@ -216,9 +219,11 @@ impl GuardianV2Extension {
score_progress.invalidate(tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("thread_context_error"),
);
event_sink.emit_warning(ExtensionWarning {
@@ -265,9 +270,11 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("configuration_error"),
);
self.event_sink.emit_warning(ExtensionWarning {
@@ -317,9 +324,11 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
outcome,
/*classification_risk*/ None,
failure_reason,
);
return;
@@ -339,9 +348,11 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("input_too_large"),
);
return;
@@ -350,9 +361,11 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("action_serialization_error"),
);
event_sink.emit_warning(ExtensionWarning {
@@ -395,15 +408,18 @@ impl GuardianV2Extension {
score_progress.fail_closed(sampled_at, tool_call_index);
record_classification(
metrics.as_deref(),
probe_comparison.as_ref(),
context_mode,
classification_started_at.elapsed(),
"failure",
/*classification_risk*/ None,
Some("permission_resolution_error"),
);
return;
};
let score_authorization = ScoreAuthorization::current(&thread, &permissions).await;
let classification = Classification {
probe_comparison,
reservation,
classification_started_at,
decisions_sampler: input
@@ -0,0 +1,65 @@
//! Logs existing probe flags alongside async results without waiting for other review stages.
//! Action IDs join these records to tool observations and synchronous review outcomes.
use std::time::Duration;
use codex_core::context::GuardianContextMode;
use codex_extension_api::ToolCallSource;
use codex_extension_api::ToolStartInput;
pub(super) struct ProbeComparison {
probe: Option<bool>,
scope: &'static str,
model: String,
thread_id: String,
turn_id: String,
call_id: String,
}
impl ProbeComparison {
pub(super) fn capture(input: &ToolStartInput<'_>) -> Self {
Self {
probe: input.guardian_probe,
scope: match input.source {
ToolCallSource::Direct => "tool_item",
ToolCallSource::CodeMode { .. } => "enclosing_item",
},
model: input.sampling_model.to_owned(),
thread_id: input.thread_store.level_id().to_owned(),
turn_id: input.turn_id.to_owned(),
call_id: input.call_id.to_owned(),
}
}
pub(super) fn record(
&self,
context_mode: GuardianContextMode,
outcome: &str,
risk: Option<&str>,
duration: Duration,
failure_reason: Option<&str>,
) {
tracing::event!(
target: "codex_otel.log_only",
tracing::Level::INFO,
event.name = "codex.guardian_probe_comparison",
stage = "async_scorer",
thread_id = %self.thread_id,
turn_id = %self.turn_id,
call_id = %self.call_id,
model = %self.model,
guardian_context_mode = context_mode.as_str(),
scope = self.scope,
probe_review = match self.probe {
Some(true) => "required",
Some(false) => "not_required",
None => "not_reported",
},
async_outcome = outcome,
async_risk = risk.unwrap_or("not_reported"),
failure_reason,
duration_ms = u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
"Guardian probe comparison"
);
}
}