Reconsider denials in attached Guardian reviews (#62374)

## Why

The denial reconsideration feature only applied to standalone reviews, so
attached Guardian reviews could deny an action after a single verdict even
when the feature was enabled.

## What changed

Move denial reconsideration into the shared verdict runner so attached
reviews can also take one additional sample on a denial. Keep the first
denial and its rationale if the second sample runs past the review deadline.

## Testing

Extend the app-server integration test to verify that attached reviews
sample denials exactly twice with identical input, can approve on the
second sample, and retain the first denial and its rationale on timeout.

GitOrigin-RevId: 06cc730d08dc755863f715362328aa48d15678f0
This commit is contained in:
felixxia-oai
2026-11-18 21:58:11 +00:00
committed by copyberry
parent d7c289ec25
commit fecf5cc672
3 changed files with 147 additions and 68 deletions
@@ -14,6 +14,7 @@ use test_case::test_case;
enum ReviewCase {
V3,
V3WithReconsideration,
V3RoutingFailure,
EphemeralV3,
Native,
@@ -49,22 +50,31 @@ fn received_user_instruction(request: &responses::ResponsesRequest, text: &str)
}
#[test_case(ReviewCase::V3; "v3_follows_subscribed_revocation_and_reapproval")]
#[test_case(ReviewCase::V3WithReconsideration; "v3_resamples_a_terminal_denial_once")]
#[test_case(ReviewCase::V3RoutingFailure; "v3_denies_when_pending_routing_fails")]
#[test_case(ReviewCase::Native; "native_without_v3")]
#[test_case(ReviewCase::EphemeralV3; "ephemeral_uses_native_with_v3")]
#[tokio::test]
async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Result<()> {
skip_if_no_network!(Ok(()));
let uses_board = matches!(case, ReviewCase::V3 | ReviewCase::V3RoutingFailure);
let reconsider = matches!(case, ReviewCase::V3WithReconsideration);
let uses_board = matches!(
case,
ReviewCase::V3 | ReviewCase::V3WithReconsideration | ReviewCase::V3RoutingFailure
);
let routing_failure = matches!(case, ReviewCase::V3RoutingFailure);
let server_enabled = !matches!(case, ReviewCase::Native);
let server = responses::start_mock_server().await;
const REVOCATION: &str = "I revoke permission for the release. Do not deploy.";
const REAPPROVAL: &str =
"I understand the release risk. I explicitly approve this release action.";
const FINAL_REVOCATION: &str = "Stop the release again. Do not deploy.";
const FIRST_DENIAL_REASON: &str = "The user explicitly stopped this deployment.";
let next_call = Arc::new(Mutex::new(None));
let actor_call = Arc::clone(&next_call);
let ask_to_subscribe = AtomicBool::new(uses_board);
let deny_first_reapproval = AtomicBool::new(reconsider);
let first_final_sample = AtomicBool::new(true);
let requests = responses::mount_response_handler(&server, move |request| {
let body = request.body_json();
let arguments = &body["text"]["format"]["schema"]["properties"];
@@ -102,20 +112,31 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
responses::ev_completed("subscribe"),
]));
}
let decision = if received_user_instruction(&request, REVOCATION)
&& !received_user_instruction(&request, REAPPROVAL)
{
"deny"
let final_revocation = received_user_instruction(&request, FINAL_REVOCATION);
let denied = if final_revocation {
first_final_sample.swap(false, Ordering::SeqCst)
} else if received_user_instruction(&request, REAPPROVAL) {
deny_first_reapproval.swap(false, Ordering::SeqCst)
} else {
"allow"
received_user_instruction(&request, REVOCATION)
};
let text =
json!({"outcome":decision,"rationale":"Checked the requested release action."})
.to_string();
responses::sse(vec![
let decision = if denied { "deny" } else { "allow" };
let rationale = if final_revocation && denied {
FIRST_DENIAL_REASON
} else {
"Checked the requested release action."
};
let text = json!({"outcome":decision,"rationale":rationale}).to_string();
let response = responses::sse_response(responses::sse(vec![
responses::ev_assistant_message("verdict", &text),
responses::ev_completed_with_tokens("verdict", /*total_tokens*/ 31),
])
]));
return if final_revocation && !denied {
// Only the second verdict request stalls past the shared review deadline.
response.set_delay(TIMEOUT)
} else {
response
};
} else if let Some(call_id) = actor_call.lock().expect("next actor call").take() {
responses::sse(vec![
responses::ev_function_call_with_namespace(
@@ -171,6 +192,13 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
.with_extra_config(
"[apps.connector_openai_browser]\ndefault_tools_approval_mode = \"prompt\"",
);
config = if reconsider {
config
.enable_feature(Feature::GuardianDenialReconsideration)
.with_root_config("auto_review.timeout_ms = 10000")
} else {
config.disable_feature(Feature::GuardianDenialReconsideration)
};
if uses_board {
config = config
.with_extra_config("[analytics]\nenabled = true")
@@ -214,8 +242,17 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
),
]);
}
if reconsider {
turns.push((
"release-reconsideration-timeout",
FINAL_REVOCATION,
GuardianApprovalReviewStatus::Denied,
));
}
let mut previous_reviewer = None;
let mut previous_review_id = None;
let mut denied_turns = Vec::new();
let mut seen_requests = 0;
for (call_id, prompt, status) in turns {
// Keep the revocation out of the board until the pending action requests its review.
let gate = if prompt == REVOCATION {
@@ -258,10 +295,11 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
timeout(TIMEOUT, app.read_notification("turn/completed")).await??;
assert_eq!(completed.turn.id, started.turn.id);
let calls = requests.requests();
let assessments: Vec<_> = calls
let assessments: Vec<_> = calls[seen_requests..]
.iter()
.filter(|r| expects_field(r, "outcome"))
.collect();
seen_requests = calls.len();
assert!(!calls.iter().any(|r| expects_field(r, "channels")));
if routing_failure {
let denied = wait_for_mcp_tool_call_completed(&mut app, call_id).await?;
@@ -302,6 +340,25 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
(&thread.id, &completed.turn.id, status)
);
previous_review_id = Some(review.review_id);
if status == GuardianApprovalReviewStatus::Denied {
denied_turns.push(review.turn_id);
}
if prompt == FINAL_REVOCATION {
assert_eq!(
review.review.rationale.as_deref(),
Some(FIRST_DENIAL_REASON)
);
}
if prompt == REVOCATION || prompt == REAPPROVAL || prompt == FINAL_REVOCATION {
if reconsider {
let [first, second] = assessments.as_slice() else {
anyhow::bail!("expected exactly two verdict samples for {call_id}");
};
assert_eq!(first.input(), second.input());
} else {
assert_eq!(assessments.len(), 1);
}
}
// A subscription tool call can add a sampling step. Every assessment must
// use the one reviewer route chosen for this actor.
assert!(assessments.iter().all(|r| {
@@ -392,9 +449,15 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re
&[codex_protocol::ThreadId::from_string(&thread.id)?],
)
.await;
let record: Value =
serde_json::from_slice(&failures.attachment.expect("denial feedback").buffer)?;
assert_eq!(record["status"], "denied");
let attachment = failures.attachment.expect("denial feedback");
let mut records =
serde_json::Deserializer::from_slice(&attachment.buffer).into_iter::<Value>();
for turn_id in denied_turns {
let record = records.next().expect("feedback for each denied turn")?;
assert_eq!(record["reviewed_turn_id"], turn_id);
assert_eq!(record["status"], "denied");
}
assert!(records.next().is_none(), "unexpected extra feedback record");
}
app.shutdown_gracefully().await?;
Ok(())
+54 -37
View File
@@ -15,6 +15,7 @@ use codex_analytics::GuardianReviewSessionKind;
use codex_guardian_context::ComposedContext;
use codex_guardian_reviewer::GuardianReviewSessionOutcome;
use codex_guardian_reviewer::SessionDisposition;
use codex_protocol::protocol::GuardianAssessmentOutcome;
use codex_protocol::user_input::UserInput;
use tokio::sync::Mutex;
@@ -84,7 +85,7 @@ impl AttachedApproval {
let mut reservation = None;
let mut analytics = GuardianReviewAnalyticsResult::without_session();
let mut usage_at_start = None;
let operation = async {
let prepare = async {
let first = self.first.lock().await.take();
let (authorization, model) = match first {
Some(first) => first,
@@ -97,7 +98,8 @@ impl AttachedApproval {
&approval.context,
cancel,
)
.await?,
.await
.map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?,
),
};
let model = reservation.insert(model);
@@ -113,18 +115,21 @@ impl AttachedApproval {
);
if let Some(stale) = authorization.invalidation(parent, Some(cancel)).await {
analytics.context_changed = stale.context_changed;
return anyhow::Ok((stale.outcome, SessionDisposition::Discard));
return Err(stale.outcome);
}
let action = codex_guardian_reviewer::build_guardian_planned_action(
&approval.request,
reasons.retry.clone().or_else(|| reasons.approval.clone()),
approval.context.enclosing_invocation_id.as_deref(),
)?;
)
.map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?;
let permissions = super::super::super::permissions::for_request(
&approval.request,
Some(&approval.context),
)?;
let action_context = ComposedContext::for_pending_action(action, permissions.as_ref())?;
)
.map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?;
let action_context = ComposedContext::for_pending_action(action, permissions.as_ref())
.map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?;
let request = review_turn(
parent,
&approval.context,
@@ -149,38 +154,50 @@ impl AttachedApproval {
if matches!(kind, GuardianReviewSessionKind::TrunkReused) {
ensure_guardian_followup_reminder(runtime).await;
}
let result = async {
prepare_verdict_session(
runtime,
&approval.context,
approval.category,
&config.node_repl_policy,
)
.await
.map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?;
runtime
.run_sync_verdict(
&approval.context,
turn,
deadline,
Some(cancel),
&mut analytics,
)
.await
}
.await;
if let Some(stale) = authorization.invalidation(parent, Some(cancel)).await {
analytics.context_changed |= stale.context_changed;
return Ok((stale.outcome, SessionDisposition::Discard));
}
Ok(match result {
Ok((turn, _)) => (turn.outcome, turn.disposition),
Err(outcome) => (outcome, SessionDisposition::Discard),
})
};
let result = run_before_review_deadline(deadline, Some(cancel), operation)
prepare_verdict_session(
runtime,
&approval.context,
approval.category,
&config.node_repl_policy,
)
.await
.and_then(|inner| inner.map_err(GuardianReviewSessionOutcome::PromptBuildFailed));
.map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?;
Ok((runtime, authorization, turn))
};
let prepared = run_before_review_deadline(deadline, Some(cancel), prepare).await;
let result = async {
let (runtime, authorization, turn) = prepared??;
// Let the runner interrupt and drain the turn before it finalizes a provisional denial.
let (turn, _) = runtime
.run_sync_verdict(&approval.context, turn, deadline, Some(cancel), &mut analytics)
.await?;
let stale = tokio::select! {
biased;
_ = cancel.cancelled() => return Err(GuardianReviewSessionOutcome::Aborted),
_ = deadline.wait() => {
// An interrupted second sample may still carry the first denial.
let recovered_denial = match &turn.outcome {
GuardianReviewSessionOutcome::Completed(Ok(Some(response))) if !turn.turn_completed => {
response.assessment().is_ok_and(|assessment| {
assessment.outcome == GuardianAssessmentOutcome::Deny
})
}
_ => false,
};
if !recovered_denial {
return Err(GuardianReviewSessionOutcome::TimedOut);
}
None
}
stale = authorization.invalidation(parent, Some(cancel)) => stale,
};
if let Some(stale) = stale {
analytics.context_changed |= stale.context_changed;
return Err(stale.outcome);
}
Ok((turn.outcome, turn.disposition))
}
.await;
let (outcome, disposition) =
result.unwrap_or_else(|outcome| (outcome, SessionDisposition::Discard));
let Some(model) = reservation else {
+15 -16
View File
@@ -591,14 +591,6 @@ async fn run_review_on_session(
items,
params.schema.clone(),
);
let store = &review_session.session.services.thread_extension_data;
if params
.spawn_config
.features
.enabled(Feature::GuardianDenialReconsideration)
{
store.insert(codex_guardian_reviewer::DenialResample::default());
}
let result = review_session
.run_sync_verdict(
&params.parent_context,
@@ -615,7 +607,6 @@ async fn run_review_on_session(
&mut analytics_result,
)
.await;
let resample = store.remove::<codex_guardian_reviewer::DenialResample>();
let (turn_result, transcript_admitted) = match result {
Ok(completed) => completed,
Err(outcome) => {
@@ -626,12 +617,6 @@ async fn run_review_on_session(
};
}
};
let turn_result = match resample {
Some(resample) => {
resample.finish(turn_result, &params.parent_context.turn().session_telemetry)
}
None => turn_result,
};
let completed_review = matches!(
turn_result.outcome,
GuardianReviewSessionOutcome::Completed(_)
@@ -800,6 +785,13 @@ impl GuardianReviewSession {
analytics: &mut GuardianReviewAnalyticsResult,
) -> Result<(ReviewTurnResult, bool), GuardianReviewSessionOutcome> {
let store = &self.session.services.thread_extension_data;
if self
.reuse_key
.features
.enabled(Feature::GuardianDenialReconsideration)
{
store.insert(codex_guardian_reviewer::DenialResample::default());
}
store.insert(turn.evidence);
store.insert(super::PendingReviewRouting(context.parent_response.clone()));
let mut observation =
@@ -826,7 +818,14 @@ impl GuardianReviewSession {
.await;
store.remove::<super::PendingReviewRouting>();
let admitted = store.remove::<PendingReviewContext>().is_none();
result.map(|turn| (turn, admitted))
let resample = store.remove::<codex_guardian_reviewer::DenialResample>();
result.map(|turn| {
let turn = match resample {
Some(resample) => resample.finish(turn, &context.turn().session_telemetry),
None => turn,
};
(turn, admitted)
})
}
pub(super) fn analytics(