diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v3_sync_tests.rs b/codex-rs/app-server/tests/suite/v2/guardian_v3_sync_tests.rs index f5d9d90b11..8ee3ac0355 100644 --- a/codex-rs/app-server/tests/suite/v2/guardian_v3_sync_tests.rs +++ b/codex-rs/app-server/tests/suite/v2/guardian_v3_sync_tests.rs @@ -14,6 +14,7 @@ use test_case::test_case; enum ReviewCase { V3, + V3WithReconsideration, V3RoutingFailure, EphemeralV3, Native, @@ -49,22 +50,31 @@ fn received_user_instruction(request: &responses::ResponsesRequest, text: &str) } #[test_case(ReviewCase::V3; "v3_follows_subscribed_revocation_and_reapproval")] +#[test_case(ReviewCase::V3WithReconsideration; "v3_resamples_a_terminal_denial_once")] #[test_case(ReviewCase::V3RoutingFailure; "v3_denies_when_pending_routing_fails")] #[test_case(ReviewCase::Native; "native_without_v3")] #[test_case(ReviewCase::EphemeralV3; "ephemeral_uses_native_with_v3")] #[tokio::test] async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Result<()> { skip_if_no_network!(Ok(())); - let uses_board = matches!(case, ReviewCase::V3 | ReviewCase::V3RoutingFailure); + let reconsider = matches!(case, ReviewCase::V3WithReconsideration); + let uses_board = matches!( + case, + ReviewCase::V3 | ReviewCase::V3WithReconsideration | ReviewCase::V3RoutingFailure + ); let routing_failure = matches!(case, ReviewCase::V3RoutingFailure); let server_enabled = !matches!(case, ReviewCase::Native); let server = responses::start_mock_server().await; const REVOCATION: &str = "I revoke permission for the release. Do not deploy."; const REAPPROVAL: &str = "I understand the release risk. I explicitly approve this release action."; + const FINAL_REVOCATION: &str = "Stop the release again. Do not deploy."; + const FIRST_DENIAL_REASON: &str = "The user explicitly stopped this deployment."; let next_call = Arc::new(Mutex::new(None)); let actor_call = Arc::clone(&next_call); let ask_to_subscribe = AtomicBool::new(uses_board); + let deny_first_reapproval = AtomicBool::new(reconsider); + let first_final_sample = AtomicBool::new(true); let requests = responses::mount_response_handler(&server, move |request| { let body = request.body_json(); let arguments = &body["text"]["format"]["schema"]["properties"]; @@ -102,20 +112,31 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re responses::ev_completed("subscribe"), ])); } - let decision = if received_user_instruction(&request, REVOCATION) - && !received_user_instruction(&request, REAPPROVAL) - { - "deny" + let final_revocation = received_user_instruction(&request, FINAL_REVOCATION); + let denied = if final_revocation { + first_final_sample.swap(false, Ordering::SeqCst) + } else if received_user_instruction(&request, REAPPROVAL) { + deny_first_reapproval.swap(false, Ordering::SeqCst) } else { - "allow" + received_user_instruction(&request, REVOCATION) }; - let text = - json!({"outcome":decision,"rationale":"Checked the requested release action."}) - .to_string(); - responses::sse(vec![ + let decision = if denied { "deny" } else { "allow" }; + let rationale = if final_revocation && denied { + FIRST_DENIAL_REASON + } else { + "Checked the requested release action." + }; + let text = json!({"outcome":decision,"rationale":rationale}).to_string(); + let response = responses::sse_response(responses::sse(vec![ responses::ev_assistant_message("verdict", &text), responses::ev_completed_with_tokens("verdict", /*total_tokens*/ 31), - ]) + ])); + return if final_revocation && !denied { + // Only the second verdict request stalls past the shared review deadline. + response.set_delay(TIMEOUT) + } else { + response + }; } else if let Some(call_id) = actor_call.lock().expect("next actor call").take() { responses::sse(vec![ responses::ev_function_call_with_namespace( @@ -171,6 +192,13 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re .with_extra_config( "[apps.connector_openai_browser]\ndefault_tools_approval_mode = \"prompt\"", ); + config = if reconsider { + config + .enable_feature(Feature::GuardianDenialReconsideration) + .with_root_config("auto_review.timeout_ms = 10000") + } else { + config.disable_feature(Feature::GuardianDenialReconsideration) + }; if uses_board { config = config .with_extra_config("[analytics]\nenabled = true") @@ -214,8 +242,17 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re ), ]); } + if reconsider { + turns.push(( + "release-reconsideration-timeout", + FINAL_REVOCATION, + GuardianApprovalReviewStatus::Denied, + )); + } let mut previous_reviewer = None; let mut previous_review_id = None; + let mut denied_turns = Vec::new(); + let mut seen_requests = 0; for (call_id, prompt, status) in turns { // Keep the revocation out of the board until the pending action requests its review. let gate = if prompt == REVOCATION { @@ -258,10 +295,11 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re timeout(TIMEOUT, app.read_notification("turn/completed")).await??; assert_eq!(completed.turn.id, started.turn.id); let calls = requests.requests(); - let assessments: Vec<_> = calls + let assessments: Vec<_> = calls[seen_requests..] .iter() .filter(|r| expects_field(r, "outcome")) .collect(); + seen_requests = calls.len(); assert!(!calls.iter().any(|r| expects_field(r, "channels"))); if routing_failure { let denied = wait_for_mcp_tool_call_completed(&mut app, call_id).await?; @@ -302,6 +340,25 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re (&thread.id, &completed.turn.id, status) ); previous_review_id = Some(review.review_id); + if status == GuardianApprovalReviewStatus::Denied { + denied_turns.push(review.turn_id); + } + if prompt == FINAL_REVOCATION { + assert_eq!( + review.review.rationale.as_deref(), + Some(FIRST_DENIAL_REASON) + ); + } + if prompt == REVOCATION || prompt == REAPPROVAL || prompt == FINAL_REVOCATION { + if reconsider { + let [first, second] = assessments.as_slice() else { + anyhow::bail!("expected exactly two verdict samples for {call_id}"); + }; + assert_eq!(first.input(), second.input()); + } else { + assert_eq!(assessments.len(), 1); + } + } // A subscription tool call can add a sampling step. Every assessment must // use the one reviewer route chosen for this actor. assert!(assessments.iter().all(|r| { @@ -392,9 +449,15 @@ async fn pending_mcp_action_uses_the_configured_reviewer(case: ReviewCase) -> Re &[codex_protocol::ThreadId::from_string(&thread.id)?], ) .await; - let record: Value = - serde_json::from_slice(&failures.attachment.expect("denial feedback").buffer)?; - assert_eq!(record["status"], "denied"); + let attachment = failures.attachment.expect("denial feedback"); + let mut records = + serde_json::Deserializer::from_slice(&attachment.buffer).into_iter::(); + for turn_id in denied_turns { + let record = records.next().expect("feedback for each denied turn")?; + assert_eq!(record["reviewed_turn_id"], turn_id); + assert_eq!(record["status"], "denied"); + } + assert!(records.next().is_none(), "unexpected extra feedback record"); } app.shutdown_gracefully().await?; Ok(()) diff --git a/codex-rs/core/src/guardian/attached_review.rs b/codex-rs/core/src/guardian/attached_review.rs index 58e461ba35..0efe33c212 100644 --- a/codex-rs/core/src/guardian/attached_review.rs +++ b/codex-rs/core/src/guardian/attached_review.rs @@ -15,6 +15,7 @@ use codex_analytics::GuardianReviewSessionKind; use codex_guardian_context::ComposedContext; use codex_guardian_reviewer::GuardianReviewSessionOutcome; use codex_guardian_reviewer::SessionDisposition; +use codex_protocol::protocol::GuardianAssessmentOutcome; use codex_protocol::user_input::UserInput; use tokio::sync::Mutex; @@ -84,7 +85,7 @@ impl AttachedApproval { let mut reservation = None; let mut analytics = GuardianReviewAnalyticsResult::without_session(); let mut usage_at_start = None; - let operation = async { + let prepare = async { let first = self.first.lock().await.take(); let (authorization, model) = match first { Some(first) => first, @@ -97,7 +98,8 @@ impl AttachedApproval { &approval.context, cancel, ) - .await?, + .await + .map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?, ), }; let model = reservation.insert(model); @@ -113,18 +115,21 @@ impl AttachedApproval { ); if let Some(stale) = authorization.invalidation(parent, Some(cancel)).await { analytics.context_changed = stale.context_changed; - return anyhow::Ok((stale.outcome, SessionDisposition::Discard)); + return Err(stale.outcome); } let action = codex_guardian_reviewer::build_guardian_planned_action( &approval.request, reasons.retry.clone().or_else(|| reasons.approval.clone()), approval.context.enclosing_invocation_id.as_deref(), - )?; + ) + .map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?; let permissions = super::super::super::permissions::for_request( &approval.request, Some(&approval.context), - )?; - let action_context = ComposedContext::for_pending_action(action, permissions.as_ref())?; + ) + .map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?; + let action_context = ComposedContext::for_pending_action(action, permissions.as_ref()) + .map_err(|err| GuardianReviewSessionOutcome::PromptBuildFailed(err.into()))?; let request = review_turn( parent, &approval.context, @@ -149,38 +154,50 @@ impl AttachedApproval { if matches!(kind, GuardianReviewSessionKind::TrunkReused) { ensure_guardian_followup_reminder(runtime).await; } - let result = async { - prepare_verdict_session( - runtime, - &approval.context, - approval.category, - &config.node_repl_policy, - ) - .await - .map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?; - runtime - .run_sync_verdict( - &approval.context, - turn, - deadline, - Some(cancel), - &mut analytics, - ) - .await - } - .await; - if let Some(stale) = authorization.invalidation(parent, Some(cancel)).await { - analytics.context_changed |= stale.context_changed; - return Ok((stale.outcome, SessionDisposition::Discard)); - } - Ok(match result { - Ok((turn, _)) => (turn.outcome, turn.disposition), - Err(outcome) => (outcome, SessionDisposition::Discard), - }) - }; - let result = run_before_review_deadline(deadline, Some(cancel), operation) + prepare_verdict_session( + runtime, + &approval.context, + approval.category, + &config.node_repl_policy, + ) .await - .and_then(|inner| inner.map_err(GuardianReviewSessionOutcome::PromptBuildFailed)); + .map_err(GuardianReviewSessionOutcome::PromptBuildFailed)?; + Ok((runtime, authorization, turn)) + }; + let prepared = run_before_review_deadline(deadline, Some(cancel), prepare).await; + let result = async { + let (runtime, authorization, turn) = prepared??; + // Let the runner interrupt and drain the turn before it finalizes a provisional denial. + let (turn, _) = runtime + .run_sync_verdict(&approval.context, turn, deadline, Some(cancel), &mut analytics) + .await?; + let stale = tokio::select! { + biased; + _ = cancel.cancelled() => return Err(GuardianReviewSessionOutcome::Aborted), + _ = deadline.wait() => { + // An interrupted second sample may still carry the first denial. + let recovered_denial = match &turn.outcome { + GuardianReviewSessionOutcome::Completed(Ok(Some(response))) if !turn.turn_completed => { + response.assessment().is_ok_and(|assessment| { + assessment.outcome == GuardianAssessmentOutcome::Deny + }) + } + _ => false, + }; + if !recovered_denial { + return Err(GuardianReviewSessionOutcome::TimedOut); + } + None + } + stale = authorization.invalidation(parent, Some(cancel)) => stale, + }; + if let Some(stale) = stale { + analytics.context_changed |= stale.context_changed; + return Err(stale.outcome); + } + Ok((turn.outcome, turn.disposition)) + } + .await; let (outcome, disposition) = result.unwrap_or_else(|outcome| (outcome, SessionDisposition::Discard)); let Some(model) = reservation else { diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 3f27b19a00..4e5cda0e86 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -591,14 +591,6 @@ async fn run_review_on_session( items, params.schema.clone(), ); - let store = &review_session.session.services.thread_extension_data; - if params - .spawn_config - .features - .enabled(Feature::GuardianDenialReconsideration) - { - store.insert(codex_guardian_reviewer::DenialResample::default()); - } let result = review_session .run_sync_verdict( ¶ms.parent_context, @@ -615,7 +607,6 @@ async fn run_review_on_session( &mut analytics_result, ) .await; - let resample = store.remove::(); let (turn_result, transcript_admitted) = match result { Ok(completed) => completed, Err(outcome) => { @@ -626,12 +617,6 @@ async fn run_review_on_session( }; } }; - let turn_result = match resample { - Some(resample) => { - resample.finish(turn_result, ¶ms.parent_context.turn().session_telemetry) - } - None => turn_result, - }; let completed_review = matches!( turn_result.outcome, GuardianReviewSessionOutcome::Completed(_) @@ -800,6 +785,13 @@ impl GuardianReviewSession { analytics: &mut GuardianReviewAnalyticsResult, ) -> Result<(ReviewTurnResult, bool), GuardianReviewSessionOutcome> { let store = &self.session.services.thread_extension_data; + if self + .reuse_key + .features + .enabled(Feature::GuardianDenialReconsideration) + { + store.insert(codex_guardian_reviewer::DenialResample::default()); + } store.insert(turn.evidence); store.insert(super::PendingReviewRouting(context.parent_response.clone())); let mut observation = @@ -826,7 +818,14 @@ impl GuardianReviewSession { .await; store.remove::(); let admitted = store.remove::().is_none(); - result.map(|turn| (turn, admitted)) + let resample = store.remove::(); + result.map(|turn| { + let turn = match resample { + Some(resample) => resample.finish(turn, &context.turn().session_telemetry), + None => turn, + }; + (turn, admitted) + }) } pub(super) fn analytics(