test: cover Guardian reviewer retries end to end

Co-authored-by: Codex noreply@openai.com
This commit is contained in:
viyatb-oai
2026-06-08 21:26:41 -07:00
parent 025e22c318
commit e46d7f8f9b
2 changed files with 142 additions and 9 deletions

View File

@@ -2099,11 +2099,13 @@ async fn guardian_review_retries_reviewer_availability_failure() -> anyhow::Resu
let request_log = mount_response_sequence(
&server,
vec![
wiremock::ResponseTemplate::new(503).set_body_json(serde_json::json!({
"error": {
"message": "reviewer capacity temporarily unavailable"
}
})),
wiremock::ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(core_test_support::responses::sse_failed(
"resp-guardian-failed",
"server_error",
"reviewer capacity temporarily unavailable",
)),
wiremock::ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(sse(vec![
@@ -2137,12 +2139,12 @@ async fn guardian_review_retries_reviewer_availability_failure() -> anyhow::Resu
assert_eq!(decision, ReviewDecision::Approved);
let requests = request_log.requests();
assert_eq!(requests.len(), 2);
let retry_request = requests[1].body_json().to_string();
assert!(retry_request.contains(PRIOR_RETRY_REASON));
assert!(retry_request.contains(
let retry_request = &requests[1];
assert!(retry_request.body_contains_text(PRIOR_RETRY_REASON));
assert!(retry_request.body_contains_text(
"The previous automatic approval review was temporarily unavailable. Retry the review once before blocking the action."
));
assert!(!retry_request.contains("reviewer capacity temporarily unavailable"));
assert!(!retry_request.body_contains_text("reviewer capacity temporarily unavailable"));
Ok(())
}

View File

@@ -14,8 +14,11 @@ use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_function_call;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_response_sequence;
use core_test_support::responses::mount_sse_sequence;
use core_test_support::responses::sse;
use core_test_support::responses::sse_failed;
use core_test_support::responses::sse_response;
use core_test_support::responses::start_mock_server;
use core_test_support::skip_if_no_network;
use core_test_support::skip_if_sandbox;
@@ -30,6 +33,134 @@ use std::os::unix::fs::PermissionsExt;
use std::time::Duration;
use tempfile::TempDir;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn guardian_retries_a_transient_reviewer_failure() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = start_mock_server().await;
let approval_policy = AskForApproval::OnRequest;
let sandbox_policy = SandboxPolicy::WorkspaceWrite {
writable_roots: vec![],
network_access: false,
exclude_tmpdir_env_var: true,
exclude_slash_tmp: true,
};
let sandbox_policy_for_config = sandbox_policy.clone();
let mut builder = test_codex().with_config(move |config| {
config.permissions.approval_policy = Constrained::allow_any(approval_policy);
config
.set_legacy_sandbox_policy(sandbox_policy_for_config)
.expect("set sandbox policy");
});
let test = builder.build(&server).await?;
let output_file = test.cwd.path().join("guardian-review-retry.txt");
let command = format!("printf guardian-retried > {}", output_file.display());
let justification = "Exercise Guardian retry routing.";
let tool_args = json!({
"cmd": command,
"yield_time_ms": 1_000_u64,
"sandbox_permissions": SandboxPermissions::RequireEscalated,
"justification": justification,
});
let responses = mount_response_sequence(
&server,
vec![
sse_response(sse(vec![
ev_response_created("resp-parent-tool"),
ev_function_call(
"exec-call",
"exec_command",
&serde_json::to_string(&tool_args)?,
),
ev_completed("resp-parent-tool"),
])),
sse_response(sse_failed(
"resp-guardian-failed",
"server_error",
"reviewer capacity temporarily unavailable",
)),
sse_response(sse(vec![
ev_response_created("resp-guardian-review"),
ev_assistant_message(
"msg-guardian-review",
&json!({
"risk_level": "low",
"user_authorization": "high",
"outcome": "allow",
"rationale": "The command writes a marker file in the workspace.",
})
.to_string(),
),
ev_completed("resp-guardian-review"),
])),
sse_response(sse(vec![
ev_response_created("resp-parent-done"),
ev_assistant_message("msg-parent-done", "done"),
ev_completed("resp-parent-done"),
])),
],
)
.await;
test.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "run a command that requires Guardian review".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
responsesapi_client_metadata: None,
additional_context: Default::default(),
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides {
environments: Some(local_selections(test.config.cwd.clone())),
approval_policy: Some(approval_policy),
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
sandbox_policy: Some(sandbox_policy),
..Default::default()
},
})
.await?;
let mut retry_warnings = Vec::new();
loop {
match wait_for_event(&test.codex, |_| true).await {
EventMsg::GuardianWarning(warning)
if warning
.message
.contains("Automatic approval review is temporarily unavailable") =>
{
retry_warnings.push(warning.message);
}
EventMsg::TurnComplete(_) => break,
_ => {}
}
}
assert_eq!(
retry_warnings,
vec![
"Automatic approval review is temporarily unavailable. Retrying once before blocking the action."
]
);
let retry_request = responses
.requests()
.into_iter()
.find(|request| {
request.body_contains_text(
"The previous automatic approval review was temporarily unavailable. Retry the review once before blocking the action.",
)
})
.expect("expected retried Guardian review request");
assert!(retry_request.body_contains_text(&command));
assert!(retry_request.body_contains_text(justification));
assert!(!retry_request.body_contains_text("reviewer capacity temporarily unavailable"));
assert_eq!(fs::read_to_string(&output_file)?, "guardian-retried");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn guardian_review_session_does_not_inherit_legacy_notify() -> Result<()> {
skip_if_no_network!(Ok(()));