mirror of
https://github.com/openai/codex.git
synced 2026-09-06 15:29:32 +00:00
test: cover Guardian reviewer retries end to end
Co-authored-by: Codex noreply@openai.com
This commit is contained in:
@@ -2099,11 +2099,13 @@ async fn guardian_review_retries_reviewer_availability_failure() -> anyhow::Resu
|
||||
let request_log = mount_response_sequence(
|
||||
&server,
|
||||
vec![
|
||||
wiremock::ResponseTemplate::new(503).set_body_json(serde_json::json!({
|
||||
"error": {
|
||||
"message": "reviewer capacity temporarily unavailable"
|
||||
}
|
||||
})),
|
||||
wiremock::ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(core_test_support::responses::sse_failed(
|
||||
"resp-guardian-failed",
|
||||
"server_error",
|
||||
"reviewer capacity temporarily unavailable",
|
||||
)),
|
||||
wiremock::ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(sse(vec![
|
||||
@@ -2137,12 +2139,12 @@ async fn guardian_review_retries_reviewer_availability_failure() -> anyhow::Resu
|
||||
assert_eq!(decision, ReviewDecision::Approved);
|
||||
let requests = request_log.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
let retry_request = requests[1].body_json().to_string();
|
||||
assert!(retry_request.contains(PRIOR_RETRY_REASON));
|
||||
assert!(retry_request.contains(
|
||||
let retry_request = &requests[1];
|
||||
assert!(retry_request.body_contains_text(PRIOR_RETRY_REASON));
|
||||
assert!(retry_request.body_contains_text(
|
||||
"The previous automatic approval review was temporarily unavailable. Retry the review once before blocking the action."
|
||||
));
|
||||
assert!(!retry_request.contains("reviewer capacity temporarily unavailable"));
|
||||
assert!(!retry_request.body_contains_text("reviewer capacity temporarily unavailable"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -14,8 +14,11 @@ use core_test_support::responses::ev_assistant_message;
|
||||
use core_test_support::responses::ev_completed;
|
||||
use core_test_support::responses::ev_function_call;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
use core_test_support::responses::mount_response_sequence;
|
||||
use core_test_support::responses::mount_sse_sequence;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::responses::sse_failed;
|
||||
use core_test_support::responses::sse_response;
|
||||
use core_test_support::responses::start_mock_server;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::skip_if_sandbox;
|
||||
@@ -30,6 +33,134 @@ use std::os::unix::fs::PermissionsExt;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_retries_a_transient_reviewer_failure() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
skip_if_sandbox!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let approval_policy = AskForApproval::OnRequest;
|
||||
let sandbox_policy = SandboxPolicy::WorkspaceWrite {
|
||||
writable_roots: vec![],
|
||||
network_access: false,
|
||||
exclude_tmpdir_env_var: true,
|
||||
exclude_slash_tmp: true,
|
||||
};
|
||||
let sandbox_policy_for_config = sandbox_policy.clone();
|
||||
let mut builder = test_codex().with_config(move |config| {
|
||||
config.permissions.approval_policy = Constrained::allow_any(approval_policy);
|
||||
config
|
||||
.set_legacy_sandbox_policy(sandbox_policy_for_config)
|
||||
.expect("set sandbox policy");
|
||||
});
|
||||
let test = builder.build(&server).await?;
|
||||
|
||||
let output_file = test.cwd.path().join("guardian-review-retry.txt");
|
||||
let command = format!("printf guardian-retried > {}", output_file.display());
|
||||
let justification = "Exercise Guardian retry routing.";
|
||||
let tool_args = json!({
|
||||
"cmd": command,
|
||||
"yield_time_ms": 1_000_u64,
|
||||
"sandbox_permissions": SandboxPermissions::RequireEscalated,
|
||||
"justification": justification,
|
||||
});
|
||||
let responses = mount_response_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse_response(sse(vec![
|
||||
ev_response_created("resp-parent-tool"),
|
||||
ev_function_call(
|
||||
"exec-call",
|
||||
"exec_command",
|
||||
&serde_json::to_string(&tool_args)?,
|
||||
),
|
||||
ev_completed("resp-parent-tool"),
|
||||
])),
|
||||
sse_response(sse_failed(
|
||||
"resp-guardian-failed",
|
||||
"server_error",
|
||||
"reviewer capacity temporarily unavailable",
|
||||
)),
|
||||
sse_response(sse(vec![
|
||||
ev_response_created("resp-guardian-review"),
|
||||
ev_assistant_message(
|
||||
"msg-guardian-review",
|
||||
&json!({
|
||||
"risk_level": "low",
|
||||
"user_authorization": "high",
|
||||
"outcome": "allow",
|
||||
"rationale": "The command writes a marker file in the workspace.",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
ev_completed("resp-guardian-review"),
|
||||
])),
|
||||
sse_response(sse(vec![
|
||||
ev_response_created("resp-parent-done"),
|
||||
ev_assistant_message("msg-parent-done", "done"),
|
||||
ev_completed("resp-parent-done"),
|
||||
])),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
test.codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "run a command that requires Guardian review".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides {
|
||||
environments: Some(local_selections(test.config.cwd.clone())),
|
||||
approval_policy: Some(approval_policy),
|
||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
||||
sandbox_policy: Some(sandbox_policy),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
|
||||
let mut retry_warnings = Vec::new();
|
||||
loop {
|
||||
match wait_for_event(&test.codex, |_| true).await {
|
||||
EventMsg::GuardianWarning(warning)
|
||||
if warning
|
||||
.message
|
||||
.contains("Automatic approval review is temporarily unavailable") =>
|
||||
{
|
||||
retry_warnings.push(warning.message);
|
||||
}
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
retry_warnings,
|
||||
vec![
|
||||
"Automatic approval review is temporarily unavailable. Retrying once before blocking the action."
|
||||
]
|
||||
);
|
||||
let retry_request = responses
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|request| {
|
||||
request.body_contains_text(
|
||||
"The previous automatic approval review was temporarily unavailable. Retry the review once before blocking the action.",
|
||||
)
|
||||
})
|
||||
.expect("expected retried Guardian review request");
|
||||
assert!(retry_request.body_contains_text(&command));
|
||||
assert!(retry_request.body_contains_text(justification));
|
||||
assert!(!retry_request.body_contains_text("reviewer capacity temporarily unavailable"));
|
||||
assert_eq!(fs::read_to_string(&output_file)?, "guardian-retried");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_review_session_does_not_inherit_legacy_notify() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
Reference in New Issue
Block a user