mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Let Guardian V2 satisfy required model reviews (#39658)
## What changed - Allow the Guardian V2 approval monitor to handle reviews for models that require automatic review. - Preserve full Guardian review when Guardian V2 is disabled by configuration or managed requirements. - Cover required-model routing for low- and high-risk actions, plus both Guardian V2 disable paths. GitOrigin-RevId: d1201cc3dbd7c4a07de633388b14f17ada9feb39
This commit is contained in:
@@ -45,6 +45,7 @@ use super::mcp_tool::TEST_TOOL_NAME;
|
||||
use super::mcp_tool::start_mcp_server;
|
||||
|
||||
const TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MODEL: &str = "mock-model";
|
||||
const USER_CONTEXT: &str = "The user authorized reading the existing project files.";
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -65,6 +66,12 @@ enum GuardianRisk {
|
||||
High,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ModelReviewRequirement {
|
||||
Optional,
|
||||
Required,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ThreadLifecycle {
|
||||
New,
|
||||
@@ -166,6 +173,7 @@ async fn luna_websocket(
|
||||
async fn guardian_v2_routes_tool_approvals(
|
||||
risk: GuardianRisk,
|
||||
lifecycle: ThreadLifecycle,
|
||||
requirement: ModelReviewRequirement,
|
||||
) -> Result<()> {
|
||||
let (luna_score, expected_guardian_reviews) = match risk {
|
||||
GuardianRisk::Low => (0.25, 1),
|
||||
@@ -197,10 +205,24 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
let (mcp_server_url, mcp_server_handle) = start_mcp_server().await?;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
let (reviewer_config, requested_reviewer) = match requirement {
|
||||
ModelReviewRequirement::Optional => (
|
||||
"approvals_reviewer = \"auto_review\"",
|
||||
ApprovalsReviewer::AutoReview,
|
||||
),
|
||||
ModelReviewRequirement::Required => {
|
||||
std::fs::write(
|
||||
codex_home.path().join("requirements.toml"),
|
||||
format!("[auto_review]\nrequired_on_models = [\"{MODEL}\"]\n"),
|
||||
)?;
|
||||
("approvals_reviewer = \"user\"", ApprovalsReviewer::User)
|
||||
}
|
||||
};
|
||||
MockResponsesConfig::new(&responses_url)
|
||||
.with_model(MODEL)
|
||||
.with_provider_config("supports_websockets = false")
|
||||
.with_approval_policy("on-request")
|
||||
.with_root_config("approvals_reviewer = \"auto_review\"")
|
||||
.with_root_config(reviewer_config)
|
||||
.with_extra_config(&format!(
|
||||
"[mcp_servers.{TEST_SERVER_NAME}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"prompt\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}"
|
||||
))
|
||||
@@ -225,14 +247,18 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
.await?;
|
||||
let thread = match lifecycle {
|
||||
ThreadLifecycle::New => {
|
||||
app_server
|
||||
let started = app_server
|
||||
.start_thread(ThreadStartParams {
|
||||
approval_policy: Some(AskForApproval::OnRequest),
|
||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
||||
approvals_reviewer: Some(requested_reviewer),
|
||||
..Default::default()
|
||||
})
|
||||
.await?
|
||||
.thread
|
||||
.await?;
|
||||
assert_eq!(
|
||||
(started.model.as_str(), started.approvals_reviewer),
|
||||
(MODEL, ApprovalsReviewer::AutoReview)
|
||||
);
|
||||
started.thread
|
||||
}
|
||||
ThreadLifecycle::Resume => {
|
||||
let original_thread_id = original_thread_id.expect("resumed thread should exist");
|
||||
@@ -274,7 +300,10 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
approval_policy: Some(AskForApproval::OnRequest),
|
||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
||||
approvals_reviewer: match requirement {
|
||||
ModelReviewRequirement::Optional => Some(ApprovalsReviewer::AutoReview),
|
||||
ModelReviewRequirement::Required => None,
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
@@ -372,29 +401,76 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_v2_low_risk_actions_skip_subsequent_reviews() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::New).await
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::Low,
|
||||
ThreadLifecycle::New,
|
||||
ModelReviewRequirement::Optional,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_v2_high_risk_actions_require_full_reviews() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(GuardianRisk::High, ThreadLifecycle::New).await
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::High,
|
||||
ThreadLifecycle::New,
|
||||
ModelReviewRequirement::Optional,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_v2_threshold_score_requires_full_reviews() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(GuardianRisk::Threshold, ThreadLifecycle::New).await
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::Threshold,
|
||||
ThreadLifecycle::New,
|
||||
ModelReviewRequirement::Optional,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_v2_required_model_honors_low_risk_assessment() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::Low,
|
||||
ThreadLifecycle::New,
|
||||
ModelReviewRequirement::Required,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn guardian_v2_required_model_high_risk_requires_full_review() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::High,
|
||||
ThreadLifecycle::New,
|
||||
ModelReviewRequirement::Required,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn resumed_thread_starts_without_guardian_score() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::Resume).await
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::Low,
|
||||
ThreadLifecycle::Resume,
|
||||
ModelReviewRequirement::Optional,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn forked_thread_starts_without_guardian_score() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::Fork).await
|
||||
guardian_v2_routes_tool_approvals(
|
||||
GuardianRisk::Low,
|
||||
ThreadLifecycle::Fork,
|
||||
ModelReviewRequirement::Optional,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -313,11 +313,13 @@ async fn run_guardian_review(
|
||||
options: GuardianReviewOptions,
|
||||
) -> ReviewDecision {
|
||||
let turn = Arc::clone(context.turn());
|
||||
if !turn
|
||||
// Required models must use Guardian, but an enabled V2 monitor can satisfy the review.
|
||||
if (!turn
|
||||
.config
|
||||
.config_layer_stack
|
||||
.requirements()
|
||||
.auto_review_required_for_model(&turn.model_info.slug)
|
||||
|| turn.config.features.enabled(Feature::GuardianV2))
|
||||
&& reasons.retry.is_none()
|
||||
&& options
|
||||
.external_cancel
|
||||
|
||||
@@ -2,7 +2,10 @@ use codex_core::TurnInputRequest;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use codex_config::test_support::CloudConfigBundleFixture;
|
||||
use codex_core::config::Constrained;
|
||||
use codex_extension_api::ApprovalReviewContributor;
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::ExtensionFuture;
|
||||
@@ -29,6 +32,7 @@ use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::ReviewDecision;
|
||||
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||
use codex_protocol::request_permissions::PermissionGrantScope;
|
||||
use codex_protocol::request_permissions::RequestPermissionProfile;
|
||||
use codex_protocol::request_permissions::RequestPermissionsResponse;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::TempDirExt;
|
||||
@@ -50,6 +54,7 @@ use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_with_timeout;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use test_case::test_case;
|
||||
use tokio::time::timeout;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
@@ -155,6 +160,144 @@ async fn approval_review_contributor_skips_existing_guardian_model_call() -> Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum GuardianV2DisableSource {
|
||||
Config,
|
||||
ManagedRequirements,
|
||||
}
|
||||
|
||||
#[test_case(GuardianV2DisableSource::Config; "disabled in config")]
|
||||
#[test_case(GuardianV2DisableSource::ManagedRequirements; "disabled by managed policy")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn required_model_bypasses_extension_approval_when_guardian_v2_is_disabled(
|
||||
disable_source: GuardianV2DisableSource,
|
||||
) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
skip_if_wine_exec!(Ok(()), "request_permissions requires host-native paths");
|
||||
|
||||
let server = MockServer::start().await;
|
||||
let model = "gpt-5.4";
|
||||
let call_id = "required-model-permissions";
|
||||
let reason = "request network access under mandatory Guardian review";
|
||||
let responses = mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_response_created("resp-required-model-parent"),
|
||||
ev_function_call(
|
||||
call_id,
|
||||
"request_permissions",
|
||||
&json!({
|
||||
"reason": reason,
|
||||
"permissions": { "network": { "enabled": true } },
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
ev_completed("resp-required-model-parent"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-required-model-review"),
|
||||
ev_assistant_message(
|
||||
"msg-required-model-review",
|
||||
&json!({
|
||||
"risk_level": "high",
|
||||
"user_authorization": "low",
|
||||
"outcome": "deny",
|
||||
"rationale": "The requested network access is not authorized.",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
ev_completed("resp-required-model-review"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-required-model-followup"),
|
||||
ev_assistant_message("msg-required-model-followup", "done"),
|
||||
ev_completed("resp-required-model-followup"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let (guardian_v2_config, reviewer_requirements) = match disable_source {
|
||||
GuardianV2DisableSource::Config => ("false", ""),
|
||||
GuardianV2DisableSource::ManagedRequirements => {
|
||||
("true", "allowed_approvals_reviewers = [\"auto_review\"]\n")
|
||||
}
|
||||
};
|
||||
let requirements =
|
||||
format!("{reviewer_requirements}[auto_review]\nrequired_on_models = [\"{model}\"]\n");
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.approval_review_contributor(Arc::new(ApprovedReviewContributor));
|
||||
let mut builder = test_codex()
|
||||
.with_model(model)
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_pre_build_hook(move |home| {
|
||||
std::fs::write(
|
||||
home.join("config.toml"),
|
||||
format!("[features]\nguardianv2 = {guardian_v2_config}\n"),
|
||||
)
|
||||
.expect("Guardian v2 configuration should be written");
|
||||
})
|
||||
.with_cloud_config_bundle(
|
||||
CloudConfigBundleFixture::loader_with_enterprise_requirement(requirements),
|
||||
)
|
||||
.with_config(|config| {
|
||||
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
||||
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
|
||||
config
|
||||
.permissions
|
||||
.set_permission_profile(PermissionProfile::read_only())
|
||||
.expect("set read-only permission profile");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::RequestPermissionsTool)
|
||||
.expect("test config should allow feature update");
|
||||
});
|
||||
let test = builder.build_with_auto_env(&server).await?;
|
||||
assert!(!test.config.features.enabled(Feature::GuardianV2));
|
||||
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: reason.into(),
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await?;
|
||||
let event = wait_for_event(&test.codex, |event| {
|
||||
matches!(
|
||||
event,
|
||||
EventMsg::RequestPermissions(_) | EventMsg::TurnComplete(_)
|
||||
)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(event, EventMsg::TurnComplete(_)),
|
||||
"required-model review should not prompt the user: {event:?}"
|
||||
);
|
||||
|
||||
let output = responses
|
||||
.function_call_output_text(call_id)
|
||||
.context("expected request_permissions output")?;
|
||||
let actual: RequestPermissionsResponse = serde_json::from_str(&output)?;
|
||||
assert_eq!(
|
||||
actual,
|
||||
RequestPermissionsResponse {
|
||||
permissions: RequestPermissionProfile::default(),
|
||||
scope: PermissionGrantScope::Turn,
|
||||
strict_auto_review: false,
|
||||
}
|
||||
);
|
||||
let requests = responses.requests();
|
||||
let guardian_request = requests
|
||||
.iter()
|
||||
.find(|request| {
|
||||
request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian")
|
||||
})
|
||||
.context("expected full Guardian review for the required model")?;
|
||||
assert!(guardian_request.body_contains_text(reason));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn remote_model_override_uses_catalog_model_for_strict_auto_review() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
Reference in New Issue
Block a user