From 7895f77eb0d02cbed551d3b1ff20dd7d99afc6dc Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Mon, 1 Jun 2026 15:22:12 -0700 Subject: [PATCH] codex: stabilize guardian selector integration test --- codex-rs/core/src/guardian/tests.rs | 87 +++++++++++++++++ .../tests/suite/model_runtime_selectors.rs | 95 ------------------- 2 files changed, 87 insertions(+), 95 deletions(-) diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 40a07d3802..85fcbf005f 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -25,6 +25,8 @@ use codex_model_provider::create_model_provider; use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_4_MODEL_ID; use codex_model_provider_info::AMAZON_BEDROCK_PROVIDER_ID; use codex_model_provider_info::ModelProviderInfo; +use codex_models_manager::manager::RefreshStrategy; +use codex_models_manager::model_info::model_info_from_slug; use codex_network_proxy::NetworkProxyConfig; use codex_protocol::ThreadId; use codex_protocol::approvals::NetworkApprovalProtocol; @@ -32,6 +34,8 @@ use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::models::ContentItem; use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::ModelVisibility; +use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -45,6 +49,7 @@ use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::GuardianAssessmentStatus; use codex_protocol::protocol::GuardianRiskLevel; use codex_protocol::protocol::GuardianUserAuthorization; +use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TurnCompleteEvent; @@ -55,6 +60,7 @@ use core_test_support::context_snapshot::ContextSnapshotOptions; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_models_once; use core_test_support::responses::mount_response_once; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; @@ -67,6 +73,7 @@ use core_test_support::test_path_buf; use insta::Settings; use insta::assert_snapshot; use pretty_assertions::assert_eq; +use serde_json::Value; use std::collections::BTreeMap; use std::collections::HashMap; use std::sync::Arc; @@ -1377,6 +1384,86 @@ async fn guardian_review_uses_preferred_review_model_without_model_catalog_overr Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_stays_disabled_when_model_selects_multi_agent_v2() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let guardian_model = "guardian-multi-agent-v2"; + let mut model = model_info_from_slug(guardian_model); + model.visibility = ModelVisibility::List; + model.used_fallback_model_metadata = false; + model.multi_agent_version = Some(MultiAgentVersion::V2); + let models_mock = mount_models_once( + &server, + ModelsResponse { + models: vec![model], + }, + ) + .await; + let request_log = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-guardian"), + ev_assistant_message("msg-guardian", "{\"outcome\":\"allow\"}"), + ev_completed("resp-guardian"), + ]), + ) + .await; + + let (session, mut turn) = guardian_test_session_and_turn(&server).await; + let _ = session + .services + .models_manager + .list_models(RefreshStrategy::Online) + .await; + Arc::get_mut(&mut turn) + .expect("turn should be unique") + .model_info + .auto_review_model_override = Some(guardian_model.to_string()); + seed_guardian_parent_history(&session, &turn).await; + + let outcome = run_guardian_review_session_for_test( + Arc::clone(&session), + turn, + GuardianApprovalRequest::Shell { + id: "shell-1".to_string(), + command: vec!["git".to_string(), "push".to_string()], + cwd: test_path_buf("/repo/codex-rs/core").abs(), + sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, + additional_permissions: None, + justification: None, + }, + Some("Sandbox denied outbound git push to github.com.".to_string()), + guardian_output_schema(), + /*external_cancel*/ None, + ) + .await; + let (GuardianReviewOutcome::Completed(_), _) = outcome else { + panic!("expected guardian assessment"); + }; + let request_body = request_log.single_request().body_json(); + let has_spawn_agent = request_body + .get("tools") + .and_then(Value::as_array) + .is_some_and(|tools| { + tools + .iter() + .any(|tool| tool.get("name").and_then(Value::as_str) == Some("spawn_agent")) + }); + + assert_eq!( + ( + models_mock.requests().len(), + request_body.get("model").and_then(Value::as_str), + has_spawn_agent, + ), + (1, Some(guardian_model), false) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn guardian_review_request_layout_matches_model_visible_request_snapshot() -> anyhow::Result<()> { diff --git a/codex-rs/core/tests/suite/model_runtime_selectors.rs b/codex-rs/core/tests/suite/model_runtime_selectors.rs index 0dca29e307..6b17a4628f 100644 --- a/codex-rs/core/tests/suite/model_runtime_selectors.rs +++ b/codex-rs/core/tests/suite/model_runtime_selectors.rs @@ -1,20 +1,16 @@ use anyhow::Result; use anyhow::bail; use codex_core::config::Config; -use codex_core::config::Constrained; -use codex_core::sandboxing::SandboxPermissions; use codex_features::Feature; use codex_login::CodexAuth; use codex_models_manager::manager::RefreshStrategy; use codex_models_manager::manager::SharedModelsManager; use codex_models_manager::model_info::model_info_from_slug; -use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; use codex_protocol::openai_models::ModelVisibility; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ToolMode; -use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::Op; @@ -28,7 +24,6 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_models_once; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_once_match; -use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; use core_test_support::submit_thread_settings; @@ -45,8 +40,6 @@ use wiremock::Request; const CHILD_PROMPT: &str = "inspect the child runtime"; const CHILD_MODEL: &str = "test-multi-agent-child"; -const GUARDIAN_REASON: &str = "Allow a narrowly scoped network request"; -const GUARDIAN_ROOT_PROMPT: &str = "request an escalated command"; const ROOT_MODEL: &str = "test-multi-agent-root"; const ROOT_PROMPT: &str = "spawn a child"; const SPAWN_CALL_ID: &str = "spawn-call-1"; @@ -322,91 +315,3 @@ async fn remote_multi_agent_selector_overrides_features_and_child_model_info() - Ok(()) } - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn guardian_stays_disabled_when_model_selects_multi_agent_v2() -> Result<()> { - skip_if_no_network!(Ok(())); - - let server = wiremock::MockServer::start().await; - let mut model = remote_model(ROOT_MODEL); - model.multi_agent_version = Some(MultiAgentVersion::V2); - let models_mock = mount_models_once( - &server, - ModelsResponse { - models: vec![model], - }, - ) - .await; - let exec_args = serde_json::to_string(&json!({ - "cmd": "true", - "sandbox_permissions": SandboxPermissions::RequireEscalated, - "justification": GUARDIAN_REASON, - }))?; - let request_log = mount_sse_sequence( - &server, - vec![ - sse(vec![ - ev_response_created("resp-root"), - ev_function_call("exec-call", "exec_command", &exec_args), - ev_completed("resp-root"), - ]), - sse(vec![ - ev_response_created("resp-guardian"), - ev_assistant_message( - "msg-guardian", - &json!({ - "risk_level": "low", - "user_authorization": "high", - "outcome": "deny", - "rationale": "Keep the test command from executing.", - }) - .to_string(), - ), - ev_completed("resp-guardian"), - ]), - sse(vec![ - ev_response_created("resp-root-followup"), - ev_completed("resp-root-followup"), - ]), - ], - ) - .await; - - let mut builder = test_codex() - .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) - .with_config(|config| { - config.model = Some(ROOT_MODEL.to_string()); - config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); - config.approvals_reviewer = ApprovalsReviewer::AutoReview; - config - .features - .disable(Feature::Apps) - .expect("test config should allow feature update"); - }); - let test = builder.build(&server).await?; - test.submit_turn(GUARDIAN_ROOT_PROMPT).await?; - wait_for_event(&test.codex, |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; - - let requests = request_log.requests(); - let [root_request, guardian_request, _root_followup_request] = requests.as_slice() else { - panic!("expected root, guardian, and root follow-up requests"); - }; - assert_eq!( - ( - models_mock.requests().len(), - test.codex.multi_agent_version(), - tool_names(&root_request.body_json()).contains(&"spawn_agent".to_string()), - guardian_request.body_contains_text(GUARDIAN_REASON), - guardian_request.body_json()["prompt_cache_key"] - .as_str() - .is_some_and(|key| key.starts_with("guardian:")), - tool_names(&guardian_request.body_json()).contains(&"spawn_agent".to_string()), - ), - (1, Some(MultiAgentVersion::V2), true, true, true, false) - ); - - Ok(()) -}