From ead489ac4d06b13a69909173469a17be126348a4 Mon Sep 17 00:00:00 2001 From: XMLIU-oai Date: Thu, 19 Nov 2026 23:44:26 +0000 Subject: [PATCH] Retry Guardian model discovery in the background (#62627) ## Why A failed catalog fetch can leave an explicitly configured Guardian provider without a classifier. Waiting for another fetch during tool dispatch would delay the tool call. ## What changed When a tool call is observed and background scoring is allowed, retry initializing the Guardian catalog without blocking dispatch. Allow only one background retry at a time. Check catalog readiness against the current credentials and schedule initialization if auth needs to be resolved first. ## Testing Add a regression test that holds the catalog retry open while the first tool call completes, then verifies that a later call is scored once discovery succeeds. Cover full-access sessions and mandatory review requirements, where background recovery is skipped. GitOrigin-RevId: 2f07bb6bdcfbf22e512f0bd223845f149fa186b5 --- codex-rs/config/src/guardian.rs | 11 +- codex-rs/config/src/guardian_tests.rs | 2 + .../suite/guardian_cached_score_tests.rs | 182 ++++++++++++++++++ .../src/async_scorer/observation.rs | 74 +++++-- .../guardian-v2/src/async_scorer/sampler.rs | 21 ++ codex-rs/login/src/auth/manager.rs | 6 + .../model-provider/src/models_endpoint.rs | 8 + .../src/models_endpoint_tests.rs | 17 ++ codex-rs/models-manager/src/manager.rs | 28 +++ 9 files changed, 327 insertions(+), 22 deletions(-) diff --git a/codex-rs/config/src/guardian.rs b/codex-rs/config/src/guardian.rs index 189a3918cb..9ed7e9fb86 100644 --- a/codex-rs/config/src/guardian.rs +++ b/codex-rs/config/src/guardian.rs @@ -85,13 +85,16 @@ impl GuardianPolicyLoader { } impl ConfigRequirements { + pub fn disables_all_guardian_scoring(&self) -> bool { + [ApprovalsReviewer::User, ApprovalsReviewer::AutoReview] + .iter() + .any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err()) + } + /// Applies the supplied administrator requirements to an already resolved model policy. /// The reviewer allowlist applies even when model metadata is unavailable. pub fn constrain_guardian_policy(&self, policy: &mut GuardianModelPolicy, model: Option<&str>) { - if [ApprovalsReviewer::User, ApprovalsReviewer::AutoReview] - .iter() - .any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err()) - { + if self.disables_all_guardian_scoring() { policy.disable_scoring(); } else if model.is_some_and(|model| self.auto_review_required_for_model(model)) { let computer_use = policy.computer_use; diff --git a/codex-rs/config/src/guardian_tests.rs b/codex-rs/config/src/guardian_tests.rs index 8f32aababb..bc173f3e4c 100644 --- a/codex-rs/config/src/guardian_tests.rs +++ b/codex-rs/config/src/guardian_tests.rs @@ -69,6 +69,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() { loader.resolve(Some(&model), &ConfigRequirements::default()), catalog ); + assert!(!ConfigRequirements::default().disables_all_guardian_scoring()); for reviewer in [ApprovalsReviewer::AutoReview, ApprovalsReviewer::User] { let restricted: ConfigRequirements = crate::ConfigRequirementsWithSources { allowed_approvals_reviewers: Some(Sourced::new( @@ -79,6 +80,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() { } .try_into() .unwrap(); + assert!(restricted.disables_all_guardian_scoring()); // Reuse the same cached base across tightening and relaxation, including // decisions without model metadata. for model in [None, Some(&model)] { diff --git a/codex-rs/core/tests/suite/guardian_cached_score_tests.rs b/codex-rs/core/tests/suite/guardian_cached_score_tests.rs index 7b13779049..29bd2be328 100644 --- a/codex-rs/core/tests/suite/guardian_cached_score_tests.rs +++ b/codex-rs/core/tests/suite/guardian_cached_score_tests.rs @@ -9,6 +9,7 @@ use anyhow::Result; use codex_config::FeatureRequirementsToml; use codex_config::RequirementSource; use codex_config::Sourced; +use codex_config::test_support::CloudConfigBundleFixture; use codex_core::TurnInputRequest; use codex_core::config::Config; use codex_core::config::Constrained; @@ -169,6 +170,187 @@ async fn compacted_cold_resume_respects_guardian_provider_override( Ok(()) } +#[test_case(AskForApproval::OnRequest, PermissionProfile::read_only(), None, true; "on_request_restricted")] +#[test_case(AskForApproval::Never, PermissionProfile::read_only(), None, true; "never_restricted")] +#[test_case(AskForApproval::Never, PermissionProfile::Disabled, None, false; "full_access")] +#[test_case(AskForApproval::Never, PermissionProfile::read_only(), Some("allowed_approvals_reviewers = ['auto_review']"), false; "mandatory_review")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn explicit_guardian_recovers_without_delaying_tool_dispatch( + approval_policy: AskForApproval, + profile: PermissionProfile, + reviewer_requirements: Option<&str>, + should_recover: bool, +) -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_target_windows!(Ok(()), "exec_command requires a POSIX executor"); + let actor = responses::start_mock_server().await; + let guardian = responses::start_mock_server().await; + let guardian_url = format!("{}/v1", guardian.uri()); + let catalog = MockServer::start().await; + let catalog_url = format!("{}/v1/models", catalog.uri()); + Mock::given(method("GET")) + .and(path("/v1/models")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(1) + .expect(u64::from(should_recover)) + .mount(&catalog) + .await; + let mut classifier = codex_models_manager::model_info::model_info_from_slug(LUNA_MODEL); + classifier.supported_in_api = true; + classifier.use_responses_lite = false; + classifier.guardian = Some(GuardianModelPolicy { + shell: Some(GuardianReviewMode::Adaptive), + sandboxed_exec_commands: Some(true), + ..Default::default() + }); + let response = ResponseTemplate::new(200).set_body_json(ModelsResponse { + models: vec![classifier], + }); + let gate = tokio_util::sync::CancellationToken::new(); + let pending = gate.clone(); + let started = Arc::new(tokio::sync::Notify::new()); + let notify = Arc::clone(&started); + let _retry = Mock::given(method("GET")) + .and(path("/v1/models")) + .respond_with(move |_: &Request| { + notify.notify_one(); + futures::executor::block_on(pending.cancelled()); + response.clone() + }) + .with_priority(6) + .expect(u64::from(should_recover)) + .mount_as_scoped(&catalog) + .await; + let release = gate.drop_guard(); + let scorer = responses::mount_sse_once( + &guardian, + sse(vec![ + ev_response_created("score"), + ev_assistant_message("score", "low"), + ev_completed("score"), + ]), + ) + .await; + let builder = test_codex() + .with_model(BYO_ACTOR_MODEL) + .with_config(move |config| { + configure_explicit_guardian_provider(config, &guardian_url); + let provider = Arc::make_mut( + config + .guardian_model_provider + .as_mut() + .expect("Guardian provider"), + ); + provider.info.model_catalog_url = Some(catalog_url.into()); + provider.info.request_max_retries = Some(0); + config.approvals_reviewer = ApprovalsReviewer::AutoReview; + config + .features + .enable(Feature::GuardianApproval) + .expect("enable Guardian approval"); + config + .features + .enable(Feature::GuardianV2) + .expect("enable Guardian V2"); + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config + .permissions + .set_permission_profile(profile) + .expect("set permission profile"); + }); + let mut builder = if let Some(requirements) = reviewer_requirements { + builder.with_cloud_config_bundle( + CloudConfigBundleFixture::loader_with_enterprise_requirement(requirements.to_owned()), + ) + } else { + builder + }; + let test = builder.build_with_auto_env(&actor).await?; + + // The first benign tool must finish while discovery's retry remains unanswered. + for body in [ + sse(vec![ + ev_function_call("first", "exec_command", r#"{"cmd":"printf first"}"#), + ev_completed("first"), + ]), + sse(vec![ + ev_assistant_message("done-first", "Done."), + ev_completed("done-first"), + ]), + ] { + responses::mount_sse_once(&actor, body).await; + } + if !should_recover { + test.submit_text_turn("Print first.").await?; + drop(release); + test.codex.shutdown_and_wait().await?; + assert!( + catalog + .received_requests() + .await + .expect("catalog requests") + .is_empty() + ); + assert!(scorer.requests().is_empty()); + return Ok(()); + } + let (turn, retry) = tokio::join!( + timeout( + Duration::from_secs(10), + test.submit_text_turn("Print first.") + ), + timeout(Duration::from_secs(10), started.notified()), + ); + turn.context("tool dispatch waited for /models")??; + retry.context("Guardian never started recovery")?; + assert!( + scorer.requests().is_empty(), + "unavailable catalog must not start classification" + ); + drop(release); + let inference = test + .codex + .thread_extension_data() + .get::() + .expect("Guardian inference"); + inference + .models_manager + .ensure_initialized(inference.http_client_factory.clone()) + .await; + + for body in [ + sse(vec![ + ev_function_call("second", "exec_command", r#"{"cmd":"printf second"}"#), + ev_completed("second"), + ]), + sse(vec![ + ev_assistant_message("done-second", "Done."), + ev_completed("done-second"), + ]), + ] { + responses::mount_sse_once(&actor, body).await; + } + test.submit_text_turn("Print second.").await?; + timeout(Duration::from_secs(10), async { + while scorer.requests().is_empty() { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(scorer.single_request().body_json()["model"], LUNA_MODEL); + test.codex.shutdown_and_wait().await?; + assert_eq!( + catalog + .received_requests() + .await + .expect("catalog requests") + .len(), + 2, + "a ready catalog must not be retried" + ); + Ok(()) +} + fn pin_guardian_parent_compaction_reuse(config: &mut Config) { config.features = ManagedFeatures::from_configured_with_warnings( config.features.get().clone(), diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs index 132276b56f..e8b2e11a00 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/observation.rs @@ -6,6 +6,7 @@ use std::time::Instant; use std::time::SystemTime; use codex_analytics::AnalyticsEventsClient; +use codex_core::GuardianInferenceContext; use codex_core::context::GuardianContextMode; use codex_core::context::GuardianReviewEvidence; use codex_core::context::NodeReplReviewEvidence; @@ -28,6 +29,7 @@ use super::authorization::ScoreAuthorization; use super::classification::Classification; use super::config::GuardianV2Config; use super::conversation::ConversationBackend; +use super::coverage::background_scoring_allowed; use super::coverage::scores_tool; use super::extension::GuardianV2Extension; use super::metrics::record_classification; @@ -99,6 +101,47 @@ impl GuardianV2Extension { let parent_model = input.thread_store.get::(); let probe_comparison = Some(super::probe_shadow::ProbeComparison::capture(&input)); let has_explicit_guardian_provider = config.guardian_model_provider.is_some(); + let lookup_thread = || async { + let parsed_thread_id = ThreadId::from_string(input.thread_store.level_id()) + .map_err(|error| error.to_string())?; + let manager = self + .thread_manager + .upgrade() + .ok_or_else(|| "thread manager is unavailable".to_string())?; + let thread = manager + .get_thread(parsed_thread_id) + .await + .map_err(|error| error.to_string())?; + Ok::<_, String>((manager, thread)) + }; + let needs_recovery = has_explicit_guardian_provider + && input + .thread_store + .get::() + .is_some_and(|context| context.models_manager.needs_initialization()); + let mut early_scoring_allowed = None; + let early_thread_context = if needs_recovery { + let context = lookup_thread().await; + if let Ok((_, thread)) = &context { + let snapshot = thread.config_snapshot().await; + let reviewer = thread.approvals_reviewer_for_turn(input.turn_id).await; + let scoring_allowed = background_scoring_allowed(reviewer, || snapshot.full_access); + early_scoring_allowed = Some(scoring_allowed); + if scoring_allowed + && !thread + .config() + .await + .config_layer_stack + .requirements() + .disables_all_guardian_scoring() + { + drop(sampler.retry_catalog_initialization_in_background()); + } + } + Some(context) + } else { + None + }; let guardian_classifier_model = match GuardianV2Config::explicit_guardian_classifier_model(&config, input.thread_store) .await @@ -223,20 +266,10 @@ impl GuardianV2Extension { .turn_store .get::() .map(|id| id.0.clone()); - let thread_context: Result<_, String> = async { - let parsed_thread_id = - ThreadId::from_string(&thread_id).map_err(|error| error.to_string())?; - let manager = self - .thread_manager - .upgrade() - .ok_or_else(|| "thread manager is unavailable".to_string())?; - let thread = manager - .get_thread(parsed_thread_id) - .await - .map_err(|error| error.to_string())?; - Ok((manager, thread)) - } - .await; + let thread_context = match early_thread_context { + Some(context) => context, + None => lookup_thread().await, + }; let (manager, thread) = match thread_context { Ok(context) => context, Err(error) => { @@ -259,10 +292,15 @@ impl GuardianV2Extension { } }; // Use the live reviewer, not the startup config or per-app reviewer overrides. - let snapshot = thread.config_snapshot().await; - if snapshot.full_access - || thread.approvals_reviewer_for_turn(input.turn_id).await == ApprovalsReviewer::User - { + let skip_scoring = match early_scoring_allowed { + Some(allowed) => !allowed, + None => { + thread.config_snapshot().await.full_access + || thread.approvals_reviewer_for_turn(input.turn_id).await + == ApprovalsReviewer::User + } + }; + if skip_scoring { // A skipped call invalidates older scores, including ones still in flight. score_progress.invalidate(tool_call_index); return; diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs b/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs index 4c3fa00449..a869e16b99 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs @@ -33,6 +33,7 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::SessionSource; use codex_protocol::turn_input::CyberAccessProgram; use thiserror::Error; +use tokio::sync::Mutex; use tokio::sync::oneshot; use uuid::Uuid; @@ -144,6 +145,7 @@ pub enum LunaSamplerError { pub struct LunaSampler { config: Arc, connections: Arc, + catalog_initialization_lock: Arc>, } impl LunaSampler { @@ -161,6 +163,7 @@ impl LunaSampler { Self { connections: ConnectionPool::new(Arc::clone(&config)), config, + catalog_initialization_lock: Arc::new(Mutex::new(())), } } @@ -172,6 +175,24 @@ impl LunaSampler { } } + /// Retry catalog discovery without delaying tool dispatch or overlapping retries. + pub(super) fn retry_catalog_initialization_in_background( + self: &Arc, + ) -> Option> { + let catalog = self.config.guardian_catalog.as_ref()?; + if !catalog.needs_initialization() { + return None; + } + let guard = Arc::clone(&self.catalog_initialization_lock) + .try_lock_owned() + .ok()?; + let sampler = Arc::clone(self); + Some(tokio::spawn(async move { + let _guard = guard; + sampler.ensure_catalog_initialized().await; + })) + } + pub(super) fn max_input_tokens(&self) -> usize { self.config.max_input_tokens } diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index fe33c39ae4..ee6c571378 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -2696,6 +2696,12 @@ impl AuthManager { })) } + /// Whether cached auth needs a proactive refresh. + pub fn needs_proactive_refresh(&self) -> bool { + self.auth_cached() + .is_some_and(|auth| Self::proactive_refresh_reason(&auth).is_some()) + } + /// Current cached auth (clone). May be `None` if not logged in or load failed. /// For managed ChatGPT auth that needs a proactive refresh, first performs /// a guarded reload and then refreshes only if the on-disk auth is unchanged. diff --git a/codex-rs/model-provider/src/models_endpoint.rs b/codex-rs/model-provider/src/models_endpoint.rs index 2900c337e9..d6533b6e89 100644 --- a/codex-rs/model-provider/src/models_endpoint.rs +++ b/codex-rs/model-provider/src/models_endpoint.rs @@ -342,6 +342,14 @@ impl ModelsEndpointClient for OpenAiModelsEndpoint { crate::models_identity::identity(&self.provider_info, auth.as_ref()).ok() } + fn auth_requires_resolution(&self) -> bool { + self.has_command_auth() + || self + .auth_manager + .as_ref() + .is_some_and(|auth| auth.needs_proactive_refresh()) + } + fn has_command_auth(&self) -> bool { self.provider_info.has_command_auth() } diff --git a/codex-rs/model-provider/src/models_endpoint_tests.rs b/codex-rs/model-provider/src/models_endpoint_tests.rs index 9e323dd523..1a05c9b11a 100644 --- a/codex-rs/model-provider/src/models_endpoint_tests.rs +++ b/codex-rs/model-provider/src/models_endpoint_tests.rs @@ -436,6 +436,7 @@ async fn command_auth_refresh_fetches_a_catalog_for_the_current_credentials() { ); // The cached identity still matches here; resolving command auth must // detect the next token before deciding whether a refresh is needed. + assert!(manager.needs_initialization()); manager .refresh_after_auth_change(HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)) .await; @@ -510,7 +511,13 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() { .expect(1) .mount(&server) .await; + let use_external_auth = auth.api_key() == Some("test-key"); let auth_manager = AuthManager::from_auth_for_testing(auth); + let auth_manager = if use_external_auth { + AuthManager::from_api_key("test-key", auth_manager.runtime_config()) + } else { + auth_manager + }; // The parallel residency test must not change this catalog's cache identity. let provider = ModelProviderInfo { model_catalog_url: Some(format!("{}/codex/models?deployment=one", server.uri()).into()), @@ -554,6 +561,16 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() { expected ); } + if use_external_auth { + assert!(!manager.needs_initialization()); + auth_manager + .set_external_auth(Arc::new(RotatingAuth(std::sync::atomic::AtomicUsize::new( + 100, + )))) + .await + .unwrap(); + assert!(manager.needs_initialization()); + } } } diff --git a/codex-rs/models-manager/src/manager.rs b/codex-rs/models-manager/src/manager.rs index 57fa79fb2b..ab41426085 100644 --- a/codex-rs/models-manager/src/manager.rs +++ b/codex-rs/models-manager/src/manager.rs @@ -68,6 +68,11 @@ pub trait ModelsEndpointClient: fmt::Debug + Send + Sync { /// Return `None` when identity is unavailable; cached catalog reuse is then disabled. fn identity(&self) -> Option; + /// Whether auth must be resolved before checking catalog identity. + fn auth_requires_resolution(&self) -> bool { + true + } + /// Returns whether this provider can authenticate command-scoped requests. fn has_command_auth(&self) -> bool; @@ -163,6 +168,12 @@ pub struct RefreshedModelCatalog { /// Coordinates model discovery plus cached metadata on disk. pub trait ModelsManager: fmt::Debug + Send + Sync { + /// Whether to schedule initialization, without waiting on the catalog lock or auth. + /// Return true when readiness is unknown or credentials must be resolved first. + fn needs_initialization(&self) -> bool { + true + } + /// Initialize a newly selected catalog without replacing an already-loaded snapshot. /// Stateful managers must check readiness for their current provider and credentials; /// a successful empty catalog is initialized too. Static catalogs need no work. @@ -526,6 +537,19 @@ impl StaticModelsManager { } impl ModelsManager for OpenAiModelsManager { + fn needs_initialization(&self) -> bool { + if self.endpoint_client.auth_requires_resolution() { + return true; + } + if self.api_key_discovery_disabled() { + return false; + } + !self + .remote_models + .try_read() + .is_ok_and(|catalog| self.catalog_is_initialized(&catalog)) + } + fn ensure_initialized( &self, http_client_factory: HttpClientFactory, @@ -1071,6 +1095,10 @@ impl OpenAiModelsManager { } impl ModelsManager for StaticModelsManager { + fn needs_initialization(&self) -> bool { + false + } + fn ensure_initialized( &self, _http_client_factory: HttpClientFactory,