Retry Guardian model discovery in the background (#62627)

## Why

A failed catalog fetch can leave an explicitly configured Guardian provider
without a classifier. Waiting for another fetch during tool dispatch would
delay the tool call.

## What changed

When a tool call is observed and background scoring is allowed, retry
initializing the Guardian catalog without blocking dispatch. Allow only one
background retry at a time. Check catalog readiness against the current
credentials and schedule initialization if auth needs to be resolved first.

## Testing

Add a regression test that holds the catalog retry open while the first tool
call completes, then verifies that a later call is scored once discovery
succeeds. Cover full-access sessions and mandatory review requirements, where
background recovery is skipped.

GitOrigin-RevId: 2f07bb6bdcfbf22e512f0bd223845f149fa186b5
This commit is contained in:
XMLIU-oai
2026-11-20 00:15:13 +00:00
committed by copyberry
parent d0264b756e
commit ead489ac4d
9 changed files with 327 additions and 22 deletions
+7 -4
View File
@@ -85,13 +85,16 @@ impl GuardianPolicyLoader {
}
impl ConfigRequirements {
pub fn disables_all_guardian_scoring(&self) -> bool {
[ApprovalsReviewer::User, ApprovalsReviewer::AutoReview]
.iter()
.any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err())
}
/// Applies the supplied administrator requirements to an already resolved model policy.
/// The reviewer allowlist applies even when model metadata is unavailable.
pub fn constrain_guardian_policy(&self, policy: &mut GuardianModelPolicy, model: Option<&str>) {
if [ApprovalsReviewer::User, ApprovalsReviewer::AutoReview]
.iter()
.any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err())
{
if self.disables_all_guardian_scoring() {
policy.disable_scoring();
} else if model.is_some_and(|model| self.auto_review_required_for_model(model)) {
let computer_use = policy.computer_use;
+2
View File
@@ -69,6 +69,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() {
loader.resolve(Some(&model), &ConfigRequirements::default()),
catalog
);
assert!(!ConfigRequirements::default().disables_all_guardian_scoring());
for reviewer in [ApprovalsReviewer::AutoReview, ApprovalsReviewer::User] {
let restricted: ConfigRequirements = crate::ConfigRequirementsWithSources {
allowed_approvals_reviewers: Some(Sourced::new(
@@ -79,6 +80,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() {
}
.try_into()
.unwrap();
assert!(restricted.disables_all_guardian_scoring());
// Reuse the same cached base across tightening and relaxation, including
// decisions without model metadata.
for model in [None, Some(&model)] {
@@ -9,6 +9,7 @@ use anyhow::Result;
use codex_config::FeatureRequirementsToml;
use codex_config::RequirementSource;
use codex_config::Sourced;
use codex_config::test_support::CloudConfigBundleFixture;
use codex_core::TurnInputRequest;
use codex_core::config::Config;
use codex_core::config::Constrained;
@@ -169,6 +170,187 @@ async fn compacted_cold_resume_respects_guardian_provider_override(
Ok(())
}
#[test_case(AskForApproval::OnRequest, PermissionProfile::read_only(), None, true; "on_request_restricted")]
#[test_case(AskForApproval::Never, PermissionProfile::read_only(), None, true; "never_restricted")]
#[test_case(AskForApproval::Never, PermissionProfile::Disabled, None, false; "full_access")]
#[test_case(AskForApproval::Never, PermissionProfile::read_only(), Some("allowed_approvals_reviewers = ['auto_review']"), false; "mandatory_review")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn explicit_guardian_recovers_without_delaying_tool_dispatch(
approval_policy: AskForApproval,
profile: PermissionProfile,
reviewer_requirements: Option<&str>,
should_recover: bool,
) -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_target_windows!(Ok(()), "exec_command requires a POSIX executor");
let actor = responses::start_mock_server().await;
let guardian = responses::start_mock_server().await;
let guardian_url = format!("{}/v1", guardian.uri());
let catalog = MockServer::start().await;
let catalog_url = format!("{}/v1/models", catalog.uri());
Mock::given(method("GET"))
.and(path("/v1/models"))
.respond_with(ResponseTemplate::new(503))
.up_to_n_times(1)
.expect(u64::from(should_recover))
.mount(&catalog)
.await;
let mut classifier = codex_models_manager::model_info::model_info_from_slug(LUNA_MODEL);
classifier.supported_in_api = true;
classifier.use_responses_lite = false;
classifier.guardian = Some(GuardianModelPolicy {
shell: Some(GuardianReviewMode::Adaptive),
sandboxed_exec_commands: Some(true),
..Default::default()
});
let response = ResponseTemplate::new(200).set_body_json(ModelsResponse {
models: vec![classifier],
});
let gate = tokio_util::sync::CancellationToken::new();
let pending = gate.clone();
let started = Arc::new(tokio::sync::Notify::new());
let notify = Arc::clone(&started);
let _retry = Mock::given(method("GET"))
.and(path("/v1/models"))
.respond_with(move |_: &Request| {
notify.notify_one();
futures::executor::block_on(pending.cancelled());
response.clone()
})
.with_priority(6)
.expect(u64::from(should_recover))
.mount_as_scoped(&catalog)
.await;
let release = gate.drop_guard();
let scorer = responses::mount_sse_once(
&guardian,
sse(vec![
ev_response_created("score"),
ev_assistant_message("score", "low"),
ev_completed("score"),
]),
)
.await;
let builder = test_codex()
.with_model(BYO_ACTOR_MODEL)
.with_config(move |config| {
configure_explicit_guardian_provider(config, &guardian_url);
let provider = Arc::make_mut(
config
.guardian_model_provider
.as_mut()
.expect("Guardian provider"),
);
provider.info.model_catalog_url = Some(catalog_url.into());
provider.info.request_max_retries = Some(0);
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
config
.features
.enable(Feature::GuardianApproval)
.expect("enable Guardian approval");
config
.features
.enable(Feature::GuardianV2)
.expect("enable Guardian V2");
config.permissions.approval_policy = Constrained::allow_any(approval_policy);
config
.permissions
.set_permission_profile(profile)
.expect("set permission profile");
});
let mut builder = if let Some(requirements) = reviewer_requirements {
builder.with_cloud_config_bundle(
CloudConfigBundleFixture::loader_with_enterprise_requirement(requirements.to_owned()),
)
} else {
builder
};
let test = builder.build_with_auto_env(&actor).await?;
// The first benign tool must finish while discovery's retry remains unanswered.
for body in [
sse(vec![
ev_function_call("first", "exec_command", r#"{"cmd":"printf first"}"#),
ev_completed("first"),
]),
sse(vec![
ev_assistant_message("done-first", "Done."),
ev_completed("done-first"),
]),
] {
responses::mount_sse_once(&actor, body).await;
}
if !should_recover {
test.submit_text_turn("Print first.").await?;
drop(release);
test.codex.shutdown_and_wait().await?;
assert!(
catalog
.received_requests()
.await
.expect("catalog requests")
.is_empty()
);
assert!(scorer.requests().is_empty());
return Ok(());
}
let (turn, retry) = tokio::join!(
timeout(
Duration::from_secs(10),
test.submit_text_turn("Print first.")
),
timeout(Duration::from_secs(10), started.notified()),
);
turn.context("tool dispatch waited for /models")??;
retry.context("Guardian never started recovery")?;
assert!(
scorer.requests().is_empty(),
"unavailable catalog must not start classification"
);
drop(release);
let inference = test
.codex
.thread_extension_data()
.get::<codex_core::GuardianInferenceContext>()
.expect("Guardian inference");
inference
.models_manager
.ensure_initialized(inference.http_client_factory.clone())
.await;
for body in [
sse(vec![
ev_function_call("second", "exec_command", r#"{"cmd":"printf second"}"#),
ev_completed("second"),
]),
sse(vec![
ev_assistant_message("done-second", "Done."),
ev_completed("done-second"),
]),
] {
responses::mount_sse_once(&actor, body).await;
}
test.submit_text_turn("Print second.").await?;
timeout(Duration::from_secs(10), async {
while scorer.requests().is_empty() {
tokio::task::yield_now().await;
}
})
.await?;
assert_eq!(scorer.single_request().body_json()["model"], LUNA_MODEL);
test.codex.shutdown_and_wait().await?;
assert_eq!(
catalog
.received_requests()
.await
.expect("catalog requests")
.len(),
2,
"a ready catalog must not be retried"
);
Ok(())
}
fn pin_guardian_parent_compaction_reuse(config: &mut Config) {
config.features = ManagedFeatures::from_configured_with_warnings(
config.features.get().clone(),
@@ -6,6 +6,7 @@ use std::time::Instant;
use std::time::SystemTime;
use codex_analytics::AnalyticsEventsClient;
use codex_core::GuardianInferenceContext;
use codex_core::context::GuardianContextMode;
use codex_core::context::GuardianReviewEvidence;
use codex_core::context::NodeReplReviewEvidence;
@@ -28,6 +29,7 @@ use super::authorization::ScoreAuthorization;
use super::classification::Classification;
use super::config::GuardianV2Config;
use super::conversation::ConversationBackend;
use super::coverage::background_scoring_allowed;
use super::coverage::scores_tool;
use super::extension::GuardianV2Extension;
use super::metrics::record_classification;
@@ -99,6 +101,47 @@ impl GuardianV2Extension {
let parent_model = input.thread_store.get::<ModelInfo>();
let probe_comparison = Some(super::probe_shadow::ProbeComparison::capture(&input));
let has_explicit_guardian_provider = config.guardian_model_provider.is_some();
let lookup_thread = || async {
let parsed_thread_id = ThreadId::from_string(input.thread_store.level_id())
.map_err(|error| error.to_string())?;
let manager = self
.thread_manager
.upgrade()
.ok_or_else(|| "thread manager is unavailable".to_string())?;
let thread = manager
.get_thread(parsed_thread_id)
.await
.map_err(|error| error.to_string())?;
Ok::<_, String>((manager, thread))
};
let needs_recovery = has_explicit_guardian_provider
&& input
.thread_store
.get::<GuardianInferenceContext>()
.is_some_and(|context| context.models_manager.needs_initialization());
let mut early_scoring_allowed = None;
let early_thread_context = if needs_recovery {
let context = lookup_thread().await;
if let Ok((_, thread)) = &context {
let snapshot = thread.config_snapshot().await;
let reviewer = thread.approvals_reviewer_for_turn(input.turn_id).await;
let scoring_allowed = background_scoring_allowed(reviewer, || snapshot.full_access);
early_scoring_allowed = Some(scoring_allowed);
if scoring_allowed
&& !thread
.config()
.await
.config_layer_stack
.requirements()
.disables_all_guardian_scoring()
{
drop(sampler.retry_catalog_initialization_in_background());
}
}
Some(context)
} else {
None
};
let guardian_classifier_model =
match GuardianV2Config::explicit_guardian_classifier_model(&config, input.thread_store)
.await
@@ -223,20 +266,10 @@ impl GuardianV2Extension {
.turn_store
.get::<codex_api::ResponseId>()
.map(|id| id.0.clone());
let thread_context: Result<_, String> = async {
let parsed_thread_id =
ThreadId::from_string(&thread_id).map_err(|error| error.to_string())?;
let manager = self
.thread_manager
.upgrade()
.ok_or_else(|| "thread manager is unavailable".to_string())?;
let thread = manager
.get_thread(parsed_thread_id)
.await
.map_err(|error| error.to_string())?;
Ok((manager, thread))
}
.await;
let thread_context = match early_thread_context {
Some(context) => context,
None => lookup_thread().await,
};
let (manager, thread) = match thread_context {
Ok(context) => context,
Err(error) => {
@@ -259,10 +292,15 @@ impl GuardianV2Extension {
}
};
// Use the live reviewer, not the startup config or per-app reviewer overrides.
let snapshot = thread.config_snapshot().await;
if snapshot.full_access
|| thread.approvals_reviewer_for_turn(input.turn_id).await == ApprovalsReviewer::User
{
let skip_scoring = match early_scoring_allowed {
Some(allowed) => !allowed,
None => {
thread.config_snapshot().await.full_access
|| thread.approvals_reviewer_for_turn(input.turn_id).await
== ApprovalsReviewer::User
}
};
if skip_scoring {
// A skipped call invalidates older scores, including ones still in flight.
score_progress.invalidate(tool_call_index);
return;
@@ -33,6 +33,7 @@ use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::SessionSource;
use codex_protocol::turn_input::CyberAccessProgram;
use thiserror::Error;
use tokio::sync::Mutex;
use tokio::sync::oneshot;
use uuid::Uuid;
@@ -144,6 +145,7 @@ pub enum LunaSamplerError {
pub struct LunaSampler {
config: Arc<LunaSamplerConfig>,
connections: Arc<ConnectionPool>,
catalog_initialization_lock: Arc<Mutex<()>>,
}
impl LunaSampler {
@@ -161,6 +163,7 @@ impl LunaSampler {
Self {
connections: ConnectionPool::new(Arc::clone(&config)),
config,
catalog_initialization_lock: Arc::new(Mutex::new(())),
}
}
@@ -172,6 +175,24 @@ impl LunaSampler {
}
}
/// Retry catalog discovery without delaying tool dispatch or overlapping retries.
pub(super) fn retry_catalog_initialization_in_background(
self: &Arc<Self>,
) -> Option<tokio::task::JoinHandle<()>> {
let catalog = self.config.guardian_catalog.as_ref()?;
if !catalog.needs_initialization() {
return None;
}
let guard = Arc::clone(&self.catalog_initialization_lock)
.try_lock_owned()
.ok()?;
let sampler = Arc::clone(self);
Some(tokio::spawn(async move {
let _guard = guard;
sampler.ensure_catalog_initialized().await;
}))
}
pub(super) fn max_input_tokens(&self) -> usize {
self.config.max_input_tokens
}
+6
View File
@@ -2696,6 +2696,12 @@ impl AuthManager {
}))
}
/// Whether cached auth needs a proactive refresh.
pub fn needs_proactive_refresh(&self) -> bool {
self.auth_cached()
.is_some_and(|auth| Self::proactive_refresh_reason(&auth).is_some())
}
/// Current cached auth (clone). May be `None` if not logged in or load failed.
/// For managed ChatGPT auth that needs a proactive refresh, first performs
/// a guarded reload and then refreshes only if the on-disk auth is unchanged.
@@ -342,6 +342,14 @@ impl ModelsEndpointClient for OpenAiModelsEndpoint {
crate::models_identity::identity(&self.provider_info, auth.as_ref()).ok()
}
fn auth_requires_resolution(&self) -> bool {
self.has_command_auth()
|| self
.auth_manager
.as_ref()
.is_some_and(|auth| auth.needs_proactive_refresh())
}
fn has_command_auth(&self) -> bool {
self.provider_info.has_command_auth()
}
@@ -436,6 +436,7 @@ async fn command_auth_refresh_fetches_a_catalog_for_the_current_credentials() {
);
// The cached identity still matches here; resolving command auth must
// detect the next token before deciding whether a refresh is needed.
assert!(manager.needs_initialization());
manager
.refresh_after_auth_change(HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault))
.await;
@@ -510,7 +511,13 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() {
.expect(1)
.mount(&server)
.await;
let use_external_auth = auth.api_key() == Some("test-key");
let auth_manager = AuthManager::from_auth_for_testing(auth);
let auth_manager = if use_external_auth {
AuthManager::from_api_key("test-key", auth_manager.runtime_config())
} else {
auth_manager
};
// The parallel residency test must not change this catalog's cache identity.
let provider = ModelProviderInfo {
model_catalog_url: Some(format!("{}/codex/models?deployment=one", server.uri()).into()),
@@ -554,6 +561,16 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() {
expected
);
}
if use_external_auth {
assert!(!manager.needs_initialization());
auth_manager
.set_external_auth(Arc::new(RotatingAuth(std::sync::atomic::AtomicUsize::new(
100,
))))
.await
.unwrap();
assert!(manager.needs_initialization());
}
}
}
+28
View File
@@ -68,6 +68,11 @@ pub trait ModelsEndpointClient: fmt::Debug + Send + Sync {
/// Return `None` when identity is unavailable; cached catalog reuse is then disabled.
fn identity(&self) -> Option<String>;
/// Whether auth must be resolved before checking catalog identity.
fn auth_requires_resolution(&self) -> bool {
true
}
/// Returns whether this provider can authenticate command-scoped requests.
fn has_command_auth(&self) -> bool;
@@ -163,6 +168,12 @@ pub struct RefreshedModelCatalog {
/// Coordinates model discovery plus cached metadata on disk.
pub trait ModelsManager: fmt::Debug + Send + Sync {
/// Whether to schedule initialization, without waiting on the catalog lock or auth.
/// Return true when readiness is unknown or credentials must be resolved first.
fn needs_initialization(&self) -> bool {
true
}
/// Initialize a newly selected catalog without replacing an already-loaded snapshot.
/// Stateful managers must check readiness for their current provider and credentials;
/// a successful empty catalog is initialized too. Static catalogs need no work.
@@ -526,6 +537,19 @@ impl StaticModelsManager {
}
impl ModelsManager for OpenAiModelsManager {
fn needs_initialization(&self) -> bool {
if self.endpoint_client.auth_requires_resolution() {
return true;
}
if self.api_key_discovery_disabled() {
return false;
}
!self
.remote_models
.try_read()
.is_ok_and(|catalog| self.catalog_is_initialized(&catalog))
}
fn ensure_initialized(
&self,
http_client_factory: HttpClientFactory,
@@ -1071,6 +1095,10 @@ impl OpenAiModelsManager {
}
impl ModelsManager for StaticModelsManager {
fn needs_initialization(&self) -> bool {
false
}
fn ensure_initialized(
&self,
_http_client_factory: HttpClientFactory,