mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
Retry Guardian model discovery in the background (#62627)
## Why A failed catalog fetch can leave an explicitly configured Guardian provider without a classifier. Waiting for another fetch during tool dispatch would delay the tool call. ## What changed When a tool call is observed and background scoring is allowed, retry initializing the Guardian catalog without blocking dispatch. Allow only one background retry at a time. Check catalog readiness against the current credentials and schedule initialization if auth needs to be resolved first. ## Testing Add a regression test that holds the catalog retry open while the first tool call completes, then verifies that a later call is scored once discovery succeeds. Cover full-access sessions and mandatory review requirements, where background recovery is skipped. GitOrigin-RevId: 2f07bb6bdcfbf22e512f0bd223845f149fa186b5
This commit is contained in:
@@ -85,13 +85,16 @@ impl GuardianPolicyLoader {
|
||||
}
|
||||
|
||||
impl ConfigRequirements {
|
||||
pub fn disables_all_guardian_scoring(&self) -> bool {
|
||||
[ApprovalsReviewer::User, ApprovalsReviewer::AutoReview]
|
||||
.iter()
|
||||
.any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err())
|
||||
}
|
||||
|
||||
/// Applies the supplied administrator requirements to an already resolved model policy.
|
||||
/// The reviewer allowlist applies even when model metadata is unavailable.
|
||||
pub fn constrain_guardian_policy(&self, policy: &mut GuardianModelPolicy, model: Option<&str>) {
|
||||
if [ApprovalsReviewer::User, ApprovalsReviewer::AutoReview]
|
||||
.iter()
|
||||
.any(|reviewer| self.approvals_reviewer.can_set(reviewer).is_err())
|
||||
{
|
||||
if self.disables_all_guardian_scoring() {
|
||||
policy.disable_scoring();
|
||||
} else if model.is_some_and(|model| self.auto_review_required_for_model(model)) {
|
||||
let computer_use = policy.computer_use;
|
||||
|
||||
@@ -69,6 +69,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() {
|
||||
loader.resolve(Some(&model), &ConfigRequirements::default()),
|
||||
catalog
|
||||
);
|
||||
assert!(!ConfigRequirements::default().disables_all_guardian_scoring());
|
||||
for reviewer in [ApprovalsReviewer::AutoReview, ApprovalsReviewer::User] {
|
||||
let restricted: ConfigRequirements = crate::ConfigRequirementsWithSources {
|
||||
allowed_approvals_reviewers: Some(Sourced::new(
|
||||
@@ -79,6 +80,7 @@ fn legacy_scope_applies_only_to_models_without_catalog_policy() {
|
||||
}
|
||||
.try_into()
|
||||
.unwrap();
|
||||
assert!(restricted.disables_all_guardian_scoring());
|
||||
// Reuse the same cached base across tightening and relaxation, including
|
||||
// decisions without model metadata.
|
||||
for model in [None, Some(&model)] {
|
||||
|
||||
@@ -9,6 +9,7 @@ use anyhow::Result;
|
||||
use codex_config::FeatureRequirementsToml;
|
||||
use codex_config::RequirementSource;
|
||||
use codex_config::Sourced;
|
||||
use codex_config::test_support::CloudConfigBundleFixture;
|
||||
use codex_core::TurnInputRequest;
|
||||
use codex_core::config::Config;
|
||||
use codex_core::config::Constrained;
|
||||
@@ -169,6 +170,187 @@ async fn compacted_cold_resume_respects_guardian_provider_override(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case(AskForApproval::OnRequest, PermissionProfile::read_only(), None, true; "on_request_restricted")]
|
||||
#[test_case(AskForApproval::Never, PermissionProfile::read_only(), None, true; "never_restricted")]
|
||||
#[test_case(AskForApproval::Never, PermissionProfile::Disabled, None, false; "full_access")]
|
||||
#[test_case(AskForApproval::Never, PermissionProfile::read_only(), Some("allowed_approvals_reviewers = ['auto_review']"), false; "mandatory_review")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn explicit_guardian_recovers_without_delaying_tool_dispatch(
|
||||
approval_policy: AskForApproval,
|
||||
profile: PermissionProfile,
|
||||
reviewer_requirements: Option<&str>,
|
||||
should_recover: bool,
|
||||
) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
skip_if_target_windows!(Ok(()), "exec_command requires a POSIX executor");
|
||||
let actor = responses::start_mock_server().await;
|
||||
let guardian = responses::start_mock_server().await;
|
||||
let guardian_url = format!("{}/v1", guardian.uri());
|
||||
let catalog = MockServer::start().await;
|
||||
let catalog_url = format!("{}/v1/models", catalog.uri());
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/models"))
|
||||
.respond_with(ResponseTemplate::new(503))
|
||||
.up_to_n_times(1)
|
||||
.expect(u64::from(should_recover))
|
||||
.mount(&catalog)
|
||||
.await;
|
||||
let mut classifier = codex_models_manager::model_info::model_info_from_slug(LUNA_MODEL);
|
||||
classifier.supported_in_api = true;
|
||||
classifier.use_responses_lite = false;
|
||||
classifier.guardian = Some(GuardianModelPolicy {
|
||||
shell: Some(GuardianReviewMode::Adaptive),
|
||||
sandboxed_exec_commands: Some(true),
|
||||
..Default::default()
|
||||
});
|
||||
let response = ResponseTemplate::new(200).set_body_json(ModelsResponse {
|
||||
models: vec![classifier],
|
||||
});
|
||||
let gate = tokio_util::sync::CancellationToken::new();
|
||||
let pending = gate.clone();
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let notify = Arc::clone(&started);
|
||||
let _retry = Mock::given(method("GET"))
|
||||
.and(path("/v1/models"))
|
||||
.respond_with(move |_: &Request| {
|
||||
notify.notify_one();
|
||||
futures::executor::block_on(pending.cancelled());
|
||||
response.clone()
|
||||
})
|
||||
.with_priority(6)
|
||||
.expect(u64::from(should_recover))
|
||||
.mount_as_scoped(&catalog)
|
||||
.await;
|
||||
let release = gate.drop_guard();
|
||||
let scorer = responses::mount_sse_once(
|
||||
&guardian,
|
||||
sse(vec![
|
||||
ev_response_created("score"),
|
||||
ev_assistant_message("score", "low"),
|
||||
ev_completed("score"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let builder = test_codex()
|
||||
.with_model(BYO_ACTOR_MODEL)
|
||||
.with_config(move |config| {
|
||||
configure_explicit_guardian_provider(config, &guardian_url);
|
||||
let provider = Arc::make_mut(
|
||||
config
|
||||
.guardian_model_provider
|
||||
.as_mut()
|
||||
.expect("Guardian provider"),
|
||||
);
|
||||
provider.info.model_catalog_url = Some(catalog_url.into());
|
||||
provider.info.request_max_retries = Some(0);
|
||||
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
||||
config
|
||||
.features
|
||||
.enable(Feature::GuardianApproval)
|
||||
.expect("enable Guardian approval");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::GuardianV2)
|
||||
.expect("enable Guardian V2");
|
||||
config.permissions.approval_policy = Constrained::allow_any(approval_policy);
|
||||
config
|
||||
.permissions
|
||||
.set_permission_profile(profile)
|
||||
.expect("set permission profile");
|
||||
});
|
||||
let mut builder = if let Some(requirements) = reviewer_requirements {
|
||||
builder.with_cloud_config_bundle(
|
||||
CloudConfigBundleFixture::loader_with_enterprise_requirement(requirements.to_owned()),
|
||||
)
|
||||
} else {
|
||||
builder
|
||||
};
|
||||
let test = builder.build_with_auto_env(&actor).await?;
|
||||
|
||||
// The first benign tool must finish while discovery's retry remains unanswered.
|
||||
for body in [
|
||||
sse(vec![
|
||||
ev_function_call("first", "exec_command", r#"{"cmd":"printf first"}"#),
|
||||
ev_completed("first"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_assistant_message("done-first", "Done."),
|
||||
ev_completed("done-first"),
|
||||
]),
|
||||
] {
|
||||
responses::mount_sse_once(&actor, body).await;
|
||||
}
|
||||
if !should_recover {
|
||||
test.submit_text_turn("Print first.").await?;
|
||||
drop(release);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
assert!(
|
||||
catalog
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("catalog requests")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(scorer.requests().is_empty());
|
||||
return Ok(());
|
||||
}
|
||||
let (turn, retry) = tokio::join!(
|
||||
timeout(
|
||||
Duration::from_secs(10),
|
||||
test.submit_text_turn("Print first.")
|
||||
),
|
||||
timeout(Duration::from_secs(10), started.notified()),
|
||||
);
|
||||
turn.context("tool dispatch waited for /models")??;
|
||||
retry.context("Guardian never started recovery")?;
|
||||
assert!(
|
||||
scorer.requests().is_empty(),
|
||||
"unavailable catalog must not start classification"
|
||||
);
|
||||
drop(release);
|
||||
let inference = test
|
||||
.codex
|
||||
.thread_extension_data()
|
||||
.get::<codex_core::GuardianInferenceContext>()
|
||||
.expect("Guardian inference");
|
||||
inference
|
||||
.models_manager
|
||||
.ensure_initialized(inference.http_client_factory.clone())
|
||||
.await;
|
||||
|
||||
for body in [
|
||||
sse(vec![
|
||||
ev_function_call("second", "exec_command", r#"{"cmd":"printf second"}"#),
|
||||
ev_completed("second"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_assistant_message("done-second", "Done."),
|
||||
ev_completed("done-second"),
|
||||
]),
|
||||
] {
|
||||
responses::mount_sse_once(&actor, body).await;
|
||||
}
|
||||
test.submit_text_turn("Print second.").await?;
|
||||
timeout(Duration::from_secs(10), async {
|
||||
while scorer.requests().is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(scorer.single_request().body_json()["model"], LUNA_MODEL);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
assert_eq!(
|
||||
catalog
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("catalog requests")
|
||||
.len(),
|
||||
2,
|
||||
"a ready catalog must not be retried"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pin_guardian_parent_compaction_reuse(config: &mut Config) {
|
||||
config.features = ManagedFeatures::from_configured_with_warnings(
|
||||
config.features.get().clone(),
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::time::Instant;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_core::GuardianInferenceContext;
|
||||
use codex_core::context::GuardianContextMode;
|
||||
use codex_core::context::GuardianReviewEvidence;
|
||||
use codex_core::context::NodeReplReviewEvidence;
|
||||
@@ -28,6 +29,7 @@ use super::authorization::ScoreAuthorization;
|
||||
use super::classification::Classification;
|
||||
use super::config::GuardianV2Config;
|
||||
use super::conversation::ConversationBackend;
|
||||
use super::coverage::background_scoring_allowed;
|
||||
use super::coverage::scores_tool;
|
||||
use super::extension::GuardianV2Extension;
|
||||
use super::metrics::record_classification;
|
||||
@@ -99,6 +101,47 @@ impl GuardianV2Extension {
|
||||
let parent_model = input.thread_store.get::<ModelInfo>();
|
||||
let probe_comparison = Some(super::probe_shadow::ProbeComparison::capture(&input));
|
||||
let has_explicit_guardian_provider = config.guardian_model_provider.is_some();
|
||||
let lookup_thread = || async {
|
||||
let parsed_thread_id = ThreadId::from_string(input.thread_store.level_id())
|
||||
.map_err(|error| error.to_string())?;
|
||||
let manager = self
|
||||
.thread_manager
|
||||
.upgrade()
|
||||
.ok_or_else(|| "thread manager is unavailable".to_string())?;
|
||||
let thread = manager
|
||||
.get_thread(parsed_thread_id)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok::<_, String>((manager, thread))
|
||||
};
|
||||
let needs_recovery = has_explicit_guardian_provider
|
||||
&& input
|
||||
.thread_store
|
||||
.get::<GuardianInferenceContext>()
|
||||
.is_some_and(|context| context.models_manager.needs_initialization());
|
||||
let mut early_scoring_allowed = None;
|
||||
let early_thread_context = if needs_recovery {
|
||||
let context = lookup_thread().await;
|
||||
if let Ok((_, thread)) = &context {
|
||||
let snapshot = thread.config_snapshot().await;
|
||||
let reviewer = thread.approvals_reviewer_for_turn(input.turn_id).await;
|
||||
let scoring_allowed = background_scoring_allowed(reviewer, || snapshot.full_access);
|
||||
early_scoring_allowed = Some(scoring_allowed);
|
||||
if scoring_allowed
|
||||
&& !thread
|
||||
.config()
|
||||
.await
|
||||
.config_layer_stack
|
||||
.requirements()
|
||||
.disables_all_guardian_scoring()
|
||||
{
|
||||
drop(sampler.retry_catalog_initialization_in_background());
|
||||
}
|
||||
}
|
||||
Some(context)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let guardian_classifier_model =
|
||||
match GuardianV2Config::explicit_guardian_classifier_model(&config, input.thread_store)
|
||||
.await
|
||||
@@ -223,20 +266,10 @@ impl GuardianV2Extension {
|
||||
.turn_store
|
||||
.get::<codex_api::ResponseId>()
|
||||
.map(|id| id.0.clone());
|
||||
let thread_context: Result<_, String> = async {
|
||||
let parsed_thread_id =
|
||||
ThreadId::from_string(&thread_id).map_err(|error| error.to_string())?;
|
||||
let manager = self
|
||||
.thread_manager
|
||||
.upgrade()
|
||||
.ok_or_else(|| "thread manager is unavailable".to_string())?;
|
||||
let thread = manager
|
||||
.get_thread(parsed_thread_id)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok((manager, thread))
|
||||
}
|
||||
.await;
|
||||
let thread_context = match early_thread_context {
|
||||
Some(context) => context,
|
||||
None => lookup_thread().await,
|
||||
};
|
||||
let (manager, thread) = match thread_context {
|
||||
Ok(context) => context,
|
||||
Err(error) => {
|
||||
@@ -259,10 +292,15 @@ impl GuardianV2Extension {
|
||||
}
|
||||
};
|
||||
// Use the live reviewer, not the startup config or per-app reviewer overrides.
|
||||
let snapshot = thread.config_snapshot().await;
|
||||
if snapshot.full_access
|
||||
|| thread.approvals_reviewer_for_turn(input.turn_id).await == ApprovalsReviewer::User
|
||||
{
|
||||
let skip_scoring = match early_scoring_allowed {
|
||||
Some(allowed) => !allowed,
|
||||
None => {
|
||||
thread.config_snapshot().await.full_access
|
||||
|| thread.approvals_reviewer_for_turn(input.turn_id).await
|
||||
== ApprovalsReviewer::User
|
||||
}
|
||||
};
|
||||
if skip_scoring {
|
||||
// A skipped call invalidates older scores, including ones still in flight.
|
||||
score_progress.invalidate(tool_call_index);
|
||||
return;
|
||||
|
||||
@@ -33,6 +33,7 @@ use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::turn_input::CyberAccessProgram;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::oneshot;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -144,6 +145,7 @@ pub enum LunaSamplerError {
|
||||
pub struct LunaSampler {
|
||||
config: Arc<LunaSamplerConfig>,
|
||||
connections: Arc<ConnectionPool>,
|
||||
catalog_initialization_lock: Arc<Mutex<()>>,
|
||||
}
|
||||
|
||||
impl LunaSampler {
|
||||
@@ -161,6 +163,7 @@ impl LunaSampler {
|
||||
Self {
|
||||
connections: ConnectionPool::new(Arc::clone(&config)),
|
||||
config,
|
||||
catalog_initialization_lock: Arc::new(Mutex::new(())),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +175,24 @@ impl LunaSampler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry catalog discovery without delaying tool dispatch or overlapping retries.
|
||||
pub(super) fn retry_catalog_initialization_in_background(
|
||||
self: &Arc<Self>,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
let catalog = self.config.guardian_catalog.as_ref()?;
|
||||
if !catalog.needs_initialization() {
|
||||
return None;
|
||||
}
|
||||
let guard = Arc::clone(&self.catalog_initialization_lock)
|
||||
.try_lock_owned()
|
||||
.ok()?;
|
||||
let sampler = Arc::clone(self);
|
||||
Some(tokio::spawn(async move {
|
||||
let _guard = guard;
|
||||
sampler.ensure_catalog_initialized().await;
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn max_input_tokens(&self) -> usize {
|
||||
self.config.max_input_tokens
|
||||
}
|
||||
|
||||
@@ -2696,6 +2696,12 @@ impl AuthManager {
|
||||
}))
|
||||
}
|
||||
|
||||
/// Whether cached auth needs a proactive refresh.
|
||||
pub fn needs_proactive_refresh(&self) -> bool {
|
||||
self.auth_cached()
|
||||
.is_some_and(|auth| Self::proactive_refresh_reason(&auth).is_some())
|
||||
}
|
||||
|
||||
/// Current cached auth (clone). May be `None` if not logged in or load failed.
|
||||
/// For managed ChatGPT auth that needs a proactive refresh, first performs
|
||||
/// a guarded reload and then refreshes only if the on-disk auth is unchanged.
|
||||
|
||||
@@ -342,6 +342,14 @@ impl ModelsEndpointClient for OpenAiModelsEndpoint {
|
||||
crate::models_identity::identity(&self.provider_info, auth.as_ref()).ok()
|
||||
}
|
||||
|
||||
fn auth_requires_resolution(&self) -> bool {
|
||||
self.has_command_auth()
|
||||
|| self
|
||||
.auth_manager
|
||||
.as_ref()
|
||||
.is_some_and(|auth| auth.needs_proactive_refresh())
|
||||
}
|
||||
|
||||
fn has_command_auth(&self) -> bool {
|
||||
self.provider_info.has_command_auth()
|
||||
}
|
||||
|
||||
@@ -436,6 +436,7 @@ async fn command_auth_refresh_fetches_a_catalog_for_the_current_credentials() {
|
||||
);
|
||||
// The cached identity still matches here; resolving command auth must
|
||||
// detect the next token before deciding whether a refresh is needed.
|
||||
assert!(manager.needs_initialization());
|
||||
manager
|
||||
.refresh_after_auth_change(HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault))
|
||||
.await;
|
||||
@@ -510,7 +511,13 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() {
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let use_external_auth = auth.api_key() == Some("test-key");
|
||||
let auth_manager = AuthManager::from_auth_for_testing(auth);
|
||||
let auth_manager = if use_external_auth {
|
||||
AuthManager::from_api_key("test-key", auth_manager.runtime_config())
|
||||
} else {
|
||||
auth_manager
|
||||
};
|
||||
// The parallel residency test must not change this catalog's cache identity.
|
||||
let provider = ModelProviderInfo {
|
||||
model_catalog_url: Some(format!("{}/codex/models?deployment=one", server.uri()).into()),
|
||||
@@ -554,6 +561,16 @@ async fn explicit_catalog_reuses_auth_headers_and_query_parameters() {
|
||||
expected
|
||||
);
|
||||
}
|
||||
if use_external_auth {
|
||||
assert!(!manager.needs_initialization());
|
||||
auth_manager
|
||||
.set_external_auth(Arc::new(RotatingAuth(std::sync::atomic::AtomicUsize::new(
|
||||
100,
|
||||
))))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(manager.needs_initialization());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -68,6 +68,11 @@ pub trait ModelsEndpointClient: fmt::Debug + Send + Sync {
|
||||
/// Return `None` when identity is unavailable; cached catalog reuse is then disabled.
|
||||
fn identity(&self) -> Option<String>;
|
||||
|
||||
/// Whether auth must be resolved before checking catalog identity.
|
||||
fn auth_requires_resolution(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Returns whether this provider can authenticate command-scoped requests.
|
||||
fn has_command_auth(&self) -> bool;
|
||||
|
||||
@@ -163,6 +168,12 @@ pub struct RefreshedModelCatalog {
|
||||
|
||||
/// Coordinates model discovery plus cached metadata on disk.
|
||||
pub trait ModelsManager: fmt::Debug + Send + Sync {
|
||||
/// Whether to schedule initialization, without waiting on the catalog lock or auth.
|
||||
/// Return true when readiness is unknown or credentials must be resolved first.
|
||||
fn needs_initialization(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Initialize a newly selected catalog without replacing an already-loaded snapshot.
|
||||
/// Stateful managers must check readiness for their current provider and credentials;
|
||||
/// a successful empty catalog is initialized too. Static catalogs need no work.
|
||||
@@ -526,6 +537,19 @@ impl StaticModelsManager {
|
||||
}
|
||||
|
||||
impl ModelsManager for OpenAiModelsManager {
|
||||
fn needs_initialization(&self) -> bool {
|
||||
if self.endpoint_client.auth_requires_resolution() {
|
||||
return true;
|
||||
}
|
||||
if self.api_key_discovery_disabled() {
|
||||
return false;
|
||||
}
|
||||
!self
|
||||
.remote_models
|
||||
.try_read()
|
||||
.is_ok_and(|catalog| self.catalog_is_initialized(&catalog))
|
||||
}
|
||||
|
||||
fn ensure_initialized(
|
||||
&self,
|
||||
http_client_factory: HttpClientFactory,
|
||||
@@ -1071,6 +1095,10 @@ impl OpenAiModelsManager {
|
||||
}
|
||||
|
||||
impl ModelsManager for StaticModelsManager {
|
||||
fn needs_initialization(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn ensure_initialized(
|
||||
&self,
|
||||
_http_client_factory: HttpClientFactory,
|
||||
|
||||
Reference in New Issue
Block a user