Reuse the parsed bundled model catalog (#62525)

## Why

Fallback lookups repeatedly parse the embedded model catalog even though
its contents are fixed for the lifetime of the process.

## What changed

Parse the bundled catalog once on first use. Borrow the shared catalog
for fallback lookups and copy it when building an owned snapshot or
merging remote models.

## Testing

Extend the API key discovery test to verify that a bundled model lookup
applies a request's `base_instructions` override and that a subsequent
lookup returns the original bundled model.

GitOrigin-RevId: 27c90b65667724312673c5d5ae01a3fc6b5b1416
This commit is contained in:
jif
2026-11-19 14:59:56 +00:00
committed by copyberry
parent 39c29c3bb6
commit 097558c1bd
2 changed files with 40 additions and 5 deletions
@@ -45,6 +45,34 @@ async fn api_key_discovery_flag_controls_fetches_and_cached_catalogs() {
let bundled = load_remote_models_from_file().unwrap();
let config = ModelsManagerConfig::default();
// Per-request overrides must not change the shared bundled fallback used by later lookups.
let bundled_model = bundled.first().expect("bundled model");
let overrides = ModelsManagerConfig {
base_instructions: Some("Instructions for this request only".to_string()),
..Default::default()
};
let mut expected_model = bundled_model.clone();
expected_model
.model_messages
.get_or_insert_default()
.instructions_template = overrides.base_instructions.clone();
assert_eq!(
manager
.get_model_info(&bundled_model.slug, &overrides)
.await,
CatalogModel {
model: expected_model.into(),
origin: ModelOrigin::Bundled,
}
);
assert_eq!(
manager.get_model_info(&bundled_model.slug, &config).await,
CatalogModel {
model: bundled_model.clone().into(),
origin: ModelOrigin::Bundled,
}
);
for strategy in [
RefreshStrategy::Online,
RefreshStrategy::OnlineIfUncached,
+12 -5
View File
@@ -28,6 +28,7 @@ use std::future::Future;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::LazyLock;
use std::time::Duration;
use tokio::sync::Mutex;
use tokio::sync::MutexGuard;
@@ -41,6 +42,11 @@ use tracing::info;
const MODEL_CACHE_FILE: &str = "models_cache.json";
const DEFAULT_MODEL_CACHE_TTL: Duration = Duration::from_secs(300);
// The embedded catalog cannot change during this process. Preserve the existing empty fallback
// on parse failure, and share the parsed data until a caller needs an owned catalog to mutate.
static BUNDLED_MODELS: LazyLock<Vec<ModelInfo>> =
LazyLock::new(|| load_remote_models_from_file().unwrap_or_default());
/// Remote endpoint used by the OpenAI-compatible model manager.
///
/// Implementations own provider-specific auth and transport details. The model
@@ -352,9 +358,9 @@ enum CatalogSource {
}
impl CatalogSource {
fn fallback_models(&self) -> Option<Vec<ModelInfo>> {
fn fallback_models(&self) -> Option<&'static [ModelInfo]> {
match self {
Self::Default => Some(load_remote_models_from_file().unwrap_or_default()),
Self::Default => Some(BUNDLED_MODELS.as_slice()),
Self::ExplicitProvider(_) => None,
}
}
@@ -481,7 +487,7 @@ impl OpenAiModelsManager {
etag: None,
client_version: Some(crate::client_version_to_whole()),
identity: endpoint_client.identity(),
models: load_remote_models_from_file().unwrap_or_default(),
models: BUNDLED_MODELS.to_vec(),
})),
settings,
endpoint_client,
@@ -730,7 +736,7 @@ impl OpenAiModelsManager {
{
SelectedCatalog::CurrentSnapshot(Cow::Borrowed(&entry.models))
} else {
SelectedCatalog::Fallback(Cow::Owned(
SelectedCatalog::Fallback(Cow::Borrowed(
self.catalog_source.fallback_models().unwrap_or_default(),
))
}
@@ -986,7 +992,8 @@ impl OpenAiModelsManager {
.auth_mode()
.is_some_and(AuthMode::has_chatgpt_account)
}));
if !remote_only && let Some(mut models) = self.catalog_source.fallback_models() {
if !remote_only && let Some(models) = self.catalog_source.fallback_models() {
let mut models = models.to_vec();
for model in std::mem::take(&mut entry.models) {
if let Some(index) = models
.iter()