From 87fccdc1bc0bc9af04db844a23d9c025f3b82000 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 15 Nov 2026 20:16:37 +0000 Subject: [PATCH] Defer plugin manifest loading until marketplace entries are selected (#61686) ## Why Marketplace listings read plugin manifests for entries later discarded by policy, product visibility, or duplicate selection across project scopes. This causes unnecessary reads and can emit parse warnings for plugins that never appear in the listing. ## What changed Separate marketplace entry validation from plugin manifest loading. Apply marketplace policy, visibility, and path and plugin ID deduplication before loading metadata, preserving first-source and first-scope precedence. ## Testing Add a regression test covering repeated marketplace paths and shadowed plugin IDs across scopes. Verify that the selected malformed manifest is read once, the discarded manifest is never read, and the visible plugin IDs and local versions are preserved. GitOrigin-RevId: 00b1d50593fb6546aafdc2089e6fccec1b4e5f42 --- codex-rs/core-plugins/src/manager.rs | 106 ++++--- codex-rs/core-plugins/src/manager_tests.rs | 93 ++++++ codex-rs/core-plugins/src/marketplace.rs | 285 ++++++++++-------- .../core-plugins/src/marketplace_context.rs | 16 +- 4 files changed, 330 insertions(+), 170 deletions(-) diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs index 663c892f7f..cac09dd438 100644 --- a/codex-rs/core-plugins/src/manager.rs +++ b/codex-rs/core-plugins/src/manager.rs @@ -52,6 +52,7 @@ use crate::marketplace::MarketplaceError; use crate::marketplace::MarketplaceInterface; use crate::marketplace::MarketplaceListError; use crate::marketplace::MarketplaceListOutcome; +use crate::marketplace::MarketplacePlugin; use crate::marketplace::MarketplacePluginAuthPolicy; use crate::marketplace::MarketplacePluginManifestFallback; use crate::marketplace::MarketplacePluginPolicy; @@ -60,6 +61,7 @@ use crate::marketplace::ResolvedMarketplacePlugin; use crate::marketplace::find_installable_marketplace_plugin; use crate::marketplace::find_marketplace_plugin; use crate::marketplace::home_dir; +use crate::marketplace::list_marketplace_manifests; use crate::marketplace::list_marketplaces_with_cache; use crate::marketplace::plugin_interface_with_marketplace_category; use crate::marketplace_policy::MarketplacePolicy; @@ -605,6 +607,13 @@ pub struct ConfiguredMarketplaceListOutcome { pub errors: Vec, } +/// Catalog paths and plugin IDs already claimed by earlier project scopes in this request. +#[derive(Default)] +struct MarketplaceSelection { + paths: HashSet, + plugin_ids: HashSet, +} + #[derive(Default)] struct ConfiguredPluginStates { installed: HashSet, @@ -2583,6 +2592,7 @@ impl PluginsManager { additional_roots, include_openai_curated, &plugin_states, + /*selection*/ None, ) } @@ -2592,43 +2602,68 @@ impl PluginsManager { additional_roots: &[AbsolutePathBuf], include_openai_curated: bool, plugin_states: &ConfiguredPluginStates, + mut selection: Option<&mut MarketplaceSelection>, ) -> Result { let marketplace_roots = self.marketplace_roots(config, additional_roots, include_openai_curated); - let marketplace_outcome = self.list_marketplaces_with_policy(config, &marketplace_roots)?; + let policy = MarketplacePolicy::from_requirements(config.config_layer_stack.requirements()); let mut seen_plugin_keys = HashSet::new(); - let marketplaces = marketplace_outcome - .marketplaces - .into_iter() - .filter_map(|marketplace| { - let marketplace_name = marketplace.name.clone(); - let plugins = marketplace - .plugins + let (marketplaces, errors) = list_marketplace_manifests( + &marketplace_roots, + home_dir().as_deref(), + |path, manifest| { + policy + .validate_install( + &config.config_layer_stack, + self.codex_home.as_path(), + path, + &manifest.name, + ) + .ok()?; + let (marketplace_name, interface, entries) = manifest.into_entries(path); + let entries = entries + .filter_map(|entry| { + // The first valid source claims its ID before product visibility. + let key = entry.id.as_key(); + if !seen_plugin_keys.insert(key.clone()) + || !self.restriction_product_matches(entry.policy.products.as_deref()) + { + return None; + } + Some((key, entry)) + }) + .collect::>(); + // A visible entry claims the path even if an earlier project claimed its ID. + if entries.is_empty() + || selection + .as_deref_mut() + .is_some_and(|s| !s.paths.insert(path.clone())) + { + return None; + } + let plugins = entries .into_iter() - .filter_map(|plugin| { - let plugin_key = format!("{}@{marketplace_name}", plugin.name); - if !seen_plugin_keys.insert(plugin_key.clone()) { - return None; - } - if !self.restriction_product_matches(plugin.policy.products.as_deref()) { - return None; - } - let plugin_id = - PluginId::new(plugin.name.clone(), marketplace_name.clone()).ok(); + .filter(|(key, _)| { + selection + .as_deref_mut() + .is_none_or(|s| s.plugin_ids.insert(key.clone())) + }) + .map(|(plugin_key, entry)| { + let plugin_id = entry.id.clone(); + let plugin = MarketplacePlugin::from( + entry.load_manifest(&self.store.manifest_cache), + ); let installed = plugin_states.installed.contains(&plugin_key); - let installed_version = installed.then_some(()).and_then(|_| { - plugin_id - .as_ref() - .and_then(|plugin_id| self.store.active_plugin_version(plugin_id)) - }); + let installed_version = installed + .then(|| self.store.active_plugin_version(&plugin_id)) + .flatten(); let enabled = plugin_states.enabled.contains(&plugin_key); let mut interface = plugin.interface; let mut local_version = plugin.local_version; - let manifest_fallback = plugin.manifest_fallback.clone(); + let manifest_fallback = plugin.manifest_fallback; if installed && plugin.source.is_install_materialized() - && let Some(plugin_id) = plugin_id.as_ref() - && let Some(plugin_root) = self.store.active_plugin_root(plugin_id) + && let Some(plugin_root) = self.store.active_plugin_root(&plugin_id) && let Some(manifest) = self .store .manifest_cache @@ -2645,10 +2680,7 @@ impl PluginsManager { ); } - Some(ConfiguredMarketplacePlugin { - // Enabled state is keyed by `@`, so duplicate - // plugin entries from duplicate marketplace files intentionally - // resolve to the first discovered source. + ConfiguredMarketplacePlugin { id: plugin_key, installed_version, installed, @@ -2660,22 +2692,22 @@ impl PluginsManager { keywords: plugin.keywords, interface, manifest_fallback, - }) + } }) .collect::>(); (!plugins.is_empty()).then_some(ConfiguredMarketplace { - name: marketplace.name, - path: marketplace.path, - interface: marketplace.interface, + name: marketplace_name, + path: path.clone(), + interface, plugins, }) - }) - .collect(); + }, + ); Ok(ConfiguredMarketplaceListOutcome { marketplaces, - errors: marketplace_outcome.errors, + errors, }) } diff --git a/codex-rs/core-plugins/src/manager_tests.rs b/codex-rs/core-plugins/src/manager_tests.rs index 3da0355c8e..b5d596b6be 100644 --- a/codex-rs/core-plugins/src/manager_tests.rs +++ b/codex-rs/core-plugins/src/manager_tests.rs @@ -387,6 +387,99 @@ path = {:?} assert_eq!(blocked_outcome.marketplaces, Vec::new()); } +#[tracing_test::traced_test] +#[tokio::test] +async fn list_marketplaces_skips_manifests_for_discarded_rows_without_changing_precedence() { + let tmp = TempDir::new().unwrap(); + let shared = tmp.path().join("a-shared"); + let repo = tmp.path().join("b-repo"); + write_file( + &shared.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "team", + "plugins": [ + {"name": "shared", "source": "./shared"}, + {"name": "shadowed", "source": "./shadowed", "policy": {"products": []}} + ] +}"#, + ); + write_file(&shared.join("shared/.codex-plugin/plugin.json"), "{"); + write_file( + &repo.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "team", + "plugins": [ + {"name": "shadowed", "source": "./shadowed"}, + {"name": "repo-only", "source": "./repo-only"} + ] +}"#, + ); + write_file(&repo.join("shadowed/.codex-plugin/plugin.json"), "{"); + write_plugin_with_version(&repo, "repo-only", "repo-only", Some("2.0.0")); + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + &format!( + r#" +[features] +plugins = true + +[marketplaces.team] +source_type = "local" +source = {shared:?} +"# + ), + ); + let config = load_config(tmp.path(), &repo).await; + let context = PluginMarketplaceContext { + global_config: config.clone(), + scopes: vec![ + PluginMarketplaceScope { + cwd: Some(AbsolutePathBuf::try_from(shared).unwrap()), + config: config.clone(), + }, + PluginMarketplaceScope { + cwd: Some(AbsolutePathBuf::try_from(repo).unwrap()), + config, + }, + ], + load_errors: Vec::new(), + }; + let result = test_plugins_manager(tmp.path().to_path_buf()) + .list_marketplaces_for_context(&context, /*include_openai_curated*/ false) + .unwrap(); + + assert_eq!(result.errors, Vec::new()); + assert_eq!( + result + .marketplaces + .iter() + .filter(|marketplace| marketplace.name == "team") + .flat_map(|marketplace| &marketplace.plugins) + .map(|plugin| (plugin.id.as_str(), plugin.local_version.as_deref())) + .collect::>(), + vec![("shared@team", None), ("repo-only@team", Some("2.0.0"))], + ); + logs_assert(|lines| { + let reads = lines + .iter() + .filter(|line| { + line.contains("failed to parse plugin manifest") && line.contains("a-shared") + }) + .count(); + let shadowed_reads = lines + .iter() + .filter(|line| { + line.contains("failed to parse plugin manifest") && line.contains("b-repo") + }) + .count(); + (reads == 1 && shadowed_reads == 0) + .then_some(()) + .ok_or_else(|| { + format!("expected one shared and zero shadowed manifest reads, got {reads} and {shadowed_reads}") + }) + }); +} + fn write_auth_projection_plugin(codex_home: &Path, name: &str, include_app: bool) { let plugin_root = codex_home .join("plugins/cache") diff --git a/codex-rs/core-plugins/src/marketplace.rs b/codex-rs/core-plugins/src/marketplace.rs index 81d5e100f6..75706cc76c 100644 --- a/codex-rs/core-plugins/src/marketplace.rs +++ b/codex-rs/core-plugins/src/marketplace.rs @@ -154,7 +154,8 @@ impl MarketplacePluginSource { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +#[serde(default)] pub struct MarketplacePluginPolicy { pub installation: MarketplacePluginInstallPolicy, pub authentication: MarketplacePluginAuthPolicy, @@ -257,13 +258,11 @@ pub fn find_marketplace_plugin( continue; } - if let Some(plugin) = resolve_marketplace_plugin_entry( - marketplace_path, - &marketplace_name, - plugin, - &ManifestCache::disabled(), - )? { - return Ok(plugin); + if let Some(entry) = plugin + .validate(marketplace_path, &marketplace_name) + .map_err(|PluginIdError::Invalid(message)| MarketplaceError::InvalidPlugin(message))? + { + return Ok(entry.load_manifest(&ManifestCache::disabled())); } } @@ -383,38 +382,15 @@ fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> // TODO: allow standalone callers to share a cache across marketplace loads. pub fn load_marketplace(path: &AbsolutePathBuf) -> Result { - load_marketplace_with_cache(path, &ManifestCache::disabled()) + Ok(load_raw_marketplace_manifest(path)?.resolve(path, &ManifestCache::disabled())) } -fn load_marketplace_with_cache( - path: &AbsolutePathBuf, - manifest_cache: &ManifestCache, -) -> Result { - let marketplace = load_raw_marketplace_manifest(path)?; - let mut plugins = Vec::new(); - - for plugin in marketplace.plugins { - let plugin = - match resolve_marketplace_plugin_entry(path, &marketplace.name, plugin, manifest_cache) - { - Ok(Some(plugin)) => plugin, - Ok(None) => continue, - Err(MarketplaceError::InvalidPlugin(message)) => { - warn!( - path = %path.display(), - marketplace = %marketplace.name, - error = %message, - "skipping invalid marketplace plugin" - ); - continue; - } - Err(err) => return Err(err), - }; - +impl From for MarketplacePlugin { + fn from(plugin: ResolvedMarketplacePlugin) -> Self { let manifest_fallback = plugin .manifest_fallback - .contents_if_has_metadata() - .map(|_| plugin.manifest_fallback.clone()); + .has_metadata + .then_some(plugin.manifest_fallback); let local_version = plugin .manifest .as_ref() @@ -424,7 +400,7 @@ fn load_marketplace_with_cache( .map(|manifest| manifest.keywords) .unwrap_or_default(); - plugins.push(MarketplacePlugin { + Self { name: plugin.plugin_id.plugin_name, local_version, source: plugin.source, @@ -432,15 +408,8 @@ fn load_marketplace_with_cache( interface: plugin.interface, keywords, manifest_fallback, - }); + } } - - Ok(Marketplace { - name: marketplace.name, - path: path.clone(), - interface: resolve_marketplace_interface(marketplace.interface), - plugins, - }) } // TODO: pass an owner-provided cache from remaining callers; the manager already does. @@ -457,26 +426,42 @@ pub(crate) fn list_marketplaces_with_cache( home_dir: Option<&Path>, manifest_cache: &ManifestCache, ) -> Result { - let mut outcome = MarketplaceListOutcome::default(); + let (marketplaces, errors) = + list_marketplace_manifests(additional_roots, home_dir, |path, manifest| { + Some(manifest.resolve(path, manifest_cache)) + }); + Ok(MarketplaceListOutcome { + marketplaces, + errors, + }) +} - for marketplace_path in discover_marketplace_paths_from_roots(additional_roots, home_dir) { - match load_marketplace_with_cache(&marketplace_path, manifest_cache) { - Ok(marketplace) => outcome.marketplaces.push(marketplace), +pub(crate) fn list_marketplace_manifests( + additional_roots: &[AbsolutePathBuf], + home_dir: Option<&Path>, + mut list: impl FnMut(&AbsolutePathBuf, RawMarketplaceManifest) -> Option, +) -> (Vec, Vec) { + let mut marketplaces = Vec::new(); + let mut errors = Vec::new(); + + for path in discover_marketplace_paths_from_roots(additional_roots, home_dir) { + match load_raw_marketplace_manifest(&path) { + Ok(manifest) => marketplaces.extend(list(&path, manifest)), Err(err) => { warn!( - path = %marketplace_path.display(), + path = %path.display(), error = %err, "skipping marketplace that failed to load" ); - outcome.errors.push(MarketplaceListError { - path: marketplace_path, + errors.push(MarketplaceListError { + path, message: err.to_string(), }); } } } - Ok(outcome) + (marketplaces, errors) } fn discover_marketplace_paths_from_roots( @@ -536,65 +521,63 @@ fn load_raw_marketplace_manifest( }) } -fn resolve_marketplace_plugin_entry( - marketplace_path: &AbsolutePathBuf, - marketplace_name: &str, - plugin: RawMarketplaceManifestPlugin, - manifest_cache: &ManifestCache, -) -> Result, MarketplaceError> { - let RawMarketplaceManifestPlugin { - name, - source, - policy, - category, - manifest_fields, - } = plugin; - let Some(source) = resolve_supported_plugin_source(marketplace_path, &name, source) else { - return Ok(None); - }; - let manifest_fallback = - marketplace_plugin_manifest_fallback(&name, category.as_deref(), &manifest_fields); +/// A catalog entry with a validated ID and source. Its plugin manifest has not been read yet. +pub(crate) struct MarketplacePluginEntry { + pub(crate) id: PluginId, + pub(crate) policy: MarketplacePluginPolicy, + source: MarketplacePluginSource, + category: Option, + manifest_fields: JsonMap, +} - let manifest = match &source { - MarketplacePluginSource::Local { path } => { - if codex_utils_plugins::find_plugin_manifest_path(path.as_path()).is_some() { - manifest_cache - .load(path.as_path()) - .map(|loaded| loaded.manifest) - } else if manifest_fallback.has_metadata { - manifest_fallback.parse_for_plugin_root(path.as_path()) - } else { - None +impl MarketplacePluginEntry { + pub(crate) fn load_manifest(self, manifest_cache: &ManifestCache) -> ResolvedMarketplacePlugin { + let Self { + id, + policy, + source, + category, + manifest_fields, + } = self; + let name = &id.plugin_name; + let manifest_fallback = + marketplace_plugin_manifest_fallback(name, category.as_deref(), &manifest_fields); + + let manifest = match &source { + MarketplacePluginSource::Local { path } => { + if codex_utils_plugins::find_plugin_manifest_path(path.as_path()).is_some() { + manifest_cache + .load(path.as_path()) + .map(|loaded| loaded.manifest) + } else if manifest_fallback.has_metadata { + manifest_fallback.parse_for_plugin_root(path.as_path()) + } else { + None + } } - } - MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } - if manifest_fallback.has_metadata => - { - manifest_fallback.parse_for_listing() - } - MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } => None, - }; - let interface = plugin_interface_with_marketplace_category( - manifest - .as_ref() - .and_then(|manifest| manifest.interface.clone()), - category, - ); + MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } + if manifest_fallback.has_metadata => + { + manifest_fallback.parse_for_listing() + } + MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } => None, + }; + let interface = plugin_interface_with_marketplace_category( + manifest + .as_ref() + .and_then(|manifest| manifest.interface.clone()), + category, + ); - Ok(Some(ResolvedMarketplacePlugin { - plugin_id: PluginId::new(name, marketplace_name.to_string()).map_err(|err| match err { - PluginIdError::Invalid(message) => MarketplaceError::InvalidPlugin(message), - })?, - source, - policy: MarketplacePluginPolicy { - installation: policy.installation, - authentication: policy.authentication, - products: policy.products, - }, - interface, - manifest, - manifest_fallback, - })) + ResolvedMarketplacePlugin { + plugin_id: id, + source, + policy, + interface, + manifest, + manifest_fallback, + } + } } fn resolve_supported_plugin_source( @@ -1003,13 +986,59 @@ pub fn marketplace_root_dir( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -struct RawMarketplaceManifest { - name: String, +pub(crate) struct RawMarketplaceManifest { + pub(crate) name: String, #[serde(default)] interface: Option, plugins: Vec, } +impl RawMarketplaceManifest { + fn resolve(self, path: &AbsolutePathBuf, manifest_cache: &ManifestCache) -> Marketplace { + let (name, interface, entries) = self.into_entries(path); + Marketplace { + name, + path: path.clone(), + interface, + plugins: entries + .map(|entry| entry.load_manifest(manifest_cache).into()) + .collect(), + } + } + + /// Yields valid entries in catalog order without reading plugin manifests, so callers + /// can apply precedence and visibility rules before loading metadata. + pub(crate) fn into_entries( + self, + path: &AbsolutePathBuf, + ) -> ( + String, + Option, + impl Iterator, + ) { + let marketplace_name = self.name.clone(); + let entries = self.plugins.into_iter().filter_map(move |plugin| { + match plugin.validate(path, &marketplace_name) { + Ok(entry) => entry, + Err(err) => { + warn!( + path = %path.display(), + marketplace = %marketplace_name, + error = %err, + "skipping invalid marketplace plugin" + ); + None + } + } + }); + ( + self.name, + resolve_marketplace_interface(self.interface), + entries, + ) + } +} + #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase")] struct RawMarketplaceManifestInterface { @@ -1023,7 +1052,7 @@ struct RawMarketplaceManifestPlugin { name: String, source: RawMarketplaceManifestPluginSource, #[serde(default)] - policy: RawMarketplaceManifestPluginPolicy, + policy: MarketplacePluginPolicy, #[serde(default)] category: Option, #[serde(default)] @@ -1031,14 +1060,30 @@ struct RawMarketplaceManifestPlugin { manifest_fields: JsonMap, } -#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawMarketplaceManifestPluginPolicy { - #[serde(default)] - installation: MarketplacePluginInstallPolicy, - #[serde(default)] - authentication: MarketplacePluginAuthPolicy, - products: Option>, +impl RawMarketplaceManifestPlugin { + fn validate( + self, + marketplace_path: &AbsolutePathBuf, + marketplace_name: &str, + ) -> Result, PluginIdError> { + let Self { + name, + source, + policy, + category, + manifest_fields, + } = self; + let Some(source) = resolve_supported_plugin_source(marketplace_path, &name, source) else { + return Ok(None); + }; + Ok(Some(MarketplacePluginEntry { + id: PluginId::new(name, marketplace_name.to_string())?, + policy, + source, + category, + manifest_fields, + })) + } } #[derive(Debug, Deserialize)] diff --git a/codex-rs/core-plugins/src/marketplace_context.rs b/codex-rs/core-plugins/src/marketplace_context.rs index af8c1d6008..107da6a477 100644 --- a/codex-rs/core-plugins/src/marketplace_context.rs +++ b/codex-rs/core-plugins/src/marketplace_context.rs @@ -50,8 +50,7 @@ impl PluginMarketplaceContext { errors: self.load_errors.clone(), ..Default::default() }; - let mut seen_marketplace_paths = HashSet::new(); - let mut seen_plugin_ids = HashSet::new(); + let mut selection = MarketplaceSelection::default(); let mut seen_error_paths = combined .errors .iter() @@ -68,18 +67,9 @@ impl PluginMarketplaceContext { scope.cwd.as_slice(), include_openai_curated, &plugin_states, + Some(&mut selection), )?; - for mut marketplace in outcome.marketplaces { - if !seen_marketplace_paths.insert(marketplace.path.clone()) { - continue; - } - marketplace - .plugins - .retain(|plugin| seen_plugin_ids.insert(plugin.id.clone())); - if !marketplace.plugins.is_empty() { - combined.marketplaces.push(marketplace); - } - } + combined.marketplaces.extend(outcome.marketplaces); combined.errors.extend( outcome .errors