Reuse plugin catalog membership snapshots

This commit is contained in:
Matthew Zeng
2026-06-15 21:52:05 -07:00
parent f0ce10bbf3
commit 030adac644
9 changed files with 443 additions and 20 deletions
@@ -7,6 +7,8 @@ use codex_utils_absolute_path::AbsolutePathBuf;
use tracing::warn;
use crate::marketplace::find_marketplace_manifest_path;
use crate::marketplace_upgrade::installed_marketplace_revision;
use crate::plugin_catalog_revision::PluginCatalogRevision;
pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces";
@@ -18,6 +20,16 @@ pub fn installed_marketplace_roots_from_layer_stack(
config_layer_stack: &ConfigLayerStack,
codex_home: &Path,
) -> Vec<AbsolutePathBuf> {
installed_marketplace_roots_with_revisions_from_layer_stack(config_layer_stack, codex_home)
.into_iter()
.map(|(root, _revision)| root)
.collect()
}
pub(crate) fn installed_marketplace_roots_with_revisions_from_layer_stack(
config_layer_stack: &ConfigLayerStack,
codex_home: &Path,
) -> Vec<(AbsolutePathBuf, Option<PluginCatalogRevision>)> {
let Some(user_config) = config_layer_stack.effective_user_config() else {
return Vec::new();
};
@@ -52,11 +64,31 @@ pub fn installed_marketplace_roots_from_layer_stack(
marketplace,
&default_install_root,
)?;
find_marketplace_manifest_path(&path).map(|_| path)
find_marketplace_manifest_path(&path)?;
let root = AbsolutePathBuf::try_from(path).ok()?;
let revision = if marketplace.get("source_type").and_then(toml::Value::as_str)
== Some("git")
{
(|| {
let source = marketplace.get("source").and_then(toml::Value::as_str)?;
let ref_name = marketplace.get("ref").and_then(toml::Value::as_str);
let sparse_paths = match marketplace.get("sparse_paths") {
Some(paths) => paths
.as_array()?
.iter()
.map(|path| path.as_str().map(ToString::to_string))
.collect::<Option<Vec<_>>>()?,
None => Vec::new(),
};
installed_marketplace_revision(root.as_path(), source, ref_name, &sparse_paths)
})()
} else {
None
};
Some((root, revision))
})
.filter_map(|path| AbsolutePathBuf::try_from(path).ok())
.collect::<Vec<_>>();
roots.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path()));
roots.sort_unstable_by(|(left, _), (right, _)| left.as_path().cmp(right.as_path()));
roots
}
+2
View File
@@ -8,6 +8,8 @@ pub mod marketplace_add;
pub mod marketplace_remove;
pub mod marketplace_upgrade;
mod plugin_bundle_archive;
mod plugin_catalog;
mod plugin_catalog_revision;
mod provider;
pub mod remote;
pub mod remote_bundle;
+82 -17
View File
@@ -1,6 +1,6 @@
use super::PluginLoadOutcome;
use crate::OPENAI_CURATED_MARKETPLACE_NAME;
use crate::installed_marketplaces::installed_marketplace_roots_from_layer_stack;
use crate::installed_marketplaces::installed_marketplace_roots_with_revisions_from_layer_stack;
use crate::loader::PluginHookLoadOutcome;
use crate::loader::configured_curated_plugin_ids_from_codex_home;
use crate::loader::curated_plugin_cache_version;
@@ -30,18 +30,24 @@ use crate::marketplace::MarketplacePluginSource;
use crate::marketplace::ResolvedMarketplacePlugin;
use crate::marketplace::find_installable_marketplace_plugin;
use crate::marketplace::find_marketplace_plugin;
use crate::marketplace::list_marketplaces;
use crate::marketplace::home_dir;
use crate::marketplace::plugin_interface_with_marketplace_category;
use crate::marketplace_upgrade::ConfiguredMarketplaceUpgradeError;
use crate::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome;
use crate::marketplace_upgrade::configured_git_marketplace_names;
use crate::marketplace_upgrade::upgrade_configured_git_marketplaces;
use crate::plugin_catalog::PluginCatalog;
use crate::plugin_catalog::PluginCatalogLoadMode;
use crate::plugin_catalog::PluginCatalogSnapshot;
use crate::plugin_catalog::PluginCatalogSource;
use crate::plugin_catalog_revision::PluginCatalogRevision;
use crate::remote::RemoteInstalledPlugin;
use crate::remote::RemotePluginCatalogError;
use crate::remote::RemotePluginServiceConfig;
use crate::remote_legacy::RemotePluginFetchError;
use crate::remote_legacy::RemotePluginMutationError;
use crate::startup_sync::curated_plugins_repo_path;
use crate::startup_sync::read_curated_plugins_catalog_revision;
use crate::startup_sync::read_curated_plugins_sha;
use crate::startup_sync::sync_openai_plugins_repo;
use crate::store::PluginInstallResult as StorePluginInstallResult;
@@ -344,6 +350,7 @@ pub struct PluginsManager {
non_curated_cache_refresh_state: RwLock<NonCuratedCacheRefreshState>,
enabled_outcome_cache: RwLock<EnabledOutcomeCache>,
enabled_outcome_load_semaphore: Semaphore,
plugin_catalog: PluginCatalog,
remote_installed_plugins_cache: RwLock<Option<Vec<RemoteInstalledPlugin>>>,
remote_installed_plugins_cache_refresh_state: RwLock<RemoteInstalledPluginsCacheRefreshState>,
global_remote_catalog_cache_refresh_state: RwLock<GlobalRemoteCatalogCacheRefreshState>,
@@ -398,6 +405,7 @@ impl PluginsManager {
non_curated_cache_refresh_state: RwLock::new(NonCuratedCacheRefreshState::default()),
enabled_outcome_cache: RwLock::new(EnabledOutcomeCache::default()),
enabled_outcome_load_semaphore: Semaphore::new(/*permits*/ 1),
plugin_catalog: PluginCatalog::default(),
remote_installed_plugins_cache: RwLock::new(None),
remote_installed_plugins_cache_refresh_state: RwLock::new(
RemoteInstalledPluginsCacheRefreshState::default(),
@@ -1060,12 +1068,9 @@ impl PluginsManager {
}
let (installed_plugins, enabled_plugins) = self.configured_plugin_states(config);
let mut marketplace_roots = self.marketplace_roots(config, additional_roots);
if !include_openai_curated {
let curated_repo_root = curated_plugins_repo_path(self.codex_home.as_path());
marketplace_roots.retain(|root| root.as_path() != curated_repo_root.as_path());
}
let marketplace_outcome = list_marketplaces(&marketplace_roots)?;
let marketplace_outcome = self
.plugin_catalog_snapshot_for_config(config, additional_roots, include_openai_curated)?
.marketplace_outcome();
let mut seen_plugin_keys = HashSet::new();
let marketplaces = marketplace_outcome
.marketplaces
@@ -1152,7 +1157,12 @@ impl PluginsManager {
return Ok(MarketplaceListOutcome::default());
}
list_marketplaces(&self.marketplace_roots(config, additional_roots))
self.plugin_catalog_snapshot_for_config(
config,
additional_roots,
/*include_openai_curated*/ true,
)
.map(|snapshot| snapshot.marketplace_outcome())
}
pub async fn read_plugin_for_config(
@@ -1358,6 +1368,43 @@ impl PluginsManager {
})
}
fn plugin_catalog_snapshot_for_config(
&self,
config: &PluginsConfigInput,
additional_roots: &[AbsolutePathBuf],
include_openai_curated: bool,
) -> Result<PluginCatalogSnapshot, MarketplaceError> {
self.plugin_catalog.snapshot(&self.plugin_catalog_sources(
config,
additional_roots,
include_openai_curated,
))
}
fn plugin_catalog_sources(
&self,
config: &PluginsConfigInput,
additional_roots: &[AbsolutePathBuf],
include_openai_curated: bool,
) -> Vec<PluginCatalogSource> {
let mut sources = Vec::new();
if let Some(home) = home_dir().and_then(|home| AbsolutePathBuf::try_from(home).ok()) {
sources.push(PluginCatalogSource::home(home));
}
let curated_repo_root = curated_plugins_repo_path(self.codex_home.as_path());
let mut roots = self.marketplace_roots_with_revisions(config, additional_roots);
if !include_openai_curated {
roots.retain(|(root, _revision)| root.as_path() != curated_repo_root.as_path());
}
sources.extend(roots.into_iter().map(|(root, revision)| {
PluginCatalogSource::filesystem_root(
root,
PluginCatalogLoadMode::for_revision(revision),
)
}));
sources
}
pub fn maybe_start_plugin_startup_tasks_for_config(
self: &Arc<Self>,
config: &PluginsConfigInput,
@@ -1882,15 +1929,19 @@ impl PluginsManager {
(installed_plugins, enabled_plugins)
}
fn marketplace_roots(
fn marketplace_roots_with_revisions(
&self,
config: &PluginsConfigInput,
additional_roots: &[AbsolutePathBuf],
) -> Vec<AbsolutePathBuf> {
) -> Vec<(AbsolutePathBuf, Option<PluginCatalogRevision>)> {
// Treat the curated catalog as an extra marketplace root so plugin listing can surface it
// without requiring every caller to know where it is stored.
let mut roots = additional_roots.to_vec();
roots.extend(installed_marketplace_roots_from_layer_stack(
let mut roots = additional_roots
.iter()
.cloned()
.map(|root| (root, None))
.collect::<Vec<_>>();
roots.extend(installed_marketplace_roots_with_revisions_from_layer_stack(
&config.config_layer_stack,
self.codex_home.as_path(),
));
@@ -1898,11 +1949,25 @@ impl PluginsManager {
if curated_repo_root.is_dir()
&& let Ok(curated_repo_root) = AbsolutePathBuf::try_from(curated_repo_root)
{
roots.push(curated_repo_root);
let revision = read_curated_plugins_catalog_revision(curated_repo_root.as_path());
roots.push((curated_repo_root, revision));
}
roots.sort_unstable();
roots.dedup();
roots
roots.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
let mut deduped =
Vec::<(AbsolutePathBuf, Option<PluginCatalogRevision>)>::with_capacity(roots.len());
for (root, revision) in roots {
if let Some((previous_root, previous_revision)) = deduped.last_mut()
&& *previous_root == root
{
*previous_revision = match (previous_revision.take(), revision) {
(Some(previous), Some(current)) if previous == current => Some(previous),
_ => None,
};
continue;
}
deduped.push((root, revision));
}
deduped
}
}
@@ -3,6 +3,7 @@ mod git;
use self::activation::activate_marketplace_root;
use self::activation::installed_marketplace_metadata_matches;
pub(crate) use self::activation::installed_marketplace_revision;
use self::activation::write_installed_marketplace_metadata;
use self::git::clone_git_source;
use self::git::git_remote_revision;
@@ -1,5 +1,7 @@
use super::ConfiguredGitMarketplace;
use crate::plugin_catalog_revision::PluginCatalogRevision;
use codex_config::types::MarketplaceSourceType;
use codex_utils_absolute_path::AbsolutePathBuf;
use serde::Deserialize;
use serde::Serialize;
use std::path::Path;
@@ -42,6 +44,22 @@ pub(super) fn installed_marketplace_metadata_matches(
metadata == installed_marketplace_metadata(marketplace, revision)
}
pub(crate) fn installed_marketplace_revision(
root: &Path,
source: &str,
ref_name: Option<&str>,
sparse_paths: &[String],
) -> Option<PluginCatalogRevision> {
let marker_path = AbsolutePathBuf::try_from(installed_marketplace_metadata_path(root)).ok()?;
let contents = std::fs::read_to_string(marker_path.as_path()).ok()?;
let metadata = serde_json::from_str::<InstalledMarketplaceMetadata>(&contents).ok()?;
(metadata.source_type == MarketplaceSourceType::Git
&& metadata.source == source
&& metadata.ref_name.as_deref() == ref_name
&& metadata.sparse_paths == sparse_paths)
.then(|| PluginCatalogRevision::new(marker_path, contents))
}
pub(super) fn write_installed_marketplace_metadata(
root: &Path,
marketplace: &ConfiguredGitMarketplace,
+194
View File
@@ -0,0 +1,194 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::PoisonError;
use codex_utils_absolute_path::AbsolutePathBuf;
use crate::marketplace::MarketplaceError;
use crate::marketplace::MarketplaceListOutcome;
use crate::marketplace::list_marketplaces_with_home;
use crate::plugin_catalog_revision::PluginCatalogRevision;
const MAX_REUSED_PLUGIN_CATALOG_ENTRIES: usize = 1024;
const MAX_REUSED_PLUGIN_CATALOG_SOURCES: usize = 64;
/// A marketplace membership view assembled from ordered plugin catalog sources.
#[derive(Default)]
pub(crate) struct PluginCatalogSnapshot {
outcome: MarketplaceListOutcome,
}
impl PluginCatalogSnapshot {
pub(crate) fn marketplace_outcome(&self) -> MarketplaceListOutcome {
self.outcome.clone()
}
}
/// Declares whether a source can safely reuse a previously loaded fragment.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum PluginCatalogLoadMode {
AlwaysRebuild,
ReuseIfRevisionMatches(PluginCatalogRevision),
}
impl PluginCatalogLoadMode {
pub(crate) fn for_revision(revision: Option<PluginCatalogRevision>) -> Self {
revision.map_or(Self::AlwaysRebuild, Self::ReuseIfRevisionMatches)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
enum PluginCatalogSourceLocation {
Home(AbsolutePathBuf),
FilesystemRoot(AbsolutePathBuf),
}
/// Ordered input describing one independently refreshable plugin catalog source.
pub(crate) struct PluginCatalogSource {
location: PluginCatalogSourceLocation,
load_mode: PluginCatalogLoadMode,
}
impl PluginCatalogSource {
pub(crate) fn home(home: AbsolutePathBuf) -> Self {
Self {
location: PluginCatalogSourceLocation::Home(home),
load_mode: PluginCatalogLoadMode::AlwaysRebuild,
}
}
pub(crate) fn filesystem_root(root: AbsolutePathBuf, load_mode: PluginCatalogLoadMode) -> Self {
Self {
location: PluginCatalogSourceLocation::FilesystemRoot(root),
load_mode,
}
}
}
/// Builds snapshots while reusing only sources with a proven revision contract.
#[derive(Default)]
pub(crate) struct PluginCatalog {
state: Mutex<PluginCatalogState>,
}
#[derive(Default)]
struct PluginCatalogState {
reused_fragments: HashMap<PluginCatalogSourceLocation, CachedPluginCatalogFragment>,
}
struct CachedPluginCatalogFragment {
revision: PluginCatalogRevision,
fragment: Arc<PluginCatalogFragment>,
}
struct PluginCatalogFragment {
outcome: MarketplaceListOutcome,
}
impl PluginCatalog {
pub(crate) fn snapshot(
&self,
sources: &[PluginCatalogSource],
) -> Result<PluginCatalogSnapshot, MarketplaceError> {
let mut outcome = MarketplaceListOutcome::default();
let mut seen_marketplace_paths = HashSet::new();
let mut seen_error_paths = HashSet::new();
for source in sources {
let fragment = self.fragment_for(source)?;
for marketplace in &fragment.outcome.marketplaces {
if seen_marketplace_paths.insert(marketplace.path.clone()) {
outcome.marketplaces.push(marketplace.clone());
}
}
for error in &fragment.outcome.errors {
if seen_error_paths.insert(error.path.clone()) {
outcome.errors.push(error.clone());
}
}
}
Ok(PluginCatalogSnapshot { outcome })
}
fn fragment_for(
&self,
source: &PluginCatalogSource,
) -> Result<Arc<PluginCatalogFragment>, MarketplaceError> {
let PluginCatalogLoadMode::ReuseIfRevisionMatches(revision) = &source.load_mode else {
self.state
.lock()
.unwrap_or_else(PoisonError::into_inner)
.reused_fragments
.remove(&source.location);
return self.load_fragment(source).map(Arc::new);
};
let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner);
if let Some(cached) = state.reused_fragments.get(&source.location)
&& cached.revision == *revision
{
return Ok(Arc::clone(&cached.fragment));
}
let fragment = Arc::new(self.load_fragment(source)?);
if !revision.is_current()
|| !fragment.outcome.errors.is_empty()
|| fragment.plugin_count() > MAX_REUSED_PLUGIN_CATALOG_ENTRIES
{
state.reused_fragments.remove(&source.location);
return Ok(fragment);
}
let retained_entry_count = state
.reused_fragments
.iter()
.filter(|(location, _cached)| *location != &source.location)
.map(|(_location, cached)| cached.fragment.plugin_count())
.sum::<usize>();
if retained_entry_count.saturating_add(fragment.plugin_count())
> MAX_REUSED_PLUGIN_CATALOG_ENTRIES
|| (state.reused_fragments.len() >= MAX_REUSED_PLUGIN_CATALOG_SOURCES
&& !state.reused_fragments.contains_key(&source.location))
{
state.reused_fragments.clear();
}
state.reused_fragments.insert(
source.location.clone(),
CachedPluginCatalogFragment {
revision: revision.clone(),
fragment: Arc::clone(&fragment),
},
);
Ok(fragment)
}
fn load_fragment(
&self,
source: &PluginCatalogSource,
) -> Result<PluginCatalogFragment, MarketplaceError> {
let outcome = match &source.location {
PluginCatalogSourceLocation::Home(home) => {
list_marketplaces_with_home(&[], Some(home.as_path()))?
}
PluginCatalogSourceLocation::FilesystemRoot(root) => {
list_marketplaces_with_home(std::slice::from_ref(root), /*home_dir*/ None)?
}
};
Ok(PluginCatalogFragment { outcome })
}
}
impl PluginCatalogFragment {
fn plugin_count(&self) -> usize {
self.outcome
.marketplaces
.iter()
.map(|marketplace| marketplace.plugins.len())
.sum()
}
}
#[cfg(test)]
#[path = "plugin_catalog_tests.rs"]
mod tests;
@@ -0,0 +1,29 @@
use std::fs;
use codex_utils_absolute_path::AbsolutePathBuf;
/// A revision marker materialized alongside the catalog source it identifies.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct PluginCatalogRevision {
marker_path: AbsolutePathBuf,
marker_contents: String,
}
impl PluginCatalogRevision {
pub(crate) fn new(marker_path: AbsolutePathBuf, marker_contents: String) -> Self {
Self {
marker_path,
marker_contents,
}
}
pub(crate) fn read(marker_path: AbsolutePathBuf) -> Option<Self> {
let marker_contents = fs::read_to_string(marker_path.as_path()).ok()?;
Some(Self::new(marker_path, marker_contents))
}
pub(crate) fn is_current(&self) -> bool {
fs::read_to_string(self.marker_path.as_path())
.is_ok_and(|contents| contents == self.marker_contents)
}
}
@@ -0,0 +1,72 @@
use codex_utils_absolute_path::AbsolutePathBuf;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
use super::*;
use crate::test_support::write_file;
use crate::test_support::write_openai_curated_marketplace;
fn revisioned_source(
root: &AbsolutePathBuf,
revision_path: &AbsolutePathBuf,
) -> PluginCatalogSource {
PluginCatalogSource::filesystem_root(
root.clone(),
PluginCatalogLoadMode::for_revision(PluginCatalogRevision::read(revision_path.clone())),
)
}
fn plugin_names(snapshot: PluginCatalogSnapshot) -> Vec<String> {
snapshot
.marketplace_outcome()
.marketplaces
.into_iter()
.flat_map(|marketplace| marketplace.plugins)
.map(|plugin| plugin.name)
.collect()
}
#[test]
fn matching_revision_reuses_membership_and_new_revision_reloads_it() {
let temp = tempdir().expect("tempdir");
write_openai_curated_marketplace(temp.path(), &["sample"]);
let revision_path = temp.path().join(".revision");
write_file(&revision_path, "one");
let root = AbsolutePathBuf::try_from(temp.path().to_path_buf()).expect("absolute root");
let revision_path = AbsolutePathBuf::try_from(revision_path).expect("absolute revision path");
let catalog = PluginCatalog::default();
let first = catalog
.snapshot(&[revisioned_source(&root, &revision_path)])
.expect("first snapshot");
assert_eq!(plugin_names(first), vec!["sample"]);
write_openai_curated_marketplace(temp.path(), &["sample", "second"]);
let matching_revision = catalog
.snapshot(&[revisioned_source(&root, &revision_path)])
.expect("matching revision snapshot");
assert_eq!(plugin_names(matching_revision), vec!["sample"]);
write_file(revision_path.as_path(), "two");
let changed_revision = catalog
.snapshot(&[revisioned_source(&root, &revision_path)])
.expect("changed revision snapshot");
assert_eq!(plugin_names(changed_revision), vec!["sample", "second"]);
write_openai_curated_marketplace(temp.path(), &["sample", "second", "third"]);
let rebuilt = catalog
.snapshot(&[PluginCatalogSource::filesystem_root(
root.clone(),
PluginCatalogLoadMode::AlwaysRebuild,
)])
.expect("always rebuild snapshot");
assert_eq!(plugin_names(rebuilt), vec!["sample", "second", "third"]);
let revisioned_again = catalog
.snapshot(&[revisioned_source(&root, &revision_path)])
.expect("revisioned snapshot after rebuild");
assert_eq!(
plugin_names(revisioned_again),
vec!["sample", "second", "third"]
);
}
+10
View File
@@ -8,12 +8,14 @@ use std::time::Duration;
use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC;
use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_METRIC;
use codex_utils_absolute_path::AbsolutePathBuf;
use reqwest::Client;
use serde::Deserialize;
use tempfile::TempDir;
use tracing::warn;
use zip::ZipArchive;
use crate::plugin_catalog_revision::PluginCatalogRevision;
use codex_login::default_client::build_reqwest_client;
const GITHUB_API_BASE_URL: &str = "https://api.github.com";
@@ -64,6 +66,14 @@ pub fn read_curated_plugins_sha(codex_home: &Path) -> Option<String> {
read_sha_file(curated_plugins_sha_path(codex_home).as_path())
}
pub(crate) fn read_curated_plugins_catalog_revision(
repo_path: &Path,
) -> Option<PluginCatalogRevision> {
let marker_path =
AbsolutePathBuf::try_from(curated_plugins_catalog_revision_path(repo_path)).ok()?;
PluginCatalogRevision::read(marker_path)
}
fn curated_plugins_sha_path(codex_home: &Path) -> PathBuf {
codex_home.join(CURATED_PLUGINS_SHA_FILE)
}