mirror of
https://github.com/openai/codex.git
synced 2026-10-18 17:18:52 +00:00
Cache remote plugin catalogs by scope (#34849)
## What changed - Cache global, user, and workspace remote plugin catalogs on disk with a three-hour TTL and scope- and account-specific keys. - Serve cached catalogs from `plugin/list`, refresh stale entries in the background, and warm existing caches at app-server startup without blocking list requests. - Make `forceRefetch: true` bypass cached catalog data and replace an entry only after a successful fetch. - Invalidate user and workspace catalog caches after remote plugin sharing changes. ## Testing - Cover cache freshness, scope isolation, legacy global cache compatibility, identity requirements for private catalogs, forced refetches, and non-blocking startup refreshes. GitOrigin-RevId: 7844e9dfb72e43b26d749b4afd16fd9c1793caba
This commit is contained in:
@@ -47,6 +47,7 @@ use crate::remote::RemoteInstalledPlugin;
|
||||
use crate::remote::RemoteInstalledPluginBundleSyncOutcome;
|
||||
use crate::remote::RemotePluginCatalogError;
|
||||
use crate::remote::RemotePluginMaterialization;
|
||||
use crate::remote::RemotePluginScope;
|
||||
use crate::remote::RemotePluginServiceConfig;
|
||||
use crate::remote_legacy::RemotePluginFetchError;
|
||||
use crate::remote_legacy::RemotePluginMutationError;
|
||||
@@ -90,8 +91,10 @@ use codex_tools::DiscoverableTool;
|
||||
use codex_tools::filter_request_plugin_install_discoverable_tools_for_client;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use codex_utils_plugins::PluginSkillRoot;
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::VecDeque;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
@@ -206,20 +209,40 @@ struct RemoteInstalledPluginsCacheRefreshState {
|
||||
in_flight: bool,
|
||||
}
|
||||
|
||||
struct GlobalRemoteCatalogCacheRefreshRequest {
|
||||
struct RemoteCatalogCacheRefreshRequest {
|
||||
service_config: RemotePluginServiceConfig,
|
||||
auth: Option<CodexAuth>,
|
||||
scopes: BTreeSet<RemotePluginScope>,
|
||||
mode: RemoteCatalogCacheRefreshMode,
|
||||
}
|
||||
|
||||
impl RemoteCatalogCacheRefreshRequest {
|
||||
fn has_same_cache_identity(&self, other: &Self) -> bool {
|
||||
self.service_config == other.service_config
|
||||
&& self.auth.as_ref().and_then(CodexAuth::get_account_id)
|
||||
== other.auth.as_ref().and_then(CodexAuth::get_account_id)
|
||||
&& self.auth.as_ref().and_then(CodexAuth::get_chatgpt_user_id)
|
||||
== other.auth.as_ref().and_then(CodexAuth::get_chatgpt_user_id)
|
||||
&& self.auth.as_ref().map(CodexAuth::is_workspace_account)
|
||||
== other.auth.as_ref().map(CodexAuth::is_workspace_account)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum RemoteCatalogCacheRefreshMode {
|
||||
OnlyIfStale,
|
||||
Force,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct GlobalRemoteCatalogCacheRefreshState {
|
||||
requested: Option<GlobalRemoteCatalogCacheRefreshRequest>,
|
||||
struct RemoteCatalogCacheRefreshState {
|
||||
requests: VecDeque<RemoteCatalogCacheRefreshRequest>,
|
||||
in_flight: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct PluginListBackgroundTaskOptions {
|
||||
pub refresh_global_remote_catalog_cache: bool,
|
||||
pub remote_catalog_cache_refresh_scopes: BTreeSet<RemotePluginScope>,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
@@ -391,7 +414,7 @@ pub struct PluginsManager {
|
||||
tool_suggest_metadata_cache: ToolSuggestMetadataCache,
|
||||
remote_installed_plugins_cache: RwLock<Option<Vec<RemoteInstalledPlugin>>>,
|
||||
remote_installed_plugins_cache_refresh_state: RwLock<RemoteInstalledPluginsCacheRefreshState>,
|
||||
global_remote_catalog_cache_refresh_state: RwLock<GlobalRemoteCatalogCacheRefreshState>,
|
||||
remote_catalog_cache_refresh_state: RwLock<RemoteCatalogCacheRefreshState>,
|
||||
restriction_product: Option<Product>,
|
||||
auth_mode: RwLock<Option<AuthMode>>,
|
||||
analytics_events_client: RwLock<Option<AnalyticsEventsClient>>,
|
||||
@@ -470,8 +493,8 @@ impl PluginsManager {
|
||||
remote_installed_plugins_cache_refresh_state: RwLock::new(
|
||||
RemoteInstalledPluginsCacheRefreshState::default(),
|
||||
),
|
||||
global_remote_catalog_cache_refresh_state: RwLock::new(
|
||||
GlobalRemoteCatalogCacheRefreshState::default(),
|
||||
remote_catalog_cache_refresh_state: RwLock::new(
|
||||
RemoteCatalogCacheRefreshState::default(),
|
||||
),
|
||||
restriction_product,
|
||||
auth_mode: RwLock::new(auth_mode),
|
||||
@@ -1047,18 +1070,22 @@ impl PluginsManager {
|
||||
);
|
||||
}
|
||||
|
||||
fn maybe_start_global_remote_catalog_cache_refresh(
|
||||
fn maybe_start_remote_catalog_cache_refresh(
|
||||
self: &Arc<Self>,
|
||||
config: &PluginsConfigInput,
|
||||
auth: Option<CodexAuth>,
|
||||
scopes: BTreeSet<RemotePluginScope>,
|
||||
mode: RemoteCatalogCacheRefreshMode,
|
||||
) {
|
||||
if !config.plugins_enabled || !config.remote_plugin_enabled {
|
||||
if !config.plugins_enabled || scopes.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
self.schedule_global_remote_catalog_cache_refresh(GlobalRemoteCatalogCacheRefreshRequest {
|
||||
self.schedule_remote_catalog_cache_refresh(RemoteCatalogCacheRefreshRequest {
|
||||
service_config: remote_plugin_service_config(config),
|
||||
auth,
|
||||
scopes,
|
||||
mode,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1071,9 +1098,12 @@ impl PluginsManager {
|
||||
on_effective_plugins_changed: Option<EffectivePluginsChangedCallback>,
|
||||
) {
|
||||
self.maybe_start_non_curated_plugin_cache_refresh(config, roots);
|
||||
if options.refresh_global_remote_catalog_cache {
|
||||
self.maybe_start_global_remote_catalog_cache_refresh(config, auth.clone());
|
||||
}
|
||||
self.maybe_start_remote_catalog_cache_refresh(
|
||||
config,
|
||||
auth.clone(),
|
||||
options.remote_catalog_cache_refresh_scopes,
|
||||
RemoteCatalogCacheRefreshMode::OnlyIfStale,
|
||||
);
|
||||
self.maybe_start_remote_plugin_caches_refresh(
|
||||
config,
|
||||
auth.clone(),
|
||||
@@ -2054,27 +2084,22 @@ impl PluginsManager {
|
||||
auth.clone(),
|
||||
on_effective_plugins_changed,
|
||||
);
|
||||
let mut scopes = crate::remote::cached_remote_plugin_catalog_scopes(
|
||||
manager.codex_home.as_path(),
|
||||
&remote_plugin_service_config(&config_for_remote_sync),
|
||||
auth.as_ref(),
|
||||
);
|
||||
if config_for_remote_sync.remote_plugin_enabled {
|
||||
match crate::remote::fetch_and_cache_global_remote_plugin_catalog(
|
||||
manager.codex_home.as_path(),
|
||||
&remote_plugin_service_config(&config_for_remote_sync),
|
||||
auth.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(
|
||||
RemotePluginCatalogError::AuthRequired
|
||||
| RemotePluginCatalogError::UnsupportedAuthMode,
|
||||
) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
error = %err,
|
||||
"failed to warm remote plugin catalog cache"
|
||||
);
|
||||
}
|
||||
}
|
||||
scopes.insert(RemotePluginScope::Global);
|
||||
} else {
|
||||
scopes.retain(|scope| *scope == RemotePluginScope::Workspace);
|
||||
}
|
||||
manager.maybe_start_remote_catalog_cache_refresh(
|
||||
&config_for_remote_sync,
|
||||
auth,
|
||||
scopes,
|
||||
RemoteCatalogCacheRefreshMode::Force,
|
||||
);
|
||||
});
|
||||
|
||||
let config_for_featured_plugins = config.clone();
|
||||
@@ -2232,16 +2257,35 @@ impl PluginsManager {
|
||||
});
|
||||
}
|
||||
|
||||
fn schedule_global_remote_catalog_cache_refresh(
|
||||
fn schedule_remote_catalog_cache_refresh(
|
||||
self: &Arc<Self>,
|
||||
request: GlobalRemoteCatalogCacheRefreshRequest,
|
||||
request: RemoteCatalogCacheRefreshRequest,
|
||||
) {
|
||||
let should_spawn = {
|
||||
let mut state = match self.global_remote_catalog_cache_refresh_state.write() {
|
||||
let mut state = match self.remote_catalog_cache_refresh_state.write() {
|
||||
Ok(state) => state,
|
||||
Err(err) => err.into_inner(),
|
||||
};
|
||||
state.requested = Some(request);
|
||||
if let Some(pending) = state
|
||||
.requests
|
||||
.iter_mut()
|
||||
.find(|pending| pending.has_same_cache_identity(&request))
|
||||
{
|
||||
pending.scopes.extend(request.scopes);
|
||||
pending.auth = request.auth;
|
||||
pending.mode = match (pending.mode, request.mode) {
|
||||
(RemoteCatalogCacheRefreshMode::Force, _)
|
||||
| (_, RemoteCatalogCacheRefreshMode::Force) => {
|
||||
RemoteCatalogCacheRefreshMode::Force
|
||||
}
|
||||
(
|
||||
RemoteCatalogCacheRefreshMode::OnlyIfStale,
|
||||
RemoteCatalogCacheRefreshMode::OnlyIfStale,
|
||||
) => RemoteCatalogCacheRefreshMode::OnlyIfStale,
|
||||
};
|
||||
} else {
|
||||
state.requests.push_back(request);
|
||||
}
|
||||
if state.in_flight {
|
||||
false
|
||||
} else {
|
||||
@@ -2255,7 +2299,7 @@ impl PluginsManager {
|
||||
|
||||
let manager = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
manager.run_global_remote_catalog_cache_refresh_loop().await;
|
||||
manager.run_remote_catalog_cache_refresh_loop().await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2472,14 +2516,14 @@ impl PluginsManager {
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_global_remote_catalog_cache_refresh_loop(self: Arc<Self>) {
|
||||
async fn run_remote_catalog_cache_refresh_loop(self: Arc<Self>) {
|
||||
loop {
|
||||
let request = {
|
||||
let mut state = match self.global_remote_catalog_cache_refresh_state.write() {
|
||||
let mut state = match self.remote_catalog_cache_refresh_state.write() {
|
||||
Ok(state) => state,
|
||||
Err(err) => err.into_inner(),
|
||||
};
|
||||
match state.requested.take() {
|
||||
match state.requests.pop_front() {
|
||||
Some(request) => request,
|
||||
None => {
|
||||
state.in_flight = false;
|
||||
@@ -2488,23 +2532,38 @@ impl PluginsManager {
|
||||
}
|
||||
};
|
||||
|
||||
match crate::remote::fetch_and_cache_global_remote_plugin_catalog(
|
||||
self.codex_home.as_path(),
|
||||
&request.service_config,
|
||||
request.auth.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(
|
||||
RemotePluginCatalogError::AuthRequired
|
||||
| RemotePluginCatalogError::UnsupportedAuthMode,
|
||||
) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
error = %err,
|
||||
"failed to refresh cached global remote plugin catalog"
|
||||
);
|
||||
for scope in request.scopes {
|
||||
if request.mode == RemoteCatalogCacheRefreshMode::OnlyIfStale
|
||||
&& crate::remote::has_fresh_cached_remote_plugin_catalog(
|
||||
self.codex_home.as_path(),
|
||||
&request.service_config,
|
||||
request.auth.as_ref(),
|
||||
scope,
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
match crate::remote::fetch_and_cache_remote_plugin_catalog(
|
||||
self.codex_home.as_path(),
|
||||
&request.service_config,
|
||||
request.auth.as_ref(),
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(
|
||||
RemotePluginCatalogError::AuthRequired
|
||||
| RemotePluginCatalogError::UnsupportedAuthMode,
|
||||
) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
error = %err,
|
||||
scope = ?scope,
|
||||
"failed to refresh cached remote plugin catalog"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,6 +189,18 @@ pub enum RemoteMarketplaceSource {
|
||||
SharedWithMe,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RemotePluginCatalogCacheMode {
|
||||
PreferCache,
|
||||
ForceRefetch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RemoteMarketplacesFetchOutcome {
|
||||
pub marketplaces: Vec<RemoteMarketplace>,
|
||||
pub catalog_cache_refresh_scopes: BTreeSet<RemotePluginScope>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RemoteInstalledPlugin {
|
||||
pub marketplace_name: String,
|
||||
@@ -443,6 +455,8 @@ pub enum RemotePluginScope {
|
||||
}
|
||||
|
||||
impl RemotePluginScope {
|
||||
const CATALOG_CACHE_SCOPES: [Self; 3] = [Self::Global, Self::User, Self::Workspace];
|
||||
|
||||
fn api_value(self) -> &'static str {
|
||||
match self {
|
||||
Self::Global => "GLOBAL",
|
||||
@@ -722,10 +736,12 @@ pub async fn fetch_remote_marketplaces(
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
sources: &[RemoteMarketplaceSource],
|
||||
global_catalog_cache_path: Option<&Path>,
|
||||
) -> Result<Vec<RemoteMarketplace>, RemotePluginCatalogError> {
|
||||
catalog_cache_root: Option<&Path>,
|
||||
catalog_cache_mode: RemotePluginCatalogCacheMode,
|
||||
) -> Result<RemoteMarketplacesFetchOutcome, RemotePluginCatalogError> {
|
||||
let auth = ensure_chatgpt_auth(auth)?;
|
||||
let mut marketplaces = Vec::new();
|
||||
let mut catalog_cache_refresh_scopes = BTreeSet::new();
|
||||
let needs_workspace_installed = sources.iter().any(|source| {
|
||||
matches!(
|
||||
source,
|
||||
@@ -742,61 +758,48 @@ pub async fn fetch_remote_marketplaces(
|
||||
match source {
|
||||
RemoteMarketplaceSource::Global => {
|
||||
let scope = RemotePluginScope::Global;
|
||||
if let Some(codex_home) = global_catalog_cache_path
|
||||
&& let Some(directory_plugins) =
|
||||
catalog_cache::load_cached_global_directory_plugins(
|
||||
codex_home, config, auth,
|
||||
)
|
||||
{
|
||||
let installed_plugins =
|
||||
fetch_installed_plugins_for_scope(config, auth, scope).await?;
|
||||
if let Some(marketplace) = build_remote_marketplace(
|
||||
scope.marketplace_name(),
|
||||
scope.marketplace_display_name(),
|
||||
directory_plugins,
|
||||
installed_plugins,
|
||||
/*include_installed_only*/ true,
|
||||
)? {
|
||||
marketplaces.push(marketplace);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let (directory_plugins, installed_plugins) = tokio::try_join!(
|
||||
fetch_directory_plugins_for_scope(config, auth, scope),
|
||||
fetch_directory_plugins_for_scope_with_cache(
|
||||
catalog_cache_root,
|
||||
config,
|
||||
auth,
|
||||
scope,
|
||||
catalog_cache_mode,
|
||||
),
|
||||
fetch_installed_plugins_for_scope(config, auth, scope),
|
||||
)?;
|
||||
let directory_plugins_for_cache =
|
||||
global_catalog_cache_path.map(|_| directory_plugins.clone());
|
||||
if directory_plugins.cache_refresh_needed {
|
||||
catalog_cache_refresh_scopes.insert(scope);
|
||||
}
|
||||
if let Some(marketplace) = build_remote_marketplace(
|
||||
scope.marketplace_name(),
|
||||
scope.marketplace_display_name(),
|
||||
directory_plugins,
|
||||
directory_plugins.plugins,
|
||||
installed_plugins,
|
||||
/*include_installed_only*/ true,
|
||||
)? {
|
||||
marketplaces.push(marketplace);
|
||||
}
|
||||
if let (Some(codex_home), Some(directory_plugins)) =
|
||||
(global_catalog_cache_path, directory_plugins_for_cache)
|
||||
{
|
||||
catalog_cache::write_cached_global_directory_plugins(
|
||||
codex_home,
|
||||
config,
|
||||
auth,
|
||||
&directory_plugins,
|
||||
);
|
||||
}
|
||||
}
|
||||
RemoteMarketplaceSource::CreatedByMeRemote => {
|
||||
let scope = RemotePluginScope::User;
|
||||
let (directory_plugins, installed_plugins) = tokio::try_join!(
|
||||
fetch_directory_plugins_for_scope(config, auth, scope),
|
||||
fetch_directory_plugins_for_scope_with_cache(
|
||||
catalog_cache_root,
|
||||
config,
|
||||
auth,
|
||||
scope,
|
||||
catalog_cache_mode,
|
||||
),
|
||||
fetch_installed_plugins_for_scope(config, auth, scope),
|
||||
)?;
|
||||
if directory_plugins.cache_refresh_needed {
|
||||
catalog_cache_refresh_scopes.insert(scope);
|
||||
}
|
||||
if let Some(marketplace) = build_remote_marketplace(
|
||||
scope.marketplace_name(),
|
||||
scope.marketplace_display_name(),
|
||||
directory_plugins,
|
||||
directory_plugins.plugins,
|
||||
installed_plugins,
|
||||
/*include_installed_only*/ false,
|
||||
)? {
|
||||
@@ -805,12 +808,21 @@ pub async fn fetch_remote_marketplaces(
|
||||
}
|
||||
RemoteMarketplaceSource::WorkspaceDirectory => {
|
||||
let scope = RemotePluginScope::Workspace;
|
||||
let directory_plugins =
|
||||
fetch_directory_plugins_for_scope(config, auth, scope).await?;
|
||||
let directory_plugins = fetch_directory_plugins_for_scope_with_cache(
|
||||
catalog_cache_root,
|
||||
config,
|
||||
auth,
|
||||
scope,
|
||||
catalog_cache_mode,
|
||||
)
|
||||
.await?;
|
||||
if directory_plugins.cache_refresh_needed {
|
||||
catalog_cache_refresh_scopes.insert(scope);
|
||||
}
|
||||
if let Some(marketplace) = build_remote_marketplace(
|
||||
scope.marketplace_name(),
|
||||
scope.marketplace_display_name(),
|
||||
directory_plugins,
|
||||
directory_plugins.plugins,
|
||||
workspace_installed_plugins.clone().unwrap_or_default(),
|
||||
/*include_installed_only*/ false,
|
||||
)? {
|
||||
@@ -878,7 +890,22 @@ pub async fn fetch_remote_marketplaces(
|
||||
}
|
||||
}
|
||||
|
||||
Ok(marketplaces)
|
||||
Ok(RemoteMarketplacesFetchOutcome {
|
||||
marketplaces,
|
||||
catalog_cache_refresh_scopes,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn fetch_and_cache_remote_plugin_catalog(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
scope: RemotePluginScope,
|
||||
) -> Result<(), RemotePluginCatalogError> {
|
||||
let auth = ensure_chatgpt_auth(auth)?;
|
||||
let plugins = fetch_directory_plugins_for_scope(config, auth, scope).await?;
|
||||
catalog_cache::write_cached_directory_plugins(codex_home, config, auth, scope, &plugins);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn fetch_and_cache_global_remote_plugin_catalog(
|
||||
@@ -886,11 +913,21 @@ pub async fn fetch_and_cache_global_remote_plugin_catalog(
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
) -> Result<(), RemotePluginCatalogError> {
|
||||
let auth = ensure_chatgpt_auth(auth)?;
|
||||
let plugins =
|
||||
fetch_directory_plugins_for_scope(config, auth, RemotePluginScope::Global).await?;
|
||||
catalog_cache::write_cached_global_directory_plugins(codex_home, config, auth, &plugins);
|
||||
Ok(())
|
||||
fetch_and_cache_remote_plugin_catalog(codex_home, config, auth, RemotePluginScope::Global).await
|
||||
}
|
||||
|
||||
pub fn invalidate_cached_remote_plugin_catalog_scopes(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
scopes: &[RemotePluginScope],
|
||||
) {
|
||||
let Ok(auth) = ensure_chatgpt_auth(auth) else {
|
||||
return;
|
||||
};
|
||||
for scope in scopes {
|
||||
catalog_cache::remove_cached_directory_plugins(codex_home, config, auth, *scope);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
@@ -975,15 +1012,39 @@ fn recommended_plugins_mode(response: RecommendedPluginsResponse) -> Recommended
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_cached_global_remote_plugin_catalog(
|
||||
pub(crate) fn has_fresh_cached_remote_plugin_catalog(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
scope: RemotePluginScope,
|
||||
) -> bool {
|
||||
let Ok(auth) = ensure_chatgpt_auth(auth) else {
|
||||
return false;
|
||||
};
|
||||
catalog_cache::load_cached_global_directory_plugins(codex_home, config, auth).is_some()
|
||||
catalog_cache::load_cached_directory_plugins(codex_home, config, auth, scope).is_some_and(
|
||||
|cached| {
|
||||
matches!(
|
||||
cached.freshness,
|
||||
catalog_cache::RemotePluginCatalogCacheFreshness::Fresh
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn cached_remote_plugin_catalog_scopes(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: Option<&CodexAuth>,
|
||||
) -> BTreeSet<RemotePluginScope> {
|
||||
let Ok(auth) = ensure_chatgpt_auth(auth) else {
|
||||
return BTreeSet::new();
|
||||
};
|
||||
RemotePluginScope::CATALOG_CACHE_SCOPES
|
||||
.into_iter()
|
||||
.filter(|scope| {
|
||||
catalog_cache::load_cached_directory_plugins(codex_home, config, auth, *scope).is_some()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn cached_global_remote_discoverable_plugins(
|
||||
@@ -991,17 +1052,25 @@ pub fn cached_global_remote_discoverable_plugins(
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
) -> Vec<RemoteDiscoverablePlugin> {
|
||||
catalog_cache::load_cached_global_directory_plugins(codex_home, config, auth)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|plugin| match remote_discoverable_plugin_from_directory_item(&plugin) {
|
||||
catalog_cache::load_cached_directory_plugins(
|
||||
codex_home,
|
||||
config,
|
||||
auth,
|
||||
RemotePluginScope::Global,
|
||||
)
|
||||
.map(|cached| cached.plugins)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(
|
||||
|plugin| match remote_discoverable_plugin_from_directory_item(&plugin) {
|
||||
Ok(plugin) => Some(plugin),
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "ignoring cached remote plugin recommendation entry");
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
},
|
||||
)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn fetch_openai_curated_remote_collection_marketplace(
|
||||
@@ -1768,6 +1837,42 @@ fn normalize_remote_default_prompt(prompt: &str) -> Option<String> {
|
||||
Some(prompt.to_string())
|
||||
}
|
||||
|
||||
struct DirectoryPluginsFetchOutcome {
|
||||
plugins: Vec<RemotePluginDirectoryItem>,
|
||||
cache_refresh_needed: bool,
|
||||
}
|
||||
|
||||
async fn fetch_directory_plugins_for_scope_with_cache(
|
||||
codex_home: Option<&Path>,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
scope: RemotePluginScope,
|
||||
cache_mode: RemotePluginCatalogCacheMode,
|
||||
) -> Result<DirectoryPluginsFetchOutcome, RemotePluginCatalogError> {
|
||||
if cache_mode == RemotePluginCatalogCacheMode::PreferCache
|
||||
&& let Some(codex_home) = codex_home
|
||||
&& let Some(cached) =
|
||||
catalog_cache::load_cached_directory_plugins(codex_home, config, auth, scope)
|
||||
{
|
||||
return Ok(DirectoryPluginsFetchOutcome {
|
||||
plugins: cached.plugins,
|
||||
cache_refresh_needed: matches!(
|
||||
cached.freshness,
|
||||
catalog_cache::RemotePluginCatalogCacheFreshness::Stale
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
let plugins = fetch_directory_plugins_for_scope(config, auth, scope).await?;
|
||||
if let Some(codex_home) = codex_home {
|
||||
catalog_cache::write_cached_directory_plugins(codex_home, config, auth, scope, &plugins);
|
||||
}
|
||||
Ok(DirectoryPluginsFetchOutcome {
|
||||
plugins,
|
||||
cache_refresh_needed: false,
|
||||
})
|
||||
}
|
||||
|
||||
async fn fetch_directory_plugins_for_scope(
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
@@ -1800,6 +1905,15 @@ async fn fetch_directory_plugins_for_scope_with_optional_collection(
|
||||
scope: RemotePluginScope,
|
||||
collection: Option<&str>,
|
||||
) -> Result<Vec<RemotePluginDirectoryItem>, RemotePluginCatalogError> {
|
||||
tracing::info!(
|
||||
operation = "plugins.remote_catalog.list",
|
||||
http.method = "GET",
|
||||
api.path = "ps/plugins/list",
|
||||
plugin.scope = scope.api_value(),
|
||||
plugin.collection = collection.unwrap_or_default(),
|
||||
"fetching remote plugin catalog"
|
||||
);
|
||||
|
||||
let mut plugins = Vec::new();
|
||||
let mut page_token = None;
|
||||
loop {
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
use super::RemotePluginDirectoryItem;
|
||||
use super::RemotePluginScope;
|
||||
use super::RemotePluginServiceConfig;
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_login::CodexAuth;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tracing::warn;
|
||||
|
||||
// `plugin/list` and other callers that assume all plugins are available locally will be
|
||||
// deprecated soon. Remove this catalog cache when those callers migrate to on-demand fetching.
|
||||
const REMOTE_PLUGIN_CATALOG_DISK_CACHE_SCHEMA_VERSION: u8 = 1;
|
||||
const REMOTE_PLUGIN_CATALOG_DISK_CACHE_DIR: &str = "cache/remote_plugin_catalog";
|
||||
const REMOTE_PLUGIN_CATALOG_DISK_CACHE_TTL: Duration = Duration::from_secs(60 * 60 * 3);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
struct RemotePluginCatalogCacheKey {
|
||||
@@ -16,34 +23,65 @@ struct RemotePluginCatalogCacheKey {
|
||||
account_id: Option<String>,
|
||||
chatgpt_user_id: Option<String>,
|
||||
is_workspace_account: bool,
|
||||
// Global catalogs predate scoped cache keys and must keep their existing filenames.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
scope: Option<RemotePluginScope>,
|
||||
}
|
||||
|
||||
impl RemotePluginCatalogCacheKey {
|
||||
fn global(config: &RemotePluginServiceConfig, auth: &CodexAuth) -> Self {
|
||||
Self {
|
||||
fn new(
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
scope: RemotePluginScope,
|
||||
) -> Option<Self> {
|
||||
let cache_key = Self {
|
||||
chatgpt_base_url: config.chatgpt_base_url.clone(),
|
||||
account_id: auth.get_account_id(),
|
||||
chatgpt_user_id: auth.get_chatgpt_user_id(),
|
||||
is_workspace_account: auth.is_workspace_account(),
|
||||
scope: (scope != RemotePluginScope::Global).then_some(scope),
|
||||
};
|
||||
// Preserve global catalog caching for existing header-auth clients, but never share
|
||||
// user or workspace catalogs when the auth mode cannot identify their owner.
|
||||
if !matches!(scope, RemotePluginScope::Global)
|
||||
&& cache_key.account_id.is_none()
|
||||
&& cache_key.chatgpt_user_id.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(cache_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct RemotePluginCatalogDiskCache {
|
||||
schema_version: u8,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
fetched_at: Option<DateTime<Utc>>,
|
||||
plugins: Vec<RemotePluginDirectoryItem>,
|
||||
}
|
||||
|
||||
pub(crate) fn load_cached_global_directory_plugins(
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum RemotePluginCatalogCacheFreshness {
|
||||
Fresh,
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct CachedDirectoryPlugins {
|
||||
pub plugins: Vec<RemotePluginDirectoryItem>,
|
||||
pub freshness: RemotePluginCatalogCacheFreshness,
|
||||
}
|
||||
|
||||
pub(crate) fn load_cached_directory_plugins(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
) -> Option<Vec<RemotePluginDirectoryItem>> {
|
||||
let cache_path = cache_path(
|
||||
codex_home,
|
||||
&RemotePluginCatalogCacheKey::global(config, auth),
|
||||
);
|
||||
scope: RemotePluginScope,
|
||||
) -> Option<CachedDirectoryPlugins> {
|
||||
let cache_key = RemotePluginCatalogCacheKey::new(config, auth, scope)?;
|
||||
let cache_path = cache_path(codex_home, &cache_key);
|
||||
let bytes = match std::fs::read(&cache_path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return None,
|
||||
@@ -71,31 +109,58 @@ pub(crate) fn load_cached_global_directory_plugins(
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(cache.plugins)
|
||||
let freshness = if is_fresh(cache.fetched_at, Utc::now()) {
|
||||
RemotePluginCatalogCacheFreshness::Fresh
|
||||
} else {
|
||||
RemotePluginCatalogCacheFreshness::Stale
|
||||
};
|
||||
Some(CachedDirectoryPlugins {
|
||||
plugins: cache.plugins,
|
||||
freshness,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn write_cached_global_directory_plugins(
|
||||
pub(crate) fn write_cached_directory_plugins(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
scope: RemotePluginScope,
|
||||
plugins: &[RemotePluginDirectoryItem],
|
||||
) {
|
||||
let cache_path = cache_path(
|
||||
codex_home,
|
||||
&RemotePluginCatalogCacheKey::global(config, auth),
|
||||
);
|
||||
if let Some(parent) = cache_path.parent()
|
||||
&& std::fs::create_dir_all(parent).is_err()
|
||||
{
|
||||
let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope) else {
|
||||
return;
|
||||
}
|
||||
let Ok(bytes) = serde_json::to_vec_pretty(&RemotePluginCatalogDiskCache {
|
||||
};
|
||||
let cache_path = cache_path(codex_home, &cache_key);
|
||||
let Ok(contents) = serde_json::to_string_pretty(&RemotePluginCatalogDiskCache {
|
||||
schema_version: REMOTE_PLUGIN_CATALOG_DISK_CACHE_SCHEMA_VERSION,
|
||||
fetched_at: Some(Utc::now()),
|
||||
plugins: plugins.to_vec(),
|
||||
}) else {
|
||||
return;
|
||||
};
|
||||
let _ = std::fs::write(cache_path, bytes);
|
||||
let _ = codex_utils_path::write_atomically(&cache_path, &contents);
|
||||
}
|
||||
|
||||
pub(crate) fn remove_cached_directory_plugins(
|
||||
codex_home: &Path,
|
||||
config: &RemotePluginServiceConfig,
|
||||
auth: &CodexAuth,
|
||||
scope: RemotePluginScope,
|
||||
) {
|
||||
let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope) else {
|
||||
return;
|
||||
};
|
||||
let cache_path = cache_path(codex_home, &cache_key);
|
||||
match std::fs::remove_file(&cache_path) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
cache_path = %cache_path.display(),
|
||||
"failed to remove remote plugin catalog disk cache: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_path(codex_home: &Path, cache_key: &RemotePluginCatalogCacheKey) -> PathBuf {
|
||||
@@ -109,3 +174,18 @@ fn cache_path(codex_home: &Path, cache_key: &RemotePluginCatalogCacheKey) -> Pat
|
||||
.join(REMOTE_PLUGIN_CATALOG_DISK_CACHE_DIR)
|
||||
.join(format!("{cache_key_hash:016x}.json"))
|
||||
}
|
||||
|
||||
fn is_fresh(fetched_at: Option<DateTime<Utc>>, now: DateTime<Utc>) -> bool {
|
||||
let Some(fetched_at) = fetched_at else {
|
||||
return false;
|
||||
};
|
||||
let Ok(ttl) = chrono::Duration::from_std(REMOTE_PLUGIN_CATALOG_DISK_CACHE_TTL) else {
|
||||
return false;
|
||||
};
|
||||
let age = now.signed_duration_since(fetched_at);
|
||||
age >= chrono::Duration::zero() && age <= ttl
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "catalog_cache_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
use super::*;
|
||||
use chrono::TimeDelta;
|
||||
use codex_login::AuthHeaders;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn catalog_cache_freshness_honors_ttl() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert!(!is_fresh(/*fetched_at*/ None, now));
|
||||
assert!(is_fresh(Some(now), now));
|
||||
assert!(is_fresh(Some(now - TimeDelta::hours(3)), now,));
|
||||
assert!(!is_fresh(
|
||||
Some(now - TimeDelta::hours(3) - TimeDelta::milliseconds(1)),
|
||||
now,
|
||||
));
|
||||
assert!(!is_fresh(Some(now + TimeDelta::milliseconds(1)), now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_cache_paths_are_isolated_by_scope() {
|
||||
let codex_home = Path::new("/tmp/codex-home");
|
||||
let cache_key_for_scope = |scope| RemotePluginCatalogCacheKey {
|
||||
chatgpt_base_url: "https://chatgpt.com/backend-api".to_string(),
|
||||
account_id: Some("account-id".to_string()),
|
||||
chatgpt_user_id: Some("user-id".to_string()),
|
||||
is_workspace_account: true,
|
||||
scope: (scope != RemotePluginScope::Global).then_some(scope),
|
||||
};
|
||||
|
||||
let paths = [
|
||||
RemotePluginScope::Global,
|
||||
RemotePluginScope::User,
|
||||
RemotePluginScope::Workspace,
|
||||
]
|
||||
.map(|scope| cache_path(codex_home, &cache_key_for_scope(scope)));
|
||||
|
||||
assert_ne!(paths[0], paths[1]);
|
||||
assert_ne!(paths[0], paths[2]);
|
||||
assert_ne!(paths[1], paths[2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_catalog_cache_reuses_legacy_cache_file() {
|
||||
let codex_home = tempfile::tempdir().expect("create codex home");
|
||||
let config = RemotePluginServiceConfig::new(
|
||||
"https://chatgpt.com/backend-api".to_string(),
|
||||
crate::test_support::test_http_client_factory(),
|
||||
);
|
||||
let auth = CodexAuth::Headers(AuthHeaders::new(http::HeaderMap::new()));
|
||||
let legacy_cache_path = codex_home
|
||||
.path()
|
||||
.join(REMOTE_PLUGIN_CATALOG_DISK_CACHE_DIR)
|
||||
.join("f22564d6f8ca89f6.json");
|
||||
let legacy_cache = serde_json::json!({
|
||||
"schema_version": REMOTE_PLUGIN_CATALOG_DISK_CACHE_SCHEMA_VERSION,
|
||||
"plugins": [],
|
||||
});
|
||||
let contents = serde_json::to_string_pretty(&legacy_cache).expect("serialize legacy cache");
|
||||
codex_utils_path::write_atomically(&legacy_cache_path, &contents).expect("write legacy cache");
|
||||
|
||||
let cached =
|
||||
load_cached_directory_plugins(codex_home.path(), &config, &auth, RemotePluginScope::Global)
|
||||
.expect("load legacy global cache");
|
||||
assert!(cached.plugins.is_empty());
|
||||
assert_eq!(cached.freshness, RemotePluginCatalogCacheFreshness::Stale);
|
||||
|
||||
write_cached_directory_plugins(
|
||||
codex_home.path(),
|
||||
&config,
|
||||
&auth,
|
||||
RemotePluginScope::Global,
|
||||
&[],
|
||||
);
|
||||
let refreshed_cache: RemotePluginCatalogDiskCache = serde_json::from_slice(
|
||||
&std::fs::read(&legacy_cache_path).expect("read refreshed legacy cache"),
|
||||
)
|
||||
.expect("parse refreshed legacy cache");
|
||||
assert!(refreshed_cache.fetched_at.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_auth_does_not_cache_private_catalogs_without_a_stable_identity() {
|
||||
let codex_home = tempfile::tempdir().expect("create codex home");
|
||||
let config = RemotePluginServiceConfig::new(
|
||||
"https://chatgpt.com/backend-api".to_string(),
|
||||
crate::test_support::test_http_client_factory(),
|
||||
);
|
||||
let auth = CodexAuth::Headers(AuthHeaders::new(http::HeaderMap::new()));
|
||||
|
||||
write_cached_directory_plugins(
|
||||
codex_home.path(),
|
||||
&config,
|
||||
&auth,
|
||||
RemotePluginScope::Global,
|
||||
&[],
|
||||
);
|
||||
assert!(
|
||||
load_cached_directory_plugins(codex_home.path(), &config, &auth, RemotePluginScope::Global)
|
||||
.is_some()
|
||||
);
|
||||
|
||||
for scope in [RemotePluginScope::User, RemotePluginScope::Workspace] {
|
||||
let insecure_cache_key = RemotePluginCatalogCacheKey {
|
||||
chatgpt_base_url: config.chatgpt_base_url.clone(),
|
||||
account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
is_workspace_account: false,
|
||||
scope: Some(scope),
|
||||
};
|
||||
let insecure_cache_path = cache_path(codex_home.path(), &insecure_cache_key);
|
||||
|
||||
write_cached_directory_plugins(codex_home.path(), &config, &auth, scope, &[]);
|
||||
assert!(!insecure_cache_path.exists());
|
||||
|
||||
let insecure_cache = RemotePluginCatalogDiskCache {
|
||||
schema_version: REMOTE_PLUGIN_CATALOG_DISK_CACHE_SCHEMA_VERSION,
|
||||
fetched_at: Some(Utc::now()),
|
||||
plugins: Vec::new(),
|
||||
};
|
||||
let contents = serde_json::to_string_pretty(&insecure_cache).expect("serialize cache");
|
||||
codex_utils_path::write_atomically(&insecure_cache_path, &contents)
|
||||
.expect("write insecure cache");
|
||||
|
||||
assert!(load_cached_directory_plugins(codex_home.path(), &config, &auth, scope).is_none());
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,30 @@ async fn remote_plugin_list_routes_the_complete_query_url() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_remote_plugin_catalog_scopes_returns_existing_scopes() {
|
||||
let codex_home = tempfile::tempdir().expect("create codex home");
|
||||
let config = RemotePluginServiceConfig::new(
|
||||
"https://chatgpt.com/backend-api".to_string(),
|
||||
crate::test_support::test_http_client_factory(),
|
||||
);
|
||||
let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing();
|
||||
for scope in [RemotePluginScope::Global, RemotePluginScope::Workspace] {
|
||||
catalog_cache::write_cached_directory_plugins(
|
||||
codex_home.path(),
|
||||
&config,
|
||||
&auth,
|
||||
scope,
|
||||
&[],
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
cached_remote_plugin_catalog_scopes(codex_home.path(), &config, Some(&auth)),
|
||||
BTreeSet::from([RemotePluginScope::Global, RemotePluginScope::Workspace])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_remote_marketplace_preserves_directory_order_and_appends_installed_only_plugins() {
|
||||
let directory_plugins = vec![
|
||||
|
||||
Reference in New Issue
Block a user