From 6b59cefcbb35951c197a235dc94dbe700f2fbc7c Mon Sep 17 00:00:00 2001 From: willwang-openai Date: Tue, 1 Sep 2026 22:33:59 +0000 Subject: [PATCH] Support remote marketplaces in the plugin CLI (#42150) ## What changed - Include remote catalog entries in `codex plugin list`, including their source, version, install policy, and authentication policy in JSON output. - Support adding and removing remote plugins through the existing plugin CLI. - Cache remote catalogs by scope and collection. Prefer fresh cached results, and refetch once when an add request misses a plugin in the cache. - Preserve the local curated catalog when an unfiltered remote listing fails, while surfacing errors for explicitly selected remote marketplaces. ## Testing - Cover remote listing, installation, removal, catalog fallback, cache refresh, collection isolation, and install failure behavior. GitOrigin-RevId: 09796b2c393d102e00ba9289f784d78a2e166a54 --- codex-rs/Cargo.lock | 2 + .../src/request_processors/plugins.rs | 3 + codex-rs/cli/Cargo.toml | 2 + codex-rs/cli/src/plugin_cmd.rs | 409 ++++++++-- codex-rs/cli/tests/plugin_cli.rs | 771 +++++++++++++++++- codex-rs/core-plugins/src/remote.rs | 113 ++- .../core-plugins/src/remote/catalog_cache.rs | 13 +- .../src/remote/catalog_cache_tests.rs | 51 +- codex-rs/core-plugins/src/remote_tests.rs | 141 +++- 9 files changed, 1373 insertions(+), 132 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 30063d3c3d..e217b70249 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2469,6 +2469,7 @@ dependencies = [ "codex-utils-path", "codex-windows-sandbox", "crossterm", + "flate2", "futures", "http 1.4.0", "insta", @@ -2483,6 +2484,7 @@ dependencies = [ "sqlx", "supports-color 3.0.2", "sys-locale", + "tar", "tempfile", "tokio", "tokio-tungstenite", diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index 13a9dfecca..bf3afd7bb9 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -680,8 +680,11 @@ impl PluginRequestProcessor { match codex_core_plugins::remote::fetch_openai_curated_remote_collection_marketplace( &remote_plugin_service_config, auth.as_ref(), + /*catalog_cache_root*/ None, + RemotePluginCatalogCacheMode::ForceRefetch, ) .await + .map(|outcome| outcome.marketplace) { Ok(Some(remote_marketplace)) => { data.push(remote_marketplace_to_info(remote_marketplace)); diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 2640c964e0..c94dd50662 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -114,11 +114,13 @@ app_test_support = { workspace = true } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-utils-cargo-bin = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } insta = { workspace = true } predicates = { workspace = true } pretty_assertions = { workspace = true } sqlx = { workspace = true } +tar = { workspace = true } tokio-tungstenite = { workspace = true } wiremock = { workspace = true } zstd = { workspace = true } diff --git a/codex-rs/cli/src/plugin_cmd.rs b/codex-rs/cli/src/plugin_cmd.rs index 44b3a11f01..e82200b26f 100644 --- a/codex-rs/cli/src/plugin_cmd.rs +++ b/codex-rs/cli/src/plugin_cmd.rs @@ -1,7 +1,10 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; +use anyhow::ensure; use clap::Parser; +use codex_app_server_protocol::PluginAuthPolicy; +use codex_app_server_protocol::PluginInstallPolicy; use codex_core::config::Config; use codex_core::config::find_codex_home; use codex_core::plugins_manager_for_config; @@ -11,6 +14,7 @@ use codex_core_plugins::PluginInstallOutcome; use codex_core_plugins::PluginInstallRequest; use codex_core_plugins::PluginsConfigInput; use codex_core_plugins::PluginsManager; +use codex_core_plugins::RemotePluginInstallRequest; use codex_core_plugins::allowed_configured_marketplace_names; use codex_core_plugins::installed_marketplaces::marketplace_install_root; use codex_core_plugins::installed_marketplaces::resolve_configured_marketplace_root; @@ -19,9 +23,17 @@ use codex_core_plugins::marketplace::MarketplacePluginAuthPolicy; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_core_plugins::marketplace::MarketplacePluginSource; use codex_core_plugins::marketplace::find_marketplace_manifest_path; +use codex_core_plugins::remote; +use codex_core_plugins::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; +use codex_core_plugins::remote::RemoteMarketplace; +use codex_core_plugins::remote::RemoteMarketplaceSource; +use codex_core_plugins::remote::RemotePluginCatalogCacheMode; +use codex_core_plugins::remote::RemotePluginSummary; use codex_login::AuthManager; +use codex_login::CodexAuth; use codex_plugin::PluginId; use codex_plugin::validate_plugin_segment; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; use serde::Serialize; use std::collections::HashMap; @@ -46,19 +58,19 @@ pub struct PluginCli { #[derive(Debug, clap::Subcommand)] pub enum PluginSubcommand { - /// Install a plugin from a configured marketplace snapshot. + /// Install a plugin from a configured or remote marketplace. /// /// Pass either `PLUGIN@MARKETPLACE` or pass `PLUGIN` with /// `--marketplace MARKETPLACE`. Add(AddPluginArgs), - /// List plugins available from configured marketplace snapshots. + /// List plugins available from configured and remote marketplaces. List(ListPluginsArgs), /// Add, list, upgrade, or remove configured plugin marketplaces. Marketplace(MarketplaceCli), - /// Remove an installed plugin from local config and cache. + /// Uninstall a plugin and remove its local cache. /// /// Pass either `PLUGIN@MARKETPLACE` or pass `PLUGIN` with /// `--marketplace MARKETPLACE`. @@ -75,7 +87,7 @@ pub struct AddPluginArgs { #[arg(value_name = "PLUGIN[@MARKETPLACE]")] plugin: String, - /// Configured marketplace name to use when PLUGIN does not include @MARKETPLACE. + /// Marketplace name to use when PLUGIN does not include @MARKETPLACE. #[arg(long = "marketplace", short = 'm', value_name = "MARKETPLACE")] marketplace_name: Option, @@ -90,7 +102,7 @@ pub struct AddPluginArgs { after_help = "Examples:\n codex plugin list\n codex plugin list --marketplace debug\n codex plugin list --json\n codex plugin list --available --json" )] pub struct ListPluginsArgs { - /// Only list plugins from this configured marketplace name. + /// Only list plugins from this marketplace name. #[arg(long = "marketplace", short = 'm', value_name = "MARKETPLACE")] marketplace_name: Option, @@ -126,52 +138,81 @@ pub async fn run_plugin_add( overrides: Vec<(String, toml::Value)>, args: AddPluginArgs, ) -> Result<()> { - let PluginCommandContext { - codex_home, - plugins_input, - manager, - } = load_plugin_command_context(overrides).await?; + let context = load_plugin_command_context(overrides).await?; let AddPluginArgs { plugin, marketplace_name, json, } = args; - let PluginSelection { - plugin_name, - marketplace_name, - .. - } = parse_plugin_selection(plugin, marketplace_name)?; - let marketplace = find_marketplace_for_plugin( - &manager, - codex_home.as_path(), - &plugins_input, - &marketplace_name, - &plugin_name, - )?; - let outcome = manager - .install_plugin( - &plugins_input, - PluginInstallRequest { - plugin_name, - marketplace_path: marketplace.path, - }, + let selection = parse_plugin_selection(plugin, marketplace_name)?; + let outcome = if selection.marketplace_name == REMOTE_GLOBAL_MARKETPLACE_NAME { + let mut listing = fetch_remote_marketplaces( + &context, + Some(&selection.marketplace_name), + RemotePluginCatalogCacheMode::PreferFreshCache, ) .await?; + // A newly published plugin may be missing from an otherwise fresh catalog cache. + if listing.catalog_cache_used + && !listing + .marketplaces + .iter() + .flat_map(|marketplace| &marketplace.plugins) + .any(|plugin| plugin.name == selection.plugin_name) + { + listing = fetch_remote_marketplaces( + &context, + Some(&selection.marketplace_name), + RemotePluginCatalogCacheMode::ForceRefetch, + ) + .await?; + } + let plugin = resolve_remote_plugin(listing.marketplaces, &selection)?; + context + .manager + .install_remote_plugin( + &context.plugins_input, + context.auth.as_ref(), + RemotePluginInstallRequest { + marketplace_name: selection.marketplace_name, + remote_plugin_id: plugin.remote_plugin_id, + install_attempt_id: None, + }, + /*on_effective_plugins_changed*/ None, + ) + .await? + .installed + } else { + let marketplace = find_marketplace_for_plugin( + &context.manager, + context.codex_home.as_path(), + &context.plugins_input, + &selection.marketplace_name, + &selection.plugin_name, + )?; + context + .manager + .install_plugin( + &context.plugins_input, + PluginInstallRequest { + plugin_name: selection.plugin_name, + marketplace_path: marketplace.path, + }, + ) + .await? + }; + let output = JsonPluginAddOutput::from_outcome(outcome); if json { - let output = JsonPluginAddOutput::from_outcome(outcome); println!("{}", serde_json::to_string_pretty(&output)?); return Ok(()); } println!( "Added plugin `{}` from marketplace `{}`.", - outcome.plugin_id.plugin_name, outcome.plugin_id.marketplace_name - ); - println!( - "Installed plugin root: {}", - outcome.installed_path.as_path().display() + output.name, output.marketplace_name ); + println!("Installed plugin root: {}", output.installed_path); Ok(()) } @@ -204,39 +245,70 @@ pub async fn run_plugin_list( overrides: Vec<(String, toml::Value)>, args: ListPluginsArgs, ) -> Result<()> { + let context = load_plugin_command_context(overrides).await?; + let remote_listing = fetch_remote_marketplaces( + &context, + args.marketplace_name.as_deref(), + RemotePluginCatalogCacheMode::PreferFreshCache, + ) + .await?; let PluginCommandContext { codex_home, plugins_input, manager, .. - } = load_plugin_command_context(overrides).await?; + } = context; let outcome = manager - .list_marketplaces_for_config(&plugins_input, &[], /*include_openai_curated*/ true) + .list_marketplaces_for_config( + &plugins_input, + &[], + /*include_openai_curated*/ !remote_listing.uses_global_catalog, + ) .context("failed to list marketplace plugins")?; ensure_configured_marketplace_snapshots_loaded( codex_home.as_path(), &plugins_input, &outcome.errors, - /*marketplace_name*/ None, + args.marketplace_name.as_deref(), )?; + let marketplace_sources = configured_marketplace_sources(&plugins_input, codex_home.as_path()); let marketplaces = outcome .marketplaces .into_iter() + .map(|marketplace| { + let source = marketplace_sources.get(&marketplace.name).cloned(); + PluginListMarketplace { + plugins: marketplace + .plugins + .into_iter() + .map(|plugin| { + PluginListEntry::from_configured_plugin( + &marketplace.name, + source.clone(), + plugin, + ) + }) + .collect(), + name: marketplace.name, + path: Some(marketplace.path), + } + }) + .chain( + remote_listing + .marketplaces + .into_iter() + .map(PluginListMarketplace::from), + ) .filter(|marketplace| { args.marketplace_name .as_ref() .is_none_or(|name| marketplace.name == *name) }) .collect::>(); - let marketplace_sources = configured_marketplace_sources(&plugins_input, codex_home.as_path()); if args.json { - let output = JsonPluginListOutput::from_marketplaces( - marketplaces, - args.available, - &marketplace_sources, - ); + let output = JsonPluginListOutput::from_marketplaces(marketplaces, args.available); println!("{}", serde_json::to_string_pretty(&output)?); return Ok(()); } @@ -253,7 +325,6 @@ pub async fn run_plugin_list( let mut plugin_width = "PLUGIN".len(); let mut status_width = "STATUS".len(); let mut installed_version_width = "VERSION".len(); - let mut path_width = "PATH".len(); for plugin in &marketplace.plugins { let state = if plugin.installed && plugin.enabled { @@ -263,19 +334,16 @@ pub async fn run_plugin_list( } else { "not installed" }; - let installed_version = plugin.installed_version.clone().unwrap_or_default(); + let installed_version = plugin.display_version.clone().unwrap_or_default(); let path = match &plugin.source { - codex_core_plugins::marketplace::MarketplacePluginSource::Local { path } => { - path.as_path().display().to_string() - } - codex_core_plugins::marketplace::MarketplacePluginSource::Git { - url, - path, - ref_name, - sha, + JsonPluginSource::Remote { id } => id.clone(), + JsonPluginSource::Local { path } => path.clone(), + JsonPluginSource::Git { url, ref_name, sha } + | JsonPluginSource::GitSubdir { + url, ref_name, sha, .. } => { let mut parts = vec![url.clone()]; - if let Some(path) = path { + if let JsonPluginSource::GitSubdir { path, .. } = &plugin.source { parts.push(format!("path `{path}`")); } if let Some(ref_name) = ref_name { @@ -286,7 +354,7 @@ pub async fn run_plugin_list( } parts.join(", ") } - codex_core_plugins::marketplace::MarketplacePluginSource::Npm { + JsonPluginSource::Npm { package, version, registry, @@ -301,26 +369,29 @@ pub async fn run_plugin_list( parts.join(", ") } }; - plugin_width = plugin_width.max(plugin.id.len()); + plugin_width = plugin_width.max(plugin.plugin_id.len()); status_width = status_width.max(state.len()); installed_version_width = installed_version_width.max(installed_version.len()); - path_width = path_width.max(path.len()); - rows.push((plugin.id.clone(), state, installed_version, path)); + rows.push((plugin.plugin_id.clone(), state, installed_version, path)); } if index > 0 { println!(); } println!("Marketplace `{}`", marketplace.name); - println!("{}", marketplace.path.as_path().display()); + if let Some(path) = &marketplace.path { + println!("{}", path.display()); + } else { + println!("Remote catalog"); + } println!(); println!( - "{:, + plugins: Vec, +} + +impl From for PluginListMarketplace { + fn from(marketplace: RemoteMarketplace) -> Self { + Self { + plugins: marketplace + .plugins + .into_iter() + .map(|plugin| { + let version = plugin.local_version.or(plugin.version); + PluginListEntry { + plugin_id: plugin.id, + name: plugin.name, + marketplace_name: marketplace.name.clone(), + display_version: version.clone(), + version, + installed: plugin.installed, + enabled: plugin.enabled, + source: JsonPluginSource::Remote { + id: plugin.remote_plugin_id, + }, + marketplace_source: None, + install_policy: match plugin.install_policy { + PluginInstallPolicy::NotAvailable => "NOT_AVAILABLE", + PluginInstallPolicy::Available => "AVAILABLE", + PluginInstallPolicy::InstalledByDefault => "INSTALLED_BY_DEFAULT", + }, + auth_policy: match plugin.auth_policy { + PluginAuthPolicy::OnInstall => "ON_INSTALL", + PluginAuthPolicy::OnUse => "ON_USE", + }, + } + }) + .collect(), + name: marketplace.name, + path: None, + } + } +} + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct JsonPluginListOutput { - installed: Vec, - available: Vec, + installed: Vec, + available: Vec, } impl JsonPluginListOutput { fn from_marketplaces( - marketplaces: Vec, + marketplaces: Vec, include_available: bool, - marketplace_sources: &HashMap, ) -> Self { let mut installed = Vec::new(); let mut available = Vec::new(); for marketplace in marketplaces { - let marketplace_source = marketplace_sources.get(&marketplace.name).cloned(); - for plugin in marketplace.plugins { - let entry = JsonPluginListEntry::from_configured_plugin( - &marketplace.name, - marketplace_source.clone(), - plugin, - ); + for entry in marketplace.plugins { if entry.installed { installed.push(entry); } else if include_available { @@ -370,11 +478,13 @@ impl JsonPluginListOutput { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct JsonPluginListEntry { +struct PluginListEntry { plugin_id: String, name: String, marketplace_name: String, version: Option, + #[serde(skip)] + display_version: Option, installed: bool, enabled: bool, source: JsonPluginSource, @@ -384,18 +494,20 @@ struct JsonPluginListEntry { auth_policy: &'static str, } -impl JsonPluginListEntry { +impl PluginListEntry { fn from_configured_plugin( marketplace_name: &str, marketplace_source: Option, plugin: codex_core_plugins::ConfiguredMarketplacePlugin, ) -> Self { - let version = plugin.installed_version.or(plugin.local_version); + let display_version = plugin.installed_version; + let version = display_version.clone().or(plugin.local_version); Self { plugin_id: plugin.id, name: plugin.name, marketplace_name: marketplace_name.to_string(), version, + display_version, installed: plugin.installed, enabled: plugin.enabled, source: JsonPluginSource::from_marketplace_source(plugin.source), @@ -409,6 +521,9 @@ impl JsonPluginListEntry { #[derive(Debug, Serialize)] #[serde(tag = "source", rename_all = "kebab-case")] enum JsonPluginSource { + Remote { + id: String, + }, Local { path: String, }, @@ -531,7 +646,7 @@ pub async fn run_plugin_remove( overrides: Vec<(String, toml::Value)>, args: RemovePluginArgs, ) -> Result<()> { - let PluginCommandContext { manager, .. } = load_plugin_command_context(overrides).await?; + let context = load_plugin_command_context(overrides).await?; let RemovePluginArgs { plugin, marketplace_name, @@ -539,9 +654,41 @@ pub async fn run_plugin_remove( } = args; let selection = parse_plugin_selection(plugin, marketplace_name)?; - manager - .uninstall_plugin(selection.plugin_key.clone()) - .await?; + if selection.marketplace_name == REMOTE_GLOBAL_MARKETPLACE_NAME { + ensure!( + context.plugins_input.plugins_enabled, + "remote plugins are not enabled" + ); + let auth = context.auth.as_ref(); + // Installed plugins may no longer appear in the directory or curated collection. + let marketplaces = context + .manager + .build_and_cache_remote_installed_plugin_marketplaces( + &context.plugins_input, + auth, + &[REMOTE_GLOBAL_MARKETPLACE_NAME], + /*on_effective_plugins_changed*/ None, + ) + .await?; + let plugin = resolve_remote_plugin(marketplaces, &selection)?; + let outcome = context + .manager + .uninstall_remote_plugin( + &context.plugins_input, + auth, + &plugin.remote_plugin_id, + /*on_effective_plugins_changed*/ None, + ) + .await?; + if let Some(err) = outcome.cache_removal_error { + return Err(err.into()); + } + } else { + context + .manager + .uninstall_plugin(selection.plugin_key.clone()) + .await?; + } if json { let output = JsonPluginRemoveOutput::from_selection(selection); println!("{}", serde_json::to_string_pretty(&output)?); @@ -577,7 +724,8 @@ impl JsonPluginRemoveOutput { struct PluginCommandContext { codex_home: PathBuf, plugins_input: PluginsConfigInput, - manager: PluginsManager, + manager: Arc, + auth: Option, } async fn load_plugin_command_context( @@ -588,11 +736,16 @@ async fn load_plugin_command_context( .await .context("failed to load configuration")?; let plugins_input = config.plugins_config_input(); - let manager = plugins_manager_for_config(&config, load_cli_auth_manager(&config).await?); + let auth_manager = load_cli_auth_manager(&config).await?; + let manager = Arc::new(plugins_manager_for_config( + &config, + Arc::clone(&auth_manager), + )); Ok(PluginCommandContext { codex_home: codex_home.to_path_buf(), plugins_input, manager, + auth: auth_manager.auth().await, }) } @@ -644,6 +797,96 @@ fn parse_plugin_selection( } } +#[derive(Default)] +struct RemoteMarketplaceListing { + marketplaces: Vec, + // A successful global catalog replaces the local curated catalog even when it is empty. + uses_global_catalog: bool, + catalog_cache_used: bool, +} + +async fn fetch_remote_marketplaces( + context: &PluginCommandContext, + marketplace_name: Option<&str>, + cache_mode: RemotePluginCatalogCacheMode, +) -> Result { + if marketplace_name.is_some_and(|name| name != REMOTE_GLOBAL_MARKETPLACE_NAME) { + return Ok(RemoteMarketplaceListing::default()); + } + if !context.plugins_input.plugins_enabled { + ensure!(marketplace_name.is_none(), "remote plugins are not enabled"); + return Ok(RemoteMarketplaceListing::default()); + } + let auth = context.auth.as_ref(); + if !auth.is_some_and(CodexAuth::uses_codex_backend) { + ensure!( + marketplace_name.is_none(), + "chatgpt authentication required for remote plugin catalog" + ); + return Ok(RemoteMarketplaceListing::default()); + } + let service = context.plugins_input.remote_plugin_service_config(); + let result = if context.plugins_input.remote_plugin_enabled { + remote::fetch_remote_marketplaces( + &service, + auth, + &[RemoteMarketplaceSource::Global], + /*catalog_cache_root*/ Some(context.codex_home.as_path()), + cache_mode, + ) + .await + .map(|outcome| RemoteMarketplaceListing { + marketplaces: outcome.marketplaces, + uses_global_catalog: true, + catalog_cache_used: outcome.catalog_cache_used, + }) + } else { + remote::fetch_openai_curated_remote_collection_marketplace( + &service, + auth, + /*catalog_cache_root*/ Some(context.codex_home.as_path()), + cache_mode, + ) + .await + .map(|outcome| RemoteMarketplaceListing { + marketplaces: outcome.marketplace.into_iter().collect(), + uses_global_catalog: false, + catalog_cache_used: outcome.catalog_cache_used, + }) + }; + match result { + Ok(listing) => Ok(listing), + Err(err) if marketplace_name.is_none() => { + eprintln!("Warning: failed to list remote marketplace plugins: {err}"); + Ok(RemoteMarketplaceListing::default()) + } + Err(err) => Err(err).context("failed to list remote marketplace plugins"), + } +} + +fn resolve_remote_plugin( + marketplaces: Vec, + selection: &PluginSelection, +) -> Result { + let matches = marketplaces + .into_iter() + .flat_map(|marketplace| marketplace.plugins) + .filter(|plugin| plugin.name == selection.plugin_name) + .collect::>(); + match matches.as_slice() { + [plugin] => Ok(plugin.clone()), + [] => bail!( + "plugin `{}` was not found in remote marketplace `{}`", + selection.plugin_name, + selection.marketplace_name + ), + _ => bail!( + "plugin `{}` matched multiple remote plugins", + selection.plugin_key + ), + } +} + fn find_marketplace_for_plugin( manager: &PluginsManager, codex_home: &std::path::Path, diff --git a/codex-rs/cli/tests/plugin_cli.rs b/codex-rs/cli/tests/plugin_cli.rs index c504ff94b3..b1c316860c 100644 --- a/codex-rs/cli/tests/plugin_cli.rs +++ b/codex-rs/cli/tests/plugin_cli.rs @@ -1,14 +1,33 @@ use anyhow::Result; +use anyhow::ensure; +use app_test_support::ChatGptAuthFixture; +use app_test_support::write_chatgpt_auth; use codex_config::CONFIG_TOML_FILE; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; +use codex_config::types::AuthCredentialsStoreMode; use codex_utils_absolute_path::canonicalize_existing_preserving_symlinks; +use flate2::Compression; +use flate2::write::GzEncoder; use predicates::prelude::PredicateBooleanExt; use predicates::str::contains; use pretty_assertions::assert_eq; +use serde_json::Value; use serde_json::json; use std::path::Path; +use std::process::Output; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use tempfile::TempDir; +use tokio::process::Command; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::header; +use wiremock::matchers::method; +use wiremock::matchers::path; +use wiremock::matchers::query_param; const MARKETPLACE_HEADER: &str = "MARKETPLACE"; const MARKETPLACE_LIST_HEADER: &str = "MARKETPLACE ROOT"; @@ -610,7 +629,7 @@ async fn plugin_list_prints_plugins_in_a_table() -> Result<()> { .stdout(contains("PLUGIN")) .stdout(contains("STATUS")) .stdout(contains("VERSION")) - .stdout(contains("PATH")) + .stdout(contains("SOURCE")) .stdout(contains(marketplace_manifest.display().to_string())) .stdout(contains("sample@debug")) .stdout(contains("not installed")) @@ -1106,3 +1125,753 @@ async fn plugin_add_rejects_cached_plugins_without_authorizing_marketplace_snaps Ok(()) } + +const REMOTE_ID: &str = "b1234567-89ab-4cde-8f01-234567890abc"; +const MARKETPLACE: &str = "openai-curated-remote"; +const PLUGIN_KEY: &str = "sample@openai-curated-remote"; + +fn sample_remote_plugin_bundle() -> Result> { + let mut archive = tar::Builder::new(GzEncoder::new(Vec::new(), Compression::default())); + for (path, contents) in [ + ( + ".codex-plugin/plugin.json", + r#"{"name":"sample","version":"1.2.3"}"#, + ), + ( + "skills/sample/SKILL.md", + "---\nname: sample\ndescription: Sample remote skill\n---\nSample instructions.\n", + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(contents.len() as u64); + header.set_mode(/*mode*/ 0o644); + header.set_cksum(); + archive.append_data(&mut header, path, contents.as_bytes())?; + } + Ok(archive.into_inner()?.finish()?) +} + +struct RemoteMarketplaceFixture { + home: TempDir, + server: MockServer, + plugin: Value, +} + +impl RemoteMarketplaceFixture { + async fn new() -> Result { + let home = TempDir::new()?; + let server = MockServer::start().await; + std::fs::write( + home.path().join("config.toml"), + format!( + "cli_auth_credentials_store = 'file'\nchatgpt_base_url = '{}/backend-api'\n[features]\nplugins = true\nremote_plugin = true\n", + server.uri() + ), + )?; + write_chatgpt_auth( + home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + let plugin = json!({ + "id": REMOTE_ID, "name": "sample", "scope": "GLOBAL", + "installation_policy": "AVAILABLE", "authentication_policy": "ON_USE", + "release": {"version": "1.2.3", "display_name": "Sample", "description": "Remote sample", + "interface": {}, "bundle_download_url": format!("{}/bundle.tar.gz", server.uri())} + }); + Mock::given(method("GET")) + .and(path("/backend-api/ps/plugins/list")) + .and(query_param("scope", "GLOBAL")) + .and(header("authorization", "Bearer chatgpt-token")) + .and(header("chatgpt-account-id", "account-123")) + .respond_with(ResponseTemplate::new(200).set_body_json( + json!({"plugins": [plugin.clone()], "pagination": {"next_page_token": null}}), + )) + .mount(&server) + .await; + Ok(Self { + home, + server, + plugin, + }) + } + + fn write_local_curated_marketplace(&self) -> Result { + let source = canonicalize_existing_preserving_symlinks(self.home.path())? + .join(".tmp") + .join("plugins"); + let mut manifest = json!({ + "name": "openai-curated", + "plugins": [{ + "name": "sample", + "source": {"source": "local", "path": "./plugins/sample"} + }] + }); + write_marketplace_source_with_manifest(&source, &manifest.to_string())?; + std::fs::write( + self.home.path().join(".tmp").join("plugins.sha"), + "local-curated", + )?; + manifest["name"] = json!("openai-api-curated"); + std::fs::write( + source + .join(".agents") + .join("plugins") + .join("api_marketplace.json"), + manifest.to_string(), + )?; + Ok(json!({ + "pluginId": "sample@openai-curated", "name": "sample", + "marketplaceName": "openai-curated", "version": "1.2.3", + "installed": false, "enabled": false, + "source": {"source": "local", "path": source.join("plugins").join("sample")}, + "installPolicy": "AVAILABLE", "authPolicy": "ON_INSTALL" + })) + } + + async fn run(&self, args: &[&str]) -> Result { + Ok(Command::new(codex_utils_cargo_bin::cargo_bin("codex")?) + .current_dir(self.home.path()) + .env("CODEX_HOME", self.home.path()) + .env("HOME", self.home.path()) + .env_remove("OPENAI_API_KEY") + .env_remove("CODEX_API_KEY") + .env_remove("CODEX_ACCESS_TOKEN") + .env("CODEX_TEST_ALLOW_HTTP_REMOTE_PLUGIN_BUNDLE_DOWNLOADS", "1") + .args(args) + .output() + .await?) + } + + async fn success(&self, args: &[&str]) -> Result { + let output = self.run(args).await?; + ensure!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(String::from_utf8(output.stdout)?) + } +} + +#[tokio::test] +async fn remote_plugin_listing_replaces_local_curated_catalog() -> Result<()> { + let fixture = RemoteMarketplaceFixture::new().await?; + let mut local_plugin = fixture.write_local_curated_marketplace()?; + fixture + .success(&["plugin", "add", "sample@openai-curated"]) + .await?; + local_plugin["installed"] = json!(true); + local_plugin["enabled"] = json!(true); + local_plugin["version"] = json!("local-curated"); + let mut installed_plugin = fixture.plugin.clone(); + installed_plugin["enabled"] = json!(true); + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plugins": [installed_plugin], "pagination": {"next_page_token": null} + }))) + .mount(&fixture.server) + .await; + + let listed = fixture + .success(&["plugin", "list", "--available", "--json"]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [{ + "pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE, + "version": "1.2.3", "installed": true, "enabled": true, + "source": {"source": "remote", "id": REMOTE_ID}, + "installPolicy": "AVAILABLE", "authPolicy": "ON_USE" + }], "available": []}) + ); + let table = fixture.success(&["plugin", "list"]).await?; + insta::assert_snapshot!(table, @r" + Marketplace `openai-curated-remote` + Remote catalog + + PLUGIN STATUS VERSION SOURCE + sample@openai-curated-remote installed, enabled 1.2.3 b1234567-89ab-4cde-8f01-234567890abc + "); + + fixture.server.reset().await; + let listed = fixture + .success(&["plugin", "list", "-m", "openai-curated", "--json"]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [local_plugin], "available": []}) + ); + assert!(fixture.server.received_requests().await.unwrap().is_empty()); + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_listing_preserves_local_curated_only_when_fetch_fails() -> Result<()> { + for status in [200, 403] { + let fixture = RemoteMarketplaceFixture::new().await?; + fixture.server.reset().await; + let local_plugin = fixture.write_local_curated_marketplace()?; + let response = ResponseTemplate::new(status).set_body_json(json!({ + "plugins": [], "pagination": {"next_page_token": null} + })); + let _catalog = Mock::given(path("/backend-api/ps/plugins/list")) + .respond_with(response) + .mount_as_scoped(&fixture.server) + .await; + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plugins": [], "pagination": {"next_page_token": null} + }))) + .mount(&fixture.server) + .await; + + let listed = fixture + .success(&["plugin", "list", "--available", "--json"]) + .await?; + let available = if status == 200 { + Vec::new() + } else { + vec![local_plugin] + }; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": available}), + "catalog response status: {status}" + ); + } + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_add_list_and_remove() -> Result<()> { + let fixture = RemoteMarketplaceFixture::new().await?; + let installed = Arc::new(AtomicBool::new(false)); + let installed_for_list = Arc::clone(&installed); + let mut installed_plugin = fixture.plugin.clone(); + installed_plugin["enabled"] = json!(true); + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with(move |_: &wiremock::Request| { + let plugins = if installed_for_list.load(Ordering::SeqCst) { + vec![installed_plugin.clone()] + } else { + Vec::new() + }; + ResponseTemplate::new(200) + .set_body_json(json!({"plugins": plugins, "pagination": {"next_page_token": null}})) + }) + .mount(&fixture.server) + .await; + Mock::given(path(format!("/backend-api/ps/plugins/{REMOTE_ID}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(&fixture.plugin)) + .mount(&fixture.server) + .await; + Mock::given(path("/bundle.tar.gz")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(sample_remote_plugin_bundle()?)) + .expect(1) + .mount(&fixture.server) + .await; + let installed_root = canonicalize_existing_preserving_symlinks(fixture.home.path())? + .join("plugins") + .join("cache") + .join(MARKETPLACE) + .join("sample") + .join("1.2.3"); + let manifest = installed_root.join(".codex-plugin/plugin.json"); + let installed_for_add = Arc::clone(&installed); + Mock::given(method("POST")) + .and(path(format!("/backend-api/ps/plugins/{REMOTE_ID}/install"))) + .and(query_param("includeAppsNeedingAuth", "true")) + .and(header("authorization", "Bearer chatgpt-token")) + .respond_with(move |_: &wiremock::Request| { + assert!( + manifest.is_file(), + "cache must exist before backend install" + ); + installed_for_add.store(true, Ordering::SeqCst); + ResponseTemplate::new(200).set_body_json(json!({"id": REMOTE_ID, "enabled": true})) + }) + .expect(1) + .mount(&fixture.server) + .await; + Mock::given(method("POST")) + .and(path(format!( + "/backend-api/ps/plugins/{REMOTE_ID}/uninstall" + ))) + .and(header("authorization", "Bearer chatgpt-token")) + .respond_with(move |_: &wiremock::Request| { + installed.store(false, Ordering::SeqCst); + ResponseTemplate::new(200).set_body_json(json!({"id": REMOTE_ID, "enabled": false})) + }) + .expect(1) + .mount(&fixture.server) + .await; + + let available = json!({ + "pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE, + "version": "1.2.3", "installed": false, "enabled": false, + "source": {"source": "remote", "id": REMOTE_ID}, "installPolicy": "AVAILABLE", "authPolicy": "ON_USE" + }); + let listed = fixture + .success(&["plugin", "list", "--available", "--json"]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": [available.clone()]}) + ); + let table = fixture + .success(&["plugin", "list", "-m", MARKETPLACE]) + .await?; + insta::assert_snapshot!(table, @r" + Marketplace `openai-curated-remote` + Remote catalog + + PLUGIN STATUS VERSION SOURCE + sample@openai-curated-remote not installed 1.2.3 b1234567-89ab-4cde-8f01-234567890abc + "); + let added = fixture + .success(&["plugin", "add", PLUGIN_KEY, "--json"]) + .await?; + assert_eq!( + serde_json::from_str::(&added)?, + json!({ + "pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE, + "version": "1.2.3", "installedPath": installed_root, "authPolicy": "ON_USE" + }) + ); + assert_eq!( + fixture + .server + .received_requests() + .await + .unwrap() + .iter() + .filter(|request| request.url.path() == "/backend-api/ps/plugins/list") + .count(), + 1, + "installing a cached name must not refetch the catalog" + ); + assert!(installed_root.join("skills/sample/SKILL.md").is_file()); + let mut expected = available; + expected["installed"] = json!(true); + expected["enabled"] = json!(true); + let listed = fixture + .success(&["plugin", "list", "-m", MARKETPLACE, "--json"]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [expected], "available": []}) + ); + let _delisted = Mock::given(path("/backend-api/ps/plugins/list")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"plugins": [], "pagination": {"next_page_token": null}})), + ) + .mount_as_scoped(&fixture.server) + .await; + let removed = fixture + .success(&[ + "-c", + "features.remote_plugin=false", + "plugin", + "remove", + "sample", + "-m", + MARKETPLACE, + "--json", + ]) + .await?; + assert_eq!( + serde_json::from_str::(&removed)?, + json!({"pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE}) + ); + assert!(!installed_root.exists()); + let listed = fixture.success(&["plugin", "list", "--json"]).await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": []}) + ); + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_add_refreshes_cached_catalog_on_name_miss() -> Result<()> { + for (remote_plugin_config, collection) in [ + ("features.remote_plugin=true", None), + ("features.remote_plugin=false", Some("vertical")), + ] { + let fixture = RemoteMarketplaceFixture::new().await?; + fixture.server.reset().await; + let published = Arc::new(AtomicBool::new(false)); + let published_for_list = Arc::clone(&published); + let plugin = fixture.plugin.clone(); + Mock::given(method("GET")) + .and(path("/backend-api/ps/plugins/list")) + .and(query_param("scope", "GLOBAL")) + .respond_with(move |_: &wiremock::Request| { + let plugins = if published_for_list.load(Ordering::SeqCst) { + vec![plugin.clone()] + } else { + Vec::new() + }; + ResponseTemplate::new(200).set_body_json(json!({ + "plugins": plugins, "pagination": {"next_page_token": null} + })) + }) + .expect(2) + .mount(&fixture.server) + .await; + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plugins": [], "pagination": {"next_page_token": null} + }))) + .mount(&fixture.server) + .await; + Mock::given(path(format!("/backend-api/ps/plugins/{REMOTE_ID}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(&fixture.plugin)) + .expect(1) + .mount(&fixture.server) + .await; + Mock::given(path("/bundle.tar.gz")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(sample_remote_plugin_bundle()?)) + .expect(1) + .mount(&fixture.server) + .await; + Mock::given(method("POST")) + .and(path(format!("/backend-api/ps/plugins/{REMOTE_ID}/install"))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"id": REMOTE_ID, "enabled": true})), + ) + .expect(1) + .mount(&fixture.server) + .await; + + for is_published in [false, true] { + published.store(is_published, Ordering::SeqCst); + let listed = fixture + .success(&[ + "-c", + remote_plugin_config, + "plugin", + "list", + "-m", + MARKETPLACE, + "--available", + "--json", + ]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": []}), + "normal listing must keep using the fresh catalog cache" + ); + } + let added = fixture + .success(&[ + "-c", + remote_plugin_config, + "plugin", + "add", + PLUGIN_KEY, + "--json", + ]) + .await?; + let installed_root = canonicalize_existing_preserving_symlinks(fixture.home.path())? + .join("plugins") + .join("cache") + .join(MARKETPLACE) + .join("sample") + .join("1.2.3"); + assert_eq!( + serde_json::from_str::(&added)?, + json!({ + "pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE, + "version": "1.2.3", "installedPath": installed_root, "authPolicy": "ON_USE" + }) + ); + assert!( + installed_root + .join("skills") + .join("sample") + .join("SKILL.md") + .is_file() + ); + let requests = fixture.server.received_requests().await.unwrap(); + let catalog_collections = requests + .iter() + .filter(|request| request.url.path() == "/backend-api/ps/plugins/list") + .map(|request| { + request + .url + .query_pairs() + .find_map(|(key, value)| (key == "collection").then(|| value.into_owned())) + }) + .collect::>(); + assert_eq!(catalog_collections, vec![collection.map(str::to_owned); 2]); + } + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_add_limits_catalog_refresh_without_mutation() -> Result<()> { + enum CatalogCache { + Missing, + Fresh, + Expired, + } + + for (remote_plugin_config, collection) in [ + ("features.remote_plugin=true", None), + ("features.remote_plugin=false", Some("vertical")), + ] { + for (cache, plugin_count, status, catalog_fetches, error) in [ + ( + CatalogCache::Missing, + 0, + 200, + 1, + "was not found in remote marketplace", + ), + ( + CatalogCache::Fresh, + 0, + 200, + 2, + "was not found in remote marketplace", + ), + ( + CatalogCache::Expired, + 0, + 200, + 2, + "was not found in remote marketplace", + ), + ( + CatalogCache::Missing, + 2, + 200, + 1, + "matched multiple remote plugins", + ), + ( + CatalogCache::Missing, + 0, + 403, + 1, + "failed to list remote marketplace plugins", + ), + ] { + let fixture = RemoteMarketplaceFixture::new().await?; + fixture.server.reset().await; + let mut duplicate = fixture.plugin.clone(); + duplicate["id"] = json!("c1234567-89ab-4cde-8f01-234567890abc"); + let plugins = [fixture.plugin.clone(), duplicate] + .into_iter() + .take(plugin_count) + .collect::>(); + Mock::given(path("/backend-api/ps/plugins/list")) + .and(query_param("scope", "GLOBAL")) + .respond_with(ResponseTemplate::new(status).set_body_json(json!({ + "plugins": plugins, "pagination": {"next_page_token": null} + }))) + .expect(catalog_fetches) + .mount(&fixture.server) + .await; + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plugins": [], "pagination": {"next_page_token": null} + }))) + .mount(&fixture.server) + .await; + if !matches!(cache, CatalogCache::Missing) { + let listed = fixture + .success(&[ + "-c", + remote_plugin_config, + "plugin", + "list", + "-m", + MARKETPLACE, + "--available", + "--json", + ]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": []}) + ); + } + if matches!(cache, CatalogCache::Expired) { + let cache_paths = std::fs::read_dir( + fixture + .home + .path() + .join("cache") + .join("remote_plugin_catalog"), + )? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>()?; + assert_eq!(cache_paths.len(), 1); + let cache_path = &cache_paths[0]; + let mut cached_catalog: Value = + serde_json::from_slice(&std::fs::read(cache_path)?)?; + cached_catalog["fetched_at"] = json!("2000-01-01T00:00:00Z"); + std::fs::write(cache_path, serde_json::to_vec(&cached_catalog)?)?; + } + let output = fixture + .run(&["-c", remote_plugin_config, "plugin", "add", PLUGIN_KEY]) + .await?; + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr)?; + assert!(stderr.contains(error), "{stderr}"); + let requests = fixture.server.received_requests().await.unwrap(); + let catalog_collections = requests + .iter() + .filter(|request| request.url.path() == "/backend-api/ps/plugins/list") + .map(|request| { + request + .url + .query_pairs() + .find_map(|(key, value)| (key == "collection").then(|| value.into_owned())) + }) + .collect::>(); + assert_eq!( + catalog_collections, + vec![collection.map(str::to_owned); catalog_fetches as usize] + ); + assert!(requests.iter().all(|request| { + request.method == "GET" + && matches!( + request.url.path(), + "/backend-api/ps/plugins/list" | "/backend-api/ps/plugins/installed" + ) + })); + assert!( + !fixture + .home + .path() + .join("plugins") + .join("cache") + .join(MARKETPLACE) + .exists() + ); + } + } + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_install_checks_policy_and_bundle_before_mutation() -> Result<()> { + let fixture = RemoteMarketplaceFixture::new().await?; + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"plugins": [], "pagination": {"next_page_token": null}})), + ) + .mount(&fixture.server) + .await; + for (field, value, error) in [ + ("status", "DISABLED_BY_ADMIN", "disabled by admin"), + ( + "installation_policy", + "NOT_AVAILABLE", + "not available for install", + ), + ("status", "ENABLED", "failed to read plugin bundle tar"), + ] { + let mut plugin = fixture.plugin.clone(); + plugin[field] = json!(value); + let _detail = Mock::given(path(format!("/backend-api/ps/plugins/{REMOTE_ID}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(plugin)) + .mount_as_scoped(&fixture.server) + .await; + let _bundle = Mock::given(path("/bundle.tar.gz")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"bad gzip")) + .mount_as_scoped(&fixture.server) + .await; + let output = fixture + .run(&["plugin", "add", "sample", "-m", MARKETPLACE]) + .await?; + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr)?; + assert!(stderr.contains(error), "{stderr}"); + } + let requests = fixture.server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| request.method != "POST")); + assert_eq!( + requests + .iter() + .filter(|request| request.url.path() == "/backend-api/ps/plugins/list") + .count(), + 1, + "installation policy and bundle failures must not refetch the catalog" + ); + Ok(()) +} + +#[tokio::test] +async fn remote_plugin_listing_uses_collection_when_remote_catalog_is_disabled() -> Result<()> { + let fixture = RemoteMarketplaceFixture::new().await?; + let mut local_plugin = fixture.write_local_curated_marketplace()?; + Mock::given(path("/backend-api/ps/plugins/installed")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"plugins": [], "pagination": {"next_page_token": null}})), + ) + .mount(&fixture.server) + .await; + let listed = fixture + .success(&[ + "-c", + "features.remote_plugin=false", + "plugin", + "list", + "--available", + "--json", + ]) + .await?; + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": [local_plugin.clone(), { + "pluginId": PLUGIN_KEY, "name": "sample", "marketplaceName": MARKETPLACE, + "version": "1.2.3", "installed": false, "enabled": false, + "source": {"source": "remote", "id": REMOTE_ID}, + "installPolicy": "AVAILABLE", "authPolicy": "ON_USE" + }]}) + ); + let requests = fixture.server.received_requests().await.unwrap(); + let listing = requests + .iter() + .find(|request| request.url.path().ends_with("/list")) + .unwrap(); + assert!( + listing + .url + .query_pairs() + .any(|(key, value)| key == "collection" && value == "vertical") + ); + fixture.server.reset().await; + fixture + .success(&["-c", "features.plugins=false", "plugin", "list"]) + .await?; + fixture + .success(&["plugin", "list", "-m", "local-only"]) + .await?; + assert!(fixture.server.received_requests().await.unwrap().is_empty()); + std::fs::remove_file(fixture.home.path().join("auth.json"))?; + let output = fixture.run(&["plugin", "list", "-m", MARKETPLACE]).await?; + assert!(!output.status.success()); + assert!(String::from_utf8(output.stderr)?.contains("chatgpt authentication required")); + let listed = fixture + .success(&["plugin", "list", "--available", "--json"]) + .await?; + local_plugin["pluginId"] = json!("sample@openai-api-curated"); + local_plugin["marketplaceName"] = json!("openai-api-curated"); + assert_eq!( + serde_json::from_str::(&listed)?, + json!({"installed": [], "available": [local_plugin]}) + ); + Ok(()) +} diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs index 68e5595630..18cd4ff967 100644 --- a/codex-rs/core-plugins/src/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -204,6 +204,8 @@ pub enum RemoteMarketplaceSource { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RemotePluginCatalogCacheMode { PreferCache, + /// Reuse fresh entries and synchronously refresh missing or stale entries. + PreferFreshCache, ForceRefetch, } @@ -211,6 +213,15 @@ pub enum RemotePluginCatalogCacheMode { pub struct RemoteMarketplacesFetchOutcome { pub marketplaces: Vec, pub catalog_cache_refresh_scopes: BTreeSet, + /// Whether any requested directory was served from disk cache, even if it was empty. + pub catalog_cache_used: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RemoteMarketplaceFetchOutcome { + pub marketplace: Option, + /// Whether the requested directory was served from disk cache, even if it was empty. + pub catalog_cache_used: bool, } #[derive(Debug, Clone, PartialEq)] @@ -791,6 +802,7 @@ pub async fn fetch_remote_marketplaces( let auth = ensure_chatgpt_auth(auth)?; let mut marketplaces = Vec::new(); let mut catalog_cache_refresh_scopes = BTreeSet::new(); + let mut catalog_cache_used = false; let needs_workspace_installed = sources.iter().any(|source| { matches!( source, @@ -813,6 +825,7 @@ pub async fn fetch_remote_marketplaces( config, auth, scope, + /*collection*/ None, catalog_cache_mode, ), fetch_installed_plugins_for_scope(config, auth, scope), @@ -820,6 +833,7 @@ pub async fn fetch_remote_marketplaces( if directory_plugins.cache_refresh_needed { catalog_cache_refresh_scopes.insert(scope); } + catalog_cache_used |= directory_plugins.catalog_cache_used; if let Some(marketplace) = build_remote_marketplace( scope.marketplace_name(), scope.marketplace_display_name(), @@ -838,6 +852,7 @@ pub async fn fetch_remote_marketplaces( config, auth, scope, + /*collection*/ None, catalog_cache_mode, ), fetch_installed_plugins_for_scope(config, auth, scope), @@ -845,6 +860,7 @@ pub async fn fetch_remote_marketplaces( if directory_plugins.cache_refresh_needed { catalog_cache_refresh_scopes.insert(scope); } + catalog_cache_used |= directory_plugins.catalog_cache_used; if let Some(marketplace) = build_remote_marketplace( scope.marketplace_name(), scope.marketplace_display_name(), @@ -862,12 +878,14 @@ pub async fn fetch_remote_marketplaces( config, auth, scope, + /*collection*/ None, catalog_cache_mode, ) .await?; if directory_plugins.cache_refresh_needed { catalog_cache_refresh_scopes.insert(scope); } + catalog_cache_used |= directory_plugins.catalog_cache_used; if let Some(marketplace) = build_remote_marketplace( scope.marketplace_name(), scope.marketplace_display_name(), @@ -942,6 +960,7 @@ pub async fn fetch_remote_marketplaces( Ok(RemoteMarketplacesFetchOutcome { marketplaces, catalog_cache_refresh_scopes, + catalog_cache_used, }) } @@ -953,7 +972,9 @@ pub(crate) async fn fetch_and_cache_remote_plugin_catalog( ) -> Result<(), RemotePluginCatalogError> { let auth = ensure_chatgpt_auth(auth)?; let plugins = fetch_directory_plugins_for_scope(config, auth, scope).await?; - catalog_cache::write_cached_directory_plugins(codex_home, config, auth, scope, &plugins); + catalog_cache::write_cached_directory_plugins( + codex_home, config, auth, scope, /*collection*/ None, &plugins, + ); Ok(()) } @@ -975,7 +996,18 @@ pub fn invalidate_cached_remote_plugin_catalog_scopes( return; }; for scope in scopes { - catalog_cache::remove_cached_directory_plugins(codex_home, config, auth, *scope); + catalog_cache::remove_cached_directory_plugins( + codex_home, config, auth, *scope, /*collection*/ None, + ); + if *scope == RemotePluginScope::Global { + catalog_cache::remove_cached_directory_plugins( + codex_home, + config, + auth, + *scope, + Some(OPENAI_CURATED_REMOTE_COLLECTION_KEY), + ); + } } } @@ -1094,14 +1126,15 @@ pub(crate) fn has_fresh_cached_remote_plugin_catalog( let Ok(auth) = ensure_chatgpt_auth(auth) else { return false; }; - catalog_cache::load_cached_directory_plugins(codex_home, config, auth, scope).is_some_and( - |cached| { - matches!( - cached.freshness, - catalog_cache::RemotePluginCatalogCacheFreshness::Fresh - ) - }, + catalog_cache::load_cached_directory_plugins( + codex_home, config, auth, scope, /*collection*/ None, ) + .is_some_and(|cached| { + matches!( + cached.freshness, + catalog_cache::RemotePluginCatalogCacheFreshness::Fresh + ) + }) } pub(crate) fn cached_remote_plugin_catalog_scopes( @@ -1115,7 +1148,10 @@ pub(crate) fn cached_remote_plugin_catalog_scopes( RemotePluginScope::CATALOG_CACHE_SCOPES .into_iter() .filter(|scope| { - catalog_cache::load_cached_directory_plugins(codex_home, config, auth, *scope).is_some() + catalog_cache::load_cached_directory_plugins( + codex_home, config, auth, *scope, /*collection*/ None, + ) + .is_some() }) .collect() } @@ -1130,6 +1166,7 @@ pub fn cached_global_remote_discoverable_plugins( config, auth, RemotePluginScope::Global, + /*collection*/ None, ) .map(|cached| cached.plugins) .unwrap_or_default() @@ -1149,26 +1186,34 @@ pub fn cached_global_remote_discoverable_plugins( pub async fn fetch_openai_curated_remote_collection_marketplace( config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, -) -> Result, RemotePluginCatalogError> { + catalog_cache_root: Option<&Path>, + catalog_cache_mode: RemotePluginCatalogCacheMode, +) -> Result { let auth = ensure_chatgpt_auth(auth)?; let scope = RemotePluginScope::Global; let (directory_plugins, installed_plugins) = tokio::try_join!( - fetch_directory_plugins_for_scope_with_collection( + fetch_directory_plugins_for_scope_with_cache( + catalog_cache_root, config, auth, scope, - OPENAI_CURATED_REMOTE_COLLECTION_KEY, + Some(OPENAI_CURATED_REMOTE_COLLECTION_KEY), + catalog_cache_mode, ), fetch_installed_plugins_for_scope(config, auth, scope), )?; - build_remote_marketplace( + let marketplace = build_remote_marketplace( REMOTE_GLOBAL_MARKETPLACE_NAME, REMOTE_GLOBAL_MARKETPLACE_DISPLAY_NAME, - directory_plugins, + directory_plugins.plugins, installed_plugins, /*include_installed_only*/ false, - ) + )?; + Ok(RemoteMarketplaceFetchOutcome { + marketplace, + catalog_cache_used: directory_plugins.catalog_cache_used, + }) } fn build_remote_marketplace( @@ -1974,6 +2019,7 @@ fn normalize_remote_default_prompt(prompt: &str) -> Option { struct DirectoryPluginsFetchOutcome { plugins: Vec, cache_refresh_needed: bool, + catalog_cache_used: bool, } async fn fetch_directory_plugins_for_scope_with_cache( @@ -1981,12 +2027,16 @@ async fn fetch_directory_plugins_for_scope_with_cache( config: &RemotePluginServiceConfig, auth: &CodexAuth, scope: RemotePluginScope, + collection: Option<&str>, cache_mode: RemotePluginCatalogCacheMode, ) -> Result { - if cache_mode == RemotePluginCatalogCacheMode::PreferCache + if cache_mode != RemotePluginCatalogCacheMode::ForceRefetch && let Some(codex_home) = codex_home - && let Some(cached) = - catalog_cache::load_cached_directory_plugins(codex_home, config, auth, scope) + && let Some(cached) = catalog_cache::load_cached_directory_plugins( + codex_home, config, auth, scope, collection, + ) + && (cache_mode == RemotePluginCatalogCacheMode::PreferCache + || cached.freshness == catalog_cache::RemotePluginCatalogCacheFreshness::Fresh) { return Ok(DirectoryPluginsFetchOutcome { plugins: cached.plugins, @@ -1994,16 +2044,22 @@ async fn fetch_directory_plugins_for_scope_with_cache( cached.freshness, catalog_cache::RemotePluginCatalogCacheFreshness::Stale ), + catalog_cache_used: true, }); } - let plugins = fetch_directory_plugins_for_scope(config, auth, scope).await?; + let plugins = + fetch_directory_plugins_for_scope_with_optional_collection(config, auth, scope, collection) + .await?; if let Some(codex_home) = codex_home { - catalog_cache::write_cached_directory_plugins(codex_home, config, auth, scope, &plugins); + catalog_cache::write_cached_directory_plugins( + codex_home, config, auth, scope, collection, &plugins, + ); } Ok(DirectoryPluginsFetchOutcome { plugins, cache_refresh_needed: false, + catalog_cache_used: false, }) } @@ -2018,21 +2074,6 @@ async fn fetch_directory_plugins_for_scope( .await } -async fn fetch_directory_plugins_for_scope_with_collection( - config: &RemotePluginServiceConfig, - auth: &CodexAuth, - scope: RemotePluginScope, - collection: &str, -) -> Result, RemotePluginCatalogError> { - fetch_directory_plugins_for_scope_with_optional_collection( - config, - auth, - scope, - Some(collection), - ) - .await -} - async fn fetch_directory_plugins_for_scope_with_optional_collection( config: &RemotePluginServiceConfig, auth: &CodexAuth, diff --git a/codex-rs/core-plugins/src/remote/catalog_cache.rs b/codex-rs/core-plugins/src/remote/catalog_cache.rs index 50999919c7..cf78330e25 100644 --- a/codex-rs/core-plugins/src/remote/catalog_cache.rs +++ b/codex-rs/core-plugins/src/remote/catalog_cache.rs @@ -26,6 +26,8 @@ struct RemotePluginCatalogCacheKey { // Global catalogs predate scoped cache keys and must keep their existing filenames. #[serde(skip_serializing_if = "Option::is_none")] scope: Option, + #[serde(skip_serializing_if = "Option::is_none")] + collection: Option, } impl RemotePluginCatalogCacheKey { @@ -33,6 +35,7 @@ impl RemotePluginCatalogCacheKey { config: &RemotePluginServiceConfig, auth: &CodexAuth, scope: RemotePluginScope, + collection: Option<&str>, ) -> Option { let cache_key = Self { chatgpt_base_url: config.chatgpt_base_url.clone(), @@ -40,6 +43,7 @@ impl RemotePluginCatalogCacheKey { chatgpt_user_id: auth.get_chatgpt_user_id(), is_workspace_account: auth.is_workspace_account(), scope: (scope != RemotePluginScope::Global).then_some(scope), + collection: collection.map(str::to_owned), }; // Preserve global catalog caching for existing header-auth clients, but never share // user or workspace catalogs when the auth mode cannot identify their owner. @@ -79,8 +83,9 @@ pub(crate) fn load_cached_directory_plugins( config: &RemotePluginServiceConfig, auth: &CodexAuth, scope: RemotePluginScope, + collection: Option<&str>, ) -> Option { - let cache_key = RemotePluginCatalogCacheKey::new(config, auth, scope)?; + let cache_key = RemotePluginCatalogCacheKey::new(config, auth, scope, collection)?; let cache_path = cache_path(codex_home, &cache_key); let bytes = match std::fs::read(&cache_path) { Ok(bytes) => bytes, @@ -125,9 +130,10 @@ pub(crate) fn write_cached_directory_plugins( config: &RemotePluginServiceConfig, auth: &CodexAuth, scope: RemotePluginScope, + collection: Option<&str>, plugins: &[RemotePluginDirectoryItem], ) { - let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope) else { + let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope, collection) else { return; }; let cache_path = cache_path(codex_home, &cache_key); @@ -146,8 +152,9 @@ pub(crate) fn remove_cached_directory_plugins( config: &RemotePluginServiceConfig, auth: &CodexAuth, scope: RemotePluginScope, + collection: Option<&str>, ) { - let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope) else { + let Some(cache_key) = RemotePluginCatalogCacheKey::new(config, auth, scope, collection) else { return; }; let cache_path = cache_path(codex_home, &cache_key); diff --git a/codex-rs/core-plugins/src/remote/catalog_cache_tests.rs b/codex-rs/core-plugins/src/remote/catalog_cache_tests.rs index 7039398660..049355968b 100644 --- a/codex-rs/core-plugins/src/remote/catalog_cache_tests.rs +++ b/codex-rs/core-plugins/src/remote/catalog_cache_tests.rs @@ -18,7 +18,7 @@ fn catalog_cache_freshness_honors_ttl() { } #[test] -fn catalog_cache_paths_are_isolated_by_scope() { +fn catalog_cache_paths_are_isolated_by_scope_and_collection() { let codex_home = Path::new("/tmp/codex-home"); let cache_key_for_scope = |scope| RemotePluginCatalogCacheKey { chatgpt_base_url: "https://chatgpt.com/backend-api".to_string(), @@ -26,6 +26,7 @@ fn catalog_cache_paths_are_isolated_by_scope() { chatgpt_user_id: Some("user-id".to_string()), is_workspace_account: true, scope: (scope != RemotePluginScope::Global).then_some(scope), + collection: None, }; let paths = [ @@ -38,6 +39,10 @@ fn catalog_cache_paths_are_isolated_by_scope() { assert_ne!(paths[0], paths[1]); assert_ne!(paths[0], paths[2]); assert_ne!(paths[1], paths[2]); + + let mut collection_key = cache_key_for_scope(RemotePluginScope::Global); + collection_key.collection = Some("vertical".to_string()); + assert!(!paths.contains(&cache_path(codex_home, &collection_key))); } #[test] @@ -59,9 +64,14 @@ fn global_catalog_cache_reuses_legacy_cache_file() { let contents = serde_json::to_string_pretty(&legacy_cache).expect("serialize legacy cache"); codex_utils_path::write_atomically(&legacy_cache_path, &contents).expect("write legacy cache"); - let cached = - load_cached_directory_plugins(codex_home.path(), &config, &auth, RemotePluginScope::Global) - .expect("load legacy global cache"); + let cached = load_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + RemotePluginScope::Global, + /*collection*/ None, + ) + .expect("load legacy global cache"); assert!(cached.plugins.is_empty()); assert_eq!(cached.freshness, RemotePluginCatalogCacheFreshness::Stale); @@ -70,6 +80,7 @@ fn global_catalog_cache_reuses_legacy_cache_file() { &config, &auth, RemotePluginScope::Global, + /*collection*/ None, &[], ); let refreshed_cache: RemotePluginCatalogDiskCache = serde_json::from_slice( @@ -93,11 +104,18 @@ fn header_auth_does_not_cache_private_catalogs_without_a_stable_identity() { &config, &auth, RemotePluginScope::Global, + /*collection*/ None, &[], ); assert!( - load_cached_directory_plugins(codex_home.path(), &config, &auth, RemotePluginScope::Global) - .is_some() + load_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + RemotePluginScope::Global, + /*collection*/ None, + ) + .is_some() ); for scope in [RemotePluginScope::User, RemotePluginScope::Workspace] { @@ -107,10 +125,18 @@ fn header_auth_does_not_cache_private_catalogs_without_a_stable_identity() { chatgpt_user_id: None, is_workspace_account: false, scope: Some(scope), + collection: None, }; let insecure_cache_path = cache_path(codex_home.path(), &insecure_cache_key); - write_cached_directory_plugins(codex_home.path(), &config, &auth, scope, &[]); + write_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + scope, + /*collection*/ None, + &[], + ); assert!(!insecure_cache_path.exists()); let insecure_cache = RemotePluginCatalogDiskCache { @@ -122,6 +148,15 @@ fn header_auth_does_not_cache_private_catalogs_without_a_stable_identity() { codex_utils_path::write_atomically(&insecure_cache_path, &contents) .expect("write insecure cache"); - assert!(load_cached_directory_plugins(codex_home.path(), &config, &auth, scope).is_none()); + assert!( + load_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + scope, + /*collection*/ None, + ) + .is_none() + ); } } diff --git a/codex-rs/core-plugins/src/remote_tests.rs b/codex-rs/core-plugins/src/remote_tests.rs index 6828084b24..1d8bc682ea 100644 --- a/codex-rs/core-plugins/src/remote_tests.rs +++ b/codex-rs/core-plugins/src/remote_tests.rs @@ -162,8 +162,116 @@ async fn remote_installed_plugins_paginate_across_all_scopes_without_download_ur ); } +#[tokio::test] +async fn remote_catalog_cache_modes_control_refresh_and_persist_fetched_results() { + let server = MockServer::start().await; + let plugin = directory_plugin("plugin-gmail", "gmail"); + Mock::given(method("GET")) + .and(path("/backend-api/ps/plugins/list")) + .and(query_param("scope", "GLOBAL")) + .and(query_param_is_missing("collection")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "plugins": [plugin], + "pagination": {"next_page_token": null}, + }))) + .expect(3) + .mount(&server) + .await; + let codex_home = tempfile::tempdir().expect("create codex home"); + let (config, selected_urls) = + recording_remote_plugin_service_config(format!("{}/backend-api", server.uri())); + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + + for (mode, expire_cache, expected_refresh_needed, expected_cache_used) in [ + ( + RemotePluginCatalogCacheMode::PreferFreshCache, + false, + false, + false, + ), + ( + RemotePluginCatalogCacheMode::PreferFreshCache, + false, + false, + true, + ), + (RemotePluginCatalogCacheMode::PreferCache, true, true, true), + ( + RemotePluginCatalogCacheMode::PreferFreshCache, + false, + false, + false, + ), + ( + RemotePluginCatalogCacheMode::ForceRefetch, + false, + false, + false, + ), + ] { + if expire_cache { + let cache_path = + std::fs::read_dir(codex_home.path().join("cache/remote_plugin_catalog")) + .expect("read catalog cache directory") + .next() + .expect("catalog cache exists") + .expect("read cache entry") + .path(); + let mut cached: serde_json::Value = + serde_json::from_slice(&std::fs::read(&cache_path).expect("read cache")) + .expect("decode cache"); + cached["fetched_at"] = serde_json::json!("2000-01-01T00:00:00Z"); + std::fs::write( + cache_path, + serde_json::to_vec(&cached).expect("encode cache"), + ) + .expect("expire catalog cache"); + } + + let outcome = fetch_directory_plugins_for_scope_with_cache( + Some(codex_home.path()), + &config, + &auth, + RemotePluginScope::Global, + /*collection*/ None, + mode, + ) + .await + .expect("fetch directory plugins"); + assert_eq!( + ( + outcome.plugins, + outcome.cache_refresh_needed, + outcome.catalog_cache_used, + ), + ( + vec![plugin.clone()], + expected_refresh_needed, + expected_cache_used, + ), + ); + } + + assert_eq!( + recorded_http_client_urls(&selected_urls), + vec![ + format!( + "{}/backend-api/ps/plugins/list?scope=GLOBAL&limit=200", + server.uri() + ); + 3 + ], + ); + assert!(has_fresh_cached_remote_plugin_catalog( + codex_home.path(), + &config, + Some(&auth), + RemotePluginScope::Global, + )); +} + #[test] -fn cached_remote_plugin_catalog_scopes_returns_existing_scopes() { +fn catalog_cache_invalidation_clears_global_collections_and_preserves_other_scopes() { let codex_home = tempfile::tempdir().expect("create codex home"); let config = RemotePluginServiceConfig::new( "https://chatgpt.com/backend-api".to_string(), @@ -176,6 +284,7 @@ fn cached_remote_plugin_catalog_scopes_returns_existing_scopes() { &config, &auth, scope, + /*collection*/ None, &[], ); } @@ -184,6 +293,36 @@ fn cached_remote_plugin_catalog_scopes_returns_existing_scopes() { cached_remote_plugin_catalog_scopes(codex_home.path(), &config, Some(&auth)), BTreeSet::from([RemotePluginScope::Global, RemotePluginScope::Workspace]) ); + catalog_cache::write_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + RemotePluginScope::Global, + Some(OPENAI_CURATED_REMOTE_COLLECTION_KEY), + &[], + ); + + invalidate_cached_remote_plugin_catalog_scopes( + codex_home.path(), + &config, + Some(&auth), + &[RemotePluginScope::Global], + ); + + assert_eq!( + cached_remote_plugin_catalog_scopes(codex_home.path(), &config, Some(&auth)), + BTreeSet::from([RemotePluginScope::Workspace]), + ); + assert!( + catalog_cache::load_cached_directory_plugins( + codex_home.path(), + &config, + &auth, + RemotePluginScope::Global, + Some(OPENAI_CURATED_REMOTE_COLLECTION_KEY), + ) + .is_none() + ); } #[test]