From a3e9ac4fe23b1eafd41ef6961bc9fdf935801ba6 Mon Sep 17 00:00:00 2001 From: xli-oai Date: Tue, 7 Apr 2026 18:28:43 -0700 Subject: [PATCH] Add marketplace add command --- codex-rs/cli/src/main.rs | 17 + codex-rs/cli/src/marketplace_cmd.rs | 466 +++++++++++++++++++++ codex-rs/cli/tests/marketplace_add.rs | 59 +++ codex-rs/core/src/plugins/manager.rs | 26 ++ codex-rs/core/src/plugins/manager_tests.rs | 62 +++ codex-rs/core/src/plugins/marketplace.rs | 11 + codex-rs/core/src/plugins/mod.rs | 3 + 7 files changed, 644 insertions(+) create mode 100644 codex-rs/cli/src/marketplace_cmd.rs create mode 100644 codex-rs/cli/tests/marketplace_add.rs diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 3007048a2f..0c9852f7f2 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -38,10 +38,12 @@ use supports_color::Stream; mod app_cmd; #[cfg(target_os = "macos")] mod desktop_app; +mod marketplace_cmd; mod mcp_cmd; #[cfg(not(windows))] mod wsl_paths; +use crate::marketplace_cmd::MarketplaceCli; use crate::mcp_cmd::McpCli; use codex_core::config::Config; @@ -105,6 +107,9 @@ enum Subcommand { /// Manage external MCP servers for Codex. Mcp(McpCli), + /// Manage plugin marketplaces for Codex. + Marketplace(MarketplaceCli), + /// Start Codex as an MCP server (stdio). McpServer, @@ -691,6 +696,18 @@ async fn cli_main(arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> { prepend_config_flags(&mut mcp_cli.config_overrides, root_config_overrides.clone()); mcp_cli.run().await?; } + Some(Subcommand::Marketplace(mut marketplace_cli)) => { + reject_remote_mode_for_subcommand( + root_remote.as_deref(), + root_remote_auth_token_env.as_deref(), + "marketplace", + )?; + prepend_config_flags( + &mut marketplace_cli.config_overrides, + root_config_overrides.clone(), + ); + marketplace_cli.run().await?; + } Some(Subcommand::AppServer(app_server_cli)) => { let AppServerCommand { subcommand, diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs new file mode 100644 index 0000000000..52a779c9cc --- /dev/null +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -0,0 +1,466 @@ +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use clap::Parser; +use codex_core::config::find_codex_home; +use codex_core::plugins::marketplace_install_root; +use codex_core::plugins::validate_marketplace_root; +use codex_utils_cli::CliConfigOverrides; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +#[derive(Debug, Parser)] +pub struct MarketplaceCli { + #[clap(flatten)] + pub config_overrides: CliConfigOverrides, + + #[command(subcommand)] + subcommand: MarketplaceSubcommand, +} + +#[derive(Debug, clap::Subcommand)] +enum MarketplaceSubcommand { + /// Add a marketplace repository or local marketplace directory. + Add(AddMarketplaceArgs), +} + +#[derive(Debug, Parser)] +struct AddMarketplaceArgs { + /// Marketplace source. Supports owner/repo[@ref], git URLs, SSH URLs, or local directories. + source: String, + + /// Git ref to check out. Overrides any @ref or #ref suffix in SOURCE. + #[arg(long = "ref", value_name = "REF")] + ref_name: Option, + + /// Sparse-checkout paths to use while cloning git sources. + #[arg(long = "sparse", value_name = "PATH", num_args = 1..)] + sparse_paths: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +enum MarketplaceSource { + LocalDirectory(PathBuf), + Git { + url: String, + ref_name: Option, + }, +} + +impl MarketplaceCli { + pub async fn run(self) -> Result<()> { + let MarketplaceCli { + config_overrides, + subcommand, + } = self; + + // Validate overrides now. This command writes to CODEX_HOME only; marketplace discovery + // happens from that cache root after the next plugin/list or app-server start. + config_overrides + .parse_overrides() + .map_err(anyhow::Error::msg)?; + + match subcommand { + MarketplaceSubcommand::Add(args) => run_add(args).await?, + } + + Ok(()) + } +} + +async fn run_add(args: AddMarketplaceArgs) -> Result<()> { + let AddMarketplaceArgs { + source, + ref_name, + sparse_paths, + } = args; + + let source = parse_marketplace_source(&source, ref_name)?; + if !sparse_paths.is_empty() && !matches!(source, MarketplaceSource::Git { .. }) { + bail!("--sparse can only be used with git marketplace sources"); + } + + let codex_home = find_codex_home().context("failed to resolve CODEX_HOME")?; + let install_root = marketplace_install_root(&codex_home); + fs::create_dir_all(&install_root).with_context(|| { + format!( + "failed to create marketplace install directory {}", + install_root.display() + ) + })?; + + let staged_dir = tempfile::Builder::new() + .prefix("marketplace-add-") + .tempdir_in(&install_root) + .with_context(|| { + format!( + "failed to create temporary marketplace directory in {}", + install_root.display() + ) + })?; + let staged_root = staged_dir.path().to_path_buf(); + + match &source { + MarketplaceSource::LocalDirectory(path) => { + copy_dir_recursive(path, &staged_root).with_context(|| { + format!( + "failed to copy marketplace source {} into {}", + path.display(), + staged_root.display() + ) + })?; + } + MarketplaceSource::Git { url, ref_name } => { + clone_git_source(url, ref_name.as_deref(), &sparse_paths, &staged_root)?; + } + } + + let marketplace_name = validate_marketplace_root(&staged_root) + .with_context(|| format!("failed to validate marketplace from {}", source.display()))?; + let destination = install_root.join(safe_marketplace_dir_name(&marketplace_name)?); + replace_marketplace_root(&staged_root, &destination) + .with_context(|| format!("failed to install marketplace at {}", destination.display()))?; + + println!( + "Added marketplace `{marketplace_name}` from {}.", + source.display() + ); + println!("Installed marketplace root: {}", destination.display()); + + Ok(()) +} + +fn parse_marketplace_source( + source: &str, + explicit_ref: Option, +) -> Result { + let source = source.trim(); + if source.is_empty() { + bail!("marketplace source must not be empty"); + } + + let source = expand_home(source); + let path = PathBuf::from(&source); + if path.exists() || looks_like_local_path(&source) { + if !path.exists() { + bail!( + "local marketplace source does not exist: {}", + path.display() + ); + } + if !path.is_dir() { + bail!( + "local marketplace source must be a directory containing .agents/plugins/marketplace.json: {}", + path.display() + ); + } + let path = path + .canonicalize() + .with_context(|| format!("failed to resolve {}", path.display()))?; + return Ok(MarketplaceSource::LocalDirectory(path)); + } + + let (base_source, parsed_ref) = split_source_ref(&source); + let ref_name = explicit_ref.or(parsed_ref); + + if is_ssh_git_url(&base_source) || is_http_git_url(&base_source) { + return Ok(MarketplaceSource::Git { + url: normalize_git_url(&base_source), + ref_name, + }); + } + + if looks_like_github_shorthand(&base_source) { + return Ok(MarketplaceSource::Git { + url: format!("https://github.com/{base_source}.git"), + ref_name, + }); + } + + if base_source.starts_with("http://") || base_source.starts_with("https://") { + bail!( + "URL marketplace manifests are not supported yet; pass a git repository URL or a local marketplace directory" + ); + } + + bail!("invalid marketplace source format: {source}"); +} + +fn split_source_ref(source: &str) -> (String, Option) { + if let Some((base, ref_name)) = source.rsplit_once('#') { + return (base.to_string(), non_empty_ref(ref_name)); + } + if !source.contains("://") + && !is_ssh_git_url(source) + && let Some((base, ref_name)) = source.rsplit_once('@') + { + return (base.to_string(), non_empty_ref(ref_name)); + } + (source.to_string(), None) +} + +fn non_empty_ref(ref_name: &str) -> Option { + let ref_name = ref_name.trim(); + (!ref_name.is_empty()).then(|| ref_name.to_string()) +} + +fn normalize_git_url(url: &str) -> String { + if url.starts_with("https://github.com/") && !url.ends_with(".git") { + format!("{url}.git") + } else { + url.to_string() + } +} + +fn looks_like_local_path(source: &str) -> bool { + source.starts_with("./") + || source.starts_with("../") + || source.starts_with('/') + || source.starts_with("~/") + || source == "." + || source == ".." +} + +fn expand_home(source: &str) -> String { + let Some(rest) = source.strip_prefix("~/") else { + return source.to_string(); + }; + if let Some(home) = std::env::var_os("HOME") { + return PathBuf::from(home).join(rest).display().to_string(); + } + source.to_string() +} + +fn is_ssh_git_url(source: &str) -> bool { + source.starts_with("git@") && source.contains(':') +} + +fn is_http_git_url(source: &str) -> bool { + (source.starts_with("http://") || source.starts_with("https://")) + && (source.ends_with(".git") || source.starts_with("https://github.com/")) +} + +fn looks_like_github_shorthand(source: &str) -> bool { + let mut segments = source.split('/'); + let owner = segments.next(); + let repo = segments.next(); + let extra = segments.next(); + owner.is_some_and(is_github_shorthand_segment) + && repo.is_some_and(is_github_shorthand_segment) + && extra.is_none() +} + +fn is_github_shorthand_segment(segment: &str) -> bool { + !segment.is_empty() + && segment + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) +} + +fn clone_git_source( + url: &str, + ref_name: Option<&str>, + sparse_paths: &[String], + destination: &Path, +) -> Result<()> { + let destination = destination.to_string_lossy().to_string(); + if sparse_paths.is_empty() { + run_git(&["clone", url, destination.as_str()], None)?; + if let Some(ref_name) = ref_name { + run_git(&["checkout", ref_name], Some(Path::new(&destination)))?; + } + return Ok(()); + } + + run_git( + &[ + "clone", + "--filter=blob:none", + "--no-checkout", + url, + destination.as_str(), + ], + None, + )?; + let mut sparse_args = vec!["sparse-checkout", "set"]; + sparse_args.extend(sparse_paths.iter().map(String::as_str)); + let destination = Path::new(&destination); + run_git(&sparse_args, Some(destination))?; + run_git(&["checkout", ref_name.unwrap_or("HEAD")], Some(destination))?; + Ok(()) +} + +fn run_git(args: &[&str], cwd: Option<&Path>) -> Result<()> { + let mut command = Command::new("git"); + command.args(args); + if let Some(cwd) = cwd { + command.current_dir(cwd); + } + + let output = command + .output() + .with_context(|| format!("failed to run git {}", args.join(" ")))?; + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + bail!( + "git {} failed with status {}\nstdout:\n{}\nstderr:\n{}", + args.join(" "), + output.status, + stdout.trim(), + stderr.trim() + ); +} + +fn copy_dir_recursive(source: &Path, target: &Path) -> Result<()> { + fs::create_dir_all(target)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + let source_path = entry.path(); + let target_path = target.join(entry.file_name()); + let file_type = entry.file_type()?; + + if file_type.is_dir() { + if entry.file_name().to_str() == Some(".git") { + continue; + } + copy_dir_recursive(&source_path, &target_path)?; + } else if file_type.is_file() { + fs::copy(&source_path, &target_path)?; + } else if file_type.is_symlink() { + copy_symlink_target(&source_path, &target_path)?; + } + } + Ok(()) +} + +#[cfg(unix)] +fn copy_symlink_target(source: &Path, target: &Path) -> Result<()> { + std::os::unix::fs::symlink(fs::read_link(source)?, target)?; + Ok(()) +} + +#[cfg(windows)] +fn copy_symlink_target(source: &Path, target: &Path) -> Result<()> { + let metadata = fs::metadata(source)?; + if metadata.is_dir() { + copy_dir_recursive(source, target) + } else { + fs::copy(source, target).map(|_| ()).map_err(Into::into) + } +} + +fn replace_marketplace_root(staged_root: &Path, destination: &Path) -> Result<()> { + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } + + let backup = if destination.exists() { + let backup = tempfile::Builder::new() + .prefix("marketplace-backup-") + .tempdir_in( + destination + .parent() + .context("marketplace destination has no parent")?, + )?; + let backup_root = backup.path().join("previous"); + fs::rename(destination, &backup_root)?; + Some((backup, backup_root)) + } else { + None + }; + + if let Err(err) = fs::rename(staged_root, destination) { + if let Some((_, backup_root)) = backup { + let _ = fs::rename(backup_root, destination); + } + return Err(err.into()); + } + + Ok(()) +} + +fn safe_marketplace_dir_name(marketplace_name: &str) -> Result { + let safe = marketplace_name + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') { + ch + } else { + '-' + } + }) + .collect::(); + let safe = safe.trim_matches('.').to_string(); + if safe.is_empty() || safe == ".." { + bail!("marketplace name `{marketplace_name}` cannot be used as an install directory"); + } + Ok(safe) +} + +impl MarketplaceSource { + fn display(&self) -> String { + match self { + Self::LocalDirectory(path) => path.display().to_string(), + Self::Git { url, ref_name } => { + if let Some(ref_name) = ref_name { + format!("{url}#{ref_name}") + } else { + url.clone() + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn github_shorthand_parses_ref_suffix() { + assert_eq!( + parse_marketplace_source("owner/repo@main", /* explicit_ref */ None).unwrap(), + MarketplaceSource::Git { + url: "https://github.com/owner/repo.git".to_string(), + ref_name: Some("main".to_string()), + } + ); + } + + #[test] + fn git_url_parses_fragment_ref() { + assert_eq!( + parse_marketplace_source( + "https://example.com/team/repo.git#v1", + /* explicit_ref */ None, + ) + .unwrap(), + MarketplaceSource::Git { + url: "https://example.com/team/repo.git".to_string(), + ref_name: Some("v1".to_string()), + } + ); + } + + #[test] + fn explicit_ref_overrides_source_ref() { + assert_eq!( + parse_marketplace_source( + "owner/repo@main", + /* explicit_ref */ Some("release".to_string()), + ) + .unwrap(), + MarketplaceSource::Git { + url: "https://github.com/owner/repo.git".to_string(), + ref_name: Some("release".to_string()), + } + ); + } +} diff --git a/codex-rs/cli/tests/marketplace_add.rs b/codex-rs/cli/tests/marketplace_add.rs new file mode 100644 index 0000000000..2994cb336c --- /dev/null +++ b/codex-rs/cli/tests/marketplace_add.rs @@ -0,0 +1,59 @@ +use anyhow::Result; +use codex_core::plugins::marketplace_install_root; +use codex_core::plugins::validate_marketplace_root; +use predicates::str::contains; +use pretty_assertions::assert_eq; +use std::path::Path; +use tempfile::TempDir; + +fn codex_command(codex_home: &Path) -> Result { + let mut cmd = assert_cmd::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?); + cmd.env("CODEX_HOME", codex_home); + Ok(cmd) +} + +#[tokio::test] +async fn marketplace_add_local_directory_installs_valid_marketplace_root() -> Result<()> { + let codex_home = TempDir::new()?; + let source = TempDir::new()?; + std::fs::create_dir_all(source.path().join(".agents/plugins"))?; + std::fs::create_dir_all(source.path().join("plugins/sample/.codex-plugin"))?; + std::fs::write( + source.path().join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample", + "source": { + "source": "local", + "path": "./plugins/sample" + } + } + ] +}"#, + )?; + std::fs::write( + source + .path() + .join("plugins/sample/.codex-plugin/plugin.json"), + r#"{"name":"sample"}"#, + )?; + + let mut add_cmd = codex_command(codex_home.path())?; + add_cmd + .args(["marketplace", "add", source.path().to_str().unwrap()]) + .assert() + .success() + .stdout(contains("Added marketplace `debug`")); + + let installed_root = marketplace_install_root(codex_home.path()).join("debug"); + assert_eq!(validate_marketplace_root(&installed_root)?, "debug"); + assert!( + installed_root + .join("plugins/sample/.codex-plugin/plugin.json") + .is_file() + ); + + Ok(()) +} diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 1da98b85cb..dab69d3709 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -79,6 +79,7 @@ use tracing::warn; const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; +pub const INSTALLED_MARKETPLACES_DIR: &str = "marketplaces"; pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; pub const OPENAI_CURATED_MARKETPLACE_DISPLAY_NAME: &str = "OpenAI Curated"; static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); @@ -1222,6 +1223,7 @@ impl PluginsManager { // Treat the curated catalog as an extra marketplace root so plugin listing can surface it // without requiring every caller to know where it is stored. let mut roots = additional_roots.to_vec(); + roots.extend(installed_marketplace_roots(self.codex_home.as_path())); let curated_repo_root = curated_plugins_repo_path(self.codex_home.as_path()); if curated_repo_root.is_dir() && let Ok(curated_repo_root) = AbsolutePathBuf::try_from(curated_repo_root) @@ -1234,6 +1236,30 @@ impl PluginsManager { } } +pub fn marketplace_install_root(codex_home: &Path) -> PathBuf { + codex_home.join(INSTALLED_MARKETPLACES_DIR) +} + +fn installed_marketplace_roots(codex_home: &Path) -> Vec { + let install_root = marketplace_install_root(codex_home); + let Ok(entries) = fs::read_dir(&install_root) else { + return Vec::new(); + }; + + let mut roots = entries + .filter_map(Result::ok) + .filter_map(|entry| { + let path = entry.path(); + let file_type = entry.file_type().ok()?; + (file_type.is_dir() && path.join(".agents/plugins/marketplace.json").is_file()) + .then_some(path) + }) + .filter_map(|path| AbsolutePathBuf::try_from(path).ok()) + .collect::>(); + roots.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + roots +} + #[derive(Debug, thiserror::Error)] pub enum PluginInstallError { #[error("{0}")] diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index 2f1d3ae9c6..fbe104371f 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -1504,6 +1504,68 @@ plugins = true ); } +#[tokio::test] +async fn list_marketplaces_includes_installed_marketplace_roots() { + let tmp = tempfile::tempdir().unwrap(); + let marketplace_root = marketplace_install_root(tmp.path()).join("debug"); + let plugin_root = marketplace_root.join("plugins/sample"); + + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true +"#, + ); + fs::create_dir_all(marketplace_root.join(".agents/plugins")).unwrap(); + fs::create_dir_all(plugin_root.join(".codex-plugin")).unwrap(); + fs::write( + marketplace_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample", + "source": { + "source": "local", + "path": "./plugins/sample" + } + } + ] +}"#, + ) + .unwrap(); + fs::write( + plugin_root.join(".codex-plugin/plugin.json"), + r#"{"name":"sample"}"#, + ) + .unwrap(); + + let config = load_config(tmp.path(), tmp.path()).await; + let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) + .list_marketplaces_for_config(&config, &[]) + .unwrap() + .marketplaces; + + let marketplace = marketplaces + .into_iter() + .find(|marketplace| marketplace.name == "debug") + .expect("installed marketplace should be listed"); + + assert_eq!( + marketplace.path, + AbsolutePathBuf::try_from(marketplace_root.join(".agents/plugins/marketplace.json")) + .unwrap() + ); + assert_eq!(marketplace.plugins.len(), 1); + assert_eq!(marketplace.plugins[0].id, "sample@debug"); + assert_eq!( + marketplace.plugins[0].source, + MarketplacePluginSource::Local { + path: AbsolutePathBuf::try_from(plugin_root).unwrap(), + } + ); +} + #[tokio::test] async fn list_marketplaces_uses_first_duplicate_plugin_entry() { let tmp = tempfile::tempdir().unwrap(); diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core/src/plugins/marketplace.rs index ee775262aa..a5ca96e041 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core/src/plugins/marketplace.rs @@ -211,6 +211,17 @@ pub fn list_marketplaces( list_marketplaces_with_home(additional_roots, home_dir().as_deref()) } +pub fn validate_marketplace_root(root: &Path) -> Result { + let path = AbsolutePathBuf::try_from(root.join(MARKETPLACE_RELATIVE_PATH)).map_err(|err| { + MarketplaceError::InvalidMarketplaceFile { + path: root.join(MARKETPLACE_RELATIVE_PATH), + message: format!("marketplace path must resolve to an absolute path: {err}"), + } + })?; + let marketplace = load_marketplace(&path)?; + Ok(marketplace.name) +} + pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index a86502c28c..4ca09d0d10 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -29,6 +29,7 @@ pub(crate) use injection::build_plugin_injections; pub use manager::ConfiguredMarketplace; pub use manager::ConfiguredMarketplaceListOutcome; pub use manager::ConfiguredMarketplacePlugin; +pub use manager::INSTALLED_MARKETPLACES_DIR; pub use manager::OPENAI_CURATED_MARKETPLACE_NAME; pub use manager::PluginDetail; pub use manager::PluginInstallError; @@ -43,6 +44,7 @@ pub use manager::RemotePluginSyncResult; pub use manager::installed_plugin_telemetry_metadata; pub use manager::load_plugin_apps; pub use manager::load_plugin_mcp_servers; +pub use manager::marketplace_install_root; pub use manager::plugin_telemetry_metadata_from_root; pub use manifest::PluginManifestInterface; pub(crate) use manifest::PluginManifestPaths; @@ -53,6 +55,7 @@ pub use marketplace::MarketplacePluginAuthPolicy; pub use marketplace::MarketplacePluginInstallPolicy; pub use marketplace::MarketplacePluginPolicy; pub use marketplace::MarketplacePluginSource; +pub use marketplace::validate_marketplace_root; pub use remote::RemotePluginFetchError; pub use remote::fetch_remote_featured_plugin_ids; pub(crate) use render::render_explicit_plugin_instructions;