From 662d5c0acabbc7a3f3750a00f17a3351bf935d02 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Fri, 16 Jan 2026 18:45:07 +0100 Subject: [PATCH] feat: configurable skills path --- codex-rs/Cargo.lock | 165 +++++++++++++++- codex-rs/Cargo.toml | 1 + codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/config/mod.rs | 5 + codex-rs/core/src/config/types.rs | 28 +++ codex-rs/core/src/skills/loader.rs | 281 ++++++++++++++++++++++++---- codex-rs/core/src/skills/manager.rs | 4 +- codex-rs/core/src/skills/mod.rs | 1 + codex-rs/core/src/skills/remote.rs | 160 ++++++++++++++++ 9 files changed, 601 insertions(+), 45 deletions(-) create mode 100644 codex-rs/core/src/skills/remote.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index b87f76384d..0897a0b6c2 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -344,6 +344,15 @@ dependencies = [ "wiremock", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arboard" version = "3.6.1" @@ -360,7 +369,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.52.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -789,6 +798,25 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -1354,6 +1382,7 @@ dependencies = [ "which", "wildmatch", "wiremock", + "zip", "zstd", ] @@ -2088,6 +2117,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.6.0" @@ -2166,6 +2201,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -2454,6 +2504,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + [[package]] name = "der" version = "0.7.10" @@ -2485,6 +2541,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -2829,7 +2896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2926,7 +2993,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.0.8", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3867,7 +3934,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4224,6 +4291,27 @@ dependencies = [ "url", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "maplit" version = "1.0.2" @@ -4973,6 +5061,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -5378,7 +5476,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5757,7 +5855,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5770,7 +5868,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -8088,7 +8186,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8699,6 +8797,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yansi" version = "1.0.1" @@ -8886,6 +8993,48 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap 2.12.0", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.17", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 33a664e06d..abf96eb9de 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -223,6 +223,7 @@ tracing-test = "0.2.5" tree-sitter = "0.25.10" tree-sitter-bash = "0.25" zstd = "0.13" +zip = "2.2.0" tree-sitter-highlight = "0.25.10" ts-rs = "11" tui-scrollbar = "0.2.2" diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 10b635b747..7927939af5 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -96,6 +96,7 @@ url = { workspace = true } uuid = { workspace = true, features = ["serde", "v4", "v5"] } which = { workspace = true } wildmatch = { workspace = true } +zip = { workspace = true } [features] deterministic_process_ids = [] diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 6228bc3ac3..8a6632e5f5 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -13,6 +13,7 @@ use crate::config::types::SandboxWorkspaceWrite; use crate::config::types::ScrollInputMode; use crate::config::types::ShellEnvironmentPolicy; use crate::config::types::ShellEnvironmentPolicyToml; +use crate::config::types::SkillsConfigToml; use crate::config::types::Tui; use crate::config::types::UriBasedFileOpener; use crate::config_loader::ConfigLayerStack; @@ -881,6 +882,10 @@ pub struct ConfigToml { /// output will be hyperlinked using the specified URI scheme. pub file_opener: Option, + /// Additional skill sources to load from local paths or URLs. + #[serde(default)] + pub skills: Option, + /// Collection of settings that are specific to the TUI. pub tui: Option, diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 981f9b541a..52fd4688b0 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -283,6 +283,34 @@ impl UriBasedFileOpener { } } +/// Configuration for additional skill sources. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct SkillsConfigToml { + /// Additional skill sources to load. Each entry can be a local path or a URL. + #[serde(default)] + pub sources: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +#[serde(untagged)] +pub enum SkillSourceToml { + Path(SkillSourcePathToml), + Url(SkillSourceUrlToml), +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct SkillSourcePathToml { + pub path: AbsolutePathBuf, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct SkillSourceUrlToml { + pub url: String, +} + /// Settings that govern if and what will be written to `~/.codex/history.jsonl`. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema)] #[schemars(deny_unknown_fields)] diff --git a/codex-rs/core/src/skills/loader.rs b/codex-rs/core/src/skills/loader.rs index da51fea41e..5bee23e178 100644 --- a/codex-rs/core/src/skills/loader.rs +++ b/codex-rs/core/src/skills/loader.rs @@ -1,9 +1,16 @@ use crate::config::Config; +use crate::config::ConfigToml; +use crate::config::types::SkillSourcePathToml; +use crate::config::types::SkillSourceToml; +use crate::config::types::SkillSourceUrlToml; +use crate::config_loader::ConfigLayerEntry; use crate::config_loader::ConfigLayerStack; use crate::skills::model::SkillError; use crate::skills::model::SkillInterface; use crate::skills::model::SkillLoadOutcome; use crate::skills::model::SkillMetadata; +use crate::skills::remote::ensure_remote_skill; +use crate::skills::remote::remote_cache_root_dir; use crate::skills::system::system_cache_root_dir; use codex_app_server_protocol::ConfigLayerSource; use codex_protocol::protocol::SkillScope; @@ -88,21 +95,57 @@ impl fmt::Display for SkillParseError { impl Error for SkillParseError {} pub fn load_skills(config: &Config) -> SkillLoadOutcome { - load_skills_from_roots(skill_roots(config)) + load_skills_from_roots(skill_roots(config), false) } pub(crate) struct SkillRoot { - pub(crate) path: PathBuf, pub(crate) scope: SkillScope, + pub(crate) source: SkillRootSource, } -pub(crate) fn load_skills_from_roots(roots: I) -> SkillLoadOutcome +pub(crate) enum SkillRootSource { + Local { path: PathBuf }, + Remote { url: String, cache_root: PathBuf }, +} + +impl SkillRoot { + fn local(path: PathBuf, scope: SkillScope) -> Self { + Self { + scope, + source: SkillRootSource::Local { path }, + } + } + + fn remote(url: String, cache_root: PathBuf, scope: SkillScope) -> Self { + Self { + scope, + source: SkillRootSource::Remote { url, cache_root }, + } + } +} + +pub(crate) fn load_skills_from_roots(roots: I, force_reload: bool) -> SkillLoadOutcome where I: IntoIterator, { let mut outcome = SkillLoadOutcome::default(); for root in roots { - discover_skills_under_root(&root.path, root.scope, &mut outcome); + let root_path = match root.source { + SkillRootSource::Local { path } => path, + SkillRootSource::Remote { url, cache_root } => { + match ensure_remote_skill(&cache_root, &url, force_reload) { + Ok(path) => path, + Err(err) => { + outcome.errors.push(SkillError { + path: PathBuf::from(url.as_str()), + message: format!("failed to fetch remote skill: {err}"), + }); + continue; + } + } + } + }; + discover_skills_under_root(&root_path, root.scope, &mut outcome); } let mut seen: HashSet = HashSet::new(); @@ -132,40 +175,48 @@ where fn skill_roots_from_layer_stack_inner(config_layer_stack: &ConfigLayerStack) -> Vec { let mut roots = Vec::new(); + let remote_cache_root = config_layer_stack + .get_user_layer() + .and_then(|layer| layer.config_folder()) + .map(|folder| remote_cache_root_dir(folder.as_path())); for layer in config_layer_stack.layers_high_to_low() { let Some(config_folder) = layer.config_folder() else { continue; }; + let extra_sources = skill_roots_from_layer(layer, remote_cache_root.as_ref()); match &layer.name { ConfigLayerSource::Project { .. } => { - roots.push(SkillRoot { - path: config_folder.as_path().join(SKILLS_DIR_NAME), - scope: SkillScope::Repo, - }); + roots.extend(extra_sources); + roots.push(SkillRoot::local( + config_folder.as_path().join(SKILLS_DIR_NAME), + SkillScope::Repo, + )); } ConfigLayerSource::User { .. } => { // `$CODEX_HOME/skills` (user-installed skills). - roots.push(SkillRoot { - path: config_folder.as_path().join(SKILLS_DIR_NAME), - scope: SkillScope::User, - }); + roots.extend(extra_sources); + roots.push(SkillRoot::local( + config_folder.as_path().join(SKILLS_DIR_NAME), + SkillScope::User, + )); // Embedded system skills are cached under `$CODEX_HOME/skills/.system` and are a // special case (not a config layer). - roots.push(SkillRoot { - path: system_cache_root_dir(config_folder.as_path()), - scope: SkillScope::System, - }); + roots.push(SkillRoot::local( + system_cache_root_dir(config_folder.as_path()), + SkillScope::System, + )); } ConfigLayerSource::System { .. } => { // The system config layer lives under `/etc/codex/` on Unix, so treat // `/etc/codex/skills` as admin-scoped skills. - roots.push(SkillRoot { - path: config_folder.as_path().join(SKILLS_DIR_NAME), - scope: SkillScope::Admin, - }); + roots.extend(extra_sources); + roots.push(SkillRoot::local( + config_folder.as_path().join(SKILLS_DIR_NAME), + SkillScope::Admin, + )); } ConfigLayerSource::Mdm { .. } | ConfigLayerSource::SessionFlags @@ -177,6 +228,46 @@ fn skill_roots_from_layer_stack_inner(config_layer_stack: &ConfigLayerStack) -> roots } +fn skill_roots_from_layer( + layer: &ConfigLayerEntry, + remote_cache_root: Option<&PathBuf>, +) -> Vec { + let scope = match layer.name { + ConfigLayerSource::Project { .. } => SkillScope::Repo, + ConfigLayerSource::User { .. } => SkillScope::User, + ConfigLayerSource::System { .. } => SkillScope::Admin, + ConfigLayerSource::Mdm { .. } + | ConfigLayerSource::SessionFlags + | ConfigLayerSource::LegacyManagedConfigTomlFromFile { .. } + | ConfigLayerSource::LegacyManagedConfigTomlFromMdm => { + return Vec::new(); + } + }; + + let Ok(config) = layer.config.clone().try_into::() else { + return Vec::new(); + }; + let Some(skills) = config.skills else { + return Vec::new(); + }; + + let mut roots = Vec::new(); + for source in skills.sources { + match source { + SkillSourceToml::Path(SkillSourcePathToml { path }) => { + roots.push(SkillRoot::local(path.as_path().to_path_buf(), scope)); + } + SkillSourceToml::Url(SkillSourceUrlToml { url }) => { + let Some(cache_root) = remote_cache_root else { + continue; + }; + roots.push(SkillRoot::remote(url, cache_root.clone(), scope)); + } + } + } + roots +} + fn skill_roots(config: &Config) -> Vec { skill_roots_from_layer_stack_inner(&config.config_layer_stack) } @@ -550,9 +641,17 @@ mod tests { use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; + use std::io::Write; use std::path::Path; use tempfile::TempDir; use toml::Value as TomlValue; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::method; + use wiremock::matchers::path; + use zip::ZipWriter; + use zip::write::FileOptions; const REPO_ROOT_CONFIG_DIR_NAME: &str = ".codex"; @@ -616,7 +715,15 @@ mod tests { let got = skill_roots_from_layer_stack(&stack) .into_iter() - .map(|root| (root.scope, root.path)) + .map(|root| { + let path = match root.source { + SkillRootSource::Local { path } => path, + SkillRootSource::Remote { .. } => { + panic!("unexpected remote skill root in test") + } + }; + (root.scope, path) + }) .collect::>(); assert_eq!( @@ -664,6 +771,18 @@ mod tests { path } + fn make_skill_archive(name: &str, description: &str) -> Vec { + let content = format!("---\nname: {name}\ndescription: {description}\n---\n\n# Body\n"); + let mut cursor = std::io::Cursor::new(Vec::new()); + let mut archive = ZipWriter::new(&mut cursor); + archive + .start_file(SKILLS_FILENAME, FileOptions::<()>::default()) + .unwrap(); + archive.write_all(content.as_bytes()).unwrap(); + archive.finish().unwrap(); + cursor.into_inner() + } + fn write_skill_interface_at(skill_dir: &Path, contents: &str) -> PathBuf { let path = skill_dir.join(SKILLS_TOML_FILENAME); fs::write(&path, contents).unwrap(); @@ -993,10 +1112,13 @@ icon_large = "./assets/../logo.svg" fs::create_dir_all(admin_root.path()).unwrap(); symlink_dir(shared.path(), &admin_root.path().join("shared")); - let outcome = load_skills_from_roots([SkillRoot { - path: admin_root.path().to_path_buf(), - scope: SkillScope::Admin, - }]); + let outcome = load_skills_from_roots( + [SkillRoot::local( + admin_root.path().to_path_buf(), + SkillScope::Admin, + )], + false, + ); assert!( outcome.errors.is_empty(), @@ -1538,6 +1660,98 @@ icon_large = "./assets/../logo.svg" assert_eq!(outcome.skills.len(), 0); } + #[tokio::test] + async fn loads_skills_from_configured_path_source() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let work_dir = tempfile::tempdir().expect("tempdir"); + let custom_root = tempfile::tempdir().expect("tempdir"); + + let skill_path = write_skill_at(custom_root.path(), "custom", "path-skill", "from path"); + let config_path = codex_home.path().join("config.toml"); + let custom_root_path = custom_root.path().display(); + fs::write( + &config_path, + format!("[skills]\nsources = [{{ path = \"{custom_root_path}\" }}]\n"), + ) + .unwrap(); + + let cfg = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .harness_overrides(ConfigOverrides { + cwd: Some(work_dir.path().to_path_buf()), + ..Default::default() + }) + .build() + .await + .expect("config should load"); + + let outcome = load_skills(&cfg); + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!( + outcome.skills, + vec![SkillMetadata { + name: "path-skill".to_string(), + description: "from path".to_string(), + short_description: None, + interface: None, + path: normalized(&skill_path), + scope: SkillScope::User, + }] + ); + } + + #[tokio::test] + async fn loads_skills_from_remote_source() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let work_dir = tempfile::tempdir().expect("tempdir"); + + let server = MockServer::start().await; + let skill_bytes = make_skill_archive("remote-skill", "from remote"); + Mock::given(method("GET")) + .and(path("/remote.skill")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(skill_bytes)) + .mount(&server) + .await; + + let server_uri = server.uri(); + let url = format!("{server_uri}/remote.skill"); + let config_path = codex_home.path().join("config.toml"); + fs::write( + &config_path, + format!("[skills]\nsources = [{{ url = \"{url}\" }}]\n"), + ) + .unwrap(); + + let cfg = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .harness_overrides(ConfigOverrides { + cwd: Some(work_dir.path().to_path_buf()), + ..Default::default() + }) + .build() + .await + .expect("config should load"); + + let outcome = load_skills(&cfg); + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + assert_eq!(outcome.skills[0].name, "remote-skill"); + assert!( + outcome.skills.first().is_some_and(|skill| skill + .path + .starts_with(codex_home.path().join("skills/.remote"))), + "expected remote skill path under .remote cache" + ); + } + #[tokio::test] async fn loads_skills_from_system_cache_when_present() { let codex_home = tempfile::tempdir().expect("tempdir"); @@ -1592,16 +1806,13 @@ icon_large = "./assets/../logo.svg" let _admin_skill_path = write_skill_at(admin_dir.path(), "admin", "dupe-skill", "from admin"); - let outcome = load_skills_from_roots([ - SkillRoot { - path: system_dir.path().to_path_buf(), - scope: SkillScope::System, - }, - SkillRoot { - path: admin_dir.path().to_path_buf(), - scope: SkillScope::Admin, - }, - ]); + let outcome = load_skills_from_roots( + [ + SkillRoot::local(system_dir.path().to_path_buf(), SkillScope::System), + SkillRoot::local(admin_dir.path().to_path_buf(), SkillScope::Admin), + ], + false, + ); assert!( outcome.errors.is_empty(), diff --git a/codex-rs/core/src/skills/manager.rs b/codex-rs/core/src/skills/manager.rs index bff928a565..2c47051f0a 100644 --- a/codex-rs/core/src/skills/manager.rs +++ b/codex-rs/core/src/skills/manager.rs @@ -44,7 +44,7 @@ impl SkillsManager { } let roots = skill_roots_from_layer_stack(&config.config_layer_stack); - let outcome = load_skills_from_roots(roots); + let outcome = load_skills_from_roots(roots, false); match self.cache_by_cwd.write() { Ok(mut cache) => { cache.insert(cwd.to_path_buf(), outcome.clone()); @@ -100,7 +100,7 @@ impl SkillsManager { }; let roots = skill_roots_from_layer_stack(&config_layer_stack); - let outcome = load_skills_from_roots(roots); + let outcome = load_skills_from_roots(roots, force_reload); match self.cache_by_cwd.write() { Ok(mut cache) => { cache.insert(cwd.to_path_buf(), outcome.clone()); diff --git a/codex-rs/core/src/skills/mod.rs b/codex-rs/core/src/skills/mod.rs index cf7c180502..0968b9de70 100644 --- a/codex-rs/core/src/skills/mod.rs +++ b/codex-rs/core/src/skills/mod.rs @@ -2,6 +2,7 @@ pub mod injection; pub mod loader; pub mod manager; pub mod model; +pub mod remote; pub mod render; pub mod system; diff --git a/codex-rs/core/src/skills/remote.rs b/codex-rs/core/src/skills/remote.rs new file mode 100644 index 0000000000..c557353b0f --- /dev/null +++ b/codex-rs/core/src/skills/remote.rs @@ -0,0 +1,160 @@ +use crate::default_client::create_client; +use sha2::Digest; +use sha2::Sha256; +use std::fs; +use std::future::Future; +use std::io::Cursor; +use std::path::Path; +use std::path::PathBuf; +use thiserror::Error; +use url::Url; +use zip::ZipArchive; + +const REMOTE_CACHE_DIR_NAME: &str = ".remote"; +const SOURCE_URL_MARKER_FILENAME: &str = ".codex-skill-source.url"; +const SKILL_FILENAME: &str = "SKILL.md"; + +#[derive(Debug, Error)] +pub(crate) enum RemoteSkillError { + #[error("invalid URL: {url}")] + InvalidUrl { url: String }, + #[error("unsupported URL scheme: {scheme}")] + UnsupportedScheme { scheme: String }, + #[error("request failed: {source}")] + Request { + #[from] + source: reqwest::Error, + }, + #[error("failed to extract zip: {source}")] + Zip { + #[from] + source: zip::result::ZipError, + }, + #[error("invalid zip entry: {name}")] + InvalidZipEntry { name: String }, + #[error("io error: {source}")] + Io { + #[from] + source: std::io::Error, + }, +} + +pub(crate) fn remote_cache_root_dir(codex_home: &Path) -> PathBuf { + codex_home.join("skills").join(REMOTE_CACHE_DIR_NAME) +} + +pub(crate) fn ensure_remote_skill( + cache_root: &Path, + url: &str, + force_reload: bool, +) -> Result { + let parsed = Url::parse(url).map_err(|_| RemoteSkillError::InvalidUrl { + url: url.to_string(), + })?; + let cache_dir = cache_root.join(url_hash(parsed.as_str())); + if cache_dir.exists() && !force_reload { + return Ok(cache_dir); + } + + fs::create_dir_all(cache_root)?; + if cache_dir.exists() { + fs::remove_dir_all(&cache_dir)?; + } + + let temp_dir = tempfile::Builder::new() + .prefix("skill-download-") + .tempdir_in(cache_root)?; + let temp_path = temp_dir.path().to_path_buf(); + + download_into(&parsed, &temp_path)?; + fs::write(temp_path.join(SOURCE_URL_MARKER_FILENAME), parsed.as_str())?; + + let temp_path = temp_dir.keep(); + fs::rename(&temp_path, &cache_dir)?; + Ok(cache_dir) +} + +fn download_into(url: &Url, dest: &Path) -> Result<(), RemoteSkillError> { + match url.scheme() { + "http" | "https" => {} + scheme => { + return Err(RemoteSkillError::UnsupportedScheme { + scheme: scheme.to_string(), + }); + } + } + + let client = create_client(); + let bytes = block_on(fetch_bytes(client, url.clone()))?; + + let path_lower = url.path().to_ascii_lowercase(); + if path_lower.ends_with(".skill") || path_lower.ends_with(".zip") { + extract_zip(&bytes, dest)?; + } else { + fs::create_dir_all(dest)?; + fs::write(dest.join(SKILL_FILENAME), bytes)?; + } + + Ok(()) +} + +fn extract_zip(bytes: &[u8], dest: &Path) -> Result<(), RemoteSkillError> { + fs::create_dir_all(dest)?; + let reader = Cursor::new(bytes); + let mut archive = ZipArchive::new(reader)?; + for index in 0..archive.len() { + let mut entry = archive.by_index(index)?; + let Some(entry_path) = entry.enclosed_name() else { + return Err(RemoteSkillError::InvalidZipEntry { + name: entry.name().to_string(), + }); + }; + let out_path = dest.join(entry_path); + if entry.is_dir() { + fs::create_dir_all(&out_path)?; + continue; + } + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + let mut out_file = fs::File::create(&out_path)?; + std::io::copy(&mut entry, &mut out_file)?; + } + Ok(()) +} + +async fn fetch_bytes( + client: codex_client::CodexHttpClient, + url: Url, +) -> Result, RemoteSkillError> { + let response = client.get(url).send().await?.error_for_status()?; + Ok(response.bytes().await?.to_vec()) +} + +fn block_on(future: F) -> Result +where + F: Future> + Send + 'static, + T: Send + 'static, +{ + if let Ok(handle) = tokio::runtime::Handle::try_current() { + return Ok(tokio::task::block_in_place(|| handle.block_on(future))?); + } + + let runtime = tokio::runtime::Runtime::new()?; + Ok(runtime.block_on(future)?) +} + +fn url_hash(url: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + hex_encode(&hasher.finalize()) +} + +fn hex_encode(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write; + let _ = write!(out, "{byte:02x}"); + } + out +}