Move host skill root resolution into the skills extension (#36943)

## What changed

- Move config-layer, user, system, plugin, extra, and repository skill-root
  resolution from `core-skills` into the host skills extension.
- Keep the core loader focused on loading explicit `SkillRoot` values.
- Relocate and expand tests for root precedence, deduplication, repository
  ancestry, plugin metadata, and concurrent probing.

GitOrigin-RevId: 3b95cf28101b8b4d64d54079d202154dad560aab
This commit is contained in:
felixxia-oai
2026-08-04 18:21:15 +00:00
committed by copyberry
parent d1fb77d692
commit 1a7519fa07
13 changed files with 1077 additions and 1109 deletions

2
codex-rs/Cargo.lock generated
View File

@@ -2922,7 +2922,6 @@ dependencies = [
"codex-utils-absolute-path",
"codex-utils-path-uri",
"codex-utils-plugins",
"dirs",
"dunce",
"futures",
"pretty_assertions",
@@ -4115,6 +4114,7 @@ dependencies = [
"codex-utils-path-uri",
"codex-utils-plugins",
"codex-utils-string",
"dirs",
"dunce",
"futures",
"insta",

View File

@@ -27,7 +27,6 @@ codex-skills = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-path-uri = { workspace = true }
codex-utils-plugins = { workspace = true }
dirs = { workspace = true }
dunce = { workspace = true }
futures = { workspace = true }
serde = { workspace = true, features = ["derive"] }
@@ -35,10 +34,10 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
shlex = { workspace = true }
tokio = { workspace = true, features = ["fs", "macros", "rt"] }
toml = { workspace = true }
tracing = { workspace = true }
zip = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
toml = { workspace = true }

View File

@@ -7,7 +7,6 @@ pub mod model;
pub mod remote;
mod root_loader;
mod skill_instructions;
pub mod system;
pub(crate) use invocation_utils::build_implicit_skill_path_indexes;
pub use invocation_utils::detect_implicit_skill_invocation_for_command;

View File

@@ -13,14 +13,7 @@ use crate::model::SkillLoadOutcome;
use crate::model::SkillMetadata;
use crate::model::SkillPolicy;
use crate::model::SkillToolDependency;
use crate::system::system_cache_root_dir;
use codex_config::ConfigLayerSource;
use codex_config::ConfigLayerStack;
use codex_config::default_project_root_markers;
use codex_config::merge_toml_values;
use codex_config::project_root_markers_from_config;
use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::LOCAL_FS;
use codex_protocol::protocol::Product;
use codex_protocol::protocol::SkillScope;
use codex_skills::ParsedSkillFrontmatter;
@@ -33,9 +26,7 @@ use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_absolute_path::AbsolutePathBufGuard;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::PluginIdentity;
use codex_utils_plugins::PluginSkillRoot;
use codex_utils_plugins::SkillDiscoveryMode;
use dirs::home_dir;
use discovery::DirectorySymlinkPolicy;
use discovery::DiscoveredSkill;
use discovery::HiddenDirectoryPolicy;
@@ -48,13 +39,11 @@ use futures::FutureExt;
use futures::StreamExt;
use namespace::SkillNamespaceResolver;
use serde::Deserialize;
use std::collections::HashSet;
use std::error::Error;
use std::fmt;
use std::io;
use std::sync::Arc;
use tokio::sync::Semaphore;
use toml::Value as TomlValue;
use tracing::error;
// TODO(anp): Tune this eight-scan limit after revisiting byte-based backpressure.
@@ -103,10 +92,8 @@ struct DependencyTool {
}
const SKILLS_FILENAME: &str = "SKILL.md";
const AGENTS_DIR_NAME: &str = ".agents";
const SKILLS_METADATA_DIR: &str = "agents";
const SKILLS_METADATA_FILENAME: &str = "openai.yaml";
const SKILLS_DIR_NAME: &str = "skills";
const MAX_NAME_LEN: usize = 64;
const MAX_QUALIFIED_NAME_LEN: usize = 128;
const MAX_DESCRIPTION_LEN: usize = 1024;
@@ -119,9 +106,6 @@ const MAX_DEPENDENCY_URL_LEN: usize = MAX_DESCRIPTION_LEN;
// Traversal depth from the skills root.
const MAX_SCAN_DEPTH: usize = 6;
const MAX_SKILLS_DIRS_PER_ROOT: usize = 2000;
// Keep ancestor metadata probes within one remote round trip for typical project hierarchies while
// leaving room for other startup discovery on the shared exec-server transport.
const MAX_CONCURRENT_ANCESTOR_PROBES: usize = 256;
struct ResolvedDiscoveredSkill {
skill: DiscoveredSkill,
@@ -197,277 +181,6 @@ pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot {
}
}
pub async fn skill_roots(
fs: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
let home_dir =
home_dir().and_then(|path| AbsolutePathBuf::from_absolute_path_checked(path).ok());
skill_roots_with_home_dir(
fs,
config_layer_stack,
cwd,
home_dir.as_ref(),
plugin_skill_roots,
extra_skill_roots,
)
.await
}
async fn skill_roots_with_home_dir(
fs: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
home_dir: Option<&AbsolutePathBuf>,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
let mut roots = skill_roots_from_layer_stack_inner(config_layer_stack, home_dir, fs.clone());
roots.extend(plugin_skill_roots.into_iter().map(|root| SkillRoot {
path: root.path,
scope: SkillScope::User,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: Some(root.plugin_identity),
plugin_namespace: Some(root.plugin_namespace),
plugin_root: Some(root.plugin_root),
discovery_mode: root.discovery_mode,
}));
roots.extend(extra_skill_roots.into_iter().map(|path| SkillRoot {
path,
scope: SkillScope::User,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
}));
roots.extend(repo_agents_skill_roots(fs, config_layer_stack, cwd).await);
dedupe_skill_roots_by_path(&mut roots);
roots
}
fn skill_roots_from_layer_stack_inner(
config_layer_stack: &ConfigLayerStack,
home_dir: Option<&AbsolutePathBuf>,
repo_fs: Option<Arc<dyn ExecutorFileSystem>>,
) -> Vec<SkillRoot> {
let mut roots = Vec::new();
for layer in config_layer_stack.all_layers_high_to_low() {
let Some(config_folder) = layer.config_folder() else {
continue;
};
match &layer.name {
ConfigLayerSource::Project { .. } => {
if let Some(repo_fs) = &repo_fs {
roots.push(SkillRoot {
path: config_folder.join(SKILLS_DIR_NAME),
scope: SkillScope::Repo,
file_system: Arc::clone(repo_fs),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
});
}
}
ConfigLayerSource::User { .. } => {
// Deprecated user skills location (`$CODEX_HOME/skills`), kept for backward
// compatibility.
roots.push(SkillRoot {
path: config_folder.join(SKILLS_DIR_NAME),
scope: SkillScope::User,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
});
// `$HOME/.agents/skills` (user-installed skills).
if let Some(home_dir) = home_dir {
roots.push(SkillRoot {
path: home_dir.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME),
scope: SkillScope::User,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
});
}
// Embedded system skills are cached under `$CODEX_HOME/skills/.system` and are a
// special case (not a config layer).
roots.push(SkillRoot {
path: system_cache_root_dir(&config_folder),
scope: SkillScope::System,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
});
}
ConfigLayerSource::System { .. } => {
// The system config layer lives under `/etc/codex/` on Unix, so treat
// `/etc/codex/skills` as admin-scoped skills.
roots.push(SkillRoot {
path: config_folder.join(SKILLS_DIR_NAME),
scope: SkillScope::Admin,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
});
}
ConfigLayerSource::Mdm { .. }
| ConfigLayerSource::EnterpriseManaged { .. }
| ConfigLayerSource::SessionFlags
| ConfigLayerSource::LegacyManagedConfigTomlFromFile { .. }
| ConfigLayerSource::LegacyManagedConfigTomlFromMdm => {}
}
}
roots
}
async fn repo_agents_skill_roots(
fs: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
) -> Vec<SkillRoot> {
let Some(fs) = fs else {
return Vec::new();
};
let project_root_markers = project_root_markers_from_stack(config_layer_stack);
let project_root = find_project_root(fs.as_ref(), cwd, &project_root_markers).await;
let dirs = dirs_between_project_root_and_cwd(cwd, &project_root);
let mut roots = Vec::new();
let mut results = futures::stream::iter(dirs)
.map(|dir| {
let fs = Arc::clone(&fs);
async move {
let agents_skills = dir.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME);
let agents_skills_uri = PathUri::from_abs_path(&agents_skills);
let result = fs.get_metadata(&agents_skills_uri, /*sandbox*/ None).await;
(agents_skills, result)
}
})
.buffered(MAX_CONCURRENT_ANCESTOR_PROBES);
while let Some((agents_skills, result)) = results.next().await {
match result {
Ok(metadata) if metadata.is_directory => roots.push(SkillRoot {
path: agents_skills,
scope: SkillScope::Repo,
file_system: Arc::clone(&fs),
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
}),
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
tracing::warn!(
"failed to stat repo skills root {}: {err:#}",
agents_skills.display()
);
}
}
}
roots
}
fn project_root_markers_from_stack(config_layer_stack: &ConfigLayerStack) -> Vec<String> {
let mut merged = TomlValue::Table(toml::map::Map::new());
for layer in config_layer_stack.layers_low_to_high() {
if matches!(layer.name, ConfigLayerSource::Project { .. }) {
continue;
}
merge_toml_values(&mut merged, &layer.config);
}
match project_root_markers_from_config(&merged) {
Ok(Some(markers)) => markers,
Ok(None) => default_project_root_markers(),
Err(err) => {
tracing::warn!("invalid project_root_markers: {err}");
default_project_root_markers()
}
}
}
async fn find_project_root(
fs: &dyn ExecutorFileSystem,
cwd: &AbsolutePathBuf,
project_root_markers: &[String],
) -> AbsolutePathBuf {
if project_root_markers.is_empty() {
return cwd.clone();
}
let mut probes = Vec::new();
for ancestor in cwd.ancestors() {
for marker in project_root_markers {
let marker_path = ancestor.join(marker);
probes.push((ancestor.clone(), marker_path));
}
}
let mut results = futures::stream::iter(probes)
.map(|(ancestor, marker_path)| async move {
let marker_path_uri = PathUri::from_abs_path(&marker_path);
let result = fs.get_metadata(&marker_path_uri, /*sandbox*/ None).await;
(ancestor, marker_path, result)
})
.buffered(MAX_CONCURRENT_ANCESTOR_PROBES);
while let Some((ancestor, marker_path, result)) = results.next().await {
match result {
Ok(_) => return ancestor,
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
tracing::warn!(
"failed to stat project root marker {}: {err:#}",
marker_path.display()
);
}
}
}
cwd.clone()
}
fn dirs_between_project_root_and_cwd(
cwd: &AbsolutePathBuf,
project_root: &AbsolutePathBuf,
) -> Vec<AbsolutePathBuf> {
let mut dirs = cwd
.ancestors()
.scan(false, |done, dir| {
if *done {
None
} else {
if &dir == project_root {
*done = true;
}
Some(dir)
}
})
.collect::<Vec<_>>();
dirs.reverse();
dirs
}
fn dedupe_skill_roots_by_path(roots: &mut Vec<SkillRoot>) {
let mut seen: HashSet<AbsolutePathBuf> = HashSet::new();
roots.retain(|root| seen.insert(root.path.clone()));
}
async fn canonicalize_for_skill_identity(
fs: &dyn ExecutorFileSystem,
path: &AbsolutePathBuf,
@@ -889,24 +602,6 @@ fn resolve_required_str(
resolve_str(Some(value), max_len, field)
}
#[cfg(test)]
pub(crate) async fn skill_roots_from_layer_stack(
fs: Arc<dyn ExecutorFileSystem>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
home_dir: Option<&AbsolutePathBuf>,
) -> Vec<SkillRoot> {
skill_roots_with_home_dir(
Some(fs),
config_layer_stack,
cwd,
home_dir,
Vec::new(),
Vec::new(),
)
.await
}
#[cfg(test)]
#[path = "loader_tests.rs"]
mod tests;

File diff suppressed because it is too large Load Diff

View File

@@ -1 +0,0 @@
pub(crate) use codex_skills::system_cache_root_dir;

View File

@@ -26,7 +26,6 @@ pub use codex_core_skills::injection::collect_explicit_skill_mentions;
pub use codex_core_skills::loader;
pub use codex_core_skills::model;
pub use codex_core_skills::remote;
pub use codex_core_skills::system;
pub use codex_skills::SkillMetadata;
pub use codex_skills::SkillPolicy;
pub use codex_skills_extension::HostSkillsLoadInput;

View File

@@ -26,12 +26,14 @@ codex-utils-absolute-path = { workspace = true }
codex-utils-path-uri = { workspace = true }
codex-utils-plugins = { workspace = true }
codex-utils-string = { workspace = true }
dirs = { workspace = true }
futures = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tokio = { workspace = true, features = ["sync", "time"] }
toml = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }
@@ -44,4 +46,3 @@ opentelemetry_sdk = { workspace = true }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
toml = { workspace = true }

View File

@@ -0,0 +1,302 @@
use std::collections::HashSet;
use std::io;
use std::sync::Arc;
use codex_config::ConfigLayerSource;
use codex_config::ConfigLayerStack;
use codex_config::default_project_root_markers;
use codex_config::merge_toml_values;
use codex_config::project_root_markers_from_config;
use codex_core_skills::loader::SkillRoot;
use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::LOCAL_FS;
use codex_protocol::protocol::SkillScope;
use codex_skills::system_cache_root_dir;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::PluginSkillRoot;
use codex_utils_plugins::SkillDiscoveryMode;
use dirs::home_dir;
use futures::StreamExt;
use toml::Value as TomlValue;
use crate::loader::HostSkillRoot;
const AGENTS_DIR_NAME: &str = ".agents";
const SKILLS_DIR_NAME: &str = "skills";
const MAX_CONCURRENT_ANCESTOR_PROBES: usize = 256;
pub(crate) async fn resolve_skill_roots(
repository_file_system: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
let home_dir =
home_dir().and_then(|path| AbsolutePathBuf::from_absolute_path_checked(path).ok());
resolve_skill_roots_with_home_dir(
repository_file_system,
config_layer_stack,
cwd,
home_dir.as_ref(),
plugin_skill_roots,
extra_skill_roots,
)
.await
}
async fn resolve_skill_roots_with_home_dir(
repository_file_system: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
home_dir: Option<&AbsolutePathBuf>,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
let mut roots =
roots_from_layer_stack(config_layer_stack, home_dir, repository_file_system.clone())
.into_iter()
.map(host_root_to_skill_root)
.collect::<Vec<_>>();
roots.extend(plugin_skill_roots.into_iter().map(|root| SkillRoot {
path: root.path,
scope: SkillScope::User,
file_system: Arc::clone(&LOCAL_FS),
plugin_identity: Some(root.plugin_identity),
plugin_namespace: Some(root.plugin_namespace),
plugin_root: Some(root.plugin_root),
discovery_mode: root.discovery_mode,
}));
roots.extend(
extra_skill_roots
.into_iter()
.map(|path| local_root(path, SkillScope::User))
.map(host_root_to_skill_root),
);
roots.extend(
repo_agents_skill_roots(repository_file_system, config_layer_stack, cwd)
.await
.into_iter()
.map(host_root_to_skill_root),
);
dedupe_skill_roots_by_path(&mut roots);
roots
}
fn roots_from_layer_stack(
config_layer_stack: &ConfigLayerStack,
home_dir: Option<&AbsolutePathBuf>,
repository_file_system: Option<Arc<dyn ExecutorFileSystem>>,
) -> Vec<HostSkillRoot> {
let mut roots = Vec::new();
for layer in config_layer_stack.all_layers_high_to_low() {
let Some(config_folder) = layer.config_folder() else {
continue;
};
match &layer.name {
ConfigLayerSource::Project { .. } => {
if let Some(repository_file_system) = &repository_file_system {
roots.push(HostSkillRoot {
path: config_folder.join(SKILLS_DIR_NAME),
scope: SkillScope::Repo,
file_system: Arc::clone(repository_file_system),
plugin_root: None,
});
}
}
ConfigLayerSource::User { .. } => {
// Deprecated user skills location (`$CODEX_HOME/skills`), kept for backward
// compatibility.
roots.push(local_root(
config_folder.join(SKILLS_DIR_NAME),
SkillScope::User,
));
if let Some(home_dir) = home_dir {
roots.push(local_root(
home_dir.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME),
SkillScope::User,
));
}
roots.push(local_root(
system_cache_root_dir(&config_folder),
SkillScope::System,
));
}
ConfigLayerSource::System { .. } => {
roots.push(local_root(
config_folder.join(SKILLS_DIR_NAME),
SkillScope::Admin,
));
}
ConfigLayerSource::Mdm { .. }
| ConfigLayerSource::EnterpriseManaged { .. }
| ConfigLayerSource::SessionFlags
| ConfigLayerSource::LegacyManagedConfigTomlFromFile { .. }
| ConfigLayerSource::LegacyManagedConfigTomlFromMdm => {}
}
}
roots
}
fn local_root(path: AbsolutePathBuf, scope: SkillScope) -> HostSkillRoot {
HostSkillRoot {
path,
scope,
file_system: Arc::clone(&LOCAL_FS),
plugin_root: None,
}
}
fn host_root_to_skill_root(root: HostSkillRoot) -> SkillRoot {
SkillRoot {
path: root.path,
scope: root.scope,
file_system: root.file_system,
plugin_identity: None,
plugin_namespace: None,
plugin_root: None,
discovery_mode: SkillDiscoveryMode::Recursive,
}
}
async fn repo_agents_skill_roots(
repository_file_system: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
) -> Vec<HostSkillRoot> {
let Some(repository_file_system) = repository_file_system else {
return Vec::new();
};
let project_root_markers = project_root_markers_from_stack(config_layer_stack);
let project_root =
find_project_root(repository_file_system.as_ref(), cwd, &project_root_markers).await;
let directories = dirs_between_project_root_and_cwd(cwd, &project_root);
let mut roots = Vec::new();
let mut results = futures::stream::iter(directories)
.map(|directory| {
let repository_file_system = Arc::clone(&repository_file_system);
async move {
let agents_skills = directory.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME);
let agents_skills_uri = PathUri::from_abs_path(&agents_skills);
let result = repository_file_system
.get_metadata(&agents_skills_uri, /*sandbox*/ None)
.await;
(agents_skills, result)
}
})
.buffered(MAX_CONCURRENT_ANCESTOR_PROBES);
while let Some((agents_skills, result)) = results.next().await {
match result {
Ok(metadata) if metadata.is_directory => roots.push(HostSkillRoot {
path: agents_skills,
scope: SkillScope::Repo,
file_system: Arc::clone(&repository_file_system),
plugin_root: None,
}),
Ok(_) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => {
tracing::warn!(
"failed to stat repo skills root {}: {error:#}",
agents_skills.display()
);
}
}
}
roots
}
fn project_root_markers_from_stack(config_layer_stack: &ConfigLayerStack) -> Vec<String> {
let mut merged = TomlValue::Table(toml::map::Map::new());
for layer in config_layer_stack.layers_low_to_high() {
if matches!(layer.name, ConfigLayerSource::Project { .. }) {
continue;
}
merge_toml_values(&mut merged, &layer.config);
}
match project_root_markers_from_config(&merged) {
Ok(Some(markers)) => markers,
Ok(None) => default_project_root_markers(),
Err(error) => {
tracing::warn!("invalid project_root_markers: {error}");
default_project_root_markers()
}
}
}
async fn find_project_root(
repository_file_system: &dyn ExecutorFileSystem,
cwd: &AbsolutePathBuf,
project_root_markers: &[String],
) -> AbsolutePathBuf {
if project_root_markers.is_empty() {
return cwd.clone();
}
let mut probes = Vec::new();
for ancestor in cwd.ancestors() {
for marker in project_root_markers {
probes.push((ancestor.clone(), ancestor.join(marker)));
}
}
let mut results = futures::stream::iter(probes)
.map(|(ancestor, marker_path)| async move {
let marker_path_uri = PathUri::from_abs_path(&marker_path);
let result = repository_file_system
.get_metadata(&marker_path_uri, /*sandbox*/ None)
.await;
(ancestor, marker_path, result)
})
.buffered(MAX_CONCURRENT_ANCESTOR_PROBES);
while let Some((ancestor, marker_path, result)) = results.next().await {
match result {
Ok(_) => return ancestor,
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => {
tracing::warn!(
"failed to stat project root marker {}: {error:#}",
marker_path.display()
);
}
}
}
cwd.clone()
}
fn dirs_between_project_root_and_cwd(
cwd: &AbsolutePathBuf,
project_root: &AbsolutePathBuf,
) -> Vec<AbsolutePathBuf> {
let mut directories = cwd
.ancestors()
.scan(false, |done, directory| {
if *done {
None
} else {
if &directory == project_root {
*done = true;
}
Some(directory)
}
})
.collect::<Vec<_>>();
directories.reverse();
directories
}
fn dedupe_skill_roots_by_path(roots: &mut Vec<SkillRoot>) {
let mut seen = HashSet::new();
roots.retain(|root| seen.insert(root.path.clone()));
}
#[cfg(test)]
#[path = "host_roots_tests.rs"]
mod tests;

View File

@@ -0,0 +1,668 @@
use std::fs;
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;
use codex_config::ConfigLayerEntry;
use codex_config::ConfigLayerSource;
use codex_config::ConfigLayerStack;
use codex_config::ConfigRequirementsToml;
use codex_core_skills::SkillMetadata;
use codex_core_skills::loader::MAX_CONCURRENT_ROOT_SCANS;
use codex_core_skills::loader::load_skills_from_roots;
use codex_exec_server::CopyOptions;
use codex_exec_server::CreateDirectoryOptions;
use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::ExecutorFileSystemFuture;
use codex_exec_server::FileMetadata;
use codex_exec_server::FileSystemReadStream;
use codex_exec_server::FileSystemSandboxContext;
use codex_exec_server::LOCAL_FS;
use codex_exec_server::ReadDirectoryEntry;
use codex_exec_server::RemoveOptions;
use codex_exec_server::WalkOptions;
use codex_exec_server::WalkOutcome;
use codex_protocol::protocol::SkillScope;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::PluginIdentity;
use codex_utils_plugins::PluginSkillRoot;
use codex_utils_plugins::SkillDiscoveryMode;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use tokio::sync::Notify;
use tokio::sync::Semaphore;
use super::repo_agents_skill_roots;
use super::resolve_skill_roots_with_home_dir;
use super::roots_from_layer_stack;
struct BlockingMetadataFileSystem {
inner: Arc<dyn ExecutorFileSystem>,
calls: Arc<BlockingMetadataCalls>,
}
struct BlockingMetadataCalls {
paths: Mutex<Vec<PathUri>>,
started: Notify,
release: Semaphore,
}
impl Default for BlockingMetadataCalls {
fn default() -> Self {
Self {
paths: Mutex::new(Vec::new()),
started: Notify::new(),
release: Semaphore::new(0),
}
}
}
impl ExecutorFileSystem for BlockingMetadataFileSystem {
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, PathUri> {
self.inner.canonicalize(path, sandbox)
}
fn read_file<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<u8>> {
self.inner.read_file(path, sandbox)
}
fn read_file_stream<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> {
self.inner.read_file_stream(path, sandbox)
}
fn write_file<'a>(
&'a self,
path: &'a PathUri,
contents: Vec<u8>,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.write_file(path, contents, sandbox)
}
fn create_directory<'a>(
&'a self,
path: &'a PathUri,
options: CreateDirectoryOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.create_directory(path, options, sandbox)
}
fn get_metadata<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileMetadata> {
let Ok(path_abs) = path.to_abs_path() else {
return self.inner.get_metadata(path, sandbox);
};
let repo_skill_root_suffix = Path::new(".agents").join("skills");
if !path_abs.ends_with(repo_skill_root_suffix) {
return self.inner.get_metadata(path, sandbox);
}
self.calls
.paths
.lock()
.expect("metadata paths lock")
.push(path.clone());
self.calls.started.notify_one();
Box::pin(async move {
self.calls
.release
.acquire()
.await
.expect("metadata release semaphore")
.forget();
self.inner.get_metadata(path, sandbox).await
})
}
fn read_directory<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<ReadDirectoryEntry>> {
self.inner.read_directory(path, sandbox)
}
fn walk<'a>(
&'a self,
path: &'a PathUri,
options: WalkOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, WalkOutcome> {
self.inner.walk(path, options, sandbox)
}
fn remove<'a>(
&'a self,
path: &'a PathUri,
options: RemoveOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.remove(path, options, sandbox)
}
fn copy<'a>(
&'a self,
source_path: &'a PathUri,
destination_path: &'a PathUri,
options: CopyOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner
.copy(source_path, destination_path, options, sandbox)
}
}
fn absolute(path: impl Into<std::path::PathBuf>) -> AbsolutePathBuf {
AbsolutePathBuf::try_from(path.into()).expect("absolute path")
}
fn empty_config() -> toml::Value {
toml::Value::Table(toml::map::Map::new())
}
fn stack(layers: Vec<ConfigLayerEntry>) -> ConfigLayerStack {
ConfigLayerStack::new(
layers,
Default::default(),
ConfigRequirementsToml::default(),
)
.expect("valid config stack")
}
fn user_layer(codex_home: &AbsolutePathBuf) -> ConfigLayerEntry {
ConfigLayerEntry::new(
ConfigLayerSource::User {
file: codex_home.join("config.toml"),
profile: None,
},
empty_config(),
)
}
fn project_layer(dot_codex_folder: &AbsolutePathBuf) -> ConfigLayerEntry {
ConfigLayerEntry::new(
ConfigLayerSource::Project {
dot_codex_folder: dot_codex_folder.clone(),
},
empty_config(),
)
}
fn write_skill(root: &AbsolutePathBuf, directory: &str, name: &str) -> AbsolutePathBuf {
let skill_dir = root.join(directory);
fs::create_dir_all(&skill_dir).expect("create skill directory");
let skill_path = skill_dir.join("SKILL.md");
fs::write(
&skill_path,
format!("---\nname: {name}\ndescription: {name} description\n---\n"),
)
.expect("write skill");
AbsolutePathBuf::from_absolute_path(
dunce::canonicalize(skill_path).expect("canonical skill path"),
)
.expect("absolute skill path")
}
fn expected_skill(path: AbsolutePathBuf, name: &str, scope: SkillScope) -> SkillMetadata {
SkillMetadata {
name: name.to_string(),
description: format!("{name} description"),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: path,
scope,
plugin_id: None,
remote_plugin_id: None,
}
}
#[test]
fn layer_roots_preserve_scope_precedence_and_disabled_projects() {
let temp_dir = TempDir::new().expect("temp dir");
let system_folder = absolute(temp_dir.path().join("etc/codex"));
let home_folder = absolute(temp_dir.path().join("home"));
let user_folder = home_folder.join("codex");
let project_folder = absolute(temp_dir.path().join("repo/.codex"));
let nested_project_folder = absolute(temp_dir.path().join("repo/nested/.codex"));
let config_stack = stack(vec![
ConfigLayerEntry::new(
ConfigLayerSource::System {
file: system_folder.join("config.toml"),
},
empty_config(),
),
user_layer(&user_folder),
ConfigLayerEntry::new_disabled(
ConfigLayerSource::Project {
dot_codex_folder: project_folder.clone(),
},
empty_config(),
"untrusted project",
),
project_layer(&nested_project_folder),
]);
let roots = roots_from_layer_stack(
&config_stack,
Some(&home_folder),
Some(Arc::clone(&LOCAL_FS)),
)
.into_iter()
.map(|root| (root.scope, root.path))
.collect::<Vec<_>>();
assert_eq!(
roots,
vec![
(SkillScope::Repo, nested_project_folder.join("skills")),
(SkillScope::Repo, project_folder.join("skills")),
(SkillScope::User, user_folder.join("skills")),
(SkillScope::User, home_folder.join(".agents/skills")),
(SkillScope::System, user_folder.join("skills/.system")),
(SkillScope::Admin, system_folder.join("skills")),
]
);
}
#[tokio::test]
async fn plugin_roots_preserve_plugin_resolution_metadata() {
let temp_dir = TempDir::new().expect("temp dir");
let cwd = absolute(temp_dir.path().join("workspace"));
let plugin_root = absolute(temp_dir.path().join("plugins/example"));
let skills_root = plugin_root.join("skills");
let plugin_identity = PluginIdentity {
plugin_id: "example@test".to_string(),
remote_plugin_id: Some("plugins~Plugin_example".to_string()),
};
let plugin_namespace = "example".to_string();
let roots = resolve_skill_roots_with_home_dir(
/*repository_file_system*/ None,
&stack(Vec::new()),
&cwd,
/*home_dir*/ None,
vec![PluginSkillRoot {
path: skills_root.clone(),
plugin_identity: plugin_identity.clone(),
plugin_namespace: plugin_namespace.clone(),
plugin_root: plugin_root.clone(),
discovery_mode: SkillDiscoveryMode::DirectChildren,
}],
Vec::new(),
)
.await;
assert_eq!(roots.len(), 1);
let root = &roots[0];
assert_eq!(
(
root.path.clone(),
root.scope,
root.plugin_identity.clone(),
root.plugin_namespace.clone(),
root.plugin_root.clone(),
root.discovery_mode,
),
(
skills_root,
SkillScope::User,
Some(plugin_identity),
Some(plugin_namespace),
Some(plugin_root),
SkillDiscoveryMode::DirectChildren,
)
);
assert!(Arc::ptr_eq(&root.file_system, &LOCAL_FS));
}
#[tokio::test]
async fn unique_extra_root_loads_as_recursive_user_root() {
let temp_dir = TempDir::new().expect("temp dir");
let cwd = absolute(temp_dir.path().join("workspace"));
let extra_root = absolute(temp_dir.path().join("runtime-skills"));
let skill_path = write_skill(&extra_root, "runtime", "runtime-skill");
let roots = resolve_skill_roots_with_home_dir(
/*repository_file_system*/ None,
&stack(Vec::new()),
&cwd,
/*home_dir*/ None,
Vec::new(),
vec![extra_root.clone()],
)
.await;
assert_eq!(roots.len(), 1);
let root = &roots[0];
assert_eq!(
(
root.path.clone(),
root.scope,
root.plugin_identity.clone(),
root.plugin_namespace.clone(),
root.plugin_root.clone(),
root.discovery_mode,
),
(
extra_root,
SkillScope::User,
None,
None,
None,
SkillDiscoveryMode::Recursive,
)
);
assert!(Arc::ptr_eq(&root.file_system, &LOCAL_FS));
let outcome = load_skills_from_roots(
roots,
/*plugin_skill_snapshots*/ None,
Arc::new(Semaphore::new(MAX_CONCURRENT_ROOT_SCANS)),
)
.await;
assert!(outcome.errors.is_empty());
assert_eq!(
outcome.skills,
vec![expected_skill(
skill_path,
"runtime-skill",
SkillScope::User,
)]
);
}
#[tokio::test]
async fn repo_ancestry_without_project_marker_does_not_walk_parents() {
let temp_dir = TempDir::new().expect("temp dir");
let outer = absolute(temp_dir.path().join("outer"));
let cwd = outer.join("nested/inner");
fs::create_dir_all(outer.join(".agents/skills")).expect("create outer skills");
fs::create_dir_all(cwd.join(".agents/skills")).expect("create cwd skills");
let roots = repo_agents_skill_roots(Some(Arc::clone(&LOCAL_FS)), &stack(Vec::new()), &cwd)
.await
.into_iter()
.map(|root| root.path)
.collect::<Vec<_>>();
assert_eq!(roots, vec![cwd.join(".agents/skills")]);
}
#[tokio::test]
async fn repo_ancestry_stops_at_project_root_and_preserves_root_to_cwd_order() {
let temp_dir = TempDir::new().expect("temp dir");
let outer = absolute(temp_dir.path().join("outer"));
let repository = outer.join("repo");
let nested = repository.join("nested/inner");
fs::create_dir_all(&nested).expect("create nested cwd");
fs::write(repository.join(".git"), "gitdir: fake\n").expect("write git marker");
fs::create_dir_all(outer.join(".agents/skills")).expect("create outer skills");
fs::create_dir_all(repository.join(".agents/skills")).expect("create repo skills");
fs::create_dir_all(repository.join("nested/.agents/skills")).expect("create nested skills");
let config_stack = stack(Vec::new());
let roots = repo_agents_skill_roots(Some(Arc::clone(&LOCAL_FS)), &config_stack, &nested)
.await
.into_iter()
.map(|root| root.path)
.collect::<Vec<_>>();
assert_eq!(
roots,
vec![
repository.join(".agents/skills"),
repository.join("nested/.agents/skills"),
]
);
}
#[tokio::test]
async fn resolved_project_layer_loads_skill_without_git_marker() {
let temp_dir = TempDir::new().expect("temp dir");
let workspace = absolute(temp_dir.path().join("workspace"));
let dot_codex = workspace.join(".codex");
let skill_root = dot_codex.join("skills");
fs::create_dir_all(&workspace).expect("create workspace");
let skill_path = write_skill(&skill_root, "local", "local-skill");
let config_stack = stack(vec![project_layer(&dot_codex)]);
let roots = resolve_skill_roots_with_home_dir(
Some(Arc::clone(&LOCAL_FS)),
&config_stack,
&workspace,
/*home_dir*/ None,
Vec::new(),
Vec::new(),
)
.await;
let outcome = load_skills_from_roots(
roots,
/*plugin_skill_snapshots*/ None,
Arc::new(Semaphore::new(MAX_CONCURRENT_ROOT_SCANS)),
)
.await;
assert!(outcome.errors.is_empty());
assert_eq!(
outcome.skills,
vec![expected_skill(skill_path, "local-skill", SkillScope::Repo)]
);
}
#[tokio::test]
async fn resolved_project_layer_loads_skill_when_cwd_is_file() {
let temp_dir = TempDir::new().expect("temp dir");
let repository = absolute(temp_dir.path().join("repo"));
let dot_codex = repository.join(".codex");
let skill_root = dot_codex.join("skills");
fs::create_dir_all(&repository).expect("create repository");
fs::write(repository.join(".git"), "gitdir: fake\n").expect("write git marker");
let cwd = repository.join("some-file.txt");
fs::write(&cwd, "contents").expect("write cwd file");
let skill_path = write_skill(&skill_root, "repo", "repo-skill");
let config_stack = stack(vec![project_layer(&dot_codex)]);
let roots = resolve_skill_roots_with_home_dir(
Some(Arc::clone(&LOCAL_FS)),
&config_stack,
&cwd,
/*home_dir*/ None,
Vec::new(),
Vec::new(),
)
.await;
let outcome = load_skills_from_roots(
roots,
/*plugin_skill_snapshots*/ None,
Arc::new(Semaphore::new(MAX_CONCURRENT_ROOT_SCANS)),
)
.await;
assert!(outcome.errors.is_empty());
assert_eq!(
outcome.skills,
vec![expected_skill(skill_path, "repo-skill", SkillScope::Repo)]
);
}
#[tokio::test]
async fn repo_ancestry_limits_concurrent_probes_and_preserves_order() {
const CONCURRENCY_LIMIT: usize = 256;
let temp_dir = TempDir::new().expect("temp dir");
let repository = absolute(temp_dir.path().join("repo"));
fs::create_dir_all(&repository).expect("create repository");
fs::write(repository.join(".git"), "gitdir: fake\n").expect("write git marker");
let mut directories = vec![repository.clone()];
let mut cwd = repository;
for _ in 0..CONCURRENCY_LIMIT {
cwd = cwd.join("d");
directories.push(cwd.clone());
}
fs::create_dir_all(&cwd).expect("create nested cwd");
let expected_roots = [0, CONCURRENCY_LIMIT / 2, CONCURRENCY_LIMIT].map(|index| {
let path = directories[index].join(".agents/skills");
fs::create_dir_all(&path).expect("create repo skill root");
path
});
let expected_probes = directories
.iter()
.map(|directory| PathUri::from_abs_path(&directory.join(".agents/skills")))
.collect::<Vec<_>>();
let calls = Arc::new(BlockingMetadataCalls::default());
let file_system: Arc<dyn ExecutorFileSystem> = Arc::new(BlockingMetadataFileSystem {
inner: Arc::clone(&LOCAL_FS),
calls: Arc::clone(&calls),
});
let assertions = async {
tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 5), async {
loop {
let started = calls.started.notified();
if calls.paths.lock().expect("metadata paths lock").len() >= CONCURRENCY_LIMIT {
break;
}
started.await;
}
})
.await
.expect("initial repo skill root window should start");
assert_eq!(
calls.paths.lock().expect("metadata paths lock").as_slice(),
&expected_probes[..CONCURRENCY_LIMIT]
);
calls.release.add_permits(/*n*/ 1);
tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 5), async {
loop {
let started = calls.started.notified();
if calls.paths.lock().expect("metadata paths lock").len() > CONCURRENCY_LIMIT {
break;
}
started.await;
}
})
.await
.expect("next repo skill root probe should start");
assert_eq!(
calls.paths.lock().expect("metadata paths lock").as_slice(),
expected_probes.as_slice()
);
calls.release.add_permits(expected_probes.len());
};
let config_stack = stack(Vec::new());
let (roots, ()) = tokio::join!(
repo_agents_skill_roots(Some(file_system), &config_stack, &cwd),
assertions,
);
assert_eq!(
roots.into_iter().map(|root| root.path).collect::<Vec<_>>(),
expected_roots
);
}
#[tokio::test]
async fn resolved_config_and_repo_roots_preserve_order_and_dedupe_paths_not_names() {
let temp_dir = TempDir::new().expect("temp dir");
let home_folder = absolute(temp_dir.path().join("home"));
let codex_home = home_folder.join("codex");
let system_folder = absolute(temp_dir.path().join("etc/codex"));
let repository = absolute(temp_dir.path().join("repo"));
let cwd = repository.join("nested/inner");
fs::create_dir_all(&cwd).expect("create cwd");
fs::write(repository.join(".git"), "gitdir: fake\n").expect("write git marker");
let project_dot_codex = repository.join(".codex");
let nested_project_dot_codex = repository.join("nested/.codex");
let user_skills = codex_home.join("skills");
let root_project_skill = write_skill(
&project_dot_codex.join("skills"),
"root-duplicate",
"duplicate-skill",
);
let nested_project_skill = write_skill(
&nested_project_dot_codex.join("skills"),
"nested-duplicate",
"duplicate-skill",
);
let user_skill = write_skill(&user_skills, "user-duplicate", "duplicate-skill");
let home_skill = write_skill(&home_folder.join(".agents/skills"), "home", "home-skill");
let system_skill = write_skill(&codex_home.join("skills/.system"), "system", "system-skill");
let admin_skill = write_skill(&system_folder.join("skills"), "admin", "admin-skill");
let repo_agent_skill = write_skill(
&repository.join(".agents/skills"),
"repo-agent",
"repo-agent-skill",
);
let nested_agent_skill = write_skill(
&repository.join("nested/.agents/skills"),
"nested-agent",
"nested-agent-skill",
);
let config_stack = stack(vec![
ConfigLayerEntry::new(
ConfigLayerSource::System {
file: system_folder.join("config.toml"),
},
empty_config(),
),
user_layer(&codex_home),
project_layer(&project_dot_codex),
project_layer(&nested_project_dot_codex),
]);
let roots = resolve_skill_roots_with_home_dir(
Some(Arc::clone(&LOCAL_FS)),
&config_stack,
&cwd,
Some(&home_folder),
Vec::new(),
vec![user_skills],
)
.await;
assert_eq!(roots.len(), 8);
let outcome = load_skills_from_roots(
roots,
/*plugin_skill_snapshots*/ None,
Arc::new(Semaphore::new(MAX_CONCURRENT_ROOT_SCANS)),
)
.await;
assert!(outcome.errors.is_empty());
assert_eq!(
outcome.skills,
vec![
expected_skill(root_project_skill, "duplicate-skill", SkillScope::Repo),
expected_skill(nested_project_skill, "duplicate-skill", SkillScope::Repo),
expected_skill(nested_agent_skill, "nested-agent-skill", SkillScope::Repo),
expected_skill(repo_agent_skill, "repo-agent-skill", SkillScope::Repo),
expected_skill(user_skill, "duplicate-skill", SkillScope::User),
expected_skill(home_skill, "home-skill", SkillScope::User),
expected_skill(system_skill, "system-skill", SkillScope::System),
expected_skill(admin_skill, "admin-skill", SkillScope::Admin),
]
);
}

View File

@@ -24,10 +24,11 @@ use codex_core_skills::config_rules::skill_config_rules_from_stack;
use codex_core_skills::loader::MAX_CONCURRENT_ROOT_SCANS;
use codex_core_skills::loader::SkillRoot;
use codex_core_skills::loader::load_skills_from_roots;
use codex_core_skills::loader::skill_roots;
use codex_skills::install_system_skills;
use codex_skills::system_cache_root_dir;
use crate::host_roots::resolve_skill_roots;
#[derive(Debug, Clone)]
pub struct HostSkillsLoadInput {
pub cwd: AbsolutePathBuf,
@@ -156,7 +157,7 @@ impl HostSkillsService {
input: &HostSkillsLoadInput,
fs: Option<Arc<dyn ExecutorFileSystem>>,
) -> Vec<SkillRoot> {
let mut roots = skill_roots(
let mut roots = resolve_skill_roots(
fs,
&input.config_layer_stack,
&input.cwd,
@@ -184,7 +185,7 @@ impl HostSkillsService {
return snapshot;
}
let mut roots = skill_roots(
let mut roots = resolve_skill_roots(
fs.clone(),
&input.config_layer_stack,
&input.cwd,

View File

@@ -4,6 +4,7 @@ mod config;
mod dynamic_skill_selector;
mod extension;
mod fragments;
mod host_roots;
mod host_service;
// Host loading is staged before its crate-internal runtime caller in this PR stack.
#[allow(dead_code)]

View File

@@ -6,6 +6,7 @@ mod namespace;
pub(crate) use environment::load_environment_skills_from_discovery;
pub(crate) use environment::load_environment_skills_from_root;
pub(crate) use host::HostSkillRoot;
pub(super) const SKILLS_FILENAME: &str = "SKILL.md";
pub(super) const SKILLS_METADATA_DIR: &str = "agents";