perf(skills): overlap missing-root preflight

This commit is contained in:
Adam Perry
2026-07-10 06:14:41 +00:00
parent e9df803d23
commit 80abbcdf84
5 changed files with 314 additions and 14 deletions
+129 -4
View File
@@ -40,6 +40,7 @@ use discovery::SkillMetadataDiscovery;
use discovery::discover_skills;
use futures::FutureExt;
use futures::StreamExt;
use futures::future::BoxFuture;
use namespace::SkillNamespaceResolver;
use serde::Deserialize;
use std::collections::HashSet;
@@ -190,7 +191,27 @@ pub struct SkillRoot {
pub plugin_root: Option<AbsolutePathBuf>,
}
#[derive(Default)]
pub(crate) struct SkillRootPreflight {
pub(crate) absent_paths: HashSet<AbsolutePathBuf>,
}
pub(crate) struct SkillRootResolution {
pub(crate) roots: Vec<SkillRoot>,
pub(crate) preflight: SkillRootPreflight,
}
#[derive(Clone, Copy)]
enum SkillRootPreflightMode {
Disabled,
MissingExecutorRoots,
}
impl SkillRoot {
pub(crate) fn is_host_owned(&self) -> bool {
Arc::ptr_eq(&self.file_system, &LOCAL_FS)
}
pub(crate) fn is_agents_skills_root(&self) -> bool {
self.path.file_name() == Some(std::ffi::OsStr::new(SKILLS_DIR_NAME))
&& self.path.parent().is_some_and(|parent| {
@@ -211,6 +232,19 @@ where
.await
}
pub(crate) fn load_skills_from_roots_with_preflight<'a>(
roots: Vec<SkillRoot>,
plugin_skill_snapshots: Option<&'a crate::PluginSkillSnapshots>,
preflight_absent_paths: &'a HashSet<AbsolutePathBuf>,
) -> BoxFuture<'a, SkillLoadOutcome> {
crate::root_loader::load_and_merge_skill_roots_with_preflight(
roots,
plugin_skill_snapshots,
preflight_absent_paths,
)
.boxed()
}
#[derive(Clone)]
pub(crate) struct SkillRootSnapshot {
pub(crate) root: AbsolutePathBuf,
@@ -219,7 +253,7 @@ pub(crate) struct SkillRootSnapshot {
pub(crate) file_system: Arc<dyn ExecutorFileSystem>,
}
pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot {
pub(crate) async fn load_skill_root(root: SkillRoot, preflight_absent: bool) -> SkillRootSnapshot {
let SkillRoot {
path,
scope,
@@ -228,6 +262,16 @@ pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot {
plugin_namespace,
plugin_root,
} = root;
// The hint comes from an exact metadata probe performed while repo roots were discovered. It
// avoids repeating canonicalize and walk calls for a root known to be absent at that point.
if preflight_absent {
return SkillRootSnapshot {
root: path,
skills: Vec::new(),
errors: Vec::new(),
file_system,
};
}
let root = canonicalize_for_skill_identity(file_system.as_ref(), &path).await;
let mut outcome = SkillLoadOutcome::default();
load_skills_under_root(
@@ -255,6 +299,44 @@ pub(crate) async fn skill_roots(
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
skill_root_resolution_with_mode(
fs,
config_layer_stack,
cwd,
plugin_skill_roots,
extra_skill_roots,
SkillRootPreflightMode::Disabled,
)
.await
.roots
}
pub(crate) async fn skill_root_resolution(
fs: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> SkillRootResolution {
skill_root_resolution_with_mode(
fs,
config_layer_stack,
cwd,
plugin_skill_roots,
extra_skill_roots,
SkillRootPreflightMode::MissingExecutorRoots,
)
.await
}
async fn skill_root_resolution_with_mode(
fs: Option<Arc<dyn ExecutorFileSystem>>,
config_layer_stack: &ConfigLayerStack,
cwd: &AbsolutePathBuf,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
preflight_mode: SkillRootPreflightMode,
) -> SkillRootResolution {
let home_dir =
home_dir().and_then(|path| AbsolutePathBuf::from_absolute_path_checked(path).ok());
skill_roots_with_home_dir(
@@ -264,6 +346,7 @@ pub(crate) async fn skill_roots(
home_dir.as_ref(),
plugin_skill_roots,
extra_skill_roots,
preflight_mode,
)
.await
}
@@ -275,7 +358,8 @@ async fn skill_roots_with_home_dir(
home_dir: Option<&AbsolutePathBuf>,
plugin_skill_roots: Vec<PluginSkillRoot>,
extra_skill_roots: Vec<AbsolutePathBuf>,
) -> Vec<SkillRoot> {
preflight_mode: SkillRootPreflightMode,
) -> SkillRootResolution {
let mut roots = skill_roots_from_layer_stack_inner(config_layer_stack, home_dir, fs.clone());
roots.extend(plugin_skill_roots.into_iter().map(|root| SkillRoot {
path: root.path,
@@ -293,9 +377,48 @@ async fn skill_roots_with_home_dir(
plugin_namespace: None,
plugin_root: None,
}));
roots.extend(repo_agents_skill_roots(fs, config_layer_stack, cwd).await);
dedupe_skill_roots_by_path(&mut roots);
roots
let preflight_paths = match preflight_mode {
SkillRootPreflightMode::Disabled => Vec::new(),
SkillRootPreflightMode::MissingExecutorRoots => roots
.iter()
.filter(|root| !root.is_host_owned())
.map(|root| root.path.clone())
.collect(),
};
let repo_fs = fs.clone();
let (repo_roots, preflight) = tokio::join!(
repo_agents_skill_roots(repo_fs, config_layer_stack, cwd),
preflight_skill_roots(fs, preflight_paths),
);
roots.extend(repo_roots);
dedupe_skill_roots_by_path(&mut roots);
SkillRootResolution { roots, preflight }
}
async fn preflight_skill_roots(
fs: Option<Arc<dyn ExecutorFileSystem>>,
paths: Vec<AbsolutePathBuf>,
) -> SkillRootPreflight {
let mut preflight = SkillRootPreflight::default();
let Some(fs) = fs else {
return preflight;
};
for paths in paths.chunks(FS_GET_METADATA_BATCH_MAX_PATHS) {
let path_uris = paths.iter().map(PathUri::from_abs_path).collect::<Vec<_>>();
let Ok(results) = fs.get_metadata_batch(&path_uris, /*sandbox*/ None).await else {
continue;
};
if results.len() != paths.len() {
continue;
}
for (path, result) in paths.iter().zip(results) {
if result.is_err_and(|err| err.kind() == io::ErrorKind::NotFound) {
preflight.absent_paths.insert(path.clone());
}
}
}
preflight
}
fn skill_roots_from_layer_stack_inner(
@@ -1205,8 +1328,10 @@ pub(crate) async fn skill_roots_from_layer_stack(
home_dir,
Vec::new(),
Vec::new(),
SkillRootPreflightMode::Disabled,
)
.await
.roots
}
#[cfg(test)]
+22
View File
@@ -145,6 +145,28 @@ fn normalized(path: &Path) -> AbsolutePathBuf {
.abs()
}
#[tokio::test]
async fn preflight_only_short_circuits_exact_not_found_results() {
let temp = tempfile::tempdir().expect("tempdir");
let missing = temp.path().join("missing").abs();
let existing_directory = temp.path().join("directory");
fs::create_dir(&existing_directory).expect("create directory");
let existing_file = temp.path().join("file");
fs::write(&existing_file, "contents").expect("write file");
let preflight = preflight_skill_roots(
Some(Arc::clone(&LOCAL_FS)),
vec![
missing.clone(),
existing_directory.abs(),
existing_file.abs(),
],
)
.await;
assert_eq!(preflight.absent_paths, HashSet::from([missing]));
}
#[tokio::test]
async fn skill_roots_from_layer_stack_maps_user_to_user_and_system_cache_and_system_to_admin()
-> anyhow::Result<()> {
+19 -1
View File
@@ -64,6 +64,23 @@ pub(crate) async fn load_and_merge_skill_roots<I>(
roots: I,
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
) -> SkillLoadOutcome
where
I: IntoIterator<Item = SkillRoot>,
{
let preflight_absent_paths = HashSet::new();
load_and_merge_skill_roots_with_preflight(
roots,
plugin_skill_snapshots,
&preflight_absent_paths,
)
.await
}
pub(crate) async fn load_and_merge_skill_roots_with_preflight<I>(
roots: I,
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
preflight_absent_paths: &HashSet<codex_utils_absolute_path::AbsolutePathBuf>,
) -> SkillLoadOutcome
where
I: IntoIterator<Item = SkillRoot>,
{
@@ -147,7 +164,8 @@ where
snapshot,
},
None => {
let snapshot = load_skill_root(root).await;
let preflight_absent = preflight_absent_paths.contains(&root.path);
let snapshot = load_skill_root(root, preflight_absent).await;
if let Some(plugin_skill_snapshots) = plugin_skill_snapshots
&& let Some(cache_key) = cache_key
{
@@ -31,6 +31,7 @@ use tokio::sync::Semaphore;
use super::MAX_CONCURRENT_ROOT_SCANS;
use super::PluginSkillSnapshots;
use super::load_and_merge_skill_roots;
use super::load_and_merge_skill_roots_with_preflight;
use crate::SkillError;
use crate::loader::SkillRoot;
@@ -40,6 +41,7 @@ struct ControlledFileSystem {
inner: Arc<dyn ExecutorFileSystem>,
walk_gate: Option<Arc<Semaphore>>,
walk_paths: Mutex<Vec<PathUri>>,
canonicalize_paths: Mutex<Vec<PathUri>>,
walks_started: AtomicUsize,
walk_started: Notify,
blocked_read_root: Option<PathUri>,
@@ -55,6 +57,7 @@ impl ControlledFileSystem {
inner,
walk_gate: None,
walk_paths: Mutex::new(Vec::new()),
canonicalize_paths: Mutex::new(Vec::new()),
walks_started: AtomicUsize::new(/*v*/ 0),
walk_started: Notify::new(),
blocked_read_root: None,
@@ -88,6 +91,13 @@ impl ControlledFileSystem {
self.walks_started.load(Ordering::Acquire)
}
fn canonicalize_paths(&self) -> Vec<PathUri> {
self.canonicalize_paths
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
fn walk_paths(&self) -> Vec<PathUri> {
self.walk_paths
.lock()
@@ -130,6 +140,10 @@ impl ExecutorFileSystem for ControlledFileSystem {
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, PathUri> {
self.canonicalize_paths
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(path.clone());
self.inner.canonicalize(path, sandbox)
}
@@ -296,6 +310,94 @@ fn plugin_root(
}
}
#[tokio::test]
async fn missing_root_preflight_skips_io_and_preserves_merge_order() {
let temp = tempfile::tempdir().expect("tempdir");
let first_root = temp.path().join("first");
let missing_root = temp.path().join("missing");
let last_root = temp.path().join("last");
let first_skill = write_skill(&first_root, "broken", "first missing frontmatter");
let last_skill = write_skill(&last_root, "broken", "last missing frontmatter");
let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS)));
let root_file_system: Arc<dyn ExecutorFileSystem> = file_system.clone();
let outcome = load_and_merge_skill_roots_with_preflight(
[
plain_root(&first_root, SkillScope::Repo, Arc::clone(&root_file_system)),
plain_root(
&missing_root,
SkillScope::Repo,
Arc::clone(&root_file_system),
),
plain_root(&last_root, SkillScope::User, root_file_system),
],
/*plugin_skill_snapshots*/ None,
&HashSet::from([missing_root.abs()]),
)
.await;
assert!(
!file_system
.canonicalize_paths()
.contains(&PathUri::from_abs_path(&missing_root.abs()))
);
assert_eq!(file_system.walks_started(), 2);
assert_eq!(outcome.skills, Vec::new());
assert_eq!(
outcome.errors,
vec![
SkillError {
path: dunce::canonicalize(first_skill)
.expect("canonical first skill")
.abs(),
message: "missing YAML frontmatter delimited by ---".to_string(),
},
SkillError {
path: dunce::canonicalize(last_skill)
.expect("canonical last skill")
.abs(),
message: "missing YAML frontmatter delimited by ---".to_string(),
},
]
);
}
#[tokio::test]
async fn plugin_snapshot_hit_precedes_preflight_hint() {
let temp = tempfile::tempdir().expect("tempdir");
let skills_root = temp.path().join("plugin").join("skills");
write_skill(
&skills_root,
"demo",
"---\nname: demo\ndescription: demo skill\n---\n",
);
let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS)));
let root_file_system: Arc<dyn ExecutorFileSystem> = file_system.clone();
let snapshots = PluginSkillSnapshots::for_plugin_load();
let first = load_and_merge_skill_roots(
[plugin_root(
&skills_root,
temp.path(),
Arc::clone(&root_file_system),
)],
Some(&snapshots),
)
.await;
assert_eq!(file_system.walks_started(), 1);
let hinted = load_and_merge_skill_roots_with_preflight(
[plugin_root(&skills_root, temp.path(), root_file_system)],
Some(&snapshots),
&HashSet::from([skills_root.abs()]),
)
.await;
assert_eq!(file_system.walks_started(), 1);
assert_eq!(hinted.skills, first.skills);
assert_eq!(hinted.errors, first.errors);
assert_eq!(hinted.skills.len(), 1);
}
#[tokio::test]
async fn loads_roots_unordered_but_merges_errors_in_input_order() {
const ROOT_COUNT: usize = MAX_CONCURRENT_ROOT_SCANS + 1;
+42 -9
View File
@@ -21,7 +21,8 @@ use crate::config_rules::SkillConfigRules;
use crate::config_rules::resolve_disabled_skill_paths;
use crate::config_rules::skill_config_rules_from_stack;
use crate::loader::SkillRoot;
use crate::loader::load_skills_from_roots;
use crate::loader::load_skills_from_roots_with_preflight;
use crate::loader::skill_root_resolution;
use crate::loader::skill_roots;
use crate::system::install_system_skills;
use crate::system::uninstall_system_skills;
@@ -128,7 +129,20 @@ impl SkillsService {
input: &SkillsLoadInput,
fs: Option<Arc<dyn ExecutorFileSystem>>,
) -> HostSkillsSnapshot {
let roots = self.skill_roots_for_config(input, fs).await;
let mut resolution = skill_root_resolution(
fs,
&input.config_layer_stack,
&input.cwd,
input.effective_skill_roots.clone(),
self.extra_roots(),
)
.await;
if !input.bundled_skills_enabled {
resolution
.roots
.retain(|root| root.scope != SkillScope::System);
}
let roots = resolution.roots;
let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack);
let cache_key = config_skills_cache_key(&roots, &skill_config_rules);
if let Some(snapshot) = self.cached_snapshot_for_config(&cache_key) {
@@ -136,8 +150,13 @@ impl SkillsService {
}
let snapshot = HostSkillsSnapshot::new(Arc::new(
self.build_skill_outcome(input, roots, &skill_config_rules)
.await,
self.build_skill_outcome(
input,
roots,
&skill_config_rules,
&resolution.preflight.absent_paths,
)
.await,
));
let mut cache = self
.cache_by_config
@@ -180,7 +199,7 @@ impl SkillsService {
return snapshot;
}
let mut roots = skill_roots(
let mut resolution = skill_root_resolution(
fs.clone(),
&input.config_layer_stack,
&input.cwd,
@@ -189,12 +208,20 @@ impl SkillsService {
)
.await;
if !bundled_skills_enabled_from_stack(&input.config_layer_stack) {
roots.retain(|root| root.scope != SkillScope::System);
resolution
.roots
.retain(|root| root.scope != SkillScope::System);
}
let roots = resolution.roots;
let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack);
let snapshot = HostSkillsSnapshot::new(Arc::new(
self.build_skill_outcome(input, roots, &skill_config_rules)
.await,
self.build_skill_outcome(
input,
roots,
&skill_config_rules,
&resolution.preflight.absent_paths,
)
.await,
));
if use_cwd_cache {
let mut cache = self
@@ -212,8 +239,14 @@ impl SkillsService {
input: &SkillsLoadInput,
roots: Vec<SkillRoot>,
skill_config_rules: &SkillConfigRules,
preflight_absent_paths: &HashSet<AbsolutePathBuf>,
) -> SkillLoadOutcome {
let outcome = load_skills_from_roots(roots, input.plugin_skill_snapshots.as_ref()).await;
let outcome = load_skills_from_roots_with_preflight(
roots,
input.plugin_skill_snapshots.as_ref(),
preflight_absent_paths,
)
.await;
let outcome =
crate::filter_skill_load_outcome_for_product(outcome, self.restriction_product);
let disabled_paths = resolve_disabled_skill_paths(&outcome.skills, skill_config_rules);