From 80abbcdf843ba593850c32744df4366fad3191ff Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Fri, 10 Jul 2026 06:13:46 +0000 Subject: [PATCH] perf(skills): overlap missing-root preflight --- codex-rs/core-skills/src/loader.rs | 133 +++++++++++++++++- codex-rs/core-skills/src/loader_tests.rs | 22 +++ codex-rs/core-skills/src/root_loader.rs | 20 ++- codex-rs/core-skills/src/root_loader_tests.rs | 102 ++++++++++++++ codex-rs/core-skills/src/service.rs | 51 +++++-- 5 files changed, 314 insertions(+), 14 deletions(-) diff --git a/codex-rs/core-skills/src/loader.rs b/codex-rs/core-skills/src/loader.rs index c0cc2eb5eb..02bed04494 100644 --- a/codex-rs/core-skills/src/loader.rs +++ b/codex-rs/core-skills/src/loader.rs @@ -40,6 +40,7 @@ use discovery::SkillMetadataDiscovery; use discovery::discover_skills; use futures::FutureExt; use futures::StreamExt; +use futures::future::BoxFuture; use namespace::SkillNamespaceResolver; use serde::Deserialize; use std::collections::HashSet; @@ -190,7 +191,27 @@ pub struct SkillRoot { pub plugin_root: Option, } +#[derive(Default)] +pub(crate) struct SkillRootPreflight { + pub(crate) absent_paths: HashSet, +} + +pub(crate) struct SkillRootResolution { + pub(crate) roots: Vec, + pub(crate) preflight: SkillRootPreflight, +} + +#[derive(Clone, Copy)] +enum SkillRootPreflightMode { + Disabled, + MissingExecutorRoots, +} + impl SkillRoot { + pub(crate) fn is_host_owned(&self) -> bool { + Arc::ptr_eq(&self.file_system, &LOCAL_FS) + } + pub(crate) fn is_agents_skills_root(&self) -> bool { self.path.file_name() == Some(std::ffi::OsStr::new(SKILLS_DIR_NAME)) && self.path.parent().is_some_and(|parent| { @@ -211,6 +232,19 @@ where .await } +pub(crate) fn load_skills_from_roots_with_preflight<'a>( + roots: Vec, + plugin_skill_snapshots: Option<&'a crate::PluginSkillSnapshots>, + preflight_absent_paths: &'a HashSet, +) -> BoxFuture<'a, SkillLoadOutcome> { + crate::root_loader::load_and_merge_skill_roots_with_preflight( + roots, + plugin_skill_snapshots, + preflight_absent_paths, + ) + .boxed() +} + #[derive(Clone)] pub(crate) struct SkillRootSnapshot { pub(crate) root: AbsolutePathBuf, @@ -219,7 +253,7 @@ pub(crate) struct SkillRootSnapshot { pub(crate) file_system: Arc, } -pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot { +pub(crate) async fn load_skill_root(root: SkillRoot, preflight_absent: bool) -> SkillRootSnapshot { let SkillRoot { path, scope, @@ -228,6 +262,16 @@ pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot { plugin_namespace, plugin_root, } = root; + // The hint comes from an exact metadata probe performed while repo roots were discovered. It + // avoids repeating canonicalize and walk calls for a root known to be absent at that point. + if preflight_absent { + return SkillRootSnapshot { + root: path, + skills: Vec::new(), + errors: Vec::new(), + file_system, + }; + } let root = canonicalize_for_skill_identity(file_system.as_ref(), &path).await; let mut outcome = SkillLoadOutcome::default(); load_skills_under_root( @@ -255,6 +299,44 @@ pub(crate) async fn skill_roots( plugin_skill_roots: Vec, extra_skill_roots: Vec, ) -> Vec { + skill_root_resolution_with_mode( + fs, + config_layer_stack, + cwd, + plugin_skill_roots, + extra_skill_roots, + SkillRootPreflightMode::Disabled, + ) + .await + .roots +} + +pub(crate) async fn skill_root_resolution( + fs: Option>, + config_layer_stack: &ConfigLayerStack, + cwd: &AbsolutePathBuf, + plugin_skill_roots: Vec, + extra_skill_roots: Vec, +) -> SkillRootResolution { + skill_root_resolution_with_mode( + fs, + config_layer_stack, + cwd, + plugin_skill_roots, + extra_skill_roots, + SkillRootPreflightMode::MissingExecutorRoots, + ) + .await +} + +async fn skill_root_resolution_with_mode( + fs: Option>, + config_layer_stack: &ConfigLayerStack, + cwd: &AbsolutePathBuf, + plugin_skill_roots: Vec, + extra_skill_roots: Vec, + preflight_mode: SkillRootPreflightMode, +) -> SkillRootResolution { let home_dir = home_dir().and_then(|path| AbsolutePathBuf::from_absolute_path_checked(path).ok()); skill_roots_with_home_dir( @@ -264,6 +346,7 @@ pub(crate) async fn skill_roots( home_dir.as_ref(), plugin_skill_roots, extra_skill_roots, + preflight_mode, ) .await } @@ -275,7 +358,8 @@ async fn skill_roots_with_home_dir( home_dir: Option<&AbsolutePathBuf>, plugin_skill_roots: Vec, extra_skill_roots: Vec, -) -> Vec { + preflight_mode: SkillRootPreflightMode, +) -> SkillRootResolution { let mut roots = skill_roots_from_layer_stack_inner(config_layer_stack, home_dir, fs.clone()); roots.extend(plugin_skill_roots.into_iter().map(|root| SkillRoot { path: root.path, @@ -293,9 +377,48 @@ async fn skill_roots_with_home_dir( plugin_namespace: None, plugin_root: None, })); - roots.extend(repo_agents_skill_roots(fs, config_layer_stack, cwd).await); dedupe_skill_roots_by_path(&mut roots); - roots + let preflight_paths = match preflight_mode { + SkillRootPreflightMode::Disabled => Vec::new(), + SkillRootPreflightMode::MissingExecutorRoots => roots + .iter() + .filter(|root| !root.is_host_owned()) + .map(|root| root.path.clone()) + .collect(), + }; + let repo_fs = fs.clone(); + let (repo_roots, preflight) = tokio::join!( + repo_agents_skill_roots(repo_fs, config_layer_stack, cwd), + preflight_skill_roots(fs, preflight_paths), + ); + roots.extend(repo_roots); + dedupe_skill_roots_by_path(&mut roots); + SkillRootResolution { roots, preflight } +} + +async fn preflight_skill_roots( + fs: Option>, + paths: Vec, +) -> SkillRootPreflight { + let mut preflight = SkillRootPreflight::default(); + let Some(fs) = fs else { + return preflight; + }; + for paths in paths.chunks(FS_GET_METADATA_BATCH_MAX_PATHS) { + let path_uris = paths.iter().map(PathUri::from_abs_path).collect::>(); + let Ok(results) = fs.get_metadata_batch(&path_uris, /*sandbox*/ None).await else { + continue; + }; + if results.len() != paths.len() { + continue; + } + for (path, result) in paths.iter().zip(results) { + if result.is_err_and(|err| err.kind() == io::ErrorKind::NotFound) { + preflight.absent_paths.insert(path.clone()); + } + } + } + preflight } fn skill_roots_from_layer_stack_inner( @@ -1205,8 +1328,10 @@ pub(crate) async fn skill_roots_from_layer_stack( home_dir, Vec::new(), Vec::new(), + SkillRootPreflightMode::Disabled, ) .await + .roots } #[cfg(test)] diff --git a/codex-rs/core-skills/src/loader_tests.rs b/codex-rs/core-skills/src/loader_tests.rs index 3122dbaf0b..113c693533 100644 --- a/codex-rs/core-skills/src/loader_tests.rs +++ b/codex-rs/core-skills/src/loader_tests.rs @@ -145,6 +145,28 @@ fn normalized(path: &Path) -> AbsolutePathBuf { .abs() } +#[tokio::test] +async fn preflight_only_short_circuits_exact_not_found_results() { + let temp = tempfile::tempdir().expect("tempdir"); + let missing = temp.path().join("missing").abs(); + let existing_directory = temp.path().join("directory"); + fs::create_dir(&existing_directory).expect("create directory"); + let existing_file = temp.path().join("file"); + fs::write(&existing_file, "contents").expect("write file"); + + let preflight = preflight_skill_roots( + Some(Arc::clone(&LOCAL_FS)), + vec![ + missing.clone(), + existing_directory.abs(), + existing_file.abs(), + ], + ) + .await; + + assert_eq!(preflight.absent_paths, HashSet::from([missing])); +} + #[tokio::test] async fn skill_roots_from_layer_stack_maps_user_to_user_and_system_cache_and_system_to_admin() -> anyhow::Result<()> { diff --git a/codex-rs/core-skills/src/root_loader.rs b/codex-rs/core-skills/src/root_loader.rs index d508e93b52..d7fa1fca79 100644 --- a/codex-rs/core-skills/src/root_loader.rs +++ b/codex-rs/core-skills/src/root_loader.rs @@ -64,6 +64,23 @@ pub(crate) async fn load_and_merge_skill_roots( roots: I, plugin_skill_snapshots: Option<&PluginSkillSnapshots>, ) -> SkillLoadOutcome +where + I: IntoIterator, +{ + let preflight_absent_paths = HashSet::new(); + load_and_merge_skill_roots_with_preflight( + roots, + plugin_skill_snapshots, + &preflight_absent_paths, + ) + .await +} + +pub(crate) async fn load_and_merge_skill_roots_with_preflight( + roots: I, + plugin_skill_snapshots: Option<&PluginSkillSnapshots>, + preflight_absent_paths: &HashSet, +) -> SkillLoadOutcome where I: IntoIterator, { @@ -147,7 +164,8 @@ where snapshot, }, None => { - let snapshot = load_skill_root(root).await; + let preflight_absent = preflight_absent_paths.contains(&root.path); + let snapshot = load_skill_root(root, preflight_absent).await; if let Some(plugin_skill_snapshots) = plugin_skill_snapshots && let Some(cache_key) = cache_key { diff --git a/codex-rs/core-skills/src/root_loader_tests.rs b/codex-rs/core-skills/src/root_loader_tests.rs index 540573e68e..7e32a4afbb 100644 --- a/codex-rs/core-skills/src/root_loader_tests.rs +++ b/codex-rs/core-skills/src/root_loader_tests.rs @@ -31,6 +31,7 @@ use tokio::sync::Semaphore; use super::MAX_CONCURRENT_ROOT_SCANS; use super::PluginSkillSnapshots; use super::load_and_merge_skill_roots; +use super::load_and_merge_skill_roots_with_preflight; use crate::SkillError; use crate::loader::SkillRoot; @@ -40,6 +41,7 @@ struct ControlledFileSystem { inner: Arc, walk_gate: Option>, walk_paths: Mutex>, + canonicalize_paths: Mutex>, walks_started: AtomicUsize, walk_started: Notify, blocked_read_root: Option, @@ -55,6 +57,7 @@ impl ControlledFileSystem { inner, walk_gate: None, walk_paths: Mutex::new(Vec::new()), + canonicalize_paths: Mutex::new(Vec::new()), walks_started: AtomicUsize::new(/*v*/ 0), walk_started: Notify::new(), blocked_read_root: None, @@ -88,6 +91,13 @@ impl ControlledFileSystem { self.walks_started.load(Ordering::Acquire) } + fn canonicalize_paths(&self) -> Vec { + self.canonicalize_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + fn walk_paths(&self) -> Vec { self.walk_paths .lock() @@ -130,6 +140,10 @@ impl ExecutorFileSystem for ControlledFileSystem { path: &'a PathUri, sandbox: Option<&'a FileSystemSandboxContext>, ) -> ExecutorFileSystemFuture<'a, PathUri> { + self.canonicalize_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(path.clone()); self.inner.canonicalize(path, sandbox) } @@ -296,6 +310,94 @@ fn plugin_root( } } +#[tokio::test] +async fn missing_root_preflight_skips_io_and_preserves_merge_order() { + let temp = tempfile::tempdir().expect("tempdir"); + let first_root = temp.path().join("first"); + let missing_root = temp.path().join("missing"); + let last_root = temp.path().join("last"); + let first_skill = write_skill(&first_root, "broken", "first missing frontmatter"); + let last_skill = write_skill(&last_root, "broken", "last missing frontmatter"); + let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS))); + let root_file_system: Arc = file_system.clone(); + + let outcome = load_and_merge_skill_roots_with_preflight( + [ + plain_root(&first_root, SkillScope::Repo, Arc::clone(&root_file_system)), + plain_root( + &missing_root, + SkillScope::Repo, + Arc::clone(&root_file_system), + ), + plain_root(&last_root, SkillScope::User, root_file_system), + ], + /*plugin_skill_snapshots*/ None, + &HashSet::from([missing_root.abs()]), + ) + .await; + + assert!( + !file_system + .canonicalize_paths() + .contains(&PathUri::from_abs_path(&missing_root.abs())) + ); + assert_eq!(file_system.walks_started(), 2); + assert_eq!(outcome.skills, Vec::new()); + assert_eq!( + outcome.errors, + vec![ + SkillError { + path: dunce::canonicalize(first_skill) + .expect("canonical first skill") + .abs(), + message: "missing YAML frontmatter delimited by ---".to_string(), + }, + SkillError { + path: dunce::canonicalize(last_skill) + .expect("canonical last skill") + .abs(), + message: "missing YAML frontmatter delimited by ---".to_string(), + }, + ] + ); +} + +#[tokio::test] +async fn plugin_snapshot_hit_precedes_preflight_hint() { + let temp = tempfile::tempdir().expect("tempdir"); + let skills_root = temp.path().join("plugin").join("skills"); + write_skill( + &skills_root, + "demo", + "---\nname: demo\ndescription: demo skill\n---\n", + ); + let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS))); + let root_file_system: Arc = file_system.clone(); + let snapshots = PluginSkillSnapshots::for_plugin_load(); + let first = load_and_merge_skill_roots( + [plugin_root( + &skills_root, + temp.path(), + Arc::clone(&root_file_system), + )], + Some(&snapshots), + ) + .await; + assert_eq!(file_system.walks_started(), 1); + + let hinted = load_and_merge_skill_roots_with_preflight( + [plugin_root(&skills_root, temp.path(), root_file_system)], + Some(&snapshots), + &HashSet::from([skills_root.abs()]), + ) + .await; + + assert_eq!(file_system.walks_started(), 1); + assert_eq!(hinted.skills, first.skills); + assert_eq!(hinted.errors, first.errors); + assert_eq!(hinted.skills.len(), 1); +} + #[tokio::test] async fn loads_roots_unordered_but_merges_errors_in_input_order() { const ROOT_COUNT: usize = MAX_CONCURRENT_ROOT_SCANS + 1; diff --git a/codex-rs/core-skills/src/service.rs b/codex-rs/core-skills/src/service.rs index b4e0d7396c..7a90440ebb 100644 --- a/codex-rs/core-skills/src/service.rs +++ b/codex-rs/core-skills/src/service.rs @@ -21,7 +21,8 @@ use crate::config_rules::SkillConfigRules; use crate::config_rules::resolve_disabled_skill_paths; use crate::config_rules::skill_config_rules_from_stack; use crate::loader::SkillRoot; -use crate::loader::load_skills_from_roots; +use crate::loader::load_skills_from_roots_with_preflight; +use crate::loader::skill_root_resolution; use crate::loader::skill_roots; use crate::system::install_system_skills; use crate::system::uninstall_system_skills; @@ -128,7 +129,20 @@ impl SkillsService { input: &SkillsLoadInput, fs: Option>, ) -> HostSkillsSnapshot { - let roots = self.skill_roots_for_config(input, fs).await; + let mut resolution = skill_root_resolution( + fs, + &input.config_layer_stack, + &input.cwd, + input.effective_skill_roots.clone(), + self.extra_roots(), + ) + .await; + if !input.bundled_skills_enabled { + resolution + .roots + .retain(|root| root.scope != SkillScope::System); + } + let roots = resolution.roots; let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack); let cache_key = config_skills_cache_key(&roots, &skill_config_rules); if let Some(snapshot) = self.cached_snapshot_for_config(&cache_key) { @@ -136,8 +150,13 @@ impl SkillsService { } let snapshot = HostSkillsSnapshot::new(Arc::new( - self.build_skill_outcome(input, roots, &skill_config_rules) - .await, + self.build_skill_outcome( + input, + roots, + &skill_config_rules, + &resolution.preflight.absent_paths, + ) + .await, )); let mut cache = self .cache_by_config @@ -180,7 +199,7 @@ impl SkillsService { return snapshot; } - let mut roots = skill_roots( + let mut resolution = skill_root_resolution( fs.clone(), &input.config_layer_stack, &input.cwd, @@ -189,12 +208,20 @@ impl SkillsService { ) .await; if !bundled_skills_enabled_from_stack(&input.config_layer_stack) { - roots.retain(|root| root.scope != SkillScope::System); + resolution + .roots + .retain(|root| root.scope != SkillScope::System); } + let roots = resolution.roots; let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack); let snapshot = HostSkillsSnapshot::new(Arc::new( - self.build_skill_outcome(input, roots, &skill_config_rules) - .await, + self.build_skill_outcome( + input, + roots, + &skill_config_rules, + &resolution.preflight.absent_paths, + ) + .await, )); if use_cwd_cache { let mut cache = self @@ -212,8 +239,14 @@ impl SkillsService { input: &SkillsLoadInput, roots: Vec, skill_config_rules: &SkillConfigRules, + preflight_absent_paths: &HashSet, ) -> SkillLoadOutcome { - let outcome = load_skills_from_roots(roots, input.plugin_skill_snapshots.as_ref()).await; + let outcome = load_skills_from_roots_with_preflight( + roots, + input.plugin_skill_snapshots.as_ref(), + preflight_absent_paths, + ) + .await; let outcome = crate::filter_skill_load_outcome_for_product(outcome, self.restriction_product); let disabled_paths = resolve_disabled_skill_paths(&outcome.skills, skill_config_rules);