mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
perf(skills): overlap missing-root preflight
This commit is contained in:
@@ -40,6 +40,7 @@ use discovery::SkillMetadataDiscovery;
|
||||
use discovery::discover_skills;
|
||||
use futures::FutureExt;
|
||||
use futures::StreamExt;
|
||||
use futures::future::BoxFuture;
|
||||
use namespace::SkillNamespaceResolver;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashSet;
|
||||
@@ -190,7 +191,27 @@ pub struct SkillRoot {
|
||||
pub plugin_root: Option<AbsolutePathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct SkillRootPreflight {
|
||||
pub(crate) absent_paths: HashSet<AbsolutePathBuf>,
|
||||
}
|
||||
|
||||
pub(crate) struct SkillRootResolution {
|
||||
pub(crate) roots: Vec<SkillRoot>,
|
||||
pub(crate) preflight: SkillRootPreflight,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum SkillRootPreflightMode {
|
||||
Disabled,
|
||||
MissingExecutorRoots,
|
||||
}
|
||||
|
||||
impl SkillRoot {
|
||||
pub(crate) fn is_host_owned(&self) -> bool {
|
||||
Arc::ptr_eq(&self.file_system, &LOCAL_FS)
|
||||
}
|
||||
|
||||
pub(crate) fn is_agents_skills_root(&self) -> bool {
|
||||
self.path.file_name() == Some(std::ffi::OsStr::new(SKILLS_DIR_NAME))
|
||||
&& self.path.parent().is_some_and(|parent| {
|
||||
@@ -211,6 +232,19 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn load_skills_from_roots_with_preflight<'a>(
|
||||
roots: Vec<SkillRoot>,
|
||||
plugin_skill_snapshots: Option<&'a crate::PluginSkillSnapshots>,
|
||||
preflight_absent_paths: &'a HashSet<AbsolutePathBuf>,
|
||||
) -> BoxFuture<'a, SkillLoadOutcome> {
|
||||
crate::root_loader::load_and_merge_skill_roots_with_preflight(
|
||||
roots,
|
||||
plugin_skill_snapshots,
|
||||
preflight_absent_paths,
|
||||
)
|
||||
.boxed()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SkillRootSnapshot {
|
||||
pub(crate) root: AbsolutePathBuf,
|
||||
@@ -219,7 +253,7 @@ pub(crate) struct SkillRootSnapshot {
|
||||
pub(crate) file_system: Arc<dyn ExecutorFileSystem>,
|
||||
}
|
||||
|
||||
pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot {
|
||||
pub(crate) async fn load_skill_root(root: SkillRoot, preflight_absent: bool) -> SkillRootSnapshot {
|
||||
let SkillRoot {
|
||||
path,
|
||||
scope,
|
||||
@@ -228,6 +262,16 @@ pub(crate) async fn load_skill_root(root: SkillRoot) -> SkillRootSnapshot {
|
||||
plugin_namespace,
|
||||
plugin_root,
|
||||
} = root;
|
||||
// The hint comes from an exact metadata probe performed while repo roots were discovered. It
|
||||
// avoids repeating canonicalize and walk calls for a root known to be absent at that point.
|
||||
if preflight_absent {
|
||||
return SkillRootSnapshot {
|
||||
root: path,
|
||||
skills: Vec::new(),
|
||||
errors: Vec::new(),
|
||||
file_system,
|
||||
};
|
||||
}
|
||||
let root = canonicalize_for_skill_identity(file_system.as_ref(), &path).await;
|
||||
let mut outcome = SkillLoadOutcome::default();
|
||||
load_skills_under_root(
|
||||
@@ -255,6 +299,44 @@ pub(crate) async fn skill_roots(
|
||||
plugin_skill_roots: Vec<PluginSkillRoot>,
|
||||
extra_skill_roots: Vec<AbsolutePathBuf>,
|
||||
) -> Vec<SkillRoot> {
|
||||
skill_root_resolution_with_mode(
|
||||
fs,
|
||||
config_layer_stack,
|
||||
cwd,
|
||||
plugin_skill_roots,
|
||||
extra_skill_roots,
|
||||
SkillRootPreflightMode::Disabled,
|
||||
)
|
||||
.await
|
||||
.roots
|
||||
}
|
||||
|
||||
pub(crate) async fn skill_root_resolution(
|
||||
fs: Option<Arc<dyn ExecutorFileSystem>>,
|
||||
config_layer_stack: &ConfigLayerStack,
|
||||
cwd: &AbsolutePathBuf,
|
||||
plugin_skill_roots: Vec<PluginSkillRoot>,
|
||||
extra_skill_roots: Vec<AbsolutePathBuf>,
|
||||
) -> SkillRootResolution {
|
||||
skill_root_resolution_with_mode(
|
||||
fs,
|
||||
config_layer_stack,
|
||||
cwd,
|
||||
plugin_skill_roots,
|
||||
extra_skill_roots,
|
||||
SkillRootPreflightMode::MissingExecutorRoots,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn skill_root_resolution_with_mode(
|
||||
fs: Option<Arc<dyn ExecutorFileSystem>>,
|
||||
config_layer_stack: &ConfigLayerStack,
|
||||
cwd: &AbsolutePathBuf,
|
||||
plugin_skill_roots: Vec<PluginSkillRoot>,
|
||||
extra_skill_roots: Vec<AbsolutePathBuf>,
|
||||
preflight_mode: SkillRootPreflightMode,
|
||||
) -> SkillRootResolution {
|
||||
let home_dir =
|
||||
home_dir().and_then(|path| AbsolutePathBuf::from_absolute_path_checked(path).ok());
|
||||
skill_roots_with_home_dir(
|
||||
@@ -264,6 +346,7 @@ pub(crate) async fn skill_roots(
|
||||
home_dir.as_ref(),
|
||||
plugin_skill_roots,
|
||||
extra_skill_roots,
|
||||
preflight_mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -275,7 +358,8 @@ async fn skill_roots_with_home_dir(
|
||||
home_dir: Option<&AbsolutePathBuf>,
|
||||
plugin_skill_roots: Vec<PluginSkillRoot>,
|
||||
extra_skill_roots: Vec<AbsolutePathBuf>,
|
||||
) -> Vec<SkillRoot> {
|
||||
preflight_mode: SkillRootPreflightMode,
|
||||
) -> SkillRootResolution {
|
||||
let mut roots = skill_roots_from_layer_stack_inner(config_layer_stack, home_dir, fs.clone());
|
||||
roots.extend(plugin_skill_roots.into_iter().map(|root| SkillRoot {
|
||||
path: root.path,
|
||||
@@ -293,9 +377,48 @@ async fn skill_roots_with_home_dir(
|
||||
plugin_namespace: None,
|
||||
plugin_root: None,
|
||||
}));
|
||||
roots.extend(repo_agents_skill_roots(fs, config_layer_stack, cwd).await);
|
||||
dedupe_skill_roots_by_path(&mut roots);
|
||||
roots
|
||||
let preflight_paths = match preflight_mode {
|
||||
SkillRootPreflightMode::Disabled => Vec::new(),
|
||||
SkillRootPreflightMode::MissingExecutorRoots => roots
|
||||
.iter()
|
||||
.filter(|root| !root.is_host_owned())
|
||||
.map(|root| root.path.clone())
|
||||
.collect(),
|
||||
};
|
||||
let repo_fs = fs.clone();
|
||||
let (repo_roots, preflight) = tokio::join!(
|
||||
repo_agents_skill_roots(repo_fs, config_layer_stack, cwd),
|
||||
preflight_skill_roots(fs, preflight_paths),
|
||||
);
|
||||
roots.extend(repo_roots);
|
||||
dedupe_skill_roots_by_path(&mut roots);
|
||||
SkillRootResolution { roots, preflight }
|
||||
}
|
||||
|
||||
async fn preflight_skill_roots(
|
||||
fs: Option<Arc<dyn ExecutorFileSystem>>,
|
||||
paths: Vec<AbsolutePathBuf>,
|
||||
) -> SkillRootPreflight {
|
||||
let mut preflight = SkillRootPreflight::default();
|
||||
let Some(fs) = fs else {
|
||||
return preflight;
|
||||
};
|
||||
for paths in paths.chunks(FS_GET_METADATA_BATCH_MAX_PATHS) {
|
||||
let path_uris = paths.iter().map(PathUri::from_abs_path).collect::<Vec<_>>();
|
||||
let Ok(results) = fs.get_metadata_batch(&path_uris, /*sandbox*/ None).await else {
|
||||
continue;
|
||||
};
|
||||
if results.len() != paths.len() {
|
||||
continue;
|
||||
}
|
||||
for (path, result) in paths.iter().zip(results) {
|
||||
if result.is_err_and(|err| err.kind() == io::ErrorKind::NotFound) {
|
||||
preflight.absent_paths.insert(path.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
preflight
|
||||
}
|
||||
|
||||
fn skill_roots_from_layer_stack_inner(
|
||||
@@ -1205,8 +1328,10 @@ pub(crate) async fn skill_roots_from_layer_stack(
|
||||
home_dir,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
SkillRootPreflightMode::Disabled,
|
||||
)
|
||||
.await
|
||||
.roots
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -145,6 +145,28 @@ fn normalized(path: &Path) -> AbsolutePathBuf {
|
||||
.abs()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preflight_only_short_circuits_exact_not_found_results() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let missing = temp.path().join("missing").abs();
|
||||
let existing_directory = temp.path().join("directory");
|
||||
fs::create_dir(&existing_directory).expect("create directory");
|
||||
let existing_file = temp.path().join("file");
|
||||
fs::write(&existing_file, "contents").expect("write file");
|
||||
|
||||
let preflight = preflight_skill_roots(
|
||||
Some(Arc::clone(&LOCAL_FS)),
|
||||
vec![
|
||||
missing.clone(),
|
||||
existing_directory.abs(),
|
||||
existing_file.abs(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(preflight.absent_paths, HashSet::from([missing]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skill_roots_from_layer_stack_maps_user_to_user_and_system_cache_and_system_to_admin()
|
||||
-> anyhow::Result<()> {
|
||||
|
||||
@@ -64,6 +64,23 @@ pub(crate) async fn load_and_merge_skill_roots<I>(
|
||||
roots: I,
|
||||
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
|
||||
) -> SkillLoadOutcome
|
||||
where
|
||||
I: IntoIterator<Item = SkillRoot>,
|
||||
{
|
||||
let preflight_absent_paths = HashSet::new();
|
||||
load_and_merge_skill_roots_with_preflight(
|
||||
roots,
|
||||
plugin_skill_snapshots,
|
||||
&preflight_absent_paths,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_and_merge_skill_roots_with_preflight<I>(
|
||||
roots: I,
|
||||
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
|
||||
preflight_absent_paths: &HashSet<codex_utils_absolute_path::AbsolutePathBuf>,
|
||||
) -> SkillLoadOutcome
|
||||
where
|
||||
I: IntoIterator<Item = SkillRoot>,
|
||||
{
|
||||
@@ -147,7 +164,8 @@ where
|
||||
snapshot,
|
||||
},
|
||||
None => {
|
||||
let snapshot = load_skill_root(root).await;
|
||||
let preflight_absent = preflight_absent_paths.contains(&root.path);
|
||||
let snapshot = load_skill_root(root, preflight_absent).await;
|
||||
if let Some(plugin_skill_snapshots) = plugin_skill_snapshots
|
||||
&& let Some(cache_key) = cache_key
|
||||
{
|
||||
|
||||
@@ -31,6 +31,7 @@ use tokio::sync::Semaphore;
|
||||
use super::MAX_CONCURRENT_ROOT_SCANS;
|
||||
use super::PluginSkillSnapshots;
|
||||
use super::load_and_merge_skill_roots;
|
||||
use super::load_and_merge_skill_roots_with_preflight;
|
||||
use crate::SkillError;
|
||||
use crate::loader::SkillRoot;
|
||||
|
||||
@@ -40,6 +41,7 @@ struct ControlledFileSystem {
|
||||
inner: Arc<dyn ExecutorFileSystem>,
|
||||
walk_gate: Option<Arc<Semaphore>>,
|
||||
walk_paths: Mutex<Vec<PathUri>>,
|
||||
canonicalize_paths: Mutex<Vec<PathUri>>,
|
||||
walks_started: AtomicUsize,
|
||||
walk_started: Notify,
|
||||
blocked_read_root: Option<PathUri>,
|
||||
@@ -55,6 +57,7 @@ impl ControlledFileSystem {
|
||||
inner,
|
||||
walk_gate: None,
|
||||
walk_paths: Mutex::new(Vec::new()),
|
||||
canonicalize_paths: Mutex::new(Vec::new()),
|
||||
walks_started: AtomicUsize::new(/*v*/ 0),
|
||||
walk_started: Notify::new(),
|
||||
blocked_read_root: None,
|
||||
@@ -88,6 +91,13 @@ impl ControlledFileSystem {
|
||||
self.walks_started.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn canonicalize_paths(&self) -> Vec<PathUri> {
|
||||
self.canonicalize_paths
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn walk_paths(&self) -> Vec<PathUri> {
|
||||
self.walk_paths
|
||||
.lock()
|
||||
@@ -130,6 +140,10 @@ impl ExecutorFileSystem for ControlledFileSystem {
|
||||
path: &'a PathUri,
|
||||
sandbox: Option<&'a FileSystemSandboxContext>,
|
||||
) -> ExecutorFileSystemFuture<'a, PathUri> {
|
||||
self.canonicalize_paths
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(path.clone());
|
||||
self.inner.canonicalize(path, sandbox)
|
||||
}
|
||||
|
||||
@@ -296,6 +310,94 @@ fn plugin_root(
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_root_preflight_skips_io_and_preserves_merge_order() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let first_root = temp.path().join("first");
|
||||
let missing_root = temp.path().join("missing");
|
||||
let last_root = temp.path().join("last");
|
||||
let first_skill = write_skill(&first_root, "broken", "first missing frontmatter");
|
||||
let last_skill = write_skill(&last_root, "broken", "last missing frontmatter");
|
||||
let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS)));
|
||||
let root_file_system: Arc<dyn ExecutorFileSystem> = file_system.clone();
|
||||
|
||||
let outcome = load_and_merge_skill_roots_with_preflight(
|
||||
[
|
||||
plain_root(&first_root, SkillScope::Repo, Arc::clone(&root_file_system)),
|
||||
plain_root(
|
||||
&missing_root,
|
||||
SkillScope::Repo,
|
||||
Arc::clone(&root_file_system),
|
||||
),
|
||||
plain_root(&last_root, SkillScope::User, root_file_system),
|
||||
],
|
||||
/*plugin_skill_snapshots*/ None,
|
||||
&HashSet::from([missing_root.abs()]),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
!file_system
|
||||
.canonicalize_paths()
|
||||
.contains(&PathUri::from_abs_path(&missing_root.abs()))
|
||||
);
|
||||
assert_eq!(file_system.walks_started(), 2);
|
||||
assert_eq!(outcome.skills, Vec::new());
|
||||
assert_eq!(
|
||||
outcome.errors,
|
||||
vec![
|
||||
SkillError {
|
||||
path: dunce::canonicalize(first_skill)
|
||||
.expect("canonical first skill")
|
||||
.abs(),
|
||||
message: "missing YAML frontmatter delimited by ---".to_string(),
|
||||
},
|
||||
SkillError {
|
||||
path: dunce::canonicalize(last_skill)
|
||||
.expect("canonical last skill")
|
||||
.abs(),
|
||||
message: "missing YAML frontmatter delimited by ---".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plugin_snapshot_hit_precedes_preflight_hint() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let skills_root = temp.path().join("plugin").join("skills");
|
||||
write_skill(
|
||||
&skills_root,
|
||||
"demo",
|
||||
"---\nname: demo\ndescription: demo skill\n---\n",
|
||||
);
|
||||
let file_system = Arc::new(ControlledFileSystem::new(Arc::clone(&LOCAL_FS)));
|
||||
let root_file_system: Arc<dyn ExecutorFileSystem> = file_system.clone();
|
||||
let snapshots = PluginSkillSnapshots::for_plugin_load();
|
||||
let first = load_and_merge_skill_roots(
|
||||
[plugin_root(
|
||||
&skills_root,
|
||||
temp.path(),
|
||||
Arc::clone(&root_file_system),
|
||||
)],
|
||||
Some(&snapshots),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(file_system.walks_started(), 1);
|
||||
|
||||
let hinted = load_and_merge_skill_roots_with_preflight(
|
||||
[plugin_root(&skills_root, temp.path(), root_file_system)],
|
||||
Some(&snapshots),
|
||||
&HashSet::from([skills_root.abs()]),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(file_system.walks_started(), 1);
|
||||
assert_eq!(hinted.skills, first.skills);
|
||||
assert_eq!(hinted.errors, first.errors);
|
||||
assert_eq!(hinted.skills.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn loads_roots_unordered_but_merges_errors_in_input_order() {
|
||||
const ROOT_COUNT: usize = MAX_CONCURRENT_ROOT_SCANS + 1;
|
||||
|
||||
@@ -21,7 +21,8 @@ use crate::config_rules::SkillConfigRules;
|
||||
use crate::config_rules::resolve_disabled_skill_paths;
|
||||
use crate::config_rules::skill_config_rules_from_stack;
|
||||
use crate::loader::SkillRoot;
|
||||
use crate::loader::load_skills_from_roots;
|
||||
use crate::loader::load_skills_from_roots_with_preflight;
|
||||
use crate::loader::skill_root_resolution;
|
||||
use crate::loader::skill_roots;
|
||||
use crate::system::install_system_skills;
|
||||
use crate::system::uninstall_system_skills;
|
||||
@@ -128,7 +129,20 @@ impl SkillsService {
|
||||
input: &SkillsLoadInput,
|
||||
fs: Option<Arc<dyn ExecutorFileSystem>>,
|
||||
) -> HostSkillsSnapshot {
|
||||
let roots = self.skill_roots_for_config(input, fs).await;
|
||||
let mut resolution = skill_root_resolution(
|
||||
fs,
|
||||
&input.config_layer_stack,
|
||||
&input.cwd,
|
||||
input.effective_skill_roots.clone(),
|
||||
self.extra_roots(),
|
||||
)
|
||||
.await;
|
||||
if !input.bundled_skills_enabled {
|
||||
resolution
|
||||
.roots
|
||||
.retain(|root| root.scope != SkillScope::System);
|
||||
}
|
||||
let roots = resolution.roots;
|
||||
let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack);
|
||||
let cache_key = config_skills_cache_key(&roots, &skill_config_rules);
|
||||
if let Some(snapshot) = self.cached_snapshot_for_config(&cache_key) {
|
||||
@@ -136,8 +150,13 @@ impl SkillsService {
|
||||
}
|
||||
|
||||
let snapshot = HostSkillsSnapshot::new(Arc::new(
|
||||
self.build_skill_outcome(input, roots, &skill_config_rules)
|
||||
.await,
|
||||
self.build_skill_outcome(
|
||||
input,
|
||||
roots,
|
||||
&skill_config_rules,
|
||||
&resolution.preflight.absent_paths,
|
||||
)
|
||||
.await,
|
||||
));
|
||||
let mut cache = self
|
||||
.cache_by_config
|
||||
@@ -180,7 +199,7 @@ impl SkillsService {
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
let mut roots = skill_roots(
|
||||
let mut resolution = skill_root_resolution(
|
||||
fs.clone(),
|
||||
&input.config_layer_stack,
|
||||
&input.cwd,
|
||||
@@ -189,12 +208,20 @@ impl SkillsService {
|
||||
)
|
||||
.await;
|
||||
if !bundled_skills_enabled_from_stack(&input.config_layer_stack) {
|
||||
roots.retain(|root| root.scope != SkillScope::System);
|
||||
resolution
|
||||
.roots
|
||||
.retain(|root| root.scope != SkillScope::System);
|
||||
}
|
||||
let roots = resolution.roots;
|
||||
let skill_config_rules = skill_config_rules_from_stack(&input.config_layer_stack);
|
||||
let snapshot = HostSkillsSnapshot::new(Arc::new(
|
||||
self.build_skill_outcome(input, roots, &skill_config_rules)
|
||||
.await,
|
||||
self.build_skill_outcome(
|
||||
input,
|
||||
roots,
|
||||
&skill_config_rules,
|
||||
&resolution.preflight.absent_paths,
|
||||
)
|
||||
.await,
|
||||
));
|
||||
if use_cwd_cache {
|
||||
let mut cache = self
|
||||
@@ -212,8 +239,14 @@ impl SkillsService {
|
||||
input: &SkillsLoadInput,
|
||||
roots: Vec<SkillRoot>,
|
||||
skill_config_rules: &SkillConfigRules,
|
||||
preflight_absent_paths: &HashSet<AbsolutePathBuf>,
|
||||
) -> SkillLoadOutcome {
|
||||
let outcome = load_skills_from_roots(roots, input.plugin_skill_snapshots.as_ref()).await;
|
||||
let outcome = load_skills_from_roots_with_preflight(
|
||||
roots,
|
||||
input.plugin_skill_snapshots.as_ref(),
|
||||
preflight_absent_paths,
|
||||
)
|
||||
.await;
|
||||
let outcome =
|
||||
crate::filter_skill_load_outcome_for_product(outcome, self.restriction_product);
|
||||
let disabled_paths = resolve_disabled_skill_paths(&outcome.skills, skill_config_rules);
|
||||
|
||||
Reference in New Issue
Block a user