use std::collections::HashMap; use std::collections::HashSet; use std::fmt; use std::sync::Arc; use std::sync::Mutex; use codex_utils_plugins::PluginSkillRoot; use futures::StreamExt; use crate::SkillLoadOutcome; use crate::loader::SkillRoot; use crate::loader::SkillRootSnapshot; use crate::loader::load_skill_root; use crate::model::SkillFileSystemsByPath; const MAX_CONCURRENT_ROOT_SCANS: usize = 5; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] enum RootScanPriority { AgentsSkills, Default, } struct PendingRootLoad { root: SkillRoot, root_index: usize, duplicate_root_indices: Vec, cache_key: Option, scheduling_priority: RootScanPriority, } struct CompletedRootLoad { root_index: usize, duplicate_root_indices: Vec, snapshot: SkillRootSnapshot, } /// Parsed plugin skill-root snapshots produced by one plugin load. /// /// Clones share the same snapshots. The plugins manager stores them with the corresponding loaded /// plugins and passes a clone to skill loading as an optional preload. #[derive(Clone)] pub struct PluginSkillSnapshots { snapshots_by_root: Arc>>, } impl PluginSkillSnapshots { /// Creates an empty snapshot collection for a plugin load to populate. pub fn for_plugin_load() -> Self { Self { snapshots_by_root: Arc::new(Mutex::new(HashMap::new())), } } } impl fmt::Debug for PluginSkillSnapshots { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PluginSkillSnapshots") .finish_non_exhaustive() } } pub(crate) async fn load_and_merge_skill_roots( roots: I, plugin_skill_snapshots: Option<&PluginSkillSnapshots>, ) -> SkillLoadOutcome where I: IntoIterator, { let preflight_absent_paths = HashSet::new(); load_and_merge_skill_roots_with_preflight( roots, plugin_skill_snapshots, &preflight_absent_paths, ) .await } pub(crate) async fn load_and_merge_skill_roots_with_preflight( roots: I, plugin_skill_snapshots: Option<&PluginSkillSnapshots>, preflight_absent_paths: &HashSet, ) -> SkillLoadOutcome where I: IntoIterator, { let roots = roots.into_iter().collect::>(); let root_count = roots.len(); let mut pending_loads = Vec::::with_capacity(root_count); let mut pending_load_by_cache_key = HashMap::::new(); for (root_index, root) in roots.into_iter().enumerate() { let cache_key = match ( root.plugin_id.clone(), root.plugin_namespace.clone(), root.plugin_root.clone(), ) { (Some(plugin_id), Some(plugin_namespace), Some(plugin_root)) => Some(PluginSkillRoot { path: root.path.clone(), plugin_id, plugin_namespace, plugin_root, }), _ => None, }; // Preserve sequential cache semantics: with a cache, the first occurrence scans and later // occurrences reuse its snapshot. Without a cache, duplicate roots remain independent. if plugin_skill_snapshots.is_some() && let Some(cache_key) = cache_key.as_ref() && let Some(pending_index) = pending_load_by_cache_key.get(cache_key) { pending_loads[*pending_index] .duplicate_root_indices .push(root_index); continue; } if plugin_skill_snapshots.is_some() && let Some(cache_key) = cache_key.as_ref() { pending_load_by_cache_key.insert(cache_key.clone(), pending_loads.len()); } let scheduling_priority = if root.is_agents_skills_root() { RootScanPriority::AgentsSkills } else { RootScanPriority::Default }; pending_loads.push(PendingRootLoad { root, root_index, duplicate_root_indices: Vec::new(), cache_key, scheduling_priority, }); } // Group duplicate cache keys before prioritization so the original first occurrence remains // the scan/cache owner. Sorting by the original index within each priority retains FIFO order. pending_loads.sort_by_key(|pending| (pending.scheduling_priority, pending.root_index)); // Unordered buffering starts known `.agents/skills` roots first and lets fast host and cached // roots free slots while an earlier executor root is still pending. The indexed snapshots // below restore the original merge order. let completed_loads = futures::stream::iter(pending_loads) .map(|pending| async move { let PendingRootLoad { root, root_index, duplicate_root_indices, cache_key, scheduling_priority: _, } = pending; let cached_snapshot = cache_key.as_ref().and_then(|cache_key| { let plugin_skill_snapshots = plugin_skill_snapshots?; plugin_skill_snapshots .snapshots_by_root .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(cache_key) .cloned() }); match cached_snapshot { Some(snapshot) => CompletedRootLoad { root_index, duplicate_root_indices, snapshot, }, None => { let preflight_absent = preflight_absent_paths.contains(&root.path); let snapshot = load_skill_root(root, preflight_absent).await; if let Some(plugin_skill_snapshots) = plugin_skill_snapshots && let Some(cache_key) = cache_key { plugin_skill_snapshots .snapshots_by_root .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(cache_key, snapshot.clone()); } CompletedRootLoad { root_index, duplicate_root_indices, snapshot, } } } }) .buffer_unordered(MAX_CONCURRENT_ROOT_SCANS) .collect::>() .await; let mut indexed_root_snapshots = Vec::with_capacity(root_count); for completed in completed_loads { for root_index in completed.duplicate_root_indices { indexed_root_snapshots.push((root_index, completed.snapshot.clone())); } indexed_root_snapshots.push((completed.root_index, completed.snapshot)); } indexed_root_snapshots.sort_unstable_by_key(|(root_index, _)| *root_index); let root_snapshots = indexed_root_snapshots .into_iter() .map(|(_, snapshot)| snapshot) .collect(); merge_skill_root_snapshots(root_snapshots) } fn merge_skill_root_snapshots(snapshots: Vec) -> SkillLoadOutcome { fn scope_rank(scope: codex_protocol::protocol::SkillScope) -> u8 { use codex_protocol::protocol::SkillScope; // Higher-priority scopes first (matches root scan order for dedupe). match scope { SkillScope::Repo => 0, SkillScope::User => 1, SkillScope::System => 2, SkillScope::Admin => 3, } } let mut outcome = SkillLoadOutcome::default(); let mut skill_roots = Vec::new(); let mut skill_root_by_path = HashMap::new(); let mut file_systems_by_skill_path = HashMap::new(); for snapshot in snapshots { let SkillRootSnapshot { root, skills, errors, file_system, } = snapshot; if !skills.is_empty() && !skill_roots.contains(&root) { skill_roots.push(root.clone()); } for skill in &skills { skill_root_by_path .entry(skill.path_to_skills_md.clone()) .or_insert_with(|| root.clone()); file_systems_by_skill_path .entry(skill.path_to_skills_md.clone()) .or_insert_with(|| Arc::clone(&file_system)); } outcome.skills.extend(skills); outcome.errors.extend(errors); } let mut seen = HashSet::new(); outcome .skills .retain(|skill| seen.insert(skill.path_to_skills_md.clone())); let retained_skill_paths = outcome .skills .iter() .map(|skill| skill.path_to_skills_md.clone()) .collect::>(); skill_root_by_path.retain(|path, _| retained_skill_paths.contains(path)); let used_roots = skill_root_by_path.values().cloned().collect::>(); skill_roots.retain(|root| used_roots.contains(root)); file_systems_by_skill_path.retain(|path, _| retained_skill_paths.contains(path)); outcome.skill_roots = skill_roots; outcome.skill_root_by_path = Arc::new(skill_root_by_path); outcome.file_systems_by_skill_path = SkillFileSystemsByPath::new(file_systems_by_skill_path); outcome.skills.sort_by(|a, b| { scope_rank(a.scope) .cmp(&scope_rank(b.scope)) .then_with(|| a.name.cmp(&b.name)) .then_with(|| a.path_to_skills_md.cmp(&b.path_to_skills_md)) }); outcome } #[cfg(test)] #[path = "root_loader_tests.rs"] mod tests;