Files
codex/codex-rs/core-skills/src/root_loader.rs
T

276 lines
9.5 KiB
Rust

use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt;
use std::sync::Arc;
use std::sync::Mutex;
use codex_utils_plugins::PluginSkillRoot;
use futures::StreamExt;
use crate::SkillLoadOutcome;
use crate::loader::SkillRoot;
use crate::loader::SkillRootSnapshot;
use crate::loader::load_skill_root;
use crate::model::SkillFileSystemsByPath;
const MAX_CONCURRENT_ROOT_SCANS: usize = 5;
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum RootScanPriority {
AgentsSkills,
Default,
}
struct PendingRootLoad {
root: SkillRoot,
root_index: usize,
duplicate_root_indices: Vec<usize>,
cache_key: Option<PluginSkillRoot>,
scheduling_priority: RootScanPriority,
}
struct CompletedRootLoad {
root_index: usize,
duplicate_root_indices: Vec<usize>,
snapshot: SkillRootSnapshot,
}
/// Parsed plugin skill-root snapshots produced by one plugin load.
///
/// Clones share the same snapshots. The plugins manager stores them with the corresponding loaded
/// plugins and passes a clone to skill loading as an optional preload.
#[derive(Clone)]
pub struct PluginSkillSnapshots {
snapshots_by_root: Arc<Mutex<HashMap<PluginSkillRoot, SkillRootSnapshot>>>,
}
impl PluginSkillSnapshots {
/// Creates an empty snapshot collection for a plugin load to populate.
pub fn for_plugin_load() -> Self {
Self {
snapshots_by_root: Arc::new(Mutex::new(HashMap::new())),
}
}
}
impl fmt::Debug for PluginSkillSnapshots {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PluginSkillSnapshots")
.finish_non_exhaustive()
}
}
pub(crate) async fn load_and_merge_skill_roots<I>(
roots: I,
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
) -> SkillLoadOutcome
where
I: IntoIterator<Item = SkillRoot>,
{
let preflight_absent_paths = HashSet::new();
load_and_merge_skill_roots_with_preflight(
roots,
plugin_skill_snapshots,
&preflight_absent_paths,
)
.await
}
pub(crate) async fn load_and_merge_skill_roots_with_preflight<I>(
roots: I,
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
preflight_absent_paths: &HashSet<codex_utils_absolute_path::AbsolutePathBuf>,
) -> SkillLoadOutcome
where
I: IntoIterator<Item = SkillRoot>,
{
let roots = roots.into_iter().collect::<Vec<_>>();
let root_count = roots.len();
let mut pending_loads = Vec::<PendingRootLoad>::with_capacity(root_count);
let mut pending_load_by_cache_key = HashMap::<PluginSkillRoot, usize>::new();
for (root_index, root) in roots.into_iter().enumerate() {
let cache_key = match (
root.plugin_id.clone(),
root.plugin_namespace.clone(),
root.plugin_root.clone(),
) {
(Some(plugin_id), Some(plugin_namespace), Some(plugin_root)) => Some(PluginSkillRoot {
path: root.path.clone(),
plugin_id,
plugin_namespace,
plugin_root,
}),
_ => None,
};
// Preserve sequential cache semantics: with a cache, the first occurrence scans and later
// occurrences reuse its snapshot. Without a cache, duplicate roots remain independent.
if plugin_skill_snapshots.is_some()
&& let Some(cache_key) = cache_key.as_ref()
&& let Some(pending_index) = pending_load_by_cache_key.get(cache_key)
{
pending_loads[*pending_index]
.duplicate_root_indices
.push(root_index);
continue;
}
if plugin_skill_snapshots.is_some()
&& let Some(cache_key) = cache_key.as_ref()
{
pending_load_by_cache_key.insert(cache_key.clone(), pending_loads.len());
}
let scheduling_priority = if root.is_agents_skills_root() {
RootScanPriority::AgentsSkills
} else {
RootScanPriority::Default
};
pending_loads.push(PendingRootLoad {
root,
root_index,
duplicate_root_indices: Vec::new(),
cache_key,
scheduling_priority,
});
}
// Group duplicate cache keys before prioritization so the original first occurrence remains
// the scan/cache owner. Sorting by the original index within each priority retains FIFO order.
pending_loads.sort_by_key(|pending| (pending.scheduling_priority, pending.root_index));
// Unordered buffering starts known `.agents/skills` roots first and lets fast host and cached
// roots free slots while an earlier executor root is still pending. The indexed snapshots
// below restore the original merge order.
let completed_loads = futures::stream::iter(pending_loads)
.map(|pending| async move {
let PendingRootLoad {
root,
root_index,
duplicate_root_indices,
cache_key,
scheduling_priority: _,
} = pending;
let cached_snapshot = cache_key.as_ref().and_then(|cache_key| {
let plugin_skill_snapshots = plugin_skill_snapshots?;
plugin_skill_snapshots
.snapshots_by_root
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(cache_key)
.cloned()
});
match cached_snapshot {
Some(snapshot) => CompletedRootLoad {
root_index,
duplicate_root_indices,
snapshot,
},
None => {
let preflight_absent = preflight_absent_paths.contains(&root.path);
let snapshot = load_skill_root(root, preflight_absent).await;
if let Some(plugin_skill_snapshots) = plugin_skill_snapshots
&& let Some(cache_key) = cache_key
{
plugin_skill_snapshots
.snapshots_by_root
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(cache_key, snapshot.clone());
}
CompletedRootLoad {
root_index,
duplicate_root_indices,
snapshot,
}
}
}
})
.buffer_unordered(MAX_CONCURRENT_ROOT_SCANS)
.collect::<Vec<_>>()
.await;
let mut indexed_root_snapshots = Vec::with_capacity(root_count);
for completed in completed_loads {
for root_index in completed.duplicate_root_indices {
indexed_root_snapshots.push((root_index, completed.snapshot.clone()));
}
indexed_root_snapshots.push((completed.root_index, completed.snapshot));
}
indexed_root_snapshots.sort_unstable_by_key(|(root_index, _)| *root_index);
let root_snapshots = indexed_root_snapshots
.into_iter()
.map(|(_, snapshot)| snapshot)
.collect();
merge_skill_root_snapshots(root_snapshots)
}
fn merge_skill_root_snapshots(snapshots: Vec<SkillRootSnapshot>) -> SkillLoadOutcome {
fn scope_rank(scope: codex_protocol::protocol::SkillScope) -> u8 {
use codex_protocol::protocol::SkillScope;
// Higher-priority scopes first (matches root scan order for dedupe).
match scope {
SkillScope::Repo => 0,
SkillScope::User => 1,
SkillScope::System => 2,
SkillScope::Admin => 3,
}
}
let mut outcome = SkillLoadOutcome::default();
let mut skill_roots = Vec::new();
let mut skill_root_by_path = HashMap::new();
let mut file_systems_by_skill_path = HashMap::new();
for snapshot in snapshots {
let SkillRootSnapshot {
root,
skills,
errors,
file_system,
} = snapshot;
if !skills.is_empty() && !skill_roots.contains(&root) {
skill_roots.push(root.clone());
}
for skill in &skills {
skill_root_by_path
.entry(skill.path_to_skills_md.clone())
.or_insert_with(|| root.clone());
file_systems_by_skill_path
.entry(skill.path_to_skills_md.clone())
.or_insert_with(|| Arc::clone(&file_system));
}
outcome.skills.extend(skills);
outcome.errors.extend(errors);
}
let mut seen = HashSet::new();
outcome
.skills
.retain(|skill| seen.insert(skill.path_to_skills_md.clone()));
let retained_skill_paths = outcome
.skills
.iter()
.map(|skill| skill.path_to_skills_md.clone())
.collect::<HashSet<_>>();
skill_root_by_path.retain(|path, _| retained_skill_paths.contains(path));
let used_roots = skill_root_by_path.values().cloned().collect::<HashSet<_>>();
skill_roots.retain(|root| used_roots.contains(root));
file_systems_by_skill_path.retain(|path, _| retained_skill_paths.contains(path));
outcome.skill_roots = skill_roots;
outcome.skill_root_by_path = Arc::new(skill_root_by_path);
outcome.file_systems_by_skill_path = SkillFileSystemsByPath::new(file_systems_by_skill_path);
outcome.skills.sort_by(|a, b| {
scope_rank(a.scope)
.cmp(&scope_rank(b.scope))
.then_with(|| a.name.cmp(&b.name))
.then_with(|| a.path_to_skills_md.cmp(&b.path_to_skills_md))
});
outcome
}
#[cfg(test)]
#[path = "root_loader_tests.rs"]
mod tests;