mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
276 lines
9.5 KiB
Rust
276 lines
9.5 KiB
Rust
use std::collections::HashMap;
|
|
use std::collections::HashSet;
|
|
use std::fmt;
|
|
use std::sync::Arc;
|
|
use std::sync::Mutex;
|
|
|
|
use codex_utils_plugins::PluginSkillRoot;
|
|
use futures::StreamExt;
|
|
|
|
use crate::SkillLoadOutcome;
|
|
use crate::loader::SkillRoot;
|
|
use crate::loader::SkillRootSnapshot;
|
|
use crate::loader::load_skill_root;
|
|
use crate::model::SkillFileSystemsByPath;
|
|
|
|
const MAX_CONCURRENT_ROOT_SCANS: usize = 5;
|
|
|
|
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
enum RootScanPriority {
|
|
AgentsSkills,
|
|
Default,
|
|
}
|
|
|
|
struct PendingRootLoad {
|
|
root: SkillRoot,
|
|
root_index: usize,
|
|
duplicate_root_indices: Vec<usize>,
|
|
cache_key: Option<PluginSkillRoot>,
|
|
scheduling_priority: RootScanPriority,
|
|
}
|
|
|
|
struct CompletedRootLoad {
|
|
root_index: usize,
|
|
duplicate_root_indices: Vec<usize>,
|
|
snapshot: SkillRootSnapshot,
|
|
}
|
|
|
|
/// Parsed plugin skill-root snapshots produced by one plugin load.
|
|
///
|
|
/// Clones share the same snapshots. The plugins manager stores them with the corresponding loaded
|
|
/// plugins and passes a clone to skill loading as an optional preload.
|
|
#[derive(Clone)]
|
|
pub struct PluginSkillSnapshots {
|
|
snapshots_by_root: Arc<Mutex<HashMap<PluginSkillRoot, SkillRootSnapshot>>>,
|
|
}
|
|
|
|
impl PluginSkillSnapshots {
|
|
/// Creates an empty snapshot collection for a plugin load to populate.
|
|
pub fn for_plugin_load() -> Self {
|
|
Self {
|
|
snapshots_by_root: Arc::new(Mutex::new(HashMap::new())),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl fmt::Debug for PluginSkillSnapshots {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.debug_struct("PluginSkillSnapshots")
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn load_and_merge_skill_roots<I>(
|
|
roots: I,
|
|
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
|
|
) -> SkillLoadOutcome
|
|
where
|
|
I: IntoIterator<Item = SkillRoot>,
|
|
{
|
|
let preflight_absent_paths = HashSet::new();
|
|
load_and_merge_skill_roots_with_preflight(
|
|
roots,
|
|
plugin_skill_snapshots,
|
|
&preflight_absent_paths,
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub(crate) async fn load_and_merge_skill_roots_with_preflight<I>(
|
|
roots: I,
|
|
plugin_skill_snapshots: Option<&PluginSkillSnapshots>,
|
|
preflight_absent_paths: &HashSet<codex_utils_absolute_path::AbsolutePathBuf>,
|
|
) -> SkillLoadOutcome
|
|
where
|
|
I: IntoIterator<Item = SkillRoot>,
|
|
{
|
|
let roots = roots.into_iter().collect::<Vec<_>>();
|
|
let root_count = roots.len();
|
|
let mut pending_loads = Vec::<PendingRootLoad>::with_capacity(root_count);
|
|
let mut pending_load_by_cache_key = HashMap::<PluginSkillRoot, usize>::new();
|
|
for (root_index, root) in roots.into_iter().enumerate() {
|
|
let cache_key = match (
|
|
root.plugin_id.clone(),
|
|
root.plugin_namespace.clone(),
|
|
root.plugin_root.clone(),
|
|
) {
|
|
(Some(plugin_id), Some(plugin_namespace), Some(plugin_root)) => Some(PluginSkillRoot {
|
|
path: root.path.clone(),
|
|
plugin_id,
|
|
plugin_namespace,
|
|
plugin_root,
|
|
}),
|
|
_ => None,
|
|
};
|
|
// Preserve sequential cache semantics: with a cache, the first occurrence scans and later
|
|
// occurrences reuse its snapshot. Without a cache, duplicate roots remain independent.
|
|
if plugin_skill_snapshots.is_some()
|
|
&& let Some(cache_key) = cache_key.as_ref()
|
|
&& let Some(pending_index) = pending_load_by_cache_key.get(cache_key)
|
|
{
|
|
pending_loads[*pending_index]
|
|
.duplicate_root_indices
|
|
.push(root_index);
|
|
continue;
|
|
}
|
|
|
|
if plugin_skill_snapshots.is_some()
|
|
&& let Some(cache_key) = cache_key.as_ref()
|
|
{
|
|
pending_load_by_cache_key.insert(cache_key.clone(), pending_loads.len());
|
|
}
|
|
let scheduling_priority = if root.is_agents_skills_root() {
|
|
RootScanPriority::AgentsSkills
|
|
} else {
|
|
RootScanPriority::Default
|
|
};
|
|
pending_loads.push(PendingRootLoad {
|
|
root,
|
|
root_index,
|
|
duplicate_root_indices: Vec::new(),
|
|
cache_key,
|
|
scheduling_priority,
|
|
});
|
|
}
|
|
|
|
// Group duplicate cache keys before prioritization so the original first occurrence remains
|
|
// the scan/cache owner. Sorting by the original index within each priority retains FIFO order.
|
|
pending_loads.sort_by_key(|pending| (pending.scheduling_priority, pending.root_index));
|
|
// Unordered buffering starts known `.agents/skills` roots first and lets fast host and cached
|
|
// roots free slots while an earlier executor root is still pending. The indexed snapshots
|
|
// below restore the original merge order.
|
|
let completed_loads = futures::stream::iter(pending_loads)
|
|
.map(|pending| async move {
|
|
let PendingRootLoad {
|
|
root,
|
|
root_index,
|
|
duplicate_root_indices,
|
|
cache_key,
|
|
scheduling_priority: _,
|
|
} = pending;
|
|
let cached_snapshot = cache_key.as_ref().and_then(|cache_key| {
|
|
let plugin_skill_snapshots = plugin_skill_snapshots?;
|
|
plugin_skill_snapshots
|
|
.snapshots_by_root
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.get(cache_key)
|
|
.cloned()
|
|
});
|
|
match cached_snapshot {
|
|
Some(snapshot) => CompletedRootLoad {
|
|
root_index,
|
|
duplicate_root_indices,
|
|
snapshot,
|
|
},
|
|
None => {
|
|
let preflight_absent = preflight_absent_paths.contains(&root.path);
|
|
let snapshot = load_skill_root(root, preflight_absent).await;
|
|
if let Some(plugin_skill_snapshots) = plugin_skill_snapshots
|
|
&& let Some(cache_key) = cache_key
|
|
{
|
|
plugin_skill_snapshots
|
|
.snapshots_by_root
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert(cache_key, snapshot.clone());
|
|
}
|
|
CompletedRootLoad {
|
|
root_index,
|
|
duplicate_root_indices,
|
|
snapshot,
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.buffer_unordered(MAX_CONCURRENT_ROOT_SCANS)
|
|
.collect::<Vec<_>>()
|
|
.await;
|
|
|
|
let mut indexed_root_snapshots = Vec::with_capacity(root_count);
|
|
for completed in completed_loads {
|
|
for root_index in completed.duplicate_root_indices {
|
|
indexed_root_snapshots.push((root_index, completed.snapshot.clone()));
|
|
}
|
|
indexed_root_snapshots.push((completed.root_index, completed.snapshot));
|
|
}
|
|
indexed_root_snapshots.sort_unstable_by_key(|(root_index, _)| *root_index);
|
|
let root_snapshots = indexed_root_snapshots
|
|
.into_iter()
|
|
.map(|(_, snapshot)| snapshot)
|
|
.collect();
|
|
|
|
merge_skill_root_snapshots(root_snapshots)
|
|
}
|
|
|
|
fn merge_skill_root_snapshots(snapshots: Vec<SkillRootSnapshot>) -> SkillLoadOutcome {
|
|
fn scope_rank(scope: codex_protocol::protocol::SkillScope) -> u8 {
|
|
use codex_protocol::protocol::SkillScope;
|
|
|
|
// Higher-priority scopes first (matches root scan order for dedupe).
|
|
match scope {
|
|
SkillScope::Repo => 0,
|
|
SkillScope::User => 1,
|
|
SkillScope::System => 2,
|
|
SkillScope::Admin => 3,
|
|
}
|
|
}
|
|
|
|
let mut outcome = SkillLoadOutcome::default();
|
|
let mut skill_roots = Vec::new();
|
|
let mut skill_root_by_path = HashMap::new();
|
|
let mut file_systems_by_skill_path = HashMap::new();
|
|
|
|
for snapshot in snapshots {
|
|
let SkillRootSnapshot {
|
|
root,
|
|
skills,
|
|
errors,
|
|
file_system,
|
|
} = snapshot;
|
|
if !skills.is_empty() && !skill_roots.contains(&root) {
|
|
skill_roots.push(root.clone());
|
|
}
|
|
for skill in &skills {
|
|
skill_root_by_path
|
|
.entry(skill.path_to_skills_md.clone())
|
|
.or_insert_with(|| root.clone());
|
|
file_systems_by_skill_path
|
|
.entry(skill.path_to_skills_md.clone())
|
|
.or_insert_with(|| Arc::clone(&file_system));
|
|
}
|
|
outcome.skills.extend(skills);
|
|
outcome.errors.extend(errors);
|
|
}
|
|
|
|
let mut seen = HashSet::new();
|
|
outcome
|
|
.skills
|
|
.retain(|skill| seen.insert(skill.path_to_skills_md.clone()));
|
|
let retained_skill_paths = outcome
|
|
.skills
|
|
.iter()
|
|
.map(|skill| skill.path_to_skills_md.clone())
|
|
.collect::<HashSet<_>>();
|
|
skill_root_by_path.retain(|path, _| retained_skill_paths.contains(path));
|
|
let used_roots = skill_root_by_path.values().cloned().collect::<HashSet<_>>();
|
|
skill_roots.retain(|root| used_roots.contains(root));
|
|
file_systems_by_skill_path.retain(|path, _| retained_skill_paths.contains(path));
|
|
outcome.skill_roots = skill_roots;
|
|
outcome.skill_root_by_path = Arc::new(skill_root_by_path);
|
|
outcome.file_systems_by_skill_path = SkillFileSystemsByPath::new(file_systems_by_skill_path);
|
|
|
|
outcome.skills.sort_by(|a, b| {
|
|
scope_rank(a.scope)
|
|
.cmp(&scope_rank(b.scope))
|
|
.then_with(|| a.name.cmp(&b.name))
|
|
.then_with(|| a.path_to_skills_md.cmp(&b.path_to_skills_md))
|
|
});
|
|
|
|
outcome
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "root_loader_tests.rs"]
|
|
mod tests;
|