mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Integrate discovery V2 with step-scoped skill catalogs (#55322)
## What changed - Build executor skill catalogs from discovery V2 inventories without reopening host paths or falling back to legacy discovery. - Merge Cloud and executor skills by remote plugin identity and skill name, retaining Cloud metadata and accepted provider read routes. Apply plugin selection and disablement before merging. - Share a stable catalog across each step's rendering, list/read tools, and explicit and implicit skill invocations. - Preserve unmatched executor entries when Cloud catalogs are unavailable or partial; only complete Cloud catalogs disable unmatched remote entries. - Bind discovery caches to captured environments and executor sessions, and refresh MCP when V2 inventories change. Preserve pending invalidations when forced MCP refreshes fail. ## Testing Add regression coverage for V2 inventory sharing and refresh, environment isolation, catalog merging, partial Cloud results, disabled plugins, and catalog consistency across steps and read routes. GitOrigin-RevId: 842bdcbc9e19364dd69a9abf9e48330b7425a76c
This commit is contained in:
committed by
copyberry
parent
57560e9d53
commit
a309f9206a
@@ -1,6 +1,9 @@
|
||||
//! Caches root discovery by captured executor/session, root, and sandbox.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Weak;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
@@ -11,17 +14,18 @@ use tokio::sync::Mutex;
|
||||
use crate::CapabilityRootDiscoverRequest;
|
||||
use crate::CapabilityRootDiscovery;
|
||||
use crate::CapabilityRootsDiscoverParams;
|
||||
use crate::EnvironmentManager;
|
||||
use crate::Environment;
|
||||
use crate::ExecutorCapabilityDiscoverySnapshot;
|
||||
use crate::FileSystemSandboxContext;
|
||||
use crate::ResolvedSelectedCapabilityRoot;
|
||||
|
||||
/// Thread-scoped cache shared by capability consumers using the high-level executor API.
|
||||
///
|
||||
/// A single miss batches every requested root by environment. Successful discoveries and
|
||||
/// permanent failures remain cached by root and sandbox; transient failures are retried on the
|
||||
/// permanent failures remain cached by executor/session, root, and sandbox; transient failures are retried on the
|
||||
/// next request. Recovery is reported so dependent MCP projections can be invalidated.
|
||||
#[derive(Default)]
|
||||
pub struct ExecutorCapabilityDiscoveryCache {
|
||||
environment_manager: Arc<EnvironmentManager>,
|
||||
entries: Mutex<Vec<CachedRoot>>,
|
||||
recovered_discovery: AtomicBool,
|
||||
}
|
||||
@@ -34,64 +38,136 @@ impl std::fmt::Debug for ExecutorCapabilityDiscoveryCache {
|
||||
}
|
||||
}
|
||||
|
||||
struct CachedRoot {
|
||||
#[derive(Clone, Debug)]
|
||||
struct RootDiscoveryKey {
|
||||
selected_root: SelectedCapabilityRoot,
|
||||
sandbox: Option<FileSystemSandboxContext>,
|
||||
environment: Weak<Environment>,
|
||||
session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl PartialEq for RootDiscoveryKey {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.selected_root == other.selected_root
|
||||
&& self.sandbox == other.sandbox
|
||||
&& self.environment.ptr_eq(&other.environment)
|
||||
&& self.session_id == other.session_id
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RootDiscoveryRequest {
|
||||
key: RootDiscoveryKey,
|
||||
environment: Result<Arc<Environment>, String>,
|
||||
}
|
||||
|
||||
struct CachedRoot {
|
||||
key: RootDiscoveryKey,
|
||||
result: Result<Arc<CapabilityRootDiscovery>, String>,
|
||||
// Preserve transport classification after the public snapshot reduces errors to strings.
|
||||
retryable: bool,
|
||||
}
|
||||
|
||||
impl ExecutorCapabilityDiscoveryCache {
|
||||
pub fn new(environment_manager: Arc<EnvironmentManager>) -> Self {
|
||||
Self {
|
||||
environment_manager,
|
||||
entries: Mutex::new(Vec::new()),
|
||||
recovered_discovery: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports whether a previously failed root has recovered since the last observation.
|
||||
pub fn take_recovered_discovery(&self) -> bool {
|
||||
self.recovered_discovery.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
/// Returns discoveries in the same order as `selected_roots`.
|
||||
/// Returns discoveries in root order using only the captured environment handles.
|
||||
#[tracing::instrument(
|
||||
name = "capability_roots.discovery_cache.resolve",
|
||||
skip_all,
|
||||
fields(root_count = selected_roots.len())
|
||||
fields(root_count = roots.len())
|
||||
)]
|
||||
pub async fn discover(
|
||||
&self,
|
||||
selected_roots: &[SelectedCapabilityRoot],
|
||||
roots: &[ResolvedSelectedCapabilityRoot],
|
||||
sandbox_contexts: &HashMap<String, FileSystemSandboxContext>,
|
||||
) -> Vec<Result<Arc<CapabilityRootDiscovery>, String>> {
|
||||
let missing = {
|
||||
let environments = roots
|
||||
.iter()
|
||||
.map(|root| {
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&root.selected_root().location;
|
||||
(environment_id.clone(), Arc::clone(root.environment()))
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
// Resolve a session once per captured environment, before matching cached results.
|
||||
let bindings = futures::future::join_all(environments.into_iter().map(
|
||||
|(id, environment)| async move {
|
||||
let session = environment
|
||||
.executor_session_id()
|
||||
.await
|
||||
.map_err(|error| error.to_string());
|
||||
(id, (environment, session))
|
||||
},
|
||||
))
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<HashMap<_, _>>();
|
||||
let requests = roots
|
||||
.iter()
|
||||
.map(|root| {
|
||||
let selected_root = root.selected_root();
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&selected_root.location;
|
||||
let (environment, session) = &bindings[environment_id];
|
||||
RootDiscoveryRequest {
|
||||
key: RootDiscoveryKey {
|
||||
selected_root: selected_root.clone(),
|
||||
sandbox: sandbox_contexts.get(environment_id).cloned(),
|
||||
environment: Arc::downgrade(environment),
|
||||
session_id: session.as_ref().ok().cloned().flatten(),
|
||||
},
|
||||
environment: session
|
||||
.as_ref()
|
||||
.map(|_| Arc::clone(environment))
|
||||
.map_err(Clone::clone),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let keys = requests
|
||||
.iter()
|
||||
.map(|request| request.key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let mut results = {
|
||||
let entries = self.entries.lock().await;
|
||||
selected_roots
|
||||
requests
|
||||
.iter()
|
||||
.filter(|selected_root| {
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&selected_root.location;
|
||||
let sandbox = sandbox_contexts.get(environment_id);
|
||||
!entries.iter().any(|cached| {
|
||||
cached.selected_root == **selected_root
|
||||
&& cached.sandbox.as_ref() == sandbox
|
||||
&& !cached.retryable
|
||||
.map(|request| {
|
||||
request.environment.as_ref().ok().and_then(|_| {
|
||||
entries
|
||||
.iter()
|
||||
.find(|cached| cached.key == request.key && !cached.retryable)
|
||||
.map(|cached| cached.result.clone())
|
||||
})
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let discovered = self.discover_missing(missing, sandbox_contexts).await;
|
||||
let missing = requests
|
||||
.into_iter()
|
||||
.zip(&results)
|
||||
.filter_map(|(request, result)| result.is_none().then_some(request))
|
||||
.collect();
|
||||
let discovered = self.discover_missing(missing).await;
|
||||
let mut entries = self.entries.lock().await;
|
||||
for discovered_root in discovered {
|
||||
// Reuse a concurrent success only for the same captured binding.
|
||||
let resolved = entries
|
||||
.iter()
|
||||
.find(|cached| cached.key == discovered_root.key && cached.result.is_ok())
|
||||
.map_or(&discovered_root.result, |cached| &cached.result);
|
||||
for (key, result) in keys.iter().zip(&mut results) {
|
||||
if *key == discovered_root.key && result.is_none() {
|
||||
*result = Some(resolved.clone());
|
||||
}
|
||||
}
|
||||
if let Some(cached) = entries
|
||||
.iter_mut()
|
||||
.find(|cached| cached.selected_root == discovered_root.selected_root)
|
||||
.find(|cached| cached.key.selected_root == discovered_root.key.selected_root)
|
||||
{
|
||||
if cached.sandbox != discovered_root.sandbox || cached.result.is_err() {
|
||||
if cached.key != discovered_root.key || cached.result.is_err() {
|
||||
if cached.result.is_err() && discovered_root.result.is_ok() {
|
||||
self.recovered_discovery.store(true, Ordering::Release);
|
||||
}
|
||||
@@ -101,146 +177,133 @@ impl ExecutorCapabilityDiscoveryCache {
|
||||
entries.push(discovered_root);
|
||||
}
|
||||
}
|
||||
selected_roots
|
||||
.iter()
|
||||
.map(|selected_root| {
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&selected_root.location;
|
||||
let sandbox = sandbox_contexts.get(environment_id);
|
||||
match entries.iter().find(|cached| {
|
||||
cached.selected_root == *selected_root && cached.sandbox.as_ref() == sandbox
|
||||
}) {
|
||||
Some(cached) => cached.result.clone(),
|
||||
None => Err(format!(
|
||||
keys.into_iter()
|
||||
.zip(results)
|
||||
.map(|(key, result)| {
|
||||
result.unwrap_or_else(|| {
|
||||
Err(format!(
|
||||
"selected capability root `{}` was not discovered",
|
||||
selected_root.id
|
||||
)),
|
||||
}
|
||||
key.selected_root.id
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Resolves the selected roots once and freezes their results for one model step.
|
||||
/// Freezes discoveries and sandbox contexts for the captured roots.
|
||||
pub async fn snapshot(
|
||||
&self,
|
||||
selected_roots: &[SelectedCapabilityRoot],
|
||||
roots: &[ResolvedSelectedCapabilityRoot],
|
||||
sandbox_contexts: &HashMap<String, FileSystemSandboxContext>,
|
||||
) -> ExecutorCapabilityDiscoverySnapshot {
|
||||
ExecutorCapabilityDiscoverySnapshot::new(
|
||||
selected_roots,
|
||||
self.discover(selected_roots, sandbox_contexts).await,
|
||||
sandbox_contexts.clone(),
|
||||
)
|
||||
let selected_roots = roots
|
||||
.iter()
|
||||
.map(|root| root.selected_root().clone())
|
||||
.collect::<Vec<_>>();
|
||||
let results = self.discover(roots, sandbox_contexts).await;
|
||||
ExecutorCapabilityDiscoverySnapshot::new(&selected_roots, results, sandbox_contexts.clone())
|
||||
}
|
||||
|
||||
async fn discover_missing(
|
||||
&self,
|
||||
missing: Vec<SelectedCapabilityRoot>,
|
||||
sandbox_contexts: &HashMap<String, FileSystemSandboxContext>,
|
||||
) -> Vec<CachedRoot> {
|
||||
let mut grouped = BTreeMap::<String, Vec<SelectedCapabilityRoot>>::new();
|
||||
async fn discover_missing(&self, missing: Vec<RootDiscoveryRequest>) -> Vec<CachedRoot> {
|
||||
let mut grouped = BTreeMap::<String, Vec<RootDiscoveryRequest>>::new();
|
||||
for selected_root in missing {
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&selected_root.location;
|
||||
&selected_root.key.selected_root.location;
|
||||
grouped
|
||||
.entry(environment_id.clone())
|
||||
.or_default()
|
||||
.push(selected_root);
|
||||
}
|
||||
|
||||
let batches = grouped.into_iter().flat_map(|(environment_id, roots)| {
|
||||
let batches = grouped.into_values().flat_map(|roots| {
|
||||
roots
|
||||
.chunks(crate::capability_discovery::MAX_ROOTS_PER_REQUEST)
|
||||
.map(|batch| (environment_id.clone(), batch.to_vec()))
|
||||
.map(<[RootDiscoveryRequest]>::to_vec)
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let discoveries =
|
||||
futures::future::join_all(batches.map(|(environment_id, selected_roots)| async move {
|
||||
let sandbox = sandbox_contexts.get(&environment_id).cloned();
|
||||
let Some(environment) = self.environment_manager.get_environment(&environment_id)
|
||||
else {
|
||||
let error = format!("environment `{environment_id}` is unavailable");
|
||||
let discoveries = futures::future::join_all(batches.map(|selected_roots| async move {
|
||||
let environment = match &selected_roots[0].environment {
|
||||
Ok(environment) => Arc::clone(environment),
|
||||
Err(error) => {
|
||||
let error = error.clone();
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
sandbox: sandbox.clone(),
|
||||
.map(|request| CachedRoot {
|
||||
key: request.key,
|
||||
result: Err(error.clone()),
|
||||
retryable: true,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
};
|
||||
let params = CapabilityRootsDiscoverParams {
|
||||
roots: selected_roots
|
||||
.iter()
|
||||
.map(|selected_root| {
|
||||
let CapabilityRootLocation::Environment { path, .. } =
|
||||
&selected_root.location;
|
||||
CapabilityRootDiscoverRequest {
|
||||
id: selected_root.id.clone(),
|
||||
path: path.clone(),
|
||||
sandbox: sandbox.clone(),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let response = match environment.discover_capability_roots(params).await {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
let retryable = crate::client::can_retry_connection_attempt(&error);
|
||||
let error = error.to_string();
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
sandbox: sandbox.clone(),
|
||||
result: Err(error.clone()),
|
||||
retryable,
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
};
|
||||
if response.roots.len() != selected_roots.len() {
|
||||
let error = format!(
|
||||
"exec-server returned {} capability roots for {} requests",
|
||||
response.roots.len(),
|
||||
selected_roots.len()
|
||||
);
|
||||
}
|
||||
};
|
||||
let params = CapabilityRootsDiscoverParams {
|
||||
roots: selected_roots
|
||||
.iter()
|
||||
.map(|request| {
|
||||
let selected_root = &request.key.selected_root;
|
||||
let CapabilityRootLocation::Environment { path, .. } =
|
||||
&selected_root.location;
|
||||
CapabilityRootDiscoverRequest {
|
||||
id: selected_root.id.clone(),
|
||||
path: path.clone(),
|
||||
sandbox: request.key.sandbox.clone(),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let response = match environment.discover_capability_roots(params).await {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
let retryable = crate::client::can_retry_connection_attempt(&error);
|
||||
let error = error.to_string();
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
sandbox: sandbox.clone(),
|
||||
.map(|request| CachedRoot {
|
||||
key: request.key,
|
||||
result: Err(error.clone()),
|
||||
retryable: false,
|
||||
retryable,
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
selected_roots
|
||||
};
|
||||
if response.roots.len() != selected_roots.len() {
|
||||
let error = format!(
|
||||
"exec-server returned {} capability roots for {} requests",
|
||||
response.roots.len(),
|
||||
selected_roots.len()
|
||||
);
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.zip(response.roots)
|
||||
.map(|(selected_root, discovery)| {
|
||||
let CapabilityRootLocation::Environment { path, .. } =
|
||||
&selected_root.location;
|
||||
let result = if discovery.id == selected_root.id && discovery.path == *path
|
||||
{
|
||||
Ok(Arc::new(discovery))
|
||||
} else {
|
||||
Err(format!(
|
||||
"exec-server returned mismatched capability root `{}` at {}",
|
||||
discovery.id, discovery.path
|
||||
))
|
||||
};
|
||||
CachedRoot {
|
||||
selected_root,
|
||||
sandbox: sandbox.clone(),
|
||||
result,
|
||||
retryable: false,
|
||||
}
|
||||
.map(|request| CachedRoot {
|
||||
key: request.key,
|
||||
result: Err(error.clone()),
|
||||
retryable: false,
|
||||
})
|
||||
.collect()
|
||||
}))
|
||||
.await;
|
||||
.collect();
|
||||
}
|
||||
selected_roots
|
||||
.into_iter()
|
||||
.zip(response.roots)
|
||||
.map(|(request, discovery)| {
|
||||
let selected_root = &request.key.selected_root;
|
||||
let CapabilityRootLocation::Environment { path, .. } = &selected_root.location;
|
||||
let result = if discovery.id == selected_root.id && discovery.path == *path {
|
||||
Ok(Arc::new(discovery))
|
||||
} else {
|
||||
Err(format!(
|
||||
"exec-server returned mismatched capability root `{}` at {}",
|
||||
discovery.id, discovery.path
|
||||
))
|
||||
};
|
||||
CachedRoot {
|
||||
key: request.key,
|
||||
result,
|
||||
retryable: false,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}))
|
||||
.await;
|
||||
discoveries.into_iter().flatten().collect()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user