mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
Add batched executor capability discovery (#33852)
## Why Selected capability roots can contribute plugins, MCP servers, connectors, and skills. Discovering each contribution separately requires repeated access to the executor filesystem. ## What changed - Add the `capabilityRoots/discoverV1` exec-server RPC to scan selected roots and materialize recognized plugin manifests, configuration files, skill instructions, and skill metadata in one bounded request. - Add the opt-in `executor_capability_discovery` feature, with a thread-scoped cache and per-step snapshot shared by MCP and skill discovery. - Parse MCP, connector, and skill contributions from the materialized snapshot, including serving cached skill instructions without another filesystem read. ## Testing - Cover discovery limits, manifest precedence, root-local failures, cache reuse, plugin contributions, and parity with the existing environment skill loader. GitOrigin-RevId: f98fd2321cafb58c596db02da1f83c09d8eb375d
This commit is contained in:
@@ -0,0 +1,462 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
|
||||
use codex_exec_server_protocol::CapabilityRootDiscoverRequest;
|
||||
use codex_exec_server_protocol::CapabilityRootDiscovery;
|
||||
use codex_exec_server_protocol::CapabilityRootsDiscoverParams;
|
||||
use codex_exec_server_protocol::CapabilityRootsDiscoverResponse;
|
||||
use codex_exec_server_protocol::CapabilityTextFile;
|
||||
use codex_exec_server_protocol::DISCOVERABLE_PLUGIN_MANIFEST_PATHS;
|
||||
use codex_exec_server_protocol::DiscoveredPluginFiles;
|
||||
use codex_exec_server_protocol::DiscoveredSkillFiles;
|
||||
use codex_file_system::ExecutorFileSystem;
|
||||
use codex_file_system::WalkEntryKind;
|
||||
use codex_file_system::WalkOptions;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use futures::StreamExt;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
const MAX_ROOTS_PER_REQUEST: usize = 128;
|
||||
const MAX_SCAN_DEPTH: usize = 6;
|
||||
const MAX_DIRECTORIES_PER_ROOT: usize = 2_000;
|
||||
const MAX_ENTRIES_PER_ROOT: usize = 20_000;
|
||||
const MAX_FILE_BYTES: usize = 1024 * 1024;
|
||||
const MAX_BUNDLE_BYTES_PER_ROOT: usize = 16 * 1024 * 1024;
|
||||
const MAX_CONCURRENT_ROOTS: usize = 8;
|
||||
const SKILL_FILE_NAME: &str = "SKILL.md";
|
||||
const SKILL_METADATA_PATH: &str = "agents/openai.yaml";
|
||||
const DEFAULT_MCP_CONFIG_PATH: &str = ".mcp.json";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CapabilityDiscoveryError {
|
||||
#[error("capability root discovery accepts at most {MAX_ROOTS_PER_REQUEST} roots")]
|
||||
TooManyRoots,
|
||||
}
|
||||
|
||||
/// Discovers and materializes capability manifests using one executor-local filesystem.
|
||||
///
|
||||
/// Product parsing and policy intentionally remain with the caller. This operation owns the
|
||||
/// filesystem-expensive portion: bounded traversal, recognized-file selection, and reads.
|
||||
#[tracing::instrument(
|
||||
name = "capability_roots.discover_v1",
|
||||
skip_all,
|
||||
fields(root_count = params.roots.len())
|
||||
)]
|
||||
pub async fn discover_capability_roots(
|
||||
file_system: &dyn ExecutorFileSystem,
|
||||
params: CapabilityRootsDiscoverParams,
|
||||
) -> Result<CapabilityRootsDiscoverResponse, CapabilityDiscoveryError> {
|
||||
if params.roots.len() > MAX_ROOTS_PER_REQUEST {
|
||||
return Err(CapabilityDiscoveryError::TooManyRoots);
|
||||
}
|
||||
|
||||
let roots = futures::stream::iter(params.roots)
|
||||
.map(|root| discover_root(file_system, root))
|
||||
.buffered(MAX_CONCURRENT_ROOTS)
|
||||
.collect()
|
||||
.await;
|
||||
Ok(CapabilityRootsDiscoverResponse { roots })
|
||||
}
|
||||
|
||||
async fn discover_root(
|
||||
file_system: &dyn ExecutorFileSystem,
|
||||
request: CapabilityRootDiscoverRequest,
|
||||
) -> CapabilityRootDiscovery {
|
||||
let CapabilityRootDiscoverRequest { id, path } = request;
|
||||
let mut discovery = CapabilityRootDiscovery {
|
||||
id,
|
||||
path: path.clone(),
|
||||
plugin: None,
|
||||
skills: Vec::new(),
|
||||
namespace_manifests: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
};
|
||||
|
||||
match file_system.get_metadata(&path, /*sandbox*/ None).await {
|
||||
Ok(metadata) if metadata.is_directory => {}
|
||||
Ok(_) => {
|
||||
discovery.error = Some(format!("capability root {path} is not a directory"));
|
||||
return discovery;
|
||||
}
|
||||
Err(error) => {
|
||||
discovery.error = Some(format!("failed to inspect capability root {path}: {error}"));
|
||||
return discovery;
|
||||
}
|
||||
}
|
||||
|
||||
let walk = match file_system
|
||||
.walk(
|
||||
&path,
|
||||
WalkOptions {
|
||||
max_depth: MAX_SCAN_DEPTH,
|
||||
max_directories: MAX_DIRECTORIES_PER_ROOT,
|
||||
max_entries: MAX_ENTRIES_PER_ROOT,
|
||||
follow_directory_symlinks: true,
|
||||
prune_hidden_directories: false,
|
||||
},
|
||||
/*sandbox*/ None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(walk) => walk,
|
||||
Err(error) => {
|
||||
discovery.error = Some(format!("failed to scan capability root {path}: {error}"));
|
||||
return discovery;
|
||||
}
|
||||
};
|
||||
discovery
|
||||
.warnings
|
||||
.extend(walk.errors.into_iter().map(|error| {
|
||||
format!(
|
||||
"failed to scan capability path {}: {}",
|
||||
error.path, error.message
|
||||
)
|
||||
}));
|
||||
if walk.truncated {
|
||||
discovery.warnings.push(format!(
|
||||
"capability scan reached its traversal limit (root: {path})"
|
||||
));
|
||||
}
|
||||
|
||||
let mut skill_paths = Vec::new();
|
||||
let mut namespace_manifest_paths = Vec::new();
|
||||
for entry in walk.entries {
|
||||
if entry.kind != WalkEntryKind::File {
|
||||
continue;
|
||||
}
|
||||
if entry.path.basename().as_deref() == Some(SKILL_FILE_NAME) {
|
||||
skill_paths.push(entry.path.clone());
|
||||
}
|
||||
if is_plugin_manifest_path(&entry.path) {
|
||||
namespace_manifest_paths.push(entry.path);
|
||||
}
|
||||
}
|
||||
skill_paths.sort_unstable_by_key(PathUri::to_string);
|
||||
namespace_manifest_paths.sort_unstable_by(|left, right| {
|
||||
let left_root = plugin_root_for_manifest(left).map(|path| path.to_string());
|
||||
let right_root = plugin_root_for_manifest(right).map(|path| path.to_string());
|
||||
left_root
|
||||
.cmp(&right_root)
|
||||
.then_with(|| plugin_manifest_priority(left).cmp(&plugin_manifest_priority(right)))
|
||||
});
|
||||
|
||||
let mut budget = BundleBudget::default();
|
||||
let root_manifest =
|
||||
read_first_plugin_manifest(file_system, &path, &mut budget, &mut discovery.warnings).await;
|
||||
|
||||
let inherited_manifest = match root_manifest.as_ref() {
|
||||
Some(manifest) => Some(manifest.clone()),
|
||||
None => {
|
||||
read_nearest_ancestor_manifest(file_system, &path, &mut budget, &mut discovery.warnings)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let mut seen_namespace_roots = HashSet::new();
|
||||
if let Some(manifest) = inherited_manifest {
|
||||
if let Some(plugin_root) = plugin_root_for_manifest(&manifest.path) {
|
||||
seen_namespace_roots.insert(plugin_root);
|
||||
}
|
||||
discovery.namespace_manifests.push(manifest);
|
||||
}
|
||||
for manifest_path in namespace_manifest_paths {
|
||||
let Some(plugin_root) = plugin_root_for_manifest(&manifest_path) else {
|
||||
continue;
|
||||
};
|
||||
if !seen_namespace_roots.insert(plugin_root) {
|
||||
continue;
|
||||
}
|
||||
if let Some(manifest) = read_optional_text_file(
|
||||
file_system,
|
||||
manifest_path,
|
||||
&mut budget,
|
||||
&mut discovery.warnings,
|
||||
)
|
||||
.await
|
||||
{
|
||||
discovery.namespace_manifests.push(manifest);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(manifest) = root_manifest {
|
||||
let declarations = plugin_declaration_paths(&path, &manifest, &mut discovery.warnings);
|
||||
let mcp_path = if declarations.mcp_inline {
|
||||
None
|
||||
} else {
|
||||
declarations
|
||||
.mcp_config
|
||||
.or_else(|| path.join(DEFAULT_MCP_CONFIG_PATH).ok())
|
||||
};
|
||||
let mcp_config = match mcp_path {
|
||||
Some(path) => {
|
||||
read_optional_text_file(file_system, path, &mut budget, &mut discovery.warnings)
|
||||
.await
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let apps_config = match declarations.apps_config {
|
||||
Some(path) => {
|
||||
read_optional_text_file(file_system, path, &mut budget, &mut discovery.warnings)
|
||||
.await
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
discovery.plugin = Some(DiscoveredPluginFiles {
|
||||
manifest,
|
||||
mcp_config,
|
||||
apps_config,
|
||||
});
|
||||
}
|
||||
|
||||
for skill_path in skill_paths {
|
||||
let Some(instructions) = read_optional_text_file(
|
||||
file_system,
|
||||
skill_path.clone(),
|
||||
&mut budget,
|
||||
&mut discovery.warnings,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let metadata = match skill_path
|
||||
.parent()
|
||||
.and_then(|skill_dir| skill_dir.join(SKILL_METADATA_PATH).ok())
|
||||
{
|
||||
Some(metadata_path) => {
|
||||
read_optional_text_file(
|
||||
file_system,
|
||||
metadata_path,
|
||||
&mut budget,
|
||||
&mut discovery.warnings,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
discovery.skills.push(DiscoveredSkillFiles {
|
||||
instructions,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
discovery
|
||||
}
|
||||
|
||||
async fn read_first_plugin_manifest(
|
||||
file_system: &dyn ExecutorFileSystem,
|
||||
root: &PathUri,
|
||||
budget: &mut BundleBudget,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Option<CapabilityTextFile> {
|
||||
for relative_path in DISCOVERABLE_PLUGIN_MANIFEST_PATHS {
|
||||
let Ok(path) = root.join(relative_path) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(manifest) = read_optional_text_file(file_system, path, budget, warnings).await {
|
||||
return Some(manifest);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn read_nearest_ancestor_manifest(
|
||||
file_system: &dyn ExecutorFileSystem,
|
||||
root: &PathUri,
|
||||
budget: &mut BundleBudget,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Option<CapabilityTextFile> {
|
||||
let mut ancestor = root.parent();
|
||||
while let Some(path) = ancestor {
|
||||
if let Some(manifest) =
|
||||
read_first_plugin_manifest(file_system, &path, budget, warnings).await
|
||||
{
|
||||
return Some(manifest);
|
||||
}
|
||||
ancestor = path.parent();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn read_optional_text_file(
|
||||
file_system: &dyn ExecutorFileSystem,
|
||||
path: PathUri,
|
||||
budget: &mut BundleBudget,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Option<CapabilityTextFile> {
|
||||
let metadata = match file_system.get_metadata(&path, /*sandbox*/ None).await {
|
||||
Ok(metadata) if metadata.is_file => metadata,
|
||||
Ok(_) => return None,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return None,
|
||||
Err(error) => {
|
||||
warnings.push(format!("failed to inspect capability file {path}: {error}"));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let Ok(size) = usize::try_from(metadata.size) else {
|
||||
warnings.push(format!("capability file {path} is too large"));
|
||||
return None;
|
||||
};
|
||||
if size > MAX_FILE_BYTES {
|
||||
warnings.push(format!(
|
||||
"capability file {path} exceeds the {MAX_FILE_BYTES}-byte limit"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
if !budget.can_add(size) {
|
||||
warnings.push(format!(
|
||||
"capability root bundle exceeds the {MAX_BUNDLE_BYTES_PER_ROOT}-byte limit"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
let mut stream = match file_system.read_file_stream(&path, /*sandbox*/ None).await {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
warnings.push(format!("failed to read capability file {path}: {error}"));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let mut contents = Vec::with_capacity(size);
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = match chunk {
|
||||
Ok(chunk) => chunk,
|
||||
Err(error) => {
|
||||
warnings.push(format!("failed to read capability file {path}: {error}"));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let Some(new_len) = contents.len().checked_add(chunk.len()) else {
|
||||
warnings.push(format!("capability file {path} exceeded its read limit"));
|
||||
return None;
|
||||
};
|
||||
if new_len > MAX_FILE_BYTES || !budget.can_add(new_len) {
|
||||
warnings.push(format!("capability file {path} exceeded its read limit"));
|
||||
return None;
|
||||
}
|
||||
contents.extend_from_slice(&chunk);
|
||||
}
|
||||
let contents = match String::from_utf8(contents) {
|
||||
Ok(contents) => contents,
|
||||
Err(error) => {
|
||||
warnings.push(format!("capability file {path} is not UTF-8: {error}"));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
budget.add(contents.len());
|
||||
Some(CapabilityTextFile { path, contents })
|
||||
}
|
||||
|
||||
fn is_plugin_manifest_path(path: &PathUri) -> bool {
|
||||
plugin_manifest_priority(path).is_some()
|
||||
}
|
||||
|
||||
fn plugin_manifest_priority(path: &PathUri) -> Option<usize> {
|
||||
if path.basename().as_deref() != Some("plugin.json") {
|
||||
return None;
|
||||
}
|
||||
let manifest_directory = path.parent()?.basename()?;
|
||||
DISCOVERABLE_PLUGIN_MANIFEST_PATHS
|
||||
.iter()
|
||||
.position(|relative_path| {
|
||||
relative_path.strip_suffix("/plugin.json") == Some(manifest_directory.as_str())
|
||||
})
|
||||
}
|
||||
|
||||
fn plugin_root_for_manifest(path: &PathUri) -> Option<PathUri> {
|
||||
path.parent()?.parent()
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct BundleBudget {
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl BundleBudget {
|
||||
fn can_add(&self, bytes: usize) -> bool {
|
||||
self.bytes
|
||||
.checked_add(bytes)
|
||||
.is_some_and(|total| total <= MAX_BUNDLE_BYTES_PER_ROOT)
|
||||
}
|
||||
|
||||
fn add(&mut self, bytes: usize) {
|
||||
self.bytes += bytes;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PluginDeclarationPaths {
|
||||
mcp_config: Option<PathUri>,
|
||||
mcp_inline: bool,
|
||||
apps_config: Option<PathUri>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RawPluginDeclarations {
|
||||
#[serde(default)]
|
||||
mcp_servers: Option<Value>,
|
||||
#[serde(default)]
|
||||
apps: Option<Value>,
|
||||
}
|
||||
|
||||
fn plugin_declaration_paths(
|
||||
root: &PathUri,
|
||||
manifest: &CapabilityTextFile,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> PluginDeclarationPaths {
|
||||
let declarations = match serde_json::from_str::<RawPluginDeclarations>(&manifest.contents) {
|
||||
Ok(declarations) => declarations,
|
||||
Err(_) => return PluginDeclarationPaths::default(),
|
||||
};
|
||||
PluginDeclarationPaths {
|
||||
mcp_config: declarations.mcp_servers.as_ref().and_then(|value| {
|
||||
declared_file_path(root, "mcpServers", value, &manifest.path, warnings)
|
||||
}),
|
||||
mcp_inline: declarations
|
||||
.mcp_servers
|
||||
.as_ref()
|
||||
.is_some_and(Value::is_object),
|
||||
apps_config: declarations
|
||||
.apps
|
||||
.as_ref()
|
||||
.and_then(|value| declared_file_path(root, "apps", value, &manifest.path, warnings)),
|
||||
}
|
||||
}
|
||||
|
||||
fn declared_file_path(
|
||||
root: &PathUri,
|
||||
field: &str,
|
||||
value: &Value,
|
||||
manifest_path: &PathUri,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Option<PathUri> {
|
||||
let Value::String(path) = value else {
|
||||
return None;
|
||||
};
|
||||
let Some(relative_path) = path.strip_prefix("./") else {
|
||||
warnings.push(format!(
|
||||
"ignoring {field} in {manifest_path}: path must start with `./`"
|
||||
));
|
||||
return None;
|
||||
};
|
||||
if relative_path.is_empty()
|
||||
|| relative_path
|
||||
.split(['/', '\\'])
|
||||
.any(|component| component == "..")
|
||||
{
|
||||
warnings.push(format!(
|
||||
"ignoring {field} in {manifest_path}: path must remain below the capability root"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
match root.join(relative_path) {
|
||||
Ok(path) if path.starts_with(root) => Some(path),
|
||||
Ok(_) | Err(_) => {
|
||||
warnings.push(format!(
|
||||
"ignoring {field} in {manifest_path}: path must remain below the capability root"
|
||||
));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_protocol::capabilities::CapabilityRootLocation;
|
||||
use codex_protocol::capabilities::SelectedCapabilityRoot;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::CapabilityRootDiscoverRequest;
|
||||
use crate::CapabilityRootDiscovery;
|
||||
use crate::CapabilityRootsDiscoverParams;
|
||||
use crate::EnvironmentManager;
|
||||
use crate::ExecutorCapabilityDiscoverySnapshot;
|
||||
|
||||
/// Thread-scoped cache shared by capability consumers using the high-level executor API.
|
||||
///
|
||||
/// A single miss batches every requested root by environment. The cache deliberately has no
|
||||
/// invalidation: selected roots are already treated as stable for the lifetime of a thread by the
|
||||
/// existing plugin and skill providers.
|
||||
pub struct ExecutorCapabilityDiscoveryCache {
|
||||
environment_manager: Arc<EnvironmentManager>,
|
||||
entries: Mutex<Vec<CachedRoot>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ExecutorCapabilityDiscoveryCache {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ExecutorCapabilityDiscoveryCache")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
struct CachedRoot {
|
||||
selected_root: SelectedCapabilityRoot,
|
||||
// Both successes and failures are memoized for the thread. Retrying a transient failure for
|
||||
// the same stable selected root requires explicit invalidation or a new thread.
|
||||
result: Result<Arc<CapabilityRootDiscovery>, String>,
|
||||
}
|
||||
|
||||
impl ExecutorCapabilityDiscoveryCache {
|
||||
pub fn new(environment_manager: Arc<EnvironmentManager>) -> Self {
|
||||
Self {
|
||||
environment_manager,
|
||||
entries: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns discoveries in the same order as `selected_roots`.
|
||||
#[tracing::instrument(
|
||||
name = "capability_roots.discovery_cache.resolve",
|
||||
skip_all,
|
||||
fields(root_count = selected_roots.len())
|
||||
)]
|
||||
pub async fn discover(
|
||||
&self,
|
||||
selected_roots: &[SelectedCapabilityRoot],
|
||||
) -> Vec<Result<Arc<CapabilityRootDiscovery>, String>> {
|
||||
let missing = {
|
||||
let entries = self.entries.lock().await;
|
||||
selected_roots
|
||||
.iter()
|
||||
.filter(|selected_root| {
|
||||
!entries
|
||||
.iter()
|
||||
.any(|cached| cached.selected_root == **selected_root)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let discovered = self.discover_missing(missing).await;
|
||||
let mut entries = self.entries.lock().await;
|
||||
for discovered_root in discovered {
|
||||
if !entries
|
||||
.iter()
|
||||
.any(|cached| cached.selected_root == discovered_root.selected_root)
|
||||
{
|
||||
entries.push(discovered_root);
|
||||
}
|
||||
}
|
||||
selected_roots
|
||||
.iter()
|
||||
.map(|selected_root| {
|
||||
match entries
|
||||
.iter()
|
||||
.find(|cached| cached.selected_root == *selected_root)
|
||||
{
|
||||
Some(cached) => cached.result.clone(),
|
||||
None => Err(format!(
|
||||
"selected capability root `{}` was not discovered",
|
||||
selected_root.id
|
||||
)),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Resolves the selected roots once and freezes their results for one model step.
|
||||
pub async fn snapshot(
|
||||
&self,
|
||||
selected_roots: &[SelectedCapabilityRoot],
|
||||
) -> ExecutorCapabilityDiscoverySnapshot {
|
||||
ExecutorCapabilityDiscoverySnapshot::new(
|
||||
selected_roots,
|
||||
self.discover(selected_roots).await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn discover_missing(&self, missing: Vec<SelectedCapabilityRoot>) -> Vec<CachedRoot> {
|
||||
let mut grouped = BTreeMap::<String, Vec<SelectedCapabilityRoot>>::new();
|
||||
for selected_root in missing {
|
||||
let CapabilityRootLocation::Environment { environment_id, .. } =
|
||||
&selected_root.location;
|
||||
grouped
|
||||
.entry(environment_id.clone())
|
||||
.or_default()
|
||||
.push(selected_root);
|
||||
}
|
||||
|
||||
let discoveries = futures::future::join_all(grouped.into_iter().map(
|
||||
|(environment_id, selected_roots)| async move {
|
||||
let Some(environment) = self.environment_manager.get_environment(&environment_id)
|
||||
else {
|
||||
let error = format!("environment `{environment_id}` is unavailable");
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
result: Err(error.clone()),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
};
|
||||
let params = CapabilityRootsDiscoverParams {
|
||||
roots: selected_roots
|
||||
.iter()
|
||||
.map(|selected_root| {
|
||||
let CapabilityRootLocation::Environment { path, .. } =
|
||||
&selected_root.location;
|
||||
CapabilityRootDiscoverRequest {
|
||||
id: selected_root.id.clone(),
|
||||
path: path.clone(),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let response = match environment.discover_capability_roots(params).await {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
let error = error.to_string();
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
result: Err(error.clone()),
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
};
|
||||
if response.roots.len() != selected_roots.len() {
|
||||
let error = format!(
|
||||
"exec-server returned {} capability roots for {} requests",
|
||||
response.roots.len(),
|
||||
selected_roots.len()
|
||||
);
|
||||
return selected_roots
|
||||
.into_iter()
|
||||
.map(|selected_root| CachedRoot {
|
||||
selected_root,
|
||||
result: Err(error.clone()),
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
selected_roots
|
||||
.into_iter()
|
||||
.zip(response.roots)
|
||||
.map(|(selected_root, discovery)| {
|
||||
let CapabilityRootLocation::Environment { path, .. } =
|
||||
&selected_root.location;
|
||||
let result = if discovery.id == selected_root.id && discovery.path == *path
|
||||
{
|
||||
Ok(Arc::new(discovery))
|
||||
} else {
|
||||
Err(format!(
|
||||
"exec-server returned mismatched capability root `{}` at {}",
|
||||
discovery.id, discovery.path
|
||||
))
|
||||
};
|
||||
CachedRoot {
|
||||
selected_root,
|
||||
result,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
},
|
||||
))
|
||||
.await;
|
||||
discoveries.into_iter().flatten().collect()
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,9 @@ use crate::environment::EnvironmentConnectionState;
|
||||
use crate::process::ExecProcessEvent;
|
||||
use crate::process::ExecProcessEventLog;
|
||||
use crate::process::ExecProcessEventReceiver;
|
||||
use crate::protocol::CAPABILITY_ROOTS_DISCOVER_METHOD;
|
||||
use crate::protocol::CapabilityRootsDiscoverParams;
|
||||
use crate::protocol::CapabilityRootsDiscoverResponse;
|
||||
use crate::protocol::ENVIRONMENT_INFO_METHOD;
|
||||
use crate::protocol::ENVIRONMENT_STATUS_METHOD;
|
||||
use crate::protocol::EXEC_CLOSED_METHOD;
|
||||
@@ -695,6 +698,13 @@ impl ExecServerClient {
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn discover_capability_roots(
|
||||
&self,
|
||||
params: CapabilityRootsDiscoverParams,
|
||||
) -> Result<CapabilityRootsDiscoverResponse, ExecServerError> {
|
||||
self.call(CAPABILITY_ROOTS_DISCOVER_METHOD, ¶ms).await
|
||||
}
|
||||
|
||||
pub async fn read(&self, params: ReadParams) -> Result<ReadResponse, ExecServerError> {
|
||||
self.call(EXEC_READ_METHOD, ¶ms).await
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ use codex_protocol::capabilities::CapabilityRootLocation;
|
||||
use codex_protocol::capabilities::SelectedCapabilityRoot;
|
||||
use futures::FutureExt;
|
||||
|
||||
use crate::CapabilityRootsDiscoverParams;
|
||||
use crate::CapabilityRootsDiscoverResponse;
|
||||
use crate::ExecServerError;
|
||||
use crate::ExecServerRuntimePaths;
|
||||
use crate::ExecutorFileSystem;
|
||||
@@ -713,6 +715,19 @@ impl Environment {
|
||||
}
|
||||
}
|
||||
|
||||
/// Discovers plugin and skill manifests through the environment's high-level discovery API.
|
||||
pub async fn discover_capability_roots(
|
||||
&self,
|
||||
params: CapabilityRootsDiscoverParams,
|
||||
) -> Result<CapabilityRootsDiscoverResponse, ExecServerError> {
|
||||
match &self.remote_client {
|
||||
Some(client) => client.get().await?.discover_capability_roots(params).await,
|
||||
None => crate::discover_capability_roots(self.filesystem.as_ref(), params)
|
||||
.await
|
||||
.map_err(|error| ExecServerError::Protocol(error.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts connecting a remote environment without waiting for it.
|
||||
/// Requires an active Tokio runtime when background startup is supported.
|
||||
pub fn start_connecting(&self) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod capability_discovery;
|
||||
mod capability_discovery_cache;
|
||||
mod client;
|
||||
mod client_api;
|
||||
mod client_transport;
|
||||
@@ -33,6 +35,9 @@ mod websocket_pong_watchdog;
|
||||
|
||||
use codex_exec_server_protocol as protocol;
|
||||
|
||||
pub use capability_discovery::CapabilityDiscoveryError;
|
||||
pub use capability_discovery::discover_capability_roots;
|
||||
pub use capability_discovery_cache::ExecutorCapabilityDiscoveryCache;
|
||||
pub use client::ExecServerClient;
|
||||
pub use client::ExecServerError;
|
||||
pub use client::http_client::HttpResponseBodyStream;
|
||||
@@ -43,6 +48,7 @@ pub use client_api::NoiseRendezvousConnectArgs;
|
||||
pub use client_api::NoiseRendezvousConnectBundle;
|
||||
pub use client_api::NoiseRendezvousConnectProvider;
|
||||
pub use client_api::RemoteExecServerConnectArgs;
|
||||
pub use codex_exec_server_protocol::ExecutorCapabilityDiscoverySnapshot;
|
||||
pub use codex_exec_server_protocol::ProcessId;
|
||||
pub use codex_file_system::CopyOptions;
|
||||
pub use codex_file_system::CreateDirectoryOptions;
|
||||
@@ -98,6 +104,14 @@ pub use process::ExecProcessEventReceiver;
|
||||
pub use process::ExecProcessFuture;
|
||||
pub use process::StartedExecProcess;
|
||||
pub use protocol::ByteChunk;
|
||||
pub use protocol::CAPABILITY_ROOTS_DISCOVER_METHOD;
|
||||
pub use protocol::CapabilityRootDiscoverRequest;
|
||||
pub use protocol::CapabilityRootDiscovery;
|
||||
pub use protocol::CapabilityRootsDiscoverParams;
|
||||
pub use protocol::CapabilityRootsDiscoverResponse;
|
||||
pub use protocol::CapabilityTextFile;
|
||||
pub use protocol::DiscoveredPluginFiles;
|
||||
pub use protocol::DiscoveredSkillFiles;
|
||||
pub use protocol::EnvironmentInfo;
|
||||
pub use protocol::EnvironmentStatus;
|
||||
pub use protocol::EnvironmentStatusKind;
|
||||
|
||||
@@ -4,6 +4,8 @@ use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use codex_exec_server_protocol::JSONRPCErrorError;
|
||||
|
||||
use crate::CapabilityRootsDiscoverParams;
|
||||
use crate::CapabilityRootsDiscoverResponse;
|
||||
use crate::CopyOptions;
|
||||
use crate::CreateDirectoryOptions;
|
||||
use crate::ExecServerRuntimePaths;
|
||||
@@ -65,6 +67,15 @@ impl FileSystemHandler {
|
||||
self.file_reads.close_all().await;
|
||||
}
|
||||
|
||||
pub(crate) async fn discover_capability_roots(
|
||||
&self,
|
||||
params: CapabilityRootsDiscoverParams,
|
||||
) -> Result<CapabilityRootsDiscoverResponse, JSONRPCErrorError> {
|
||||
crate::discover_capability_roots(&self.file_system, params)
|
||||
.await
|
||||
.map_err(|error| invalid_request(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) async fn open(
|
||||
&self,
|
||||
params: FsOpenParams,
|
||||
|
||||
@@ -14,6 +14,8 @@ use tokio_util::task::TaskTracker;
|
||||
use crate::ExecServerRuntimePaths;
|
||||
use crate::client::http_client::PendingReqwestHttpBodyStream;
|
||||
use crate::client::http_client::ReqwestHttpRequestRunner;
|
||||
use crate::protocol::CapabilityRootsDiscoverParams;
|
||||
use crate::protocol::CapabilityRootsDiscoverResponse;
|
||||
use crate::protocol::EnvironmentInfo;
|
||||
use crate::protocol::EnvironmentStatus;
|
||||
use crate::protocol::EnvironmentStatusKind;
|
||||
@@ -258,6 +260,14 @@ impl ExecServerHandler {
|
||||
self.file_system.read_file(params).await
|
||||
}
|
||||
|
||||
pub(crate) async fn discover_capability_roots(
|
||||
&self,
|
||||
params: CapabilityRootsDiscoverParams,
|
||||
) -> Result<CapabilityRootsDiscoverResponse, JSONRPCErrorError> {
|
||||
self.require_initialized_for("capability discovery")?;
|
||||
self.file_system.discover_capability_roots(params).await
|
||||
}
|
||||
|
||||
pub(crate) async fn fs_open(
|
||||
&self,
|
||||
params: FsOpenParams,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::protocol::CAPABILITY_ROOTS_DISCOVER_METHOD;
|
||||
use crate::protocol::CapabilityRootsDiscoverParams;
|
||||
use crate::protocol::ENVIRONMENT_INFO_METHOD;
|
||||
use crate::protocol::ENVIRONMENT_STATUS_METHOD;
|
||||
use crate::protocol::EXEC_METHOD;
|
||||
@@ -76,6 +78,12 @@ pub(crate) fn build_router() -> RpcRouter<ExecServerHandler> {
|
||||
ENVIRONMENT_STATUS_METHOD,
|
||||
|handler: Arc<ExecServerHandler>, _params: ()| async move { handler.environment_status() },
|
||||
);
|
||||
router.request(
|
||||
CAPABILITY_ROOTS_DISCOVER_METHOD,
|
||||
|handler: Arc<ExecServerHandler>, params: CapabilityRootsDiscoverParams| async move {
|
||||
handler.discover_capability_roots(params).await
|
||||
},
|
||||
);
|
||||
router.request(
|
||||
EXEC_READ_METHOD,
|
||||
|handler: Arc<ExecServerHandler>, params: ReadParams| async move {
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
mod common;
|
||||
|
||||
use codex_exec_server::CAPABILITY_ROOTS_DISCOVER_METHOD;
|
||||
use codex_exec_server::CapabilityRootDiscovery;
|
||||
use codex_exec_server::CapabilityRootsDiscoverParams;
|
||||
use codex_exec_server::CapabilityRootsDiscoverResponse;
|
||||
use codex_exec_server::InitializeParams;
|
||||
use codex_exec_server::InitializeResponse;
|
||||
use codex_exec_server_protocol::CapabilityRootDiscoverRequest;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::JSONRPCResponse;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use common::exec_server::exec_server;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn discovers_a_complete_capability_bundle_in_one_request() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
write_file(
|
||||
&root.path().join(".codex-plugin/plugin.json"),
|
||||
r#"{
|
||||
"name": "demo",
|
||||
"interface": {"displayName": "Demo Plugin"},
|
||||
"mcpServers": "./config/mcp.json",
|
||||
"apps": "./config/apps.json"
|
||||
}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join(".claude-plugin/plugin.json"),
|
||||
r#"{"name":"lower-priority-claude"}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join(".cursor-plugin/plugin.json"),
|
||||
r#"{"name":"lower-priority-cursor"}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("config/mcp.json"),
|
||||
r#"{"mcpServers":{"demo":{"command":"demo-server"}}}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("config/apps.json"),
|
||||
r#"{"apps":{"demo":{"connector_id":"connector-demo"}}}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("skills/deploy/SKILL.md"),
|
||||
"---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy instructions.\n",
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("skills/deploy/agents/openai.yaml"),
|
||||
"policy:\n allow_implicit_invocation: false\n",
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("nested/.claude-plugin/plugin.json"),
|
||||
r#"{"name":"nested"}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("nested/skills/audit/SKILL.md"),
|
||||
"---\nname: audit\ndescription: Audit the service.\n---\n",
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("nested-cursor/.cursor-plugin/plugin.json"),
|
||||
r#"{"name":"cursor-nested"}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join("nested-cursor/skills/review/SKILL.md"),
|
||||
"---\nname: review\ndescription: Review the service.\n---\n",
|
||||
)?;
|
||||
|
||||
let mut server = exec_server().await?;
|
||||
initialize(&mut server).await?;
|
||||
let root_uri = PathUri::from_host_native_path(root.path())?;
|
||||
let discovery = discover_root(&mut server, "demo@1", root_uri.clone()).await?;
|
||||
|
||||
assert_eq!(discovery.id, "demo@1");
|
||||
assert_eq!(discovery.path, root_uri);
|
||||
assert_eq!(discovery.error, None);
|
||||
assert_eq!(discovery.warnings, Vec::<String>::new());
|
||||
let plugin = discovery.plugin.as_ref().expect("root plugin");
|
||||
assert_eq!(
|
||||
plugin.manifest.path,
|
||||
root_uri.join(".codex-plugin/plugin.json")?
|
||||
);
|
||||
assert!(plugin.manifest.contents.contains("Demo Plugin"));
|
||||
assert_eq!(
|
||||
plugin.mcp_config.as_ref().map(|file| &file.path),
|
||||
Some(&root_uri.join("config/mcp.json")?)
|
||||
);
|
||||
assert_eq!(
|
||||
plugin.apps_config.as_ref().map(|file| &file.path),
|
||||
Some(&root_uri.join("config/apps.json")?)
|
||||
);
|
||||
assert_eq!(
|
||||
discovery
|
||||
.namespace_manifests
|
||||
.iter()
|
||||
.map(|file| file.path.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
root_uri.join(".codex-plugin/plugin.json")?,
|
||||
root_uri.join("nested/.claude-plugin/plugin.json")?,
|
||||
root_uri.join("nested-cursor/.cursor-plugin/plugin.json")?,
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
discovery
|
||||
.skills
|
||||
.iter()
|
||||
.map(|skill| (
|
||||
skill.instructions.path.clone(),
|
||||
skill
|
||||
.metadata
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.path.clone()),
|
||||
))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(root_uri.join("nested-cursor/skills/review/SKILL.md")?, None,),
|
||||
(root_uri.join("nested/skills/audit/SKILL.md")?, None,),
|
||||
(
|
||||
root_uri.join("skills/deploy/SKILL.md")?,
|
||||
Some(root_uri.join("skills/deploy/agents/openai.yaml")?),
|
||||
),
|
||||
]
|
||||
);
|
||||
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn discovers_cursor_plugin_without_reading_default_mcp_for_inline_servers()
|
||||
-> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
write_file(
|
||||
&root.path().join(".cursor-plugin/plugin.json"),
|
||||
r#"{"name":"cursor-demo","mcpServers":{"inline":{"command":"inline"}}}"#,
|
||||
)?;
|
||||
write_file(
|
||||
&root.path().join(".mcp.json"),
|
||||
r#"{"mcpServers":{"should-not-load":{"command":"wrong"}}}"#,
|
||||
)?;
|
||||
|
||||
let mut server = exec_server().await?;
|
||||
initialize(&mut server).await?;
|
||||
let root_uri = PathUri::from_host_native_path(root.path())?;
|
||||
let discovery = discover_root(&mut server, "cursor@1", root_uri.clone()).await?;
|
||||
|
||||
assert_eq!(discovery.error, None);
|
||||
assert_eq!(discovery.warnings, Vec::<String>::new());
|
||||
let plugin = discovery.plugin.expect("cursor plugin");
|
||||
assert_eq!(
|
||||
plugin.manifest.path,
|
||||
root_uri.join(".cursor-plugin/plugin.json")?
|
||||
);
|
||||
assert_eq!(plugin.mcp_config, None);
|
||||
assert_eq!(
|
||||
discovery
|
||||
.namespace_manifests
|
||||
.iter()
|
||||
.map(|manifest| manifest.path.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![root_uri.join(".cursor-plugin/plugin.json")?]
|
||||
);
|
||||
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn discover_root(
|
||||
server: &mut common::exec_server::ExecServerHarness,
|
||||
id: &str,
|
||||
path: PathUri,
|
||||
) -> anyhow::Result<CapabilityRootDiscovery> {
|
||||
let request_id = server
|
||||
.send_request(
|
||||
CAPABILITY_ROOTS_DISCOVER_METHOD,
|
||||
serde_json::to_value(CapabilityRootsDiscoverParams {
|
||||
roots: vec![CapabilityRootDiscoverRequest {
|
||||
id: id.to_string(),
|
||||
path,
|
||||
}],
|
||||
})?,
|
||||
)
|
||||
.await?;
|
||||
let response = server.next_event().await?;
|
||||
let JSONRPCMessage::Response(JSONRPCResponse { id, result }) = response else {
|
||||
anyhow::bail!("expected discovery response, received {response:?}");
|
||||
};
|
||||
assert_eq!(id, request_id);
|
||||
let response: CapabilityRootsDiscoverResponse = serde_json::from_value(result)?;
|
||||
let [discovery] = response.roots.as_slice() else {
|
||||
anyhow::bail!("expected exactly one discovered root");
|
||||
};
|
||||
Ok(discovery.clone())
|
||||
}
|
||||
|
||||
async fn initialize(server: &mut common::exec_server::ExecServerHarness) -> anyhow::Result<()> {
|
||||
let initialize_id = server
|
||||
.send_request(
|
||||
"initialize",
|
||||
serde_json::to_value(InitializeParams {
|
||||
client_name: "capability-discovery-test".to_string(),
|
||||
resume_session_id: None,
|
||||
})?,
|
||||
)
|
||||
.await?;
|
||||
let response = server
|
||||
.wait_for_event(|event| {
|
||||
matches!(event, JSONRPCMessage::Response(response) if response.id == initialize_id)
|
||||
})
|
||||
.await?;
|
||||
let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else {
|
||||
unreachable!("wait predicate only accepts a response");
|
||||
};
|
||||
let _: InitializeResponse = serde_json::from_value(result)?;
|
||||
server
|
||||
.send_notification("initialized", serde_json::json!({}))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_file(path: &std::path::Path, contents: &str) -> anyhow::Result<()> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("test file should have a parent"))?;
|
||||
std::fs::create_dir_all(parent)?;
|
||||
std::fs::write(path, contents)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user