Add executor capability discovery V2 infrastructure (#47683)

## What changed

- Define `capabilities/discoverV2` request and inventory types and advertise `capabilityDiscoveryV2` in executor metadata.
- Prewarm installed plugin and global skill locations at server startup, with nonfatal scan failures.
- Add sandbox-scoped discovery caching with file watcher invalidation and request-time reloads when watching is unavailable.
- Batch metadata reads through the filesystem sandbox helper and bound discovery responses to 4 MiB and 2,048 capabilities.
- Share plugin identity and installed-version selection in `codex-core-plugin-common`, and fall back to `.app.json` when a plugin does not declare an apps configuration.

The V2 discovery manager is not yet wired into RPC request dispatch.

## Testing

Add coverage for inventory metadata, sandbox cache isolation, watcher invalidation, startup scan failures, response limits, and compatibility with older executor metadata.

GitOrigin-RevId: a223a5bc15fdf251023c0c71f468fd92c3b2d847
This commit is contained in:
Anthony Tafoya
2026-09-23 22:20:34 +00:00
committed by copyberry
parent 6fa3bcaae8
commit ab7439231c
36 changed files with 2655 additions and 83 deletions
+117 -1
View File
@@ -1,16 +1,21 @@
use std::io;
use std::sync::Arc;
use std::time::Instant;
use codex_build_info::BuildInfo;
use codex_exec_server_protocol::JSONRPCMessage;
use tokio::sync::mpsc;
use tokio_util::task::AbortOnDropHandle;
use tracing::debug;
use tracing::warn;
use crate::ExecServerRuntimePaths;
use crate::LocalFileSystem;
use crate::connection::CHANNEL_CAPACITY;
use crate::connection::JsonRpcConnection;
use crate::connection::JsonRpcConnectionEvent;
use crate::discover_v2::capability_locations::CapabilityLocationRequest;
use crate::discover_v2::capability_manager::CapabilityManager;
use crate::rpc::RpcCallError;
use crate::rpc::RpcNotificationSender;
use crate::rpc::RpcServerOutboundMessage;
@@ -26,9 +31,12 @@ use crate::telemetry::ConnectionTransport;
use crate::telemetry::ExecServerTelemetry;
use crate::telemetry::ExecutorRegistration;
use codex_http_client::HttpClientFactory;
use codex_utils_path_uri::PathUri;
#[derive(Clone)]
pub(crate) struct ConnectionProcessor {
capability_manager: Arc<CapabilityManager>,
capability_prewarm: Arc<AbortOnDropHandle<()>>,
session_registry: Arc<SessionRegistry>,
runtime_paths: ExecServerRuntimePaths,
telemetry: ExecServerTelemetry,
@@ -39,13 +47,17 @@ pub(crate) struct ConnectionProcessor {
impl ConnectionProcessor {
#[cfg(test)]
pub(crate) fn new(runtime_paths: ExecServerRuntimePaths) -> Self {
Self::new_with_telemetry(
Self::new_with_location_request(
runtime_paths,
ExecServerTelemetry::default(),
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
RequestDispatchMode::Inline,
// Connection-only tests must not scan the real user home.
Err(io::Error::other(
"capability home paths not supplied by this test",
)),
)
}
@@ -54,11 +66,49 @@ impl ConnectionProcessor {
telemetry: ExecServerTelemetry,
http_client_factory: HttpClientFactory,
request_dispatch_mode: RequestDispatchMode,
) -> Self {
let request =
codex_utils_home_dir::find_codex_home().map(|codex_home| CapabilityLocationRequest {
codex_home: PathUri::from_abs_path(&codex_home),
user_home: dirs::home_dir()
.and_then(|home| PathUri::from_host_native_path(home).ok()),
});
Self::new_with_location_request(
runtime_paths,
telemetry,
http_client_factory,
request_dispatch_mode,
request,
)
}
fn new_with_location_request(
runtime_paths: ExecServerRuntimePaths,
telemetry: ExecServerTelemetry,
http_client_factory: HttpClientFactory,
request_dispatch_mode: RequestDispatchMode,
request: io::Result<CapabilityLocationRequest>,
) -> Self {
// Library callers may bypass CLI startup. Capture the version before serving clients.
let _ = BuildInfo::get();
let capability_manager =
CapabilityManager::new(LocalFileSystem::with_runtime_paths(runtime_paths.clone()));
let manager = Arc::clone(&capability_manager);
// Start before any client connects; all sessions share these inert global locations.
let prewarm = tokio::spawn(async move {
// Home resolution failures are nonfatal, just like scan failures.
let result = match request {
Ok(request) => manager.prewarm_locations(request).await.map(|_| ()),
Err(error) => Err(error),
};
if let Err(error) = result {
tracing::warn!(error_kind = ?error.kind(), "capability location prewarming unavailable");
}
});
Self {
session_registry: SessionRegistry::new(telemetry.clone()),
capability_manager,
capability_prewarm: Arc::new(AbortOnDropHandle::new(prewarm)),
runtime_paths,
telemetry,
http_client_factory,
@@ -95,6 +145,7 @@ impl ConnectionProcessor {
}
pub(crate) async fn shutdown(&self) {
self.capability_prewarm.abort();
self.session_registry.shutdown().await;
}
}
@@ -106,6 +157,8 @@ async fn run_connection(
executor_registration: Option<Arc<ExecutorRegistration>>,
) {
let ConnectionProcessor {
capability_manager: _capability_manager,
capability_prewarm: _capability_prewarm,
session_registry,
runtime_paths,
telemetry,
@@ -317,6 +370,69 @@ mod tests {
use crate::rpc_server_requests::RpcServerRequestSender;
use crate::server::session_registry::SessionRegistry;
#[tokio::test]
async fn startup_prewarms_without_a_client_connection() -> anyhow::Result<()> {
let paths = ExecServerRuntimePaths::new(
std::env::current_exe()?,
/*codex_linux_sandbox_exe*/ None,
)?;
let directory = tempfile::tempdir()?;
let request = crate::discover_v2::capability_locations::CapabilityLocationRequest {
codex_home: PathUri::from_host_native_path(directory.path().join("codex"))?,
user_home: Some(PathUri::from_host_native_path(
directory.path().join("home"),
)?),
};
let skill_root = directory.path().join("codex/skills");
std::fs::create_dir_all(&skill_root)?;
let mut processor = super::ConnectionProcessor::new_with_location_request(
paths,
crate::ExecServerTelemetry::default(),
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
crate::server::RequestDispatchMode::Inline,
Ok(request.clone()),
);
Arc::get_mut(&mut processor.capability_prewarm)
.ok_or_else(|| anyhow::anyhow!("startup task unexpectedly shared"))?
.await?;
let cloned_processor = processor.clone();
assert!(Arc::ptr_eq(
&processor.capability_manager,
&cloned_processor.capability_manager,
));
let manager = &processor.capability_manager;
// Removing the fixture proves startup retained it before any later call.
std::fs::remove_dir(&skill_root)?;
let (first, second) = tokio::try_join!(
manager.prewarm_locations(request.clone()),
manager.prewarm_locations(request.clone()),
)?;
assert!(std::ptr::eq(first, second));
assert_eq!(first.0, request);
let expected_root = request.codex_home.join("skills")?;
assert!(
first
.1
.locations
.iter()
.any(|location| location.root == expected_root)
);
let mut different_request = request.clone();
different_request.codex_home = request.codex_home.join("different")?;
let Err(error) = manager.prewarm_locations(different_request).await else {
anyhow::bail!("different home paths must not reuse cached locations");
};
assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
assert!(std::ptr::eq(
first,
manager.prewarm_locations(request).await?,
));
processor.shutdown().await;
Ok(())
}
#[tokio::test]
async fn connection_accepts_pipelined_scalar_requests() {
let registry = SessionRegistry::new(crate::ExecServerTelemetry::default());