Force executor capability rescans after V2 cache invalidation (#62675)

## Why

Invalidating Codex's discovery cache could still return stale inventory
if the executor reused its cached snapshot before its file watcher
reported a change.

## What changed

Add an optional `forceReload` flag to `capabilities/discoverV2` that
makes the executor rescan its files. Codex sends it on the next lookup
of each invalidated context. Explicit reloads also replace the matching
cache entry so subsequent ordinary requests can reuse the fresh result.

## Testing

Add coverage for discovering a newly added skill before the watcher
runs, forwarding the reload flag after invalidation, and reusing
refreshed results.

GitOrigin-RevId: c98f2b8d4772e10b74ff09dd382ad8386857158f
This commit is contained in:
Anthony Tafoya
2026-11-20 05:18:42 +00:00
committed by copyberry
parent ffa05c8bfb
commit 15c5444e87
14 changed files with 221 additions and 34 deletions
@@ -53,6 +53,7 @@ impl ExecutorDiscovery {
&selected.selection.environment_id,
selected.environment(),
DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: selected.cwd().clone(),
sandbox: Some(selected.sandbox_context(/*additional_permissions*/ None)),
},
@@ -129,6 +130,7 @@ impl Session {
continue;
}
let request = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: selected.cwd().clone(),
sandbox: Some(self.discovery_sandbox(selected, skill_turn)),
};
@@ -1123,6 +1123,7 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<
.context("remote environment")?;
let cache = codex_exec_server::ExecutorCapabilityDiscoveryV2Cache::default();
let request = codex_exec_server::DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: codex_utils_path_uri::PathUri::parse("file:///repo")?,
sandbox: None,
};
@@ -1276,6 +1277,16 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
assert_eq!(calls().await, before, "invalidation is lazy");
let _ = test.codex.current_mcp_config_and_runtime_context().await;
assert_eq!(calls().await, before + 1);
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.rfind(|request| request["method"] == "capabilities/discoverV2")
.context("refreshed discovery request")?["params"]["forceReload"],
true,
);
let observed = observed.lock().await;
let inventory = observed
.last()
@@ -1591,6 +1602,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.context("remote environment")?;
let cache = codex_exec_server::ExecutorCapabilityDiscoveryV2Cache::default();
let request = codex_exec_server::DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: codex_utils_path_uri::PathUri::from_host_native_path(root.path())?,
sandbox: None,
};
@@ -1718,13 +1730,78 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.map_err(anyhow::Error::msg)?;
// Reading the oldest entry did not promote it: eviction is FIFO, not LRU.
let reloaded = cache
.discover("other-executor", &environment, request)
.discover("other-executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert_eq!(calls().await, 19);
assert!(!Arc::ptr_eq(&other, &reloaded));
assert_eq!(other.as_ref(), reloaded.as_ref());
// Explicit requests and dirty entries reload once; ordinary reads reuse the result.
let mut previous = reloaded;
for (expected_calls, force_reload) in [(20, true), (21, false)] {
if !force_reload {
cache.invalidate();
assert_eq!(calls().await, expected_calls - 1);
}
let refreshed = cache
.discover(
"executor",
&environment,
codex_exec_server::DiscoverV2CapabilitiesRequest {
force_reload,
..request.clone()
},
)
.await
.map_err(anyhow::Error::msg)?;
assert!(!Arc::ptr_eq(&previous, &refreshed));
let ordinary = cache
.discover("executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&refreshed, &ordinary));
assert_eq!(calls().await, expected_calls);
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.rfind(|request| request["method"] == "capabilities/discoverV2")
.context("forced discovery request")?["params"]["forceReload"],
true,
);
previous = refreshed;
}
// Reloading one dirty entry leaves the others dirty; new contexts load normally.
for (cwd, force_reload) in [("overflow", true), ("new-context", false)] {
cache
.discover(
"executor",
&environment,
codex_exec_server::DiscoverV2CapabilitiesRequest {
cwd: request.cwd.join(cwd)?,
..request.clone()
},
)
.await
.map_err(anyhow::Error::msg)?;
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.rfind(|request| request["method"] == "capabilities/discoverV2")
.context("discovery request")?["params"]["forceReload"]
.as_bool()
.unwrap_or(false),
force_reload,
);
}
Ok(())
}
@@ -39,6 +39,9 @@ pub struct DiscoverV2CapabilitiesRequest {
pub cwd: PathUri,
#[serde(default)]
pub sandbox: Option<FileSystemSandboxContext>,
/// Rescan the executor's files even if its watcher has not reported a change.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub force_reload: bool,
}
/// Discovered metadata before caller policy is applied.
@@ -7,14 +7,26 @@ use pretty_assertions::assert_eq;
#[test]
fn discovery_contract_round_trips_json() -> Result<(), Box<dyn std::error::Error>> {
let request = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::parse("file:///workspace")?,
sandbox: None,
};
let wire = serde_json::to_value(&request)?;
assert!(wire.get("forceReload").is_none());
assert_eq!(
serde_json::from_value::<DiscoverV2CapabilitiesRequest>(wire)?,
request
);
let refreshed = DiscoverV2CapabilitiesRequest {
force_reload: true,
..request
};
let wire = serde_json::to_value(&refreshed)?;
assert_eq!(wire["forceReload"], true);
assert_eq!(
serde_json::from_value::<DiscoverV2CapabilitiesRequest>(wire)?,
refreshed
);
let text_file = CapabilityTextFile {
path: PathUri::parse("file:///plugin/config.json")?,
contents: "{}".to_string(),
@@ -1,5 +1,5 @@
//! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox.
//! Dirty entries are replaced on the next lookup; in-flight readers may finish with older results.
//! Reloads replace the matching entry; in-flight readers may finish with older results.
//! Reports recovery from retryable failures so Core can refresh dependent MCP projections.
use std::sync::Arc;
@@ -9,12 +9,14 @@ use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use codex_utils_path_uri::PathUri;
use tokio::sync::OnceCell;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
use crate::Environment;
use crate::ExecServerError;
use crate::FileSystemSandboxContext;
const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16;
@@ -26,12 +28,28 @@ pub struct ExecutorCapabilityDiscoveryV2Cache {
recovered_discovery: AtomicBool,
}
struct CachedEnvironmentDiscovery {
request: DiscoverV2CapabilitiesRequest,
// Reload controls how this entry is populated, not which inventory it identifies.
struct DiscoveryCacheKey {
environment: Weak<Environment>,
session_id: Option<String>,
cwd: PathUri,
sandbox: Option<FileSystemSandboxContext>,
}
impl PartialEq for DiscoveryCacheKey {
fn eq(&self, other: &Self) -> bool {
self.environment.ptr_eq(&other.environment)
&& self.session_id == other.session_id
&& self.cwd == other.cwd
&& self.sandbox == other.sandbox
}
}
struct CachedEnvironmentDiscovery {
key: DiscoveryCacheKey,
response: Arc<OnceCell<Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>>>,
retryable: Arc<AtomicBool>,
force_reload: bool,
dirty: bool,
}
@@ -46,7 +64,7 @@ impl ExecutorCapabilityDiscoveryV2Cache {
self.recovered_discovery.swap(false, Ordering::AcqRel)
}
/// Makes future lookups reload every context without interrupting existing loads.
/// Dirties cached entries so their next lookup reloads without interrupting existing loads.
pub fn invalidate(&self) {
let mut entries = self
.entries
@@ -66,56 +84,68 @@ impl ExecutorCapabilityDiscoveryV2Cache {
let environment = Arc::downgrade(environment);
for entry in entries
.iter_mut()
.filter(|entry| entry.environment.ptr_eq(&environment))
.filter(|entry| entry.key.environment.ptr_eq(&environment))
{
entry.dirty = true;
}
}
/// Reuses successes and non-retryable failures using V1's error classification.
/// Reuses successes and non-retryable failures unless the request forces a reload.
/// Retryable failures and cancelled loads leave the cell available for retry.
/// Invalidation during a load applies to the next lookup, not its current readers.
pub async fn discover(
&self,
environment_id: &str,
environment: &Arc<Environment>,
request: DiscoverV2CapabilitiesRequest,
mut request: DiscoverV2CapabilitiesRequest,
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>> {
// Resolve startup/replacement before lookup; a resumed session keeps its cache entries.
let session_id = environment.executor_session_id().await.map_err(|error| {
tracing::warn!(environment_id, "executor discovery V2 session unavailable");
Arc::new(error)
})?;
let (response, retryable) = {
let key = DiscoveryCacheKey {
environment: Arc::downgrade(environment),
session_id,
cwd: request.cwd.clone(),
sandbox: request.sandbox.clone(),
};
let (response, retryable, force_reload) = {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let environment = Arc::downgrade(environment);
if let Some(entry) = entries.iter().find(|entry| {
!entry.dirty
&& entry.environment.ptr_eq(&environment)
&& entry.session_id == session_id
&& entry.request == request
}) {
(Arc::clone(&entry.response), Arc::clone(&entry.retryable))
let matching = entries.iter().position(|entry| entry.key == key);
if let Some(entry) = matching
.map(|index| &entries[index])
.filter(|entry| !entry.dirty && !request.force_reload)
{
(
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
entry.force_reload,
)
} else {
let force_reload = request.force_reload || matching.is_some();
if let Some(index) = matching {
entries.remove(index);
}
// Recovery belongs to the environment/request, not one executor session.
let retryable = entries
.iter()
.find(|entry| {
!entry.dirty
&& entry.environment.ptr_eq(&environment)
&& entry.request == request
&& entry.key.environment.ptr_eq(&key.environment)
&& entry.key.cwd == key.cwd
&& entry.key.sandbox == key.sandbox
})
.map(|entry| Arc::clone(&entry.retryable))
.unwrap_or_default();
let entry = CachedEnvironmentDiscovery {
request: request.clone(),
environment,
session_id,
key,
response: Arc::new(OnceCell::new()),
retryable,
force_reload,
dirty: false,
};
let capacity = self
@@ -123,17 +153,21 @@ impl ExecutorCapabilityDiscoveryV2Cache {
.load(Ordering::Relaxed)
.max(MIN_CACHED_DISCOVERY_CONTEXTS);
// Existing readers retain their response cells after replacement or eviction.
entries.retain(|entry| !entry.dirty);
if entries.len() >= capacity {
entries.remove(0);
}
let load = (Arc::clone(&entry.response), Arc::clone(&entry.retryable));
let load = (
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
force_reload,
);
entries.push(entry);
load
}
};
let response = response
.get_or_try_init(|| async {
request.force_reload = force_reload;
match environment.discover_capabilities_v2(request).await {
Ok(response) => Ok(Ok(Arc::new(response))),
Err(error) => {
@@ -349,6 +349,7 @@ async fn refresh_retires_a_still_connected_old_executor() -> Result<()> {
));
let cache = crate::ExecutorCapabilityDiscoveryV2Cache::default();
let request = crate::DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: codex_utils_path_uri::PathUri::from_host_native_path(&root)?,
sandbox: None,
};
@@ -1,7 +1,6 @@
//! Retains watched global locations and sandbox-scoped response snapshots.
//! Replacement snapshots isolate in-flight loads; failed watch registration forces request-time reloads.
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
use crate::FileSystemSandboxContext;
use crate::LocalFileSystem;
@@ -80,22 +79,25 @@ impl CapabilityManager {
}
/// Reuses startup locations, retrying prewarming if the background scan failed.
pub(crate) async fn get_or_refresh_discovery(
pub(crate) async fn discover(
self: &Arc<Self>,
params: DiscoverV2CapabilitiesRequest,
sandbox: Option<FileSystemSandboxContext>,
force_reload: bool,
) -> io::Result<Arc<DiscoverV2CapabilitiesResponse>> {
let (location_request, _) = self.prewarm_locations().await?;
let location_state = self.get_or_refresh_locations(location_request).await?;
let location_state = self
.get_or_refresh_locations(location_request, force_reload)
.await?;
let discovery = {
let mut discoveries = location_state.sandbox_discoveries.lock().await;
if let Some(discovery) = discoveries
.iter()
.find(|discovery| discovery.sandbox == params.sandbox)
.find(|discovery| discovery.sandbox == sandbox)
{
Arc::clone(discovery)
} else {
let discovery = Arc::new(CachedSandboxDiscovery {
sandbox: params.sandbox,
sandbox,
response: OnceCell::new(),
});
if discoveries.len() == MAX_CACHED_SANDBOX_CONTEXTS {
@@ -130,8 +132,10 @@ impl CapabilityManager {
async fn get_or_refresh_locations(
&self,
request: &CapabilityLocationRequest,
force_reload: bool,
) -> io::Result<Arc<LocationState>> {
if let Some(location_state) = self.location_snapshot().await
if !force_reload
&& let Some(location_state) = self.location_snapshot().await
&& location_state.registration.registration_succeeded()
&& !location_state.dirty.load(Ordering::Relaxed)
{
@@ -142,7 +146,8 @@ impl CapabilityManager {
.acquire()
.await
.map_err(io::Error::other)?;
if let Some(location_state) = self.location_snapshot().await
if !force_reload
&& let Some(location_state) = self.location_snapshot().await
&& location_state.registration.registration_succeeded()
&& !location_state.dirty.load(Ordering::Relaxed)
{
@@ -152,7 +157,7 @@ impl CapabilityManager {
let locations = match prewarm_locations(&self.file_system, request).await {
Ok(locations) => locations,
Err(error) => {
if let Some(location_state) = self.location_snapshot().await {
if !force_reload && let Some(location_state) = self.location_snapshot().await {
tracing::warn!(error_kind = ?error.kind(), "capability refresh failed; retrying on the next request");
return Ok(location_state);
}
@@ -120,6 +120,7 @@ async fn discovery_preserves_inventory_and_v1_metadata() -> anyhow::Result<()> {
manager.seed_prewarmed_locations(request.clone()).await?;
let response = manager
.get_or_refresh_discovery(DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(temp.path().join("project"))?,
sandbox: None,
})
@@ -344,6 +345,7 @@ policy:
manager.seed_prewarmed_locations(request.clone()).await?;
let response = manager
.get_or_refresh_discovery(DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(temp.path().join("project"))?,
sandbox: None,
})
@@ -396,6 +398,7 @@ async fn standalone_skills_retain_their_search_roots() -> anyhow::Result<()> {
let manager = CapabilityManager::new(LocalFileSystem::unsandboxed());
manager.seed_prewarmed_locations(request.clone()).await?;
let query = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(temp.path().join("project"))?,
sandbox: None,
};
@@ -1,6 +1,7 @@
//! Covers bounded context caching, shared initialization, cancellation retry, and cached warnings.
use super::*;
use crate::DiscoverV2CapabilitiesRequest;
use crate::discover_v2::capability_locations::tests::request;
use crate::discover_v2::capability_locations::tests::write;
use codex_protocol::models::PermissionProfile;
@@ -8,6 +9,13 @@ use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
impl CapabilityManager {
pub(in crate::discover_v2) async fn get_or_refresh_discovery(
self: &Arc<Self>,
request: DiscoverV2CapabilitiesRequest,
) -> io::Result<Arc<DiscoverV2CapabilitiesResponse>> {
self.discover(request.sandbox, request.force_reload).await
}
// Supply fixture paths through the existing cell, without production override state.
pub(crate) async fn seed_prewarmed_locations(
self: &Arc<Self>,
@@ -47,6 +55,7 @@ async fn discovery_responses_are_cached_by_sandbox_context() -> anyhow::Result<(
let cwd = PathUri::from_host_native_path(directory.path().join("project"))?;
let other_cwd = PathUri::from_host_native_path(directory.path().join("other"))?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: cwd.clone(),
sandbox: None,
};
@@ -99,6 +108,39 @@ async fn discovery_responses_are_cached_by_sandbox_context() -> anyhow::Result<(
Ok(())
}
#[tokio::test]
async fn forced_discovery_reads_new_skills_before_the_watcher_runs() -> anyhow::Result<()> {
let (directory, manager) = setup().await?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(directory.path())?,
sandbox: None,
};
let original = manager.get_or_refresh_discovery(params.clone()).await?;
write(
directory.path(),
"codex/skills/added/SKILL.md",
"---\nname: added\ndescription: new skill\n---\n",
)?;
assert!(Arc::ptr_eq(
&original,
&manager.get_or_refresh_discovery(params.clone()).await?,
));
let refreshed = manager
.get_or_refresh_discovery(DiscoverV2CapabilitiesRequest {
force_reload: true,
..params.clone()
})
.await?;
assert!(refreshed.skills.iter().any(|skill| skill.name == "added"));
assert!(Arc::ptr_eq(
&refreshed,
&manager.get_or_refresh_discovery(params).await?,
));
Ok(())
}
#[test]
fn cancelled_initialization_retries_and_warning_responses_are_cached() -> anyhow::Result<()> {
tokio::runtime::Builder::new_current_thread()
@@ -108,6 +150,7 @@ fn cancelled_initialization_retries_and_warning_responses_are_cached() -> anyhow
.block_on(async {
let (directory, manager) = setup().await?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(directory.path().join("project"))?,
sandbox: None,
};
@@ -40,6 +40,7 @@ async fn watch_events_invalidate_discovery_and_no_subscription_disables_caching(
let manager = CapabilityManager::new(LocalFileSystem::unsandboxed());
manager.seed_prewarmed_locations(request.clone()).await?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(&root)?,
sandbox: None,
};
@@ -279,6 +280,7 @@ async fn failed_watch_registration_reloads_locations_and_contents() -> anyhow::R
)?;
let request = request(&root)?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::from_host_native_path(&root)?,
sandbox: None,
};
@@ -93,6 +93,7 @@ async fn discovery_checks_version_and_retries_disconnect(
ScopedSkillsConfig::default(),
);
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::parse("file:///caller/project-b")?,
sandbox: None,
};
+1 -1
View File
@@ -348,7 +348,7 @@ impl ExecServerHandler {
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, JSONRPCErrorError> {
self.require_initialized_for("capability discovery")?;
self.capability_manager
.get_or_refresh_discovery(params)
.discover(params.sandbox, params.force_reload)
.await
.map_err(|error| internal_error(error.to_string()))
}
@@ -63,6 +63,7 @@ async fn v2_rpc_refreshes_metadata_and_enforces_sandbox() -> anyhow::Result<()>
let project = tempfile::tempdir()?;
let cwd = PathUri::from_host_native_path(project.path())?;
let params = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: cwd.clone(),
sandbox: None,
};
@@ -138,6 +139,7 @@ async fn v2_rpc_refreshes_metadata_and_enforces_sandbox() -> anyhow::Result<()>
..sandbox
};
let restricted = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: cwd.clone(),
sandbox: Some(sandbox),
};
@@ -176,6 +178,7 @@ async fn v2_rpc_refreshes_metadata_and_enforces_sandbox() -> anyhow::Result<()>
sandbox.windows_sandbox_selection =
codex_exec_server::WindowsSandboxSelection::RestrictedToken;
let permitted = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: cwd.clone(),
sandbox: Some(sandbox),
};
+1
View File
@@ -196,6 +196,7 @@ async fn discovery_prewarms_before_initialize_and_recovers_without_client_opt_in
.send_request(
CAPABILITIES_DISCOVER_V2_METHOD,
serde_json::to_value(DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: info.cwd.context("startup cwd")?,
sandbox: None,
})?,