Retry executor discovery V2 failures with backoff (#62679)

## Why

Discovery could retry transport failures on successive lookups while
caching server errors until invalidation. That could repeat requests
against an unhealthy executor or leave a session without an inventory
after the server recovered.

## What changed

Cache failed discovery loads per context and return the cached error
during cooldown. The first lookup after cooldown retries, with delays
doubling from 1 second up to 30 seconds on consecutive failures.
Successful inventories are reused, canceled loads add no cooldown, and
explicit refresh clears the cooldown for the next lookup without
interrupting in-flight readers.

Compare discovery snapshots to detect inventory changes so recovered
plugins and MCP capabilities can become available without an explicit
refresh.

## Testing

Cover backoff timing, shared results for concurrent callers,
cancellation, and refresh bypassing cooldown. Exercise recovery after
failed prewarming and verify that MCP inventory updates after recovery.

GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430
This commit is contained in:
Anthony Tafoya
2026-11-20 06:05:04 +00:00
committed by copyberry
parent ea8ccc1d20
commit 2fb728da7d
7 changed files with 529 additions and 264 deletions
@@ -1,16 +1,19 @@
//! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox.
//! Reloads replace the matching entry; in-flight readers may finish with older results.
//! Reports recovery from retryable failures so Core can refresh dependent MCP projections.
//! Invalidation replaces entries with empty forced loads; in-flight readers may finish with older results.
//! Consumers detect recovery and replacement by comparing discovery snapshots.
//! Failed loads have a per-context cooldown; lookups never sleep or schedule background retries.
use std::future::Future;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_utils_path_uri::PathUri;
use tokio::sync::OnceCell;
use tokio::time::Instant;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
@@ -19,16 +22,18 @@ use crate::ExecServerError;
use crate::FileSystemSandboxContext;
const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16;
const INITIAL_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1);
const MAX_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(30);
/// Shared by turn and idle consumers until Core explicitly invalidates discovery.
#[derive(Default)]
pub struct ExecutorCapabilityDiscoveryV2Cache {
entries: Mutex<Vec<CachedEnvironmentDiscovery>>,
entries: Mutex<Vec<Arc<DiscoveryV2CacheEntry>>>,
capacity: AtomicUsize,
recovered_discovery: AtomicBool,
}
// Reload controls how this entry is populated, not which inventory it identifies.
#[derive(Clone)]
struct DiscoveryCacheKey {
environment: Weak<Environment>,
session_id: Option<String>,
@@ -45,12 +50,81 @@ impl PartialEq for DiscoveryCacheKey {
}
}
struct CachedEnvironmentDiscovery {
struct DiscoveryV2CacheEntry {
key: DiscoveryCacheKey,
response: Arc<OnceCell<Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>>>,
retryable: Arc<AtomicBool>,
force_reload: bool,
dirty: bool,
response: OnceCell<Arc<DiscoverV2CapabilitiesResponse>>,
failure: Mutex<Option<DiscoveryV2Failure>>,
}
struct DiscoveryV2Failure {
error: Arc<ExecServerError>,
attempts: u32,
next_retry_at: Instant,
}
impl DiscoveryV2CacheEntry {
fn force_reload(&self) -> Self {
// Discard the old snapshot and cooldown, but retain the forced reload through retries.
Self {
key: self.key.clone(),
force_reload: true,
response: OnceCell::new(),
failure: Mutex::new(None),
}
}
async fn get_or_try_init<F>(
&self,
discover: impl FnOnce() -> F,
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>
where
F: Future<Output = Result<DiscoverV2CapabilitiesResponse, ExecServerError>>,
{
self.response
.get_or_try_init(|| async {
// Check here so waiting callers don't immediately retry a failed request.
{
let failure_state = self
.failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(failure) = failure_state.as_ref()
&& Instant::now() < failure.next_retry_at
{
return Err(Arc::clone(&failure.error));
}
}
let result = discover().await;
let mut failure_state = self
.failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match result {
Ok(response) => {
*failure_state = None;
Ok(Arc::new(response))
}
Err(error) => {
let attempts = failure_state
.as_ref()
.map_or(1, |failure| failure.attempts.saturating_add(1));
let delay = INITIAL_DISCOVERY_RETRY_DELAY
.saturating_mul(2_u32.saturating_pow(attempts - 1))
.min(MAX_DISCOVERY_RETRY_DELAY);
let error = Arc::new(error);
*failure_state = Some(DiscoveryV2Failure {
error: Arc::clone(&error),
attempts,
next_retry_at: Instant::now() + delay,
});
Err(error)
}
}
})
.await
.cloned()
}
}
impl ExecutorCapabilityDiscoveryV2Cache {
@@ -59,19 +133,14 @@ impl ExecutorCapabilityDiscoveryV2Cache {
self.capacity.fetch_max(minimum, Ordering::Relaxed);
}
/// Reports whether a failed discovery recovered since the last observation.
pub fn take_recovered_discovery(&self) -> bool {
self.recovered_discovery.swap(false, Ordering::AcqRel)
}
/// Dirties cached entries so their next lookup reloads without interrupting existing loads.
/// Makes future lookups reload every context without interrupting existing loads.
pub fn invalidate(&self) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for entry in entries.iter_mut() {
entry.dirty = true;
*entry = Arc::new(entry.force_reload());
}
}
@@ -86,12 +155,13 @@ impl ExecutorCapabilityDiscoveryV2Cache {
.iter_mut()
.filter(|entry| entry.key.environment.ptr_eq(&environment))
{
entry.dirty = true;
*entry = Arc::new(entry.force_reload());
}
}
/// Reuses successes and non-retryable failures unless the request forces a reload.
/// Retryable failures and cancelled loads leave the cell available for retry.
/// Reuses successful inventories; failed loads retry on the first lookup after their cooldown.
/// Cancellation adds no cooldown; explicit invalidation and new sessions start fresh loads.
/// A connection error classification does not determine discovery's lifetime.
/// Invalidation during a load applies to the next lookup, not its current readers.
pub async fn discover(
&self,
@@ -110,7 +180,7 @@ impl ExecutorCapabilityDiscoveryV2Cache {
cwd: request.cwd.clone(),
sandbox: request.sandbox.clone(),
};
let (response, retryable, force_reload) = {
let entry = {
let mut entries = self
.entries
.lock()
@@ -118,77 +188,45 @@ impl ExecutorCapabilityDiscoveryV2Cache {
let matching = entries.iter().position(|entry| entry.key == key);
if let Some(entry) = matching
.map(|index| &entries[index])
.filter(|entry| !entry.dirty && !request.force_reload)
.filter(|_| !request.force_reload)
{
(
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
entry.force_reload,
)
Arc::clone(entry)
} else {
let force_reload = request.force_reload || matching.is_some();
if let Some(index) = matching {
entries.remove(index);
}
// Recovery belongs to the environment/request, not one executor session.
let retryable = entries
.iter()
.find(|entry| {
!entry.dirty
&& entry.key.environment.ptr_eq(&key.environment)
&& entry.key.cwd == key.cwd
&& entry.key.sandbox == key.sandbox
})
.map(|entry| Arc::clone(&entry.retryable))
.unwrap_or_default();
let entry = CachedEnvironmentDiscovery {
let entry = Arc::new(DiscoveryV2CacheEntry {
key,
response: Arc::new(OnceCell::new()),
retryable,
force_reload,
dirty: false,
};
force_reload: request.force_reload,
response: OnceCell::new(),
failure: Mutex::new(None),
});
let capacity = self
.capacity
.load(Ordering::Relaxed)
.max(MIN_CACHED_DISCOVERY_CONTEXTS);
// Existing readers retain their response cells after replacement or eviction.
// Existing readers retain their entries after replacement or eviction.
if entries.len() >= capacity {
entries.remove(0);
}
let load = (
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
force_reload,
);
entries.push(entry);
load
entries.push(Arc::clone(&entry));
entry
}
};
let response = response
entry
.get_or_try_init(|| async {
request.force_reload = force_reload;
match environment.discover_capabilities_v2(request).await {
Ok(response) => Ok(Ok(Arc::new(response))),
Err(error) => {
let can_retry = crate::client::can_retry_connection_attempt(&error);
request.force_reload = entry.force_reload;
environment
.discover_capabilities_v2(request)
.await
.inspect_err(|_| {
tracing::warn!(environment_id, "executor discovery V2 failed");
let error = Arc::new(error);
if can_retry {
retryable.store(true, Ordering::Release);
Err(error)
} else {
Ok(Err(error))
}
}
}
})
})
.await
.cloned()
.unwrap_or_else(Err);
if response.is_ok() && retryable.swap(false, Ordering::AcqRel) {
self.recovered_discovery.store(true, Ordering::Release);
}
response
}
}
#[cfg(test)]
#[path = "capability_discovery_v2_cache_tests.rs"]
mod tests;
@@ -0,0 +1,180 @@
#![cfg(test)]
use super::DiscoveryCacheKey;
use super::DiscoveryV2CacheEntry;
use super::ExecutorCapabilityDiscoveryV2Cache;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
use crate::Environment;
use crate::ExecServerError;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use std::cell::Cell;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::time::Duration;
use tokio::sync::OnceCell;
use tokio::sync::oneshot;
use tokio::time::advance;
fn empty_cache_entry() -> DiscoveryV2CacheEntry {
DiscoveryV2CacheEntry {
key: DiscoveryCacheKey {
environment: Weak::new(),
session_id: None,
cwd: PathUri::parse("file:///project").expect("test cwd"),
sandbox: None,
},
force_reload: false,
response: OnceCell::new(),
failure: Mutex::new(None),
}
}
#[tokio::test(start_paused = true)]
async fn repeated_failures_back_off_without_extending_cooldown_on_reads() {
let entry = empty_cache_entry();
let calls = Cell::new(0);
let fail = || async {
calls.set(calls.get() + 1);
Err(ExecServerError::Protocol("discovery failed".to_string()))
};
for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() {
let error = entry.get_or_try_init(fail).await.expect_err("failed load");
assert_eq!(calls.get(), attempt + 1);
advance(Duration::from_secs(delay) - Duration::from_millis(1)).await;
let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
assert_eq!(calls.get(), attempt + 1);
// Suppressed lookups must not move the deadline or advance the failure count.
advance(Duration::from_millis(1)).await;
}
let response = entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("recovery after cooldown");
advance(Duration::from_secs(300)).await;
let reused = entry.get_or_try_init(fail).await.expect("cached success");
assert!(Arc::ptr_eq(&response, &reused));
assert_eq!(calls.get(), 7);
}
#[tokio::test(start_paused = true)]
async fn queued_callers_share_failure_and_the_next_successful_attempt() {
let entry = empty_cache_entry();
let (release, wait) = oneshot::channel();
let first = entry.get_or_try_init(|| async {
wait.await.expect("release failed load");
Err(ExecServerError::Protocol("discovery failed".to_string()))
});
tokio::pin!(first);
assert!(futures::poll!(first.as_mut()).is_pending());
let calls = Cell::new(0);
let succeed = || async {
calls.set(calls.get() + 1);
Ok(DiscoverV2CapabilitiesResponse::default())
};
let queued = entry.get_or_try_init(succeed);
tokio::pin!(queued);
assert!(futures::poll!(queued.as_mut()).is_pending());
release.send(()).expect("release failed load");
let (first, queued) = tokio::join!(first, queued);
assert!(Arc::ptr_eq(
&first.expect_err("failed loader"),
&queued.expect_err("queued caller observes cooldown"),
));
assert_eq!(calls.get(), 0);
advance(Duration::from_secs(1)).await;
let (first, concurrent) = tokio::join!(
entry.get_or_try_init(succeed),
entry.get_or_try_init(succeed),
);
assert!(Arc::ptr_eq(
&first.expect("recovered loader"),
&concurrent.expect("shared success"),
));
assert_eq!(calls.get(), 1);
}
#[tokio::test(start_paused = true)]
async fn cancellation_does_not_add_a_failure_or_cooldown() {
let entry = empty_cache_entry();
// Exercise cancellation both before any failure and after an expired failure.
for delay in [1, 2] {
let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending));
assert!(futures::poll!(cancelled.as_mut()).is_pending());
drop(cancelled);
let attempted = Cell::new(false);
entry
.get_or_try_init(|| async {
attempted.set(true);
Err(ExecServerError::Protocol("discovery failed".to_string()))
})
.await
.expect_err("fresh attempt after cancellation");
assert!(attempted.get(), "cancellation must not start a cooldown");
advance(Duration::from_secs(delay)).await;
}
entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("cancellation did not advance the backoff");
}
#[tokio::test]
async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> {
// Local environments reject V2; their errors exercise the cache without a network fixture.
let environment = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let replacement = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let cache = ExecutorCapabilityDiscoveryV2Cache::default();
let request = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::parse("file:///project")?,
sandbox: None,
};
let error = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("local environment rejects V2");
let other_request = DiscoverV2CapabilitiesRequest {
cwd: request.cwd.join("other")?,
..request.clone()
};
let other_error = cache
.discover("executor", &environment, other_request.clone())
.await
.expect_err("independent cwd");
let replacement_error = cache
.discover("executor", &replacement, request.clone())
.await
.expect_err("independent environment");
assert!(!Arc::ptr_eq(&error, &other_error));
assert!(!Arc::ptr_eq(&error, &replacement_error));
cache.invalidate_environment(&replacement);
let suppressed = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("unrelated invalidation preserves cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
cache.invalidate_environment(&environment);
let refreshed = cache
.discover("executor", &environment, request)
.await
.expect_err("explicit refresh retries immediately");
assert!(!Arc::ptr_eq(&error, &refreshed));
cache.invalidate();
let refreshed_other = cache
.discover("executor", &environment, other_request)
.await
.expect_err("global invalidation retries immediately");
assert!(!Arc::ptr_eq(&other_error, &refreshed_other));
Ok(())
}