mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
Retry executor discovery V2 failures with backoff (#62679)
## Why Discovery could retry transport failures on successive lookups while caching server errors until invalidation. That could repeat requests against an unhealthy executor or leave a session without an inventory after the server recovered. ## What changed Cache failed discovery loads per context and return the cached error during cooldown. The first lookup after cooldown retries, with delays doubling from 1 second up to 30 seconds on consecutive failures. Successful inventories are reused, canceled loads add no cooldown, and explicit refresh clears the cooldown for the next lookup without interrupting in-flight readers. Compare discovery snapshots to detect inventory changes so recovered plugins and MCP capabilities can become available without an explicit refresh. ## Testing Cover backoff timing, shared results for concurrent callers, cancellation, and refresh bypassing cooldown. Exercise recovery after failed prewarming and verify that MCP inventory updates after recovery. GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430
This commit is contained in:
committed by
copyberry
parent
ea8ccc1d20
commit
2fb728da7d
@@ -1,16 +1,19 @@
|
||||
//! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox.
|
||||
//! Reloads replace the matching entry; in-flight readers may finish with older results.
|
||||
//! Reports recovery from retryable failures so Core can refresh dependent MCP projections.
|
||||
//! Invalidation replaces entries with empty forced loads; in-flight readers may finish with older results.
|
||||
//! Consumers detect recovery and replacement by comparing discovery snapshots.
|
||||
//! Failed loads have a per-context cooldown; lookups never sleep or schedule background retries.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::Weak;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::DiscoverV2CapabilitiesRequest;
|
||||
use crate::DiscoverV2CapabilitiesResponse;
|
||||
@@ -19,16 +22,18 @@ use crate::ExecServerError;
|
||||
use crate::FileSystemSandboxContext;
|
||||
|
||||
const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16;
|
||||
const INITIAL_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1);
|
||||
const MAX_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Shared by turn and idle consumers until Core explicitly invalidates discovery.
|
||||
#[derive(Default)]
|
||||
pub struct ExecutorCapabilityDiscoveryV2Cache {
|
||||
entries: Mutex<Vec<CachedEnvironmentDiscovery>>,
|
||||
entries: Mutex<Vec<Arc<DiscoveryV2CacheEntry>>>,
|
||||
capacity: AtomicUsize,
|
||||
recovered_discovery: AtomicBool,
|
||||
}
|
||||
|
||||
// Reload controls how this entry is populated, not which inventory it identifies.
|
||||
#[derive(Clone)]
|
||||
struct DiscoveryCacheKey {
|
||||
environment: Weak<Environment>,
|
||||
session_id: Option<String>,
|
||||
@@ -45,12 +50,81 @@ impl PartialEq for DiscoveryCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
struct CachedEnvironmentDiscovery {
|
||||
struct DiscoveryV2CacheEntry {
|
||||
key: DiscoveryCacheKey,
|
||||
response: Arc<OnceCell<Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>>>,
|
||||
retryable: Arc<AtomicBool>,
|
||||
force_reload: bool,
|
||||
dirty: bool,
|
||||
response: OnceCell<Arc<DiscoverV2CapabilitiesResponse>>,
|
||||
failure: Mutex<Option<DiscoveryV2Failure>>,
|
||||
}
|
||||
|
||||
struct DiscoveryV2Failure {
|
||||
error: Arc<ExecServerError>,
|
||||
attempts: u32,
|
||||
next_retry_at: Instant,
|
||||
}
|
||||
|
||||
impl DiscoveryV2CacheEntry {
|
||||
fn force_reload(&self) -> Self {
|
||||
// Discard the old snapshot and cooldown, but retain the forced reload through retries.
|
||||
Self {
|
||||
key: self.key.clone(),
|
||||
force_reload: true,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_or_try_init<F>(
|
||||
&self,
|
||||
discover: impl FnOnce() -> F,
|
||||
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>
|
||||
where
|
||||
F: Future<Output = Result<DiscoverV2CapabilitiesResponse, ExecServerError>>,
|
||||
{
|
||||
self.response
|
||||
.get_or_try_init(|| async {
|
||||
// Check here so waiting callers don't immediately retry a failed request.
|
||||
{
|
||||
let failure_state = self
|
||||
.failure
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(failure) = failure_state.as_ref()
|
||||
&& Instant::now() < failure.next_retry_at
|
||||
{
|
||||
return Err(Arc::clone(&failure.error));
|
||||
}
|
||||
}
|
||||
let result = discover().await;
|
||||
let mut failure_state = self
|
||||
.failure
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
match result {
|
||||
Ok(response) => {
|
||||
*failure_state = None;
|
||||
Ok(Arc::new(response))
|
||||
}
|
||||
Err(error) => {
|
||||
let attempts = failure_state
|
||||
.as_ref()
|
||||
.map_or(1, |failure| failure.attempts.saturating_add(1));
|
||||
let delay = INITIAL_DISCOVERY_RETRY_DELAY
|
||||
.saturating_mul(2_u32.saturating_pow(attempts - 1))
|
||||
.min(MAX_DISCOVERY_RETRY_DELAY);
|
||||
let error = Arc::new(error);
|
||||
*failure_state = Some(DiscoveryV2Failure {
|
||||
error: Arc::clone(&error),
|
||||
attempts,
|
||||
next_retry_at: Instant::now() + delay,
|
||||
});
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
@@ -59,19 +133,14 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
self.capacity.fetch_max(minimum, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Reports whether a failed discovery recovered since the last observation.
|
||||
pub fn take_recovered_discovery(&self) -> bool {
|
||||
self.recovered_discovery.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
/// Dirties cached entries so their next lookup reloads without interrupting existing loads.
|
||||
/// Makes future lookups reload every context without interrupting existing loads.
|
||||
pub fn invalidate(&self) {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for entry in entries.iter_mut() {
|
||||
entry.dirty = true;
|
||||
*entry = Arc::new(entry.force_reload());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,12 +155,13 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
.iter_mut()
|
||||
.filter(|entry| entry.key.environment.ptr_eq(&environment))
|
||||
{
|
||||
entry.dirty = true;
|
||||
*entry = Arc::new(entry.force_reload());
|
||||
}
|
||||
}
|
||||
|
||||
/// Reuses successes and non-retryable failures unless the request forces a reload.
|
||||
/// Retryable failures and cancelled loads leave the cell available for retry.
|
||||
/// Reuses successful inventories; failed loads retry on the first lookup after their cooldown.
|
||||
/// Cancellation adds no cooldown; explicit invalidation and new sessions start fresh loads.
|
||||
/// A connection error classification does not determine discovery's lifetime.
|
||||
/// Invalidation during a load applies to the next lookup, not its current readers.
|
||||
pub async fn discover(
|
||||
&self,
|
||||
@@ -110,7 +180,7 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
cwd: request.cwd.clone(),
|
||||
sandbox: request.sandbox.clone(),
|
||||
};
|
||||
let (response, retryable, force_reload) = {
|
||||
let entry = {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
@@ -118,77 +188,45 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
let matching = entries.iter().position(|entry| entry.key == key);
|
||||
if let Some(entry) = matching
|
||||
.map(|index| &entries[index])
|
||||
.filter(|entry| !entry.dirty && !request.force_reload)
|
||||
.filter(|_| !request.force_reload)
|
||||
{
|
||||
(
|
||||
Arc::clone(&entry.response),
|
||||
Arc::clone(&entry.retryable),
|
||||
entry.force_reload,
|
||||
)
|
||||
Arc::clone(entry)
|
||||
} else {
|
||||
let force_reload = request.force_reload || matching.is_some();
|
||||
if let Some(index) = matching {
|
||||
entries.remove(index);
|
||||
}
|
||||
// Recovery belongs to the environment/request, not one executor session.
|
||||
let retryable = entries
|
||||
.iter()
|
||||
.find(|entry| {
|
||||
!entry.dirty
|
||||
&& entry.key.environment.ptr_eq(&key.environment)
|
||||
&& entry.key.cwd == key.cwd
|
||||
&& entry.key.sandbox == key.sandbox
|
||||
})
|
||||
.map(|entry| Arc::clone(&entry.retryable))
|
||||
.unwrap_or_default();
|
||||
let entry = CachedEnvironmentDiscovery {
|
||||
let entry = Arc::new(DiscoveryV2CacheEntry {
|
||||
key,
|
||||
response: Arc::new(OnceCell::new()),
|
||||
retryable,
|
||||
force_reload,
|
||||
dirty: false,
|
||||
};
|
||||
force_reload: request.force_reload,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
});
|
||||
let capacity = self
|
||||
.capacity
|
||||
.load(Ordering::Relaxed)
|
||||
.max(MIN_CACHED_DISCOVERY_CONTEXTS);
|
||||
// Existing readers retain their response cells after replacement or eviction.
|
||||
// Existing readers retain their entries after replacement or eviction.
|
||||
if entries.len() >= capacity {
|
||||
entries.remove(0);
|
||||
}
|
||||
let load = (
|
||||
Arc::clone(&entry.response),
|
||||
Arc::clone(&entry.retryable),
|
||||
force_reload,
|
||||
);
|
||||
entries.push(entry);
|
||||
load
|
||||
entries.push(Arc::clone(&entry));
|
||||
entry
|
||||
}
|
||||
};
|
||||
let response = response
|
||||
entry
|
||||
.get_or_try_init(|| async {
|
||||
request.force_reload = force_reload;
|
||||
match environment.discover_capabilities_v2(request).await {
|
||||
Ok(response) => Ok(Ok(Arc::new(response))),
|
||||
Err(error) => {
|
||||
let can_retry = crate::client::can_retry_connection_attempt(&error);
|
||||
request.force_reload = entry.force_reload;
|
||||
environment
|
||||
.discover_capabilities_v2(request)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
tracing::warn!(environment_id, "executor discovery V2 failed");
|
||||
let error = Arc::new(error);
|
||||
if can_retry {
|
||||
retryable.store(true, Ordering::Release);
|
||||
Err(error)
|
||||
} else {
|
||||
Ok(Err(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.cloned()
|
||||
.unwrap_or_else(Err);
|
||||
if response.is_ok() && retryable.swap(false, Ordering::AcqRel) {
|
||||
self.recovered_discovery.store(true, Ordering::Release);
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "capability_discovery_v2_cache_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::DiscoveryCacheKey;
|
||||
use super::DiscoveryV2CacheEntry;
|
||||
use super::ExecutorCapabilityDiscoveryV2Cache;
|
||||
use crate::DiscoverV2CapabilitiesRequest;
|
||||
use crate::DiscoverV2CapabilitiesResponse;
|
||||
use crate::Environment;
|
||||
use crate::ExecServerError;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::cell::Cell;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::Weak;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::advance;
|
||||
|
||||
fn empty_cache_entry() -> DiscoveryV2CacheEntry {
|
||||
DiscoveryV2CacheEntry {
|
||||
key: DiscoveryCacheKey {
|
||||
environment: Weak::new(),
|
||||
session_id: None,
|
||||
cwd: PathUri::parse("file:///project").expect("test cwd"),
|
||||
sandbox: None,
|
||||
},
|
||||
force_reload: false,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_failures_back_off_without_extending_cooldown_on_reads() {
|
||||
let entry = empty_cache_entry();
|
||||
let calls = Cell::new(0);
|
||||
let fail = || async {
|
||||
calls.set(calls.get() + 1);
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
};
|
||||
for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() {
|
||||
let error = entry.get_or_try_init(fail).await.expect_err("failed load");
|
||||
assert_eq!(calls.get(), attempt + 1);
|
||||
advance(Duration::from_secs(delay) - Duration::from_millis(1)).await;
|
||||
let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown");
|
||||
assert!(Arc::ptr_eq(&error, &suppressed));
|
||||
assert_eq!(calls.get(), attempt + 1);
|
||||
// Suppressed lookups must not move the deadline or advance the failure count.
|
||||
advance(Duration::from_millis(1)).await;
|
||||
}
|
||||
|
||||
let response = entry
|
||||
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
|
||||
.await
|
||||
.expect("recovery after cooldown");
|
||||
advance(Duration::from_secs(300)).await;
|
||||
let reused = entry.get_or_try_init(fail).await.expect("cached success");
|
||||
assert!(Arc::ptr_eq(&response, &reused));
|
||||
assert_eq!(calls.get(), 7);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn queued_callers_share_failure_and_the_next_successful_attempt() {
|
||||
let entry = empty_cache_entry();
|
||||
let (release, wait) = oneshot::channel();
|
||||
let first = entry.get_or_try_init(|| async {
|
||||
wait.await.expect("release failed load");
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
});
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
let calls = Cell::new(0);
|
||||
let succeed = || async {
|
||||
calls.set(calls.get() + 1);
|
||||
Ok(DiscoverV2CapabilitiesResponse::default())
|
||||
};
|
||||
let queued = entry.get_or_try_init(succeed);
|
||||
tokio::pin!(queued);
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
release.send(()).expect("release failed load");
|
||||
let (first, queued) = tokio::join!(first, queued);
|
||||
assert!(Arc::ptr_eq(
|
||||
&first.expect_err("failed loader"),
|
||||
&queued.expect_err("queued caller observes cooldown"),
|
||||
));
|
||||
assert_eq!(calls.get(), 0);
|
||||
|
||||
advance(Duration::from_secs(1)).await;
|
||||
let (first, concurrent) = tokio::join!(
|
||||
entry.get_or_try_init(succeed),
|
||||
entry.get_or_try_init(succeed),
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
&first.expect("recovered loader"),
|
||||
&concurrent.expect("shared success"),
|
||||
));
|
||||
assert_eq!(calls.get(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_does_not_add_a_failure_or_cooldown() {
|
||||
let entry = empty_cache_entry();
|
||||
// Exercise cancellation both before any failure and after an expired failure.
|
||||
for delay in [1, 2] {
|
||||
let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending));
|
||||
assert!(futures::poll!(cancelled.as_mut()).is_pending());
|
||||
drop(cancelled);
|
||||
let attempted = Cell::new(false);
|
||||
entry
|
||||
.get_or_try_init(|| async {
|
||||
attempted.set(true);
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
})
|
||||
.await
|
||||
.expect_err("fresh attempt after cancellation");
|
||||
assert!(attempted.get(), "cancellation must not start a cooldown");
|
||||
advance(Duration::from_secs(delay)).await;
|
||||
}
|
||||
entry
|
||||
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
|
||||
.await
|
||||
.expect("cancellation did not advance the backoff");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> {
|
||||
// Local environments reject V2; their errors exercise the cache without a network fixture.
|
||||
let environment = Arc::new(Environment::create_for_tests(
|
||||
/*exec_server_url*/ None,
|
||||
)?);
|
||||
let replacement = Arc::new(Environment::create_for_tests(
|
||||
/*exec_server_url*/ None,
|
||||
)?);
|
||||
let cache = ExecutorCapabilityDiscoveryV2Cache::default();
|
||||
let request = DiscoverV2CapabilitiesRequest {
|
||||
force_reload: false,
|
||||
cwd: PathUri::parse("file:///project")?,
|
||||
sandbox: None,
|
||||
};
|
||||
let error = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.expect_err("local environment rejects V2");
|
||||
let other_request = DiscoverV2CapabilitiesRequest {
|
||||
cwd: request.cwd.join("other")?,
|
||||
..request.clone()
|
||||
};
|
||||
let other_error = cache
|
||||
.discover("executor", &environment, other_request.clone())
|
||||
.await
|
||||
.expect_err("independent cwd");
|
||||
let replacement_error = cache
|
||||
.discover("executor", &replacement, request.clone())
|
||||
.await
|
||||
.expect_err("independent environment");
|
||||
assert!(!Arc::ptr_eq(&error, &other_error));
|
||||
assert!(!Arc::ptr_eq(&error, &replacement_error));
|
||||
|
||||
cache.invalidate_environment(&replacement);
|
||||
let suppressed = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.expect_err("unrelated invalidation preserves cooldown");
|
||||
assert!(Arc::ptr_eq(&error, &suppressed));
|
||||
cache.invalidate_environment(&environment);
|
||||
let refreshed = cache
|
||||
.discover("executor", &environment, request)
|
||||
.await
|
||||
.expect_err("explicit refresh retries immediately");
|
||||
assert!(!Arc::ptr_eq(&error, &refreshed));
|
||||
cache.invalidate();
|
||||
let refreshed_other = cache
|
||||
.discover("executor", &environment, other_request)
|
||||
.await
|
||||
.expect_err("global invalidation retries immediately");
|
||||
assert!(!Arc::ptr_eq(&other_error, &refreshed_other));
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user