Files
codex/codex-rs/exec-server/src/capability_discovery_v2_cache_tests.rs
T
Anthony Tafoya 2fb728da7d Retry executor discovery V2 failures with backoff (#62679)
## Why

Discovery could retry transport failures on successive lookups while
caching server errors until invalidation. That could repeat requests
against an unhealthy executor or leave a session without an inventory
after the server recovered.

## What changed

Cache failed discovery loads per context and return the cached error
during cooldown. The first lookup after cooldown retries, with delays
doubling from 1 second up to 30 seconds on consecutive failures.
Successful inventories are reused, canceled loads add no cooldown, and
explicit refresh clears the cooldown for the next lookup without
interrupting in-flight readers.

Compare discovery snapshots to detect inventory changes so recovered
plugins and MCP capabilities can become available without an explicit
refresh.

## Testing

Cover backoff timing, shared results for concurrent callers,
cancellation, and refresh bypassing cooldown. Exercise recovery after
failed prewarming and verify that MCP inventory updates after recovery.

GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430
2026-11-20 06:05:04 +00:00

181 lines
6.6 KiB
Rust

#![cfg(test)]
use super::DiscoveryCacheKey;
use super::DiscoveryV2CacheEntry;
use super::ExecutorCapabilityDiscoveryV2Cache;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
use crate::Environment;
use crate::ExecServerError;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use std::cell::Cell;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::time::Duration;
use tokio::sync::OnceCell;
use tokio::sync::oneshot;
use tokio::time::advance;
fn empty_cache_entry() -> DiscoveryV2CacheEntry {
DiscoveryV2CacheEntry {
key: DiscoveryCacheKey {
environment: Weak::new(),
session_id: None,
cwd: PathUri::parse("file:///project").expect("test cwd"),
sandbox: None,
},
force_reload: false,
response: OnceCell::new(),
failure: Mutex::new(None),
}
}
#[tokio::test(start_paused = true)]
async fn repeated_failures_back_off_without_extending_cooldown_on_reads() {
let entry = empty_cache_entry();
let calls = Cell::new(0);
let fail = || async {
calls.set(calls.get() + 1);
Err(ExecServerError::Protocol("discovery failed".to_string()))
};
for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() {
let error = entry.get_or_try_init(fail).await.expect_err("failed load");
assert_eq!(calls.get(), attempt + 1);
advance(Duration::from_secs(delay) - Duration::from_millis(1)).await;
let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
assert_eq!(calls.get(), attempt + 1);
// Suppressed lookups must not move the deadline or advance the failure count.
advance(Duration::from_millis(1)).await;
}
let response = entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("recovery after cooldown");
advance(Duration::from_secs(300)).await;
let reused = entry.get_or_try_init(fail).await.expect("cached success");
assert!(Arc::ptr_eq(&response, &reused));
assert_eq!(calls.get(), 7);
}
#[tokio::test(start_paused = true)]
async fn queued_callers_share_failure_and_the_next_successful_attempt() {
let entry = empty_cache_entry();
let (release, wait) = oneshot::channel();
let first = entry.get_or_try_init(|| async {
wait.await.expect("release failed load");
Err(ExecServerError::Protocol("discovery failed".to_string()))
});
tokio::pin!(first);
assert!(futures::poll!(first.as_mut()).is_pending());
let calls = Cell::new(0);
let succeed = || async {
calls.set(calls.get() + 1);
Ok(DiscoverV2CapabilitiesResponse::default())
};
let queued = entry.get_or_try_init(succeed);
tokio::pin!(queued);
assert!(futures::poll!(queued.as_mut()).is_pending());
release.send(()).expect("release failed load");
let (first, queued) = tokio::join!(first, queued);
assert!(Arc::ptr_eq(
&first.expect_err("failed loader"),
&queued.expect_err("queued caller observes cooldown"),
));
assert_eq!(calls.get(), 0);
advance(Duration::from_secs(1)).await;
let (first, concurrent) = tokio::join!(
entry.get_or_try_init(succeed),
entry.get_or_try_init(succeed),
);
assert!(Arc::ptr_eq(
&first.expect("recovered loader"),
&concurrent.expect("shared success"),
));
assert_eq!(calls.get(), 1);
}
#[tokio::test(start_paused = true)]
async fn cancellation_does_not_add_a_failure_or_cooldown() {
let entry = empty_cache_entry();
// Exercise cancellation both before any failure and after an expired failure.
for delay in [1, 2] {
let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending));
assert!(futures::poll!(cancelled.as_mut()).is_pending());
drop(cancelled);
let attempted = Cell::new(false);
entry
.get_or_try_init(|| async {
attempted.set(true);
Err(ExecServerError::Protocol("discovery failed".to_string()))
})
.await
.expect_err("fresh attempt after cancellation");
assert!(attempted.get(), "cancellation must not start a cooldown");
advance(Duration::from_secs(delay)).await;
}
entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("cancellation did not advance the backoff");
}
#[tokio::test]
async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> {
// Local environments reject V2; their errors exercise the cache without a network fixture.
let environment = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let replacement = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let cache = ExecutorCapabilityDiscoveryV2Cache::default();
let request = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::parse("file:///project")?,
sandbox: None,
};
let error = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("local environment rejects V2");
let other_request = DiscoverV2CapabilitiesRequest {
cwd: request.cwd.join("other")?,
..request.clone()
};
let other_error = cache
.discover("executor", &environment, other_request.clone())
.await
.expect_err("independent cwd");
let replacement_error = cache
.discover("executor", &replacement, request.clone())
.await
.expect_err("independent environment");
assert!(!Arc::ptr_eq(&error, &other_error));
assert!(!Arc::ptr_eq(&error, &replacement_error));
cache.invalidate_environment(&replacement);
let suppressed = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("unrelated invalidation preserves cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
cache.invalidate_environment(&environment);
let refreshed = cache
.discover("executor", &environment, request)
.await
.expect_err("explicit refresh retries immediately");
assert!(!Arc::ptr_eq(&error, &refreshed));
cache.invalidate();
let refreshed_other = cache
.discover("executor", &environment, other_request)
.await
.expect_err("global invalidation retries immediately");
assert!(!Arc::ptr_eq(&other_error, &refreshed_other));
Ok(())
}