Retry executor discovery V2 failures with backoff (#62679)

## Why

Discovery could retry transport failures on successive lookups while
caching server errors until invalidation. That could repeat requests
against an unhealthy executor or leave a session without an inventory
after the server recovered.

## What changed

Cache failed discovery loads per context and return the cached error
during cooldown. The first lookup after cooldown retries, with delays
doubling from 1 second up to 30 seconds on consecutive failures.
Successful inventories are reused, canceled loads add no cooldown, and
explicit refresh clears the cooldown for the next lookup without
interrupting in-flight readers.

Compare discovery snapshots to detect inventory changes so recovered
plugins and MCP capabilities can become available without an explicit
refresh.

## Testing

Cover backoff timing, shared results for concurrent callers,
cancellation, and refresh bypassing cooldown. Exercise recovery after
failed prewarming and verify that MCP inventory updates after recovery.

GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430
This commit is contained in:
Anthony Tafoya
2026-11-20 06:05:04 +00:00
committed by copyberry
parent ea8ccc1d20
commit 2fb728da7d
7 changed files with 529 additions and 264 deletions
@@ -1,4 +1,4 @@
//! Core owns a thread discovery cache shared across turns. Explicit refresh dirties every entry.
//! Core owns a thread discovery cache shared across turns. Explicit refresh evicts every entry.
//! In-flight readers may keep their inventories; invalidation applies to the next lookup.
//! Prewarming bounds discovery before readiness without publishing MCP state or failing startup.
@@ -166,7 +166,6 @@ impl Session {
.zip(&snapshots)
.any(|(previous, current)| !previous.same_discovery(current))
});
let inventory_changed = cache.take_recovered_discovery() || inventory_changed;
(Some(snapshots), inventory_changed)
}
}
+211 -182
View File
@@ -201,6 +201,15 @@ pub(super) enum DiscoveryFailure {
}
impl Executor {
async fn discovery_count(&self) -> usize {
self.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count()
}
pub(super) async fn start(supports_v2: bool, failure: DiscoveryFailure) -> Result<Self> {
Self::start_with_version(supports_v2, failure, "0.164.0").await
}
@@ -616,37 +625,11 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
responses::sse(vec![responses::ev_completed("ready")]),
)
.await;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "Initialize both environments.".into(),
text_elements: Vec::new(),
}]))
test.submit_text_turn("Initialize both environments.")
.await?;
core_test_support::wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let _ = test.codex.current_mcp_config_and_runtime_context().await;
let second_calls = || async {
second_executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count()
};
let second_before = second_calls().await;
let calls = || async {
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count()
};
let before = calls().await;
let second_before = second_executor.discovery_count().await;
let before = executor.discovery_count().await;
let (started, captured) = oneshot::channel();
let (resume, released) = oneshot::channel();
*second_executor.discovery_pause.lock().await = Some(DiscoveryPause {
@@ -658,7 +641,7 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
tokio::time::timeout(Duration::from_secs(/*secs*/ 10), captured).await??;
// A has finished and B is still loading. Both readers may finish with their
// captured inventories; repeated refreshes must survive until the next lookup.
assert_eq!(calls().await, before + 1);
assert_eq!(executor.discovery_count().await, before + 1);
*executor.inventory.lock().await = Some(json!({"plugins":[],"skills":[],"warnings":[]}));
*second_executor.inventory.lock().await =
Some(json!({"plugins":[],"skills":[],"warnings":[]}));
@@ -689,62 +672,51 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
);
}
assert_eq!(
calls().await,
executor.discovery_count().await,
before + 1,
"refresh does not restart the current discovery"
);
assert_eq!(second_calls().await, second_before + 1);
assert_eq!(second_executor.discovery_count().await, second_before + 1);
let response = responses::mount_sse_once(
&model,
responses::sse(vec![responses::ev_completed("after-refresh")]),
)
.await;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "Use the refreshed inventories.".into(),
text_elements: Vec::new(),
}]))
test.submit_text_turn("Use the refreshed inventories.")
.await?;
core_test_support::wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
response.single_request();
let snapshots = test
.codex
.thread_extension_data()
.get::<Vec<ExecutorCapabilityDiscoveryV2Snapshot>>()
.context("refreshed inventory")?;
let inventory = snapshots
.iter()
.find(|snapshot| snapshot.environment_id == original_selection.environment_id)
.context("executor inventory")?
.result
.as_ref()
.map_err(|error| anyhow::anyhow!("{error}"))?;
assert!(inventory.plugins.is_empty(), "the next step must refresh");
let _ = test.codex.current_mcp_config_and_runtime_context().await;
assert_eq!(calls().await, before + 2);
assert_eq!(second_calls().await, second_before + 2);
assert_eq!(executor.discovery_count().await, before + 2);
assert_eq!(second_executor.discovery_count().await, second_before + 2);
assert_eq!(snapshots.len(), 2);
let second = snapshots
.iter()
.find(|snapshot| snapshot.environment_id == "second")
.context("second inventory")?;
assert!(
second
for environment_id in [original_selection.environment_id.as_str(), "second"] {
let inventory = snapshots
.iter()
.find(|snapshot| snapshot.environment_id == environment_id)
.context("refreshed environment inventory")?
.result
.as_ref()
.map_err(|error| anyhow::anyhow!("{error}"))?
.plugins
.is_empty()
);
.map_err(|error| anyhow::anyhow!("{error}"))?;
assert!(inventory.plugins.is_empty(), "the next step must refresh");
}
Ok(())
}
pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Result<()> {
skip_if_no_network!(Ok(()));
let prewarm_failed = match failure {
DiscoveryFailure::None => false,
DiscoveryFailure::Server => true,
DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => {
unreachable!("transport recovery and cancellation have separate coverage")
}
};
let executor = Executor::start(/*supports_v2*/ true, failure).await?;
let (discovery_started, discovery_captured) = oneshot::channel();
let (release_discovery, discovery_released) = oneshot::channel();
@@ -814,7 +786,7 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
selection.config = codex_protocol::protocol::EnvironmentConfigState::Ready(
environment_config_for_selection(&test.config, &selection),
);
// Freeze the prewarm deadline until sampling reaches the readiness barrier.
// Control the prewarm deadline and retry cooldown without wall-clock races.
tokio::time::pause();
let clock_guard = AbortOnDropHandle::new(tokio::spawn(async {
// Keep paused time from automatically advancing while mock HTTP/socket I/O is pending.
@@ -881,8 +853,6 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
release_discovery
.send(())
.map_err(|_| anyhow::anyhow!("preparation stopped before release"))?;
tokio::time::resume();
drop(clock_guard);
core_test_support::wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
@@ -901,26 +871,13 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
.message_input_texts("developer")
.join("\n")
.contains("inert-sentinel"),
!matches!(failure, DiscoveryFailure::Server),
"the next step retries transient failures and retains permanent errors"
!prewarm_failed,
"failed loads stay in cooldown"
);
let expected_calls = match failure {
DiscoveryFailure::None | DiscoveryFailure::Server => 1,
DiscoveryFailure::NoReply => 2,
// The two failed transport attempts belong to prewarm; the next step recovers.
DiscoveryFailure::Disconnect => 3,
DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"),
};
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count(),
expected_calls,
"only retryable prewarm failures should issue another discovery request"
executor.discovery_count().await,
1,
"failed prewarm must not retry before its cooldown expires"
);
let mut history = starting.requests();
history.extend(ready.requests());
@@ -940,31 +897,52 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
.message_input_texts("developer")
.join("\n")
.contains("inert-sentinel"),
!matches!(failure, DiscoveryFailure::Server)
!prewarm_failed
);
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count(),
expected_calls,
"unchanged turns reuse the successful inventory or cached permanent error"
executor.discovery_count().await,
1,
"unchanged turns reuse the inventory or the error during cooldown"
);
history.extend(reused.requests());
let snapshot_name = match failure {
DiscoveryFailure::None | DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply => {
"v2_discovery_readiness"
}
DiscoveryFailure::Server => "v2_discovery_readiness_permanent_failure",
DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"),
if prewarm_failed {
tokio::time::advance(Duration::from_secs(1)).await;
let recovered = responses::mount_sse_once(
&model,
responses::sse(vec![
responses::ev_response_created("recovered"),
responses::ev_completed("recovered"),
]),
)
.await;
test.submit_text_turn("Inspect the environment after discovery's cooldown.")
.await?;
assert!(
recovered
.single_request()
.message_input_texts("developer")
.join("\n")
.contains("inert-sentinel"),
"the first turn after cooldown recovers discovery"
);
assert_eq!(
executor.discovery_count().await,
2,
"recovery needs one discovery request"
);
history.extend(recovered.requests());
}
tokio::time::resume();
drop(clock_guard);
let snapshot_name = if prewarm_failed {
"v2_discovery_readiness_failure"
} else {
"v2_discovery_readiness"
};
insta::assert_snapshot!(
snapshot_name,
context_snapshot::format_request_history_snapshot(
"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.",
"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.",
&history,
&ContextSnapshotOptions::default(),
)
@@ -1021,13 +999,7 @@ async fn v2_prewarm_retains_all_selected_environments_until_the_first_step() ->
.await?;
response.single_request();
assert_eq!(
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count(),
executor.discovery_count().await,
selections.len(),
"the first step and later turns must reuse every prewarmed inventory"
);
@@ -1108,8 +1080,10 @@ async fn v2_prewarm_timeout_does_not_block_session_configured() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<()> {
#[test_case::test_case(false; "ordinary_load")]
#[test_case::test_case(true; "forced_load")]
#[tokio::test]
async fn v2_discovery_caches_successes_and_retries_failed_loads(force_reload: bool) -> Result<()> {
skip_if_no_network!(Ok(()));
for failure in [DiscoveryFailure::Server, DiscoveryFailure::Disconnect] {
let executor = Executor::start(/*supports_v2*/ true, failure).await?;
@@ -1127,45 +1101,42 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<
cwd: codex_utils_path_uri::PathUri::parse("file:///repo")?,
sandbox: None,
};
assert!(
cache
.discover("executor", &environment, request.clone())
.await
.is_err()
);
assert!(!cache.take_recovered_discovery());
let error = cache
.discover(
"executor",
&environment,
codex_exec_server::DiscoverV2CapabilitiesRequest {
force_reload,
..request.clone()
},
)
.await
.expect_err("initial discovery fails");
let suppressed = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("cooldown suppresses an immediate retry");
assert!(Arc::ptr_eq(&error, &suppressed));
// Advance only the cooldown; keep the socket fixture on real time.
tokio::time::pause();
tokio::time::advance(std::time::Duration::from_secs(1)).await;
tokio::time::resume();
let (first, concurrent) = tokio::join!(
cache.discover("executor", &environment, request.clone()),
cache.discover("executor", &environment, request.clone()),
);
let first = first.map_err(anyhow::Error::msg)?;
let concurrent = concurrent.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &concurrent));
let reused = cache
.discover("executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &reused));
let expected_calls = match failure {
DiscoveryFailure::Server => {
assert!(first.is_err());
assert!(concurrent.is_err());
let refreshed = codex_exec_server::ExecutorCapabilityDiscoveryV2Cache::default();
refreshed
.discover("executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert!(!refreshed.take_recovered_discovery());
assert!(!cache.take_recovered_discovery());
2
}
DiscoveryFailure::Disconnect => {
let first = first.map_err(anyhow::Error::msg)?;
let concurrent = concurrent.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &concurrent));
assert!(cache.take_recovered_discovery());
assert!(!cache.take_recovered_discovery());
let reused = cache
.discover("executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &reused));
assert!(!cache.take_recovered_discovery());
// Two disconnected attempts, then one shared successful retry in the same cache.
3
}
DiscoveryFailure::Server => 2,
// Two disconnected attempts, then one shared successful retry in the same cache.
DiscoveryFailure::Disconnect => 3,
DiscoveryFailure::None | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => {
unreachable!("the test only exercises startup failures")
}
@@ -1177,13 +1148,85 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count(),
expected_calls,
.map(|request| request["params"]["forceReload"].as_bool().unwrap_or(false))
.collect::<Vec<_>>(),
vec![force_reload; expected_calls],
);
}
Ok(())
}
#[tokio::test]
async fn v2_discovery_recovery_reprojects_mcp_without_explicit_invalidation() -> Result<()> {
skip_if_no_network!(Ok(()));
let executor = Executor::start(/*supports_v2*/ true, DiscoveryFailure::None).await?;
// Keep discovery failing until the test repairs the executor's response.
*executor.inventory.lock().await = Some(Value::Null);
let model = responses::start_mock_server().await;
let observed = Arc::new(Mutex::new(Vec::new()));
let recorder = Arc::new(DiscoveryRecorder {
mcp_projections: AtomicUsize::new(0),
snapshots: Arc::clone(&observed),
selected_plugins: Arc::new(Mutex::new(None)),
});
let mut extensions = ExtensionRegistryBuilder::new();
extensions.mcp_server_contributor(recorder.clone());
let test = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_exec_server_url(executor.url.clone())
.with_config(|config| {
config
.features
.enable(Feature::ExecutorCapabilityDiscoveryV2)
.expect("enable discovery V2");
})
.build_with_auto_env(&model)
.await?;
let first = responses::mount_sse_once(
&model,
responses::sse(vec![responses::ev_completed("failed-discovery")]),
)
.await;
test.submit_turn("Hello.").await?;
first.single_request();
assert!(
observed
.lock()
.await
.last()
.context("failed MCP inventory")?
.result
.is_err()
);
let before = recorder.mcp_projections.load(Ordering::Relaxed);
*executor.inventory.lock().await = None;
// Expire even the maximum cooldown without sleeping or changing executor bindings.
tokio::time::pause();
tokio::time::advance(Duration::from_secs(31)).await;
tokio::time::resume();
let second = responses::mount_sse_once(
&model,
responses::sse(vec![responses::ev_completed("recovered-discovery")]),
)
.await;
test.submit_turn("Hello again.").await?;
second.single_request();
assert!(recorder.mcp_projections.load(Ordering::Relaxed) > before);
let observed = observed.lock().await;
let recovered = observed
.last()
.context("recovered MCP inventory")?
.result
.as_ref()
.map_err(|error| anyhow::anyhow!("{error}"))?;
assert_eq!(
recovered.plugins.first().context("recovered plugin")?.id,
"inert-sentinel@market"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn v2_old_executor_retains_v1_selected_roots() -> Result<()> {
check_acquisition(/*enabled*/ true, "0.163.0").await
@@ -1247,18 +1290,9 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
})
.build_with_auto_env(&model)
.await?;
let calls = || async {
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count()
};
if acquires_v2 {
let _ = test.codex.current_mcp_config_and_runtime_context().await;
let before = calls().await;
let before = executor.discovery_count().await;
assert!(before > 0);
{
let observed = observed.lock().await;
@@ -1274,9 +1308,13 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
*executor.inventory.lock().await = None; // Serve the sentinel plugin and skill.
test.codex.invalidate_executor_discovery();
assert_eq!(calls().await, before, "invalidation is lazy");
assert_eq!(
executor.discovery_count().await,
before,
"invalidation is lazy"
);
let _ = test.codex.current_mcp_config_and_runtime_context().await;
assert_eq!(calls().await, before + 1);
assert_eq!(executor.discovery_count().await, before + 1);
assert_eq!(
executor
.requests
@@ -1303,7 +1341,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
"inert-sentinel"
);
}
let initial_calls = calls().await;
let initial_calls = executor.discovery_count().await;
let mut selection = test.executor_environment().selection().clone();
selection.environment_id = codex_exec_server::REMOTE_ENVIRONMENT_ID.to_string();
let cwd = selection.cwd.clone();
@@ -1392,7 +1430,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
"both discovery modes must render the skill catalog",
);
if acquires_v2 {
let discovery_calls = calls().await;
let discovery_calls = executor.discovery_count().await;
let mcp_projections = recorder.mcp_projections.load(Ordering::Relaxed);
let second_response = responses::mount_sse_once(
&model,
@@ -1409,7 +1447,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
.await?;
second_response.single_request();
assert_eq!(
calls().await,
executor.discovery_count().await,
discovery_calls,
"an unchanged turn must reuse the discovered inventory"
);
@@ -1429,7 +1467,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
)
);
} else {
assert_eq!(calls().await, 0);
assert_eq!(executor.discovery_count().await, 0);
assert!(observed.lock().await.is_empty());
}
let requests = executor.requests.lock().await.clone();
@@ -1606,17 +1644,8 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
cwd: codex_utils_path_uri::PathUri::from_host_native_path(root.path())?,
sandbox: None,
};
let calls = || async {
executor
.requests
.lock()
.await
.iter()
.filter(|request| request["method"] == "capabilities/discoverV2")
.count()
};
assert_eq!(
calls().await,
executor.discovery_count().await,
0,
"creating a cache must not request discovery"
);
@@ -1629,7 +1658,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
&first,
&concurrent.map_err(anyhow::Error::msg)?
));
assert_eq!(calls().await, 1);
assert_eq!(executor.discovery_count().await, 1);
let restricted = codex_exec_server::DiscoverV2CapabilitiesRequest {
sandbox: Some(
codex_exec_server::FileSystemSandboxContext::from_permission_profile(
@@ -1648,7 +1677,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &original));
assert_eq!(calls().await, 2);
assert_eq!(executor.discovery_count().await, 2);
cache
.discover(
"executor",
@@ -1661,7 +1690,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.await
.map_err(anyhow::Error::msg)?;
assert_eq!(
calls().await,
executor.discovery_count().await,
3,
"another cwd must not reuse the same inventory"
);
@@ -1670,7 +1699,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&first, &other));
assert_eq!(calls().await, 3);
assert_eq!(executor.discovery_count().await, 3);
let replacement_manager = codex_exec_server::EnvironmentManager::create_for_tests(
Some(executor.url.clone()),
/*local_runtime_paths*/ None,
@@ -1683,14 +1712,14 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.discover("executor", &replacement, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert_eq!(calls().await, 4);
assert_eq!(executor.discovery_count().await, 4);
let other_reused = cache
.discover("other-executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&other, &other_reused));
assert_eq!(
calls().await,
executor.discovery_count().await,
4,
"a new environment instance must not invalidate the original instance"
);
@@ -1715,7 +1744,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&other, &oldest));
assert_eq!(calls().await, 17);
assert_eq!(executor.discovery_count().await, 17);
cache
.discover(
@@ -1733,16 +1762,16 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.discover("other-executor", &environment, request.clone())
.await
.map_err(anyhow::Error::msg)?;
assert_eq!(calls().await, 19);
assert_eq!(executor.discovery_count().await, 19);
assert!(!Arc::ptr_eq(&other, &reloaded));
assert_eq!(other.as_ref(), reloaded.as_ref());
// Explicit requests and dirty entries reload once; ordinary reads reuse the result.
// Explicit requests and invalidated entries reload once; ordinary reads reuse the result.
let mut previous = reloaded;
for (expected_calls, force_reload) in [(20, true), (21, false)] {
if !force_reload {
cache.invalidate();
assert_eq!(calls().await, expected_calls - 1);
assert_eq!(executor.discovery_count().await, expected_calls - 1);
}
let refreshed = cache
.discover(
@@ -1761,7 +1790,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
.await
.map_err(anyhow::Error::msg)?;
assert!(Arc::ptr_eq(&refreshed, &ordinary));
assert_eq!(calls().await, expected_calls);
assert_eq!(executor.discovery_count().await, expected_calls);
assert_eq!(
executor
.requests
@@ -1775,7 +1804,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
previous = refreshed;
}
// Reloading one dirty entry leaves the others dirty; new contexts load normally.
// Reloading one invalidated entry preserves the others' reloads; new contexts load normally.
for (cwd, force_reload) in [("overflow", true), ("new-context", false)] {
cache
.discover(
+1 -3
View File
@@ -10,9 +10,7 @@ async fn executor_discovery_v2_refreshes_inventory_and_reuses_it_across_turns()
}
#[test_case::test_case(DiscoveryFailure::None; "success")]
#[test_case::test_case(DiscoveryFailure::Server; "permanent_failure")]
#[test_case::test_case(DiscoveryFailure::Disconnect; "retryable_failure")]
#[test_case::test_case(DiscoveryFailure::NoReply; "timeout")]
#[test_case::test_case(DiscoveryFailure::Server; "server_failure")]
#[tokio::test]
async fn executor_discovery_v2_prepares_inventory_while_sampling(
failure: DiscoveryFailure,
@@ -1,8 +1,8 @@
---
source: core/tests/suite/executor_discovery_v2.rs
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)"
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)"
---
Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.
Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.
## Window 1
-- request 1 (turn) --
@@ -1,8 +1,8 @@
---
source: core/tests/suite/executor_discovery_v2.rs
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)"
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)"
---
Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.
Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.
## Window 1
-- request 1 (turn) --
@@ -81,3 +81,24 @@ Scenario: Readiness waits for prewarm to finish; steps reuse successes and perma
-- request 3 (turn) --
07:message/user:
Inspect the same environment again.
-- request 4 (turn) --
08:message/developer:
<skills_instructions>
## Skills
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...urce` locators use their provider's access mechanism. [hash=b6782e3abdba03b2]
Read a skill package directly with `skills.read({"package":"<package>"})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647]
### Available skills
- inert-sentinel: Executor skill used by the acquisition test (executor package: skill://codex-home-skills/inert-sentinel)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + source locator...ource` entries use their provider's access mechanism. [hash=a9c099703b9c0121]
<OMITTED 9 LINES; ~479 TOKENS; hash=c5994539994fb066>
- If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
- Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
- Progressive disclosure applies to selecting relevant files, not partially reading a selected instruc...Do not load unrelated references, scripts, or assets. [hash=d6eeb13522175402]
- Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
- When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
</skills_instructions>
09:message/user:
Inspect the environment after discovery's cooldown.
@@ -1,16 +1,19 @@
//! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox.
//! Reloads replace the matching entry; in-flight readers may finish with older results.
//! Reports recovery from retryable failures so Core can refresh dependent MCP projections.
//! Invalidation replaces entries with empty forced loads; in-flight readers may finish with older results.
//! Consumers detect recovery and replacement by comparing discovery snapshots.
//! Failed loads have a per-context cooldown; lookups never sleep or schedule background retries.
use std::future::Future;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_utils_path_uri::PathUri;
use tokio::sync::OnceCell;
use tokio::time::Instant;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
@@ -19,16 +22,18 @@ use crate::ExecServerError;
use crate::FileSystemSandboxContext;
const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16;
const INITIAL_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1);
const MAX_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(30);
/// Shared by turn and idle consumers until Core explicitly invalidates discovery.
#[derive(Default)]
pub struct ExecutorCapabilityDiscoveryV2Cache {
entries: Mutex<Vec<CachedEnvironmentDiscovery>>,
entries: Mutex<Vec<Arc<DiscoveryV2CacheEntry>>>,
capacity: AtomicUsize,
recovered_discovery: AtomicBool,
}
// Reload controls how this entry is populated, not which inventory it identifies.
#[derive(Clone)]
struct DiscoveryCacheKey {
environment: Weak<Environment>,
session_id: Option<String>,
@@ -45,12 +50,81 @@ impl PartialEq for DiscoveryCacheKey {
}
}
struct CachedEnvironmentDiscovery {
struct DiscoveryV2CacheEntry {
key: DiscoveryCacheKey,
response: Arc<OnceCell<Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>>>,
retryable: Arc<AtomicBool>,
force_reload: bool,
dirty: bool,
response: OnceCell<Arc<DiscoverV2CapabilitiesResponse>>,
failure: Mutex<Option<DiscoveryV2Failure>>,
}
struct DiscoveryV2Failure {
error: Arc<ExecServerError>,
attempts: u32,
next_retry_at: Instant,
}
impl DiscoveryV2CacheEntry {
fn force_reload(&self) -> Self {
// Discard the old snapshot and cooldown, but retain the forced reload through retries.
Self {
key: self.key.clone(),
force_reload: true,
response: OnceCell::new(),
failure: Mutex::new(None),
}
}
async fn get_or_try_init<F>(
&self,
discover: impl FnOnce() -> F,
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>
where
F: Future<Output = Result<DiscoverV2CapabilitiesResponse, ExecServerError>>,
{
self.response
.get_or_try_init(|| async {
// Check here so waiting callers don't immediately retry a failed request.
{
let failure_state = self
.failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(failure) = failure_state.as_ref()
&& Instant::now() < failure.next_retry_at
{
return Err(Arc::clone(&failure.error));
}
}
let result = discover().await;
let mut failure_state = self
.failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match result {
Ok(response) => {
*failure_state = None;
Ok(Arc::new(response))
}
Err(error) => {
let attempts = failure_state
.as_ref()
.map_or(1, |failure| failure.attempts.saturating_add(1));
let delay = INITIAL_DISCOVERY_RETRY_DELAY
.saturating_mul(2_u32.saturating_pow(attempts - 1))
.min(MAX_DISCOVERY_RETRY_DELAY);
let error = Arc::new(error);
*failure_state = Some(DiscoveryV2Failure {
error: Arc::clone(&error),
attempts,
next_retry_at: Instant::now() + delay,
});
Err(error)
}
}
})
.await
.cloned()
}
}
impl ExecutorCapabilityDiscoveryV2Cache {
@@ -59,19 +133,14 @@ impl ExecutorCapabilityDiscoveryV2Cache {
self.capacity.fetch_max(minimum, Ordering::Relaxed);
}
/// Reports whether a failed discovery recovered since the last observation.
pub fn take_recovered_discovery(&self) -> bool {
self.recovered_discovery.swap(false, Ordering::AcqRel)
}
/// Dirties cached entries so their next lookup reloads without interrupting existing loads.
/// Makes future lookups reload every context without interrupting existing loads.
pub fn invalidate(&self) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for entry in entries.iter_mut() {
entry.dirty = true;
*entry = Arc::new(entry.force_reload());
}
}
@@ -86,12 +155,13 @@ impl ExecutorCapabilityDiscoveryV2Cache {
.iter_mut()
.filter(|entry| entry.key.environment.ptr_eq(&environment))
{
entry.dirty = true;
*entry = Arc::new(entry.force_reload());
}
}
/// Reuses successes and non-retryable failures unless the request forces a reload.
/// Retryable failures and cancelled loads leave the cell available for retry.
/// Reuses successful inventories; failed loads retry on the first lookup after their cooldown.
/// Cancellation adds no cooldown; explicit invalidation and new sessions start fresh loads.
/// A connection error classification does not determine discovery's lifetime.
/// Invalidation during a load applies to the next lookup, not its current readers.
pub async fn discover(
&self,
@@ -110,7 +180,7 @@ impl ExecutorCapabilityDiscoveryV2Cache {
cwd: request.cwd.clone(),
sandbox: request.sandbox.clone(),
};
let (response, retryable, force_reload) = {
let entry = {
let mut entries = self
.entries
.lock()
@@ -118,77 +188,45 @@ impl ExecutorCapabilityDiscoveryV2Cache {
let matching = entries.iter().position(|entry| entry.key == key);
if let Some(entry) = matching
.map(|index| &entries[index])
.filter(|entry| !entry.dirty && !request.force_reload)
.filter(|_| !request.force_reload)
{
(
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
entry.force_reload,
)
Arc::clone(entry)
} else {
let force_reload = request.force_reload || matching.is_some();
if let Some(index) = matching {
entries.remove(index);
}
// Recovery belongs to the environment/request, not one executor session.
let retryable = entries
.iter()
.find(|entry| {
!entry.dirty
&& entry.key.environment.ptr_eq(&key.environment)
&& entry.key.cwd == key.cwd
&& entry.key.sandbox == key.sandbox
})
.map(|entry| Arc::clone(&entry.retryable))
.unwrap_or_default();
let entry = CachedEnvironmentDiscovery {
let entry = Arc::new(DiscoveryV2CacheEntry {
key,
response: Arc::new(OnceCell::new()),
retryable,
force_reload,
dirty: false,
};
force_reload: request.force_reload,
response: OnceCell::new(),
failure: Mutex::new(None),
});
let capacity = self
.capacity
.load(Ordering::Relaxed)
.max(MIN_CACHED_DISCOVERY_CONTEXTS);
// Existing readers retain their response cells after replacement or eviction.
// Existing readers retain their entries after replacement or eviction.
if entries.len() >= capacity {
entries.remove(0);
}
let load = (
Arc::clone(&entry.response),
Arc::clone(&entry.retryable),
force_reload,
);
entries.push(entry);
load
entries.push(Arc::clone(&entry));
entry
}
};
let response = response
entry
.get_or_try_init(|| async {
request.force_reload = force_reload;
match environment.discover_capabilities_v2(request).await {
Ok(response) => Ok(Ok(Arc::new(response))),
Err(error) => {
let can_retry = crate::client::can_retry_connection_attempt(&error);
request.force_reload = entry.force_reload;
environment
.discover_capabilities_v2(request)
.await
.inspect_err(|_| {
tracing::warn!(environment_id, "executor discovery V2 failed");
let error = Arc::new(error);
if can_retry {
retryable.store(true, Ordering::Release);
Err(error)
} else {
Ok(Err(error))
}
}
}
})
})
.await
.cloned()
.unwrap_or_else(Err);
if response.is_ok() && retryable.swap(false, Ordering::AcqRel) {
self.recovered_discovery.store(true, Ordering::Release);
}
response
}
}
#[cfg(test)]
#[path = "capability_discovery_v2_cache_tests.rs"]
mod tests;
@@ -0,0 +1,180 @@
#![cfg(test)]
use super::DiscoveryCacheKey;
use super::DiscoveryV2CacheEntry;
use super::ExecutorCapabilityDiscoveryV2Cache;
use crate::DiscoverV2CapabilitiesRequest;
use crate::DiscoverV2CapabilitiesResponse;
use crate::Environment;
use crate::ExecServerError;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use std::cell::Cell;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::time::Duration;
use tokio::sync::OnceCell;
use tokio::sync::oneshot;
use tokio::time::advance;
fn empty_cache_entry() -> DiscoveryV2CacheEntry {
DiscoveryV2CacheEntry {
key: DiscoveryCacheKey {
environment: Weak::new(),
session_id: None,
cwd: PathUri::parse("file:///project").expect("test cwd"),
sandbox: None,
},
force_reload: false,
response: OnceCell::new(),
failure: Mutex::new(None),
}
}
#[tokio::test(start_paused = true)]
async fn repeated_failures_back_off_without_extending_cooldown_on_reads() {
let entry = empty_cache_entry();
let calls = Cell::new(0);
let fail = || async {
calls.set(calls.get() + 1);
Err(ExecServerError::Protocol("discovery failed".to_string()))
};
for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() {
let error = entry.get_or_try_init(fail).await.expect_err("failed load");
assert_eq!(calls.get(), attempt + 1);
advance(Duration::from_secs(delay) - Duration::from_millis(1)).await;
let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
assert_eq!(calls.get(), attempt + 1);
// Suppressed lookups must not move the deadline or advance the failure count.
advance(Duration::from_millis(1)).await;
}
let response = entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("recovery after cooldown");
advance(Duration::from_secs(300)).await;
let reused = entry.get_or_try_init(fail).await.expect("cached success");
assert!(Arc::ptr_eq(&response, &reused));
assert_eq!(calls.get(), 7);
}
#[tokio::test(start_paused = true)]
async fn queued_callers_share_failure_and_the_next_successful_attempt() {
let entry = empty_cache_entry();
let (release, wait) = oneshot::channel();
let first = entry.get_or_try_init(|| async {
wait.await.expect("release failed load");
Err(ExecServerError::Protocol("discovery failed".to_string()))
});
tokio::pin!(first);
assert!(futures::poll!(first.as_mut()).is_pending());
let calls = Cell::new(0);
let succeed = || async {
calls.set(calls.get() + 1);
Ok(DiscoverV2CapabilitiesResponse::default())
};
let queued = entry.get_or_try_init(succeed);
tokio::pin!(queued);
assert!(futures::poll!(queued.as_mut()).is_pending());
release.send(()).expect("release failed load");
let (first, queued) = tokio::join!(first, queued);
assert!(Arc::ptr_eq(
&first.expect_err("failed loader"),
&queued.expect_err("queued caller observes cooldown"),
));
assert_eq!(calls.get(), 0);
advance(Duration::from_secs(1)).await;
let (first, concurrent) = tokio::join!(
entry.get_or_try_init(succeed),
entry.get_or_try_init(succeed),
);
assert!(Arc::ptr_eq(
&first.expect("recovered loader"),
&concurrent.expect("shared success"),
));
assert_eq!(calls.get(), 1);
}
#[tokio::test(start_paused = true)]
async fn cancellation_does_not_add_a_failure_or_cooldown() {
let entry = empty_cache_entry();
// Exercise cancellation both before any failure and after an expired failure.
for delay in [1, 2] {
let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending));
assert!(futures::poll!(cancelled.as_mut()).is_pending());
drop(cancelled);
let attempted = Cell::new(false);
entry
.get_or_try_init(|| async {
attempted.set(true);
Err(ExecServerError::Protocol("discovery failed".to_string()))
})
.await
.expect_err("fresh attempt after cancellation");
assert!(attempted.get(), "cancellation must not start a cooldown");
advance(Duration::from_secs(delay)).await;
}
entry
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
.await
.expect("cancellation did not advance the backoff");
}
#[tokio::test]
async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> {
// Local environments reject V2; their errors exercise the cache without a network fixture.
let environment = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let replacement = Arc::new(Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let cache = ExecutorCapabilityDiscoveryV2Cache::default();
let request = DiscoverV2CapabilitiesRequest {
force_reload: false,
cwd: PathUri::parse("file:///project")?,
sandbox: None,
};
let error = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("local environment rejects V2");
let other_request = DiscoverV2CapabilitiesRequest {
cwd: request.cwd.join("other")?,
..request.clone()
};
let other_error = cache
.discover("executor", &environment, other_request.clone())
.await
.expect_err("independent cwd");
let replacement_error = cache
.discover("executor", &replacement, request.clone())
.await
.expect_err("independent environment");
assert!(!Arc::ptr_eq(&error, &other_error));
assert!(!Arc::ptr_eq(&error, &replacement_error));
cache.invalidate_environment(&replacement);
let suppressed = cache
.discover("executor", &environment, request.clone())
.await
.expect_err("unrelated invalidation preserves cooldown");
assert!(Arc::ptr_eq(&error, &suppressed));
cache.invalidate_environment(&environment);
let refreshed = cache
.discover("executor", &environment, request)
.await
.expect_err("explicit refresh retries immediately");
assert!(!Arc::ptr_eq(&error, &refreshed));
cache.invalidate();
let refreshed_other = cache
.discover("executor", &environment, other_request)
.await
.expect_err("global invalidation retries immediately");
assert!(!Arc::ptr_eq(&other_error, &refreshed_other));
Ok(())
}