From 2fb728da7d0b3e4273f19c3b4eb798a13cacb37c Mon Sep 17 00:00:00 2001 From: Anthony Tafoya Date: Fri, 20 Nov 2026 05:53:55 +0000 Subject: [PATCH] Retry executor discovery V2 failures with backoff (#62679) ## Why Discovery could retry transport failures on successive lookups while caching server errors until invalidation. That could repeat requests against an unhealthy executor or leave a session without an inventory after the server recovered. ## What changed Cache failed discovery loads per context and return the cached error during cooldown. The first lookup after cooldown retries, with delays doubling from 1 second up to 30 seconds on consecutive failures. Successful inventories are reused, canceled loads add no cooldown, and explicit refresh clears the cooldown for the next lookup without interrupting in-flight readers. Compare discovery snapshots to detect inventory changes so recovered plugins and MCP capabilities can become available without an explicit refresh. ## Testing Cover backoff timing, shared results for concurrent callers, cancellation, and refresh bypassing cooldown. Exercise recovery after failed prewarming and verify that MCP inventory updates after recovery. GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430 --- .../core/src/session/executor_discovery.rs | 3 +- .../core/tests/suite/executor_discovery_v2.rs | 393 ++++++++++-------- codex-rs/core/tests/suite/scenarios.rs | 4 +- ..._discovery_v2__v2_discovery_readiness.snap | 4 +- ...y_v2__v2_discovery_readiness_failure.snap} | 25 +- .../src/capability_discovery_v2_cache.rs | 184 ++++---- .../capability_discovery_v2_cache_tests.rs | 180 ++++++++ 7 files changed, 529 insertions(+), 264 deletions(-) rename codex-rs/core/tests/suite/snapshots/{all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap => all__suite__executor_discovery_v2__v2_discovery_readiness_failure.snap} (67%) create mode 100644 codex-rs/exec-server/src/capability_discovery_v2_cache_tests.rs diff --git a/codex-rs/core/src/session/executor_discovery.rs b/codex-rs/core/src/session/executor_discovery.rs index e0c98d11ed..f951bd1f87 100644 --- a/codex-rs/core/src/session/executor_discovery.rs +++ b/codex-rs/core/src/session/executor_discovery.rs @@ -1,4 +1,4 @@ -//! Core owns a thread discovery cache shared across turns. Explicit refresh dirties every entry. +//! Core owns a thread discovery cache shared across turns. Explicit refresh evicts every entry. //! In-flight readers may keep their inventories; invalidation applies to the next lookup. //! Prewarming bounds discovery before readiness without publishing MCP state or failing startup. @@ -166,7 +166,6 @@ impl Session { .zip(&snapshots) .any(|(previous, current)| !previous.same_discovery(current)) }); - let inventory_changed = cache.take_recovered_discovery() || inventory_changed; (Some(snapshots), inventory_changed) } } diff --git a/codex-rs/core/tests/suite/executor_discovery_v2.rs b/codex-rs/core/tests/suite/executor_discovery_v2.rs index 5c2ce9b1ce..5c5a5948cd 100644 --- a/codex-rs/core/tests/suite/executor_discovery_v2.rs +++ b/codex-rs/core/tests/suite/executor_discovery_v2.rs @@ -201,6 +201,15 @@ pub(super) enum DiscoveryFailure { } impl Executor { + async fn discovery_count(&self) -> usize { + self.requests + .lock() + .await + .iter() + .filter(|request| request["method"] == "capabilities/discoverV2") + .count() + } + pub(super) async fn start(supports_v2: bool, failure: DiscoveryFailure) -> Result { Self::start_with_version(supports_v2, failure, "0.164.0").await } @@ -616,37 +625,11 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> { responses::sse(vec![responses::ev_completed("ready")]), ) .await; - test.codex - .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { - text: "Initialize both environments.".into(), - text_elements: Vec::new(), - }])) + test.submit_text_turn("Initialize both environments.") .await?; - core_test_support::wait_for_event(&test.codex, |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; let _ = test.codex.current_mcp_config_and_runtime_context().await; - let second_calls = || async { - second_executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count() - }; - let second_before = second_calls().await; - let calls = || async { - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count() - }; - let before = calls().await; + let second_before = second_executor.discovery_count().await; + let before = executor.discovery_count().await; let (started, captured) = oneshot::channel(); let (resume, released) = oneshot::channel(); *second_executor.discovery_pause.lock().await = Some(DiscoveryPause { @@ -658,7 +641,7 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> { tokio::time::timeout(Duration::from_secs(/*secs*/ 10), captured).await??; // A has finished and B is still loading. Both readers may finish with their // captured inventories; repeated refreshes must survive until the next lookup. - assert_eq!(calls().await, before + 1); + assert_eq!(executor.discovery_count().await, before + 1); *executor.inventory.lock().await = Some(json!({"plugins":[],"skills":[],"warnings":[]})); *second_executor.inventory.lock().await = Some(json!({"plugins":[],"skills":[],"warnings":[]})); @@ -689,62 +672,51 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> { ); } assert_eq!( - calls().await, + executor.discovery_count().await, before + 1, "refresh does not restart the current discovery" ); - assert_eq!(second_calls().await, second_before + 1); + assert_eq!(second_executor.discovery_count().await, second_before + 1); let response = responses::mount_sse_once( &model, responses::sse(vec![responses::ev_completed("after-refresh")]), ) .await; - test.codex - .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { - text: "Use the refreshed inventories.".into(), - text_elements: Vec::new(), - }])) + test.submit_text_turn("Use the refreshed inventories.") .await?; - core_test_support::wait_for_event(&test.codex, |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; response.single_request(); let snapshots = test .codex .thread_extension_data() .get::>() .context("refreshed inventory")?; - let inventory = snapshots - .iter() - .find(|snapshot| snapshot.environment_id == original_selection.environment_id) - .context("executor inventory")? - .result - .as_ref() - .map_err(|error| anyhow::anyhow!("{error}"))?; - assert!(inventory.plugins.is_empty(), "the next step must refresh"); let _ = test.codex.current_mcp_config_and_runtime_context().await; - assert_eq!(calls().await, before + 2); - assert_eq!(second_calls().await, second_before + 2); + assert_eq!(executor.discovery_count().await, before + 2); + assert_eq!(second_executor.discovery_count().await, second_before + 2); assert_eq!(snapshots.len(), 2); - let second = snapshots - .iter() - .find(|snapshot| snapshot.environment_id == "second") - .context("second inventory")?; - assert!( - second + for environment_id in [original_selection.environment_id.as_str(), "second"] { + let inventory = snapshots + .iter() + .find(|snapshot| snapshot.environment_id == environment_id) + .context("refreshed environment inventory")? .result .as_ref() - .map_err(|error| anyhow::anyhow!("{error}"))? - .plugins - .is_empty() - ); + .map_err(|error| anyhow::anyhow!("{error}"))?; + assert!(inventory.plugins.is_empty(), "the next step must refresh"); + } Ok(()) } pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Result<()> { skip_if_no_network!(Ok(())); + let prewarm_failed = match failure { + DiscoveryFailure::None => false, + DiscoveryFailure::Server => true, + DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => { + unreachable!("transport recovery and cancellation have separate coverage") + } + }; let executor = Executor::start(/*supports_v2*/ true, failure).await?; let (discovery_started, discovery_captured) = oneshot::channel(); let (release_discovery, discovery_released) = oneshot::channel(); @@ -814,7 +786,7 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu selection.config = codex_protocol::protocol::EnvironmentConfigState::Ready( environment_config_for_selection(&test.config, &selection), ); - // Freeze the prewarm deadline until sampling reaches the readiness barrier. + // Control the prewarm deadline and retry cooldown without wall-clock races. tokio::time::pause(); let clock_guard = AbortOnDropHandle::new(tokio::spawn(async { // Keep paused time from automatically advancing while mock HTTP/socket I/O is pending. @@ -881,8 +853,6 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu release_discovery .send(()) .map_err(|_| anyhow::anyhow!("preparation stopped before release"))?; - tokio::time::resume(); - drop(clock_guard); core_test_support::wait_for_event(&test.codex, |event| { matches!(event, EventMsg::TurnComplete(_)) }) @@ -901,26 +871,13 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu .message_input_texts("developer") .join("\n") .contains("inert-sentinel"), - !matches!(failure, DiscoveryFailure::Server), - "the next step retries transient failures and retains permanent errors" + !prewarm_failed, + "failed loads stay in cooldown" ); - let expected_calls = match failure { - DiscoveryFailure::None | DiscoveryFailure::Server => 1, - DiscoveryFailure::NoReply => 2, - // The two failed transport attempts belong to prewarm; the next step recovers. - DiscoveryFailure::Disconnect => 3, - DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"), - }; assert_eq!( - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count(), - expected_calls, - "only retryable prewarm failures should issue another discovery request" + executor.discovery_count().await, + 1, + "failed prewarm must not retry before its cooldown expires" ); let mut history = starting.requests(); history.extend(ready.requests()); @@ -940,31 +897,52 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu .message_input_texts("developer") .join("\n") .contains("inert-sentinel"), - !matches!(failure, DiscoveryFailure::Server) + !prewarm_failed ); assert_eq!( - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count(), - expected_calls, - "unchanged turns reuse the successful inventory or cached permanent error" + executor.discovery_count().await, + 1, + "unchanged turns reuse the inventory or the error during cooldown" ); history.extend(reused.requests()); - let snapshot_name = match failure { - DiscoveryFailure::None | DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply => { - "v2_discovery_readiness" - } - DiscoveryFailure::Server => "v2_discovery_readiness_permanent_failure", - DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"), + if prewarm_failed { + tokio::time::advance(Duration::from_secs(1)).await; + let recovered = responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created("recovered"), + responses::ev_completed("recovered"), + ]), + ) + .await; + test.submit_text_turn("Inspect the environment after discovery's cooldown.") + .await?; + assert!( + recovered + .single_request() + .message_input_texts("developer") + .join("\n") + .contains("inert-sentinel"), + "the first turn after cooldown recovers discovery" + ); + assert_eq!( + executor.discovery_count().await, + 2, + "recovery needs one discovery request" + ); + history.extend(recovered.requests()); + } + tokio::time::resume(); + drop(clock_guard); + let snapshot_name = if prewarm_failed { + "v2_discovery_readiness_failure" + } else { + "v2_discovery_readiness" }; insta::assert_snapshot!( snapshot_name, context_snapshot::format_request_history_snapshot( - "Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.", + "Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.", &history, &ContextSnapshotOptions::default(), ) @@ -1021,13 +999,7 @@ async fn v2_prewarm_retains_all_selected_environments_until_the_first_step() -> .await?; response.single_request(); assert_eq!( - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count(), + executor.discovery_count().await, selections.len(), "the first step and later turns must reuse every prewarmed inventory" ); @@ -1108,8 +1080,10 @@ async fn v2_prewarm_timeout_does_not_block_session_configured() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<()> { +#[test_case::test_case(false; "ordinary_load")] +#[test_case::test_case(true; "forced_load")] +#[tokio::test] +async fn v2_discovery_caches_successes_and_retries_failed_loads(force_reload: bool) -> Result<()> { skip_if_no_network!(Ok(())); for failure in [DiscoveryFailure::Server, DiscoveryFailure::Disconnect] { let executor = Executor::start(/*supports_v2*/ true, failure).await?; @@ -1127,45 +1101,42 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result< cwd: codex_utils_path_uri::PathUri::parse("file:///repo")?, sandbox: None, }; - assert!( - cache - .discover("executor", &environment, request.clone()) - .await - .is_err() - ); - assert!(!cache.take_recovered_discovery()); + let error = cache + .discover( + "executor", + &environment, + codex_exec_server::DiscoverV2CapabilitiesRequest { + force_reload, + ..request.clone() + }, + ) + .await + .expect_err("initial discovery fails"); + let suppressed = cache + .discover("executor", &environment, request.clone()) + .await + .expect_err("cooldown suppresses an immediate retry"); + assert!(Arc::ptr_eq(&error, &suppressed)); + // Advance only the cooldown; keep the socket fixture on real time. + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(1)).await; + tokio::time::resume(); let (first, concurrent) = tokio::join!( cache.discover("executor", &environment, request.clone()), cache.discover("executor", &environment, request.clone()), ); + let first = first.map_err(anyhow::Error::msg)?; + let concurrent = concurrent.map_err(anyhow::Error::msg)?; + assert!(Arc::ptr_eq(&first, &concurrent)); + let reused = cache + .discover("executor", &environment, request.clone()) + .await + .map_err(anyhow::Error::msg)?; + assert!(Arc::ptr_eq(&first, &reused)); let expected_calls = match failure { - DiscoveryFailure::Server => { - assert!(first.is_err()); - assert!(concurrent.is_err()); - let refreshed = codex_exec_server::ExecutorCapabilityDiscoveryV2Cache::default(); - refreshed - .discover("executor", &environment, request.clone()) - .await - .map_err(anyhow::Error::msg)?; - assert!(!refreshed.take_recovered_discovery()); - assert!(!cache.take_recovered_discovery()); - 2 - } - DiscoveryFailure::Disconnect => { - let first = first.map_err(anyhow::Error::msg)?; - let concurrent = concurrent.map_err(anyhow::Error::msg)?; - assert!(Arc::ptr_eq(&first, &concurrent)); - assert!(cache.take_recovered_discovery()); - assert!(!cache.take_recovered_discovery()); - let reused = cache - .discover("executor", &environment, request.clone()) - .await - .map_err(anyhow::Error::msg)?; - assert!(Arc::ptr_eq(&first, &reused)); - assert!(!cache.take_recovered_discovery()); - // Two disconnected attempts, then one shared successful retry in the same cache. - 3 - } + DiscoveryFailure::Server => 2, + // Two disconnected attempts, then one shared successful retry in the same cache. + DiscoveryFailure::Disconnect => 3, DiscoveryFailure::None | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => { unreachable!("the test only exercises startup failures") } @@ -1177,13 +1148,85 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result< .await .iter() .filter(|request| request["method"] == "capabilities/discoverV2") - .count(), - expected_calls, + .map(|request| request["params"]["forceReload"].as_bool().unwrap_or(false)) + .collect::>(), + vec![force_reload; expected_calls], ); } Ok(()) } +#[tokio::test] +async fn v2_discovery_recovery_reprojects_mcp_without_explicit_invalidation() -> Result<()> { + skip_if_no_network!(Ok(())); + let executor = Executor::start(/*supports_v2*/ true, DiscoveryFailure::None).await?; + // Keep discovery failing until the test repairs the executor's response. + *executor.inventory.lock().await = Some(Value::Null); + let model = responses::start_mock_server().await; + let observed = Arc::new(Mutex::new(Vec::new())); + let recorder = Arc::new(DiscoveryRecorder { + mcp_projections: AtomicUsize::new(0), + snapshots: Arc::clone(&observed), + selected_plugins: Arc::new(Mutex::new(None)), + }); + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.mcp_server_contributor(recorder.clone()); + let test = test_codex() + .with_extensions(Arc::new(extensions.build())) + .with_exec_server_url(executor.url.clone()) + .with_config(|config| { + config + .features + .enable(Feature::ExecutorCapabilityDiscoveryV2) + .expect("enable discovery V2"); + }) + .build_with_auto_env(&model) + .await?; + let first = responses::mount_sse_once( + &model, + responses::sse(vec![responses::ev_completed("failed-discovery")]), + ) + .await; + test.submit_turn("Hello.").await?; + first.single_request(); + assert!( + observed + .lock() + .await + .last() + .context("failed MCP inventory")? + .result + .is_err() + ); + let before = recorder.mcp_projections.load(Ordering::Relaxed); + + *executor.inventory.lock().await = None; + // Expire even the maximum cooldown without sleeping or changing executor bindings. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(31)).await; + tokio::time::resume(); + let second = responses::mount_sse_once( + &model, + responses::sse(vec![responses::ev_completed("recovered-discovery")]), + ) + .await; + test.submit_turn("Hello again.").await?; + second.single_request(); + assert!(recorder.mcp_projections.load(Ordering::Relaxed) > before); + let observed = observed.lock().await; + let recovered = observed + .last() + .context("recovered MCP inventory")? + .result + .as_ref() + .map_err(|error| anyhow::anyhow!("{error}"))?; + assert_eq!( + recovered.plugins.first().context("recovered plugin")?.id, + "inert-sentinel@market" + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn v2_old_executor_retains_v1_selected_roots() -> Result<()> { check_acquisition(/*enabled*/ true, "0.163.0").await @@ -1247,18 +1290,9 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R }) .build_with_auto_env(&model) .await?; - let calls = || async { - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count() - }; if acquires_v2 { let _ = test.codex.current_mcp_config_and_runtime_context().await; - let before = calls().await; + let before = executor.discovery_count().await; assert!(before > 0); { let observed = observed.lock().await; @@ -1274,9 +1308,13 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R *executor.inventory.lock().await = None; // Serve the sentinel plugin and skill. test.codex.invalidate_executor_discovery(); - assert_eq!(calls().await, before, "invalidation is lazy"); + assert_eq!( + executor.discovery_count().await, + before, + "invalidation is lazy" + ); let _ = test.codex.current_mcp_config_and_runtime_context().await; - assert_eq!(calls().await, before + 1); + assert_eq!(executor.discovery_count().await, before + 1); assert_eq!( executor .requests @@ -1303,7 +1341,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R "inert-sentinel" ); } - let initial_calls = calls().await; + let initial_calls = executor.discovery_count().await; let mut selection = test.executor_environment().selection().clone(); selection.environment_id = codex_exec_server::REMOTE_ENVIRONMENT_ID.to_string(); let cwd = selection.cwd.clone(); @@ -1392,7 +1430,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R "both discovery modes must render the skill catalog", ); if acquires_v2 { - let discovery_calls = calls().await; + let discovery_calls = executor.discovery_count().await; let mcp_projections = recorder.mcp_projections.load(Ordering::Relaxed); let second_response = responses::mount_sse_once( &model, @@ -1409,7 +1447,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R .await?; second_response.single_request(); assert_eq!( - calls().await, + executor.discovery_count().await, discovery_calls, "an unchanged turn must reuse the discovered inventory" ); @@ -1429,7 +1467,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R ) ); } else { - assert_eq!(calls().await, 0); + assert_eq!(executor.discovery_count().await, 0); assert!(observed.lock().await.is_empty()); } let requests = executor.requests.lock().await.clone(); @@ -1606,17 +1644,8 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont cwd: codex_utils_path_uri::PathUri::from_host_native_path(root.path())?, sandbox: None, }; - let calls = || async { - executor - .requests - .lock() - .await - .iter() - .filter(|request| request["method"] == "capabilities/discoverV2") - .count() - }; assert_eq!( - calls().await, + executor.discovery_count().await, 0, "creating a cache must not request discovery" ); @@ -1629,7 +1658,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont &first, &concurrent.map_err(anyhow::Error::msg)? )); - assert_eq!(calls().await, 1); + assert_eq!(executor.discovery_count().await, 1); let restricted = codex_exec_server::DiscoverV2CapabilitiesRequest { sandbox: Some( codex_exec_server::FileSystemSandboxContext::from_permission_profile( @@ -1648,7 +1677,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .await .map_err(anyhow::Error::msg)?; assert!(Arc::ptr_eq(&first, &original)); - assert_eq!(calls().await, 2); + assert_eq!(executor.discovery_count().await, 2); cache .discover( "executor", @@ -1661,7 +1690,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .await .map_err(anyhow::Error::msg)?; assert_eq!( - calls().await, + executor.discovery_count().await, 3, "another cwd must not reuse the same inventory" ); @@ -1670,7 +1699,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .await .map_err(anyhow::Error::msg)?; assert!(Arc::ptr_eq(&first, &other)); - assert_eq!(calls().await, 3); + assert_eq!(executor.discovery_count().await, 3); let replacement_manager = codex_exec_server::EnvironmentManager::create_for_tests( Some(executor.url.clone()), /*local_runtime_paths*/ None, @@ -1683,14 +1712,14 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .discover("executor", &replacement, request.clone()) .await .map_err(anyhow::Error::msg)?; - assert_eq!(calls().await, 4); + assert_eq!(executor.discovery_count().await, 4); let other_reused = cache .discover("other-executor", &environment, request.clone()) .await .map_err(anyhow::Error::msg)?; assert!(Arc::ptr_eq(&other, &other_reused)); assert_eq!( - calls().await, + executor.discovery_count().await, 4, "a new environment instance must not invalidate the original instance" ); @@ -1715,7 +1744,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .await .map_err(anyhow::Error::msg)?; assert!(Arc::ptr_eq(&other, &oldest)); - assert_eq!(calls().await, 17); + assert_eq!(executor.discovery_count().await, 17); cache .discover( @@ -1733,16 +1762,16 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .discover("other-executor", &environment, request.clone()) .await .map_err(anyhow::Error::msg)?; - assert_eq!(calls().await, 19); + assert_eq!(executor.discovery_count().await, 19); assert!(!Arc::ptr_eq(&other, &reloaded)); assert_eq!(other.as_ref(), reloaded.as_ref()); - // Explicit requests and dirty entries reload once; ordinary reads reuse the result. + // Explicit requests and invalidated entries reload once; ordinary reads reuse the result. let mut previous = reloaded; for (expected_calls, force_reload) in [(20, true), (21, false)] { if !force_reload { cache.invalidate(); - assert_eq!(calls().await, expected_calls - 1); + assert_eq!(executor.discovery_count().await, expected_calls - 1); } let refreshed = cache .discover( @@ -1761,7 +1790,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont .await .map_err(anyhow::Error::msg)?; assert!(Arc::ptr_eq(&refreshed, &ordinary)); - assert_eq!(calls().await, expected_calls); + assert_eq!(executor.discovery_count().await, expected_calls); assert_eq!( executor .requests @@ -1775,7 +1804,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont previous = refreshed; } - // Reloading one dirty entry leaves the others dirty; new contexts load normally. + // Reloading one invalidated entry preserves the others' reloads; new contexts load normally. for (cwd, force_reload) in [("overflow", true), ("new-context", false)] { cache .discover( diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index c7241aced1..0f821009aa 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -10,9 +10,7 @@ async fn executor_discovery_v2_refreshes_inventory_and_reuses_it_across_turns() } #[test_case::test_case(DiscoveryFailure::None; "success")] -#[test_case::test_case(DiscoveryFailure::Server; "permanent_failure")] -#[test_case::test_case(DiscoveryFailure::Disconnect; "retryable_failure")] -#[test_case::test_case(DiscoveryFailure::NoReply; "timeout")] +#[test_case::test_case(DiscoveryFailure::Server; "server_failure")] #[tokio::test] async fn executor_discovery_v2_prepares_inventory_while_sampling( failure: DiscoveryFailure, diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap index 3bafcd9684..ad1e565367 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap @@ -1,8 +1,8 @@ --- source: core/tests/suite/executor_discovery_v2.rs -expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)" +expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)" --- -Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures. +Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown. ## Window 1 -- request 1 (turn) -- diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_failure.snap similarity index 67% rename from codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap rename to codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_failure.snap index 4bfd427974..ccaa40d2b2 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_failure.snap @@ -1,8 +1,8 @@ --- source: core/tests/suite/executor_discovery_v2.rs -expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)" +expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)" --- -Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures. +Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown. ## Window 1 -- request 1 (turn) -- @@ -81,3 +81,24 @@ Scenario: Readiness waits for prewarm to finish; steps reuse successes and perma -- request 3 (turn) -- 07:message/user: Inspect the same environment again. +-- request 4 (turn) -- +08:message/developer: + + ## Skills + A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...urce` locators use their provider's access mechanism. [hash=b6782e3abdba03b2] + Read a skill package directly with `skills.read({"package":""})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647] + ### Available skills + - inert-sentinel: Executor skill used by the acquisition test (executor package: skill://codex-home-skills/inert-sentinel) + ### How to use skills + - Discovery: The list above is the skills available in this session (name + description + source locator...ource` entries use their provider's access mechanism. [hash=a9c099703b9c0121] + + - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them. + - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why. + - Context hygiene: + - Progressive disclosure applies to selecting relevant files, not partially reading a selected instruc...Do not load unrelated references, scripts, or assets. [hash=d6eeb13522175402] + - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked. + - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice. + - Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue. + +09:message/user: + Inspect the environment after discovery's cooldown. diff --git a/codex-rs/exec-server/src/capability_discovery_v2_cache.rs b/codex-rs/exec-server/src/capability_discovery_v2_cache.rs index d8e575c8c2..4cf8121f0b 100644 --- a/codex-rs/exec-server/src/capability_discovery_v2_cache.rs +++ b/codex-rs/exec-server/src/capability_discovery_v2_cache.rs @@ -1,16 +1,19 @@ //! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox. -//! Reloads replace the matching entry; in-flight readers may finish with older results. -//! Reports recovery from retryable failures so Core can refresh dependent MCP projections. +//! Invalidation replaces entries with empty forced loads; in-flight readers may finish with older results. +//! Consumers detect recovery and replacement by comparing discovery snapshots. +//! Failed loads have a per-context cooldown; lookups never sleep or schedule background retries. +use std::future::Future; use std::sync::Arc; use std::sync::Mutex; use std::sync::Weak; -use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; +use std::time::Duration; use codex_utils_path_uri::PathUri; use tokio::sync::OnceCell; +use tokio::time::Instant; use crate::DiscoverV2CapabilitiesRequest; use crate::DiscoverV2CapabilitiesResponse; @@ -19,16 +22,18 @@ use crate::ExecServerError; use crate::FileSystemSandboxContext; const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16; +const INITIAL_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1); +const MAX_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(30); /// Shared by turn and idle consumers until Core explicitly invalidates discovery. #[derive(Default)] pub struct ExecutorCapabilityDiscoveryV2Cache { - entries: Mutex>, + entries: Mutex>>, capacity: AtomicUsize, - recovered_discovery: AtomicBool, } // Reload controls how this entry is populated, not which inventory it identifies. +#[derive(Clone)] struct DiscoveryCacheKey { environment: Weak, session_id: Option, @@ -45,12 +50,81 @@ impl PartialEq for DiscoveryCacheKey { } } -struct CachedEnvironmentDiscovery { +struct DiscoveryV2CacheEntry { key: DiscoveryCacheKey, - response: Arc, Arc>>>, - retryable: Arc, force_reload: bool, - dirty: bool, + response: OnceCell>, + failure: Mutex>, +} + +struct DiscoveryV2Failure { + error: Arc, + attempts: u32, + next_retry_at: Instant, +} + +impl DiscoveryV2CacheEntry { + fn force_reload(&self) -> Self { + // Discard the old snapshot and cooldown, but retain the forced reload through retries. + Self { + key: self.key.clone(), + force_reload: true, + response: OnceCell::new(), + failure: Mutex::new(None), + } + } + + async fn get_or_try_init( + &self, + discover: impl FnOnce() -> F, + ) -> Result, Arc> + where + F: Future>, + { + self.response + .get_or_try_init(|| async { + // Check here so waiting callers don't immediately retry a failed request. + { + let failure_state = self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(failure) = failure_state.as_ref() + && Instant::now() < failure.next_retry_at + { + return Err(Arc::clone(&failure.error)); + } + } + let result = discover().await; + let mut failure_state = self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match result { + Ok(response) => { + *failure_state = None; + Ok(Arc::new(response)) + } + Err(error) => { + let attempts = failure_state + .as_ref() + .map_or(1, |failure| failure.attempts.saturating_add(1)); + let delay = INITIAL_DISCOVERY_RETRY_DELAY + .saturating_mul(2_u32.saturating_pow(attempts - 1)) + .min(MAX_DISCOVERY_RETRY_DELAY); + let error = Arc::new(error); + *failure_state = Some(DiscoveryV2Failure { + error: Arc::clone(&error), + attempts, + next_retry_at: Instant::now() + delay, + }); + Err(error) + } + } + }) + .await + .cloned() + } } impl ExecutorCapabilityDiscoveryV2Cache { @@ -59,19 +133,14 @@ impl ExecutorCapabilityDiscoveryV2Cache { self.capacity.fetch_max(minimum, Ordering::Relaxed); } - /// Reports whether a failed discovery recovered since the last observation. - pub fn take_recovered_discovery(&self) -> bool { - self.recovered_discovery.swap(false, Ordering::AcqRel) - } - - /// Dirties cached entries so their next lookup reloads without interrupting existing loads. + /// Makes future lookups reload every context without interrupting existing loads. pub fn invalidate(&self) { let mut entries = self .entries .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); for entry in entries.iter_mut() { - entry.dirty = true; + *entry = Arc::new(entry.force_reload()); } } @@ -86,12 +155,13 @@ impl ExecutorCapabilityDiscoveryV2Cache { .iter_mut() .filter(|entry| entry.key.environment.ptr_eq(&environment)) { - entry.dirty = true; + *entry = Arc::new(entry.force_reload()); } } - /// Reuses successes and non-retryable failures unless the request forces a reload. - /// Retryable failures and cancelled loads leave the cell available for retry. + /// Reuses successful inventories; failed loads retry on the first lookup after their cooldown. + /// Cancellation adds no cooldown; explicit invalidation and new sessions start fresh loads. + /// A connection error classification does not determine discovery's lifetime. /// Invalidation during a load applies to the next lookup, not its current readers. pub async fn discover( &self, @@ -110,7 +180,7 @@ impl ExecutorCapabilityDiscoveryV2Cache { cwd: request.cwd.clone(), sandbox: request.sandbox.clone(), }; - let (response, retryable, force_reload) = { + let entry = { let mut entries = self .entries .lock() @@ -118,77 +188,45 @@ impl ExecutorCapabilityDiscoveryV2Cache { let matching = entries.iter().position(|entry| entry.key == key); if let Some(entry) = matching .map(|index| &entries[index]) - .filter(|entry| !entry.dirty && !request.force_reload) + .filter(|_| !request.force_reload) { - ( - Arc::clone(&entry.response), - Arc::clone(&entry.retryable), - entry.force_reload, - ) + Arc::clone(entry) } else { - let force_reload = request.force_reload || matching.is_some(); if let Some(index) = matching { entries.remove(index); } - // Recovery belongs to the environment/request, not one executor session. - let retryable = entries - .iter() - .find(|entry| { - !entry.dirty - && entry.key.environment.ptr_eq(&key.environment) - && entry.key.cwd == key.cwd - && entry.key.sandbox == key.sandbox - }) - .map(|entry| Arc::clone(&entry.retryable)) - .unwrap_or_default(); - let entry = CachedEnvironmentDiscovery { + let entry = Arc::new(DiscoveryV2CacheEntry { key, - response: Arc::new(OnceCell::new()), - retryable, - force_reload, - dirty: false, - }; + force_reload: request.force_reload, + response: OnceCell::new(), + failure: Mutex::new(None), + }); let capacity = self .capacity .load(Ordering::Relaxed) .max(MIN_CACHED_DISCOVERY_CONTEXTS); - // Existing readers retain their response cells after replacement or eviction. + // Existing readers retain their entries after replacement or eviction. if entries.len() >= capacity { entries.remove(0); } - let load = ( - Arc::clone(&entry.response), - Arc::clone(&entry.retryable), - force_reload, - ); - entries.push(entry); - load + entries.push(Arc::clone(&entry)); + entry } }; - let response = response + entry .get_or_try_init(|| async { - request.force_reload = force_reload; - match environment.discover_capabilities_v2(request).await { - Ok(response) => Ok(Ok(Arc::new(response))), - Err(error) => { - let can_retry = crate::client::can_retry_connection_attempt(&error); + request.force_reload = entry.force_reload; + environment + .discover_capabilities_v2(request) + .await + .inspect_err(|_| { tracing::warn!(environment_id, "executor discovery V2 failed"); - let error = Arc::new(error); - if can_retry { - retryable.store(true, Ordering::Release); - Err(error) - } else { - Ok(Err(error)) - } - } - } + }) }) .await - .cloned() - .unwrap_or_else(Err); - if response.is_ok() && retryable.swap(false, Ordering::AcqRel) { - self.recovered_discovery.store(true, Ordering::Release); - } - response } } + +#[cfg(test)] +#[path = "capability_discovery_v2_cache_tests.rs"] +mod tests; diff --git a/codex-rs/exec-server/src/capability_discovery_v2_cache_tests.rs b/codex-rs/exec-server/src/capability_discovery_v2_cache_tests.rs new file mode 100644 index 0000000000..3de1348ab0 --- /dev/null +++ b/codex-rs/exec-server/src/capability_discovery_v2_cache_tests.rs @@ -0,0 +1,180 @@ +#![cfg(test)] + +use super::DiscoveryCacheKey; +use super::DiscoveryV2CacheEntry; +use super::ExecutorCapabilityDiscoveryV2Cache; +use crate::DiscoverV2CapabilitiesRequest; +use crate::DiscoverV2CapabilitiesResponse; +use crate::Environment; +use crate::ExecServerError; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use std::cell::Cell; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::Weak; +use std::time::Duration; +use tokio::sync::OnceCell; +use tokio::sync::oneshot; +use tokio::time::advance; + +fn empty_cache_entry() -> DiscoveryV2CacheEntry { + DiscoveryV2CacheEntry { + key: DiscoveryCacheKey { + environment: Weak::new(), + session_id: None, + cwd: PathUri::parse("file:///project").expect("test cwd"), + sandbox: None, + }, + force_reload: false, + response: OnceCell::new(), + failure: Mutex::new(None), + } +} + +#[tokio::test(start_paused = true)] +async fn repeated_failures_back_off_without_extending_cooldown_on_reads() { + let entry = empty_cache_entry(); + let calls = Cell::new(0); + let fail = || async { + calls.set(calls.get() + 1); + Err(ExecServerError::Protocol("discovery failed".to_string())) + }; + for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() { + let error = entry.get_or_try_init(fail).await.expect_err("failed load"); + assert_eq!(calls.get(), attempt + 1); + advance(Duration::from_secs(delay) - Duration::from_millis(1)).await; + let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown"); + assert!(Arc::ptr_eq(&error, &suppressed)); + assert_eq!(calls.get(), attempt + 1); + // Suppressed lookups must not move the deadline or advance the failure count. + advance(Duration::from_millis(1)).await; + } + + let response = entry + .get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) }) + .await + .expect("recovery after cooldown"); + advance(Duration::from_secs(300)).await; + let reused = entry.get_or_try_init(fail).await.expect("cached success"); + assert!(Arc::ptr_eq(&response, &reused)); + assert_eq!(calls.get(), 7); +} + +#[tokio::test(start_paused = true)] +async fn queued_callers_share_failure_and_the_next_successful_attempt() { + let entry = empty_cache_entry(); + let (release, wait) = oneshot::channel(); + let first = entry.get_or_try_init(|| async { + wait.await.expect("release failed load"); + Err(ExecServerError::Protocol("discovery failed".to_string())) + }); + tokio::pin!(first); + assert!(futures::poll!(first.as_mut()).is_pending()); + let calls = Cell::new(0); + let succeed = || async { + calls.set(calls.get() + 1); + Ok(DiscoverV2CapabilitiesResponse::default()) + }; + let queued = entry.get_or_try_init(succeed); + tokio::pin!(queued); + assert!(futures::poll!(queued.as_mut()).is_pending()); + release.send(()).expect("release failed load"); + let (first, queued) = tokio::join!(first, queued); + assert!(Arc::ptr_eq( + &first.expect_err("failed loader"), + &queued.expect_err("queued caller observes cooldown"), + )); + assert_eq!(calls.get(), 0); + + advance(Duration::from_secs(1)).await; + let (first, concurrent) = tokio::join!( + entry.get_or_try_init(succeed), + entry.get_or_try_init(succeed), + ); + assert!(Arc::ptr_eq( + &first.expect("recovered loader"), + &concurrent.expect("shared success"), + )); + assert_eq!(calls.get(), 1); +} + +#[tokio::test(start_paused = true)] +async fn cancellation_does_not_add_a_failure_or_cooldown() { + let entry = empty_cache_entry(); + // Exercise cancellation both before any failure and after an expired failure. + for delay in [1, 2] { + let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending)); + assert!(futures::poll!(cancelled.as_mut()).is_pending()); + drop(cancelled); + let attempted = Cell::new(false); + entry + .get_or_try_init(|| async { + attempted.set(true); + Err(ExecServerError::Protocol("discovery failed".to_string())) + }) + .await + .expect_err("fresh attempt after cancellation"); + assert!(attempted.get(), "cancellation must not start a cooldown"); + advance(Duration::from_secs(delay)).await; + } + entry + .get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) }) + .await + .expect("cancellation did not advance the backoff"); +} + +#[tokio::test] +async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> { + // Local environments reject V2; their errors exercise the cache without a network fixture. + let environment = Arc::new(Environment::create_for_tests( + /*exec_server_url*/ None, + )?); + let replacement = Arc::new(Environment::create_for_tests( + /*exec_server_url*/ None, + )?); + let cache = ExecutorCapabilityDiscoveryV2Cache::default(); + let request = DiscoverV2CapabilitiesRequest { + force_reload: false, + cwd: PathUri::parse("file:///project")?, + sandbox: None, + }; + let error = cache + .discover("executor", &environment, request.clone()) + .await + .expect_err("local environment rejects V2"); + let other_request = DiscoverV2CapabilitiesRequest { + cwd: request.cwd.join("other")?, + ..request.clone() + }; + let other_error = cache + .discover("executor", &environment, other_request.clone()) + .await + .expect_err("independent cwd"); + let replacement_error = cache + .discover("executor", &replacement, request.clone()) + .await + .expect_err("independent environment"); + assert!(!Arc::ptr_eq(&error, &other_error)); + assert!(!Arc::ptr_eq(&error, &replacement_error)); + + cache.invalidate_environment(&replacement); + let suppressed = cache + .discover("executor", &environment, request.clone()) + .await + .expect_err("unrelated invalidation preserves cooldown"); + assert!(Arc::ptr_eq(&error, &suppressed)); + cache.invalidate_environment(&environment); + let refreshed = cache + .discover("executor", &environment, request) + .await + .expect_err("explicit refresh retries immediately"); + assert!(!Arc::ptr_eq(&error, &refreshed)); + cache.invalidate(); + let refreshed_other = cache + .discover("executor", &environment, other_request) + .await + .expect_err("global invalidation retries immediately"); + assert!(!Arc::ptr_eq(&other_error, &refreshed_other)); + Ok(()) +}