mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
Retry executor discovery V2 failures with backoff (#62679)
## Why Discovery could retry transport failures on successive lookups while caching server errors until invalidation. That could repeat requests against an unhealthy executor or leave a session without an inventory after the server recovered. ## What changed Cache failed discovery loads per context and return the cached error during cooldown. The first lookup after cooldown retries, with delays doubling from 1 second up to 30 seconds on consecutive failures. Successful inventories are reused, canceled loads add no cooldown, and explicit refresh clears the cooldown for the next lookup without interrupting in-flight readers. Compare discovery snapshots to detect inventory changes so recovered plugins and MCP capabilities can become available without an explicit refresh. ## Testing Cover backoff timing, shared results for concurrent callers, cancellation, and refresh bypassing cooldown. Exercise recovery after failed prewarming and verify that MCP inventory updates after recovery. GitOrigin-RevId: 6de34bb8957a0b24ee874e2447157bd3af562430
This commit is contained in:
committed by
copyberry
parent
ea8ccc1d20
commit
2fb728da7d
@@ -1,4 +1,4 @@
|
||||
//! Core owns a thread discovery cache shared across turns. Explicit refresh dirties every entry.
|
||||
//! Core owns a thread discovery cache shared across turns. Explicit refresh evicts every entry.
|
||||
//! In-flight readers may keep their inventories; invalidation applies to the next lookup.
|
||||
//! Prewarming bounds discovery before readiness without publishing MCP state or failing startup.
|
||||
|
||||
@@ -166,7 +166,6 @@ impl Session {
|
||||
.zip(&snapshots)
|
||||
.any(|(previous, current)| !previous.same_discovery(current))
|
||||
});
|
||||
let inventory_changed = cache.take_recovered_discovery() || inventory_changed;
|
||||
(Some(snapshots), inventory_changed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,6 +201,15 @@ pub(super) enum DiscoveryFailure {
|
||||
}
|
||||
|
||||
impl Executor {
|
||||
async fn discovery_count(&self) -> usize {
|
||||
self.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count()
|
||||
}
|
||||
|
||||
pub(super) async fn start(supports_v2: bool, failure: DiscoveryFailure) -> Result<Self> {
|
||||
Self::start_with_version(supports_v2, failure, "0.164.0").await
|
||||
}
|
||||
@@ -616,37 +625,11 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
|
||||
responses::sse(vec![responses::ev_completed("ready")]),
|
||||
)
|
||||
.await;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "Initialize both environments.".into(),
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
test.submit_text_turn("Initialize both environments.")
|
||||
.await?;
|
||||
core_test_support::wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
let _ = test.codex.current_mcp_config_and_runtime_context().await;
|
||||
let second_calls = || async {
|
||||
second_executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count()
|
||||
};
|
||||
let second_before = second_calls().await;
|
||||
let calls = || async {
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count()
|
||||
};
|
||||
let before = calls().await;
|
||||
let second_before = second_executor.discovery_count().await;
|
||||
let before = executor.discovery_count().await;
|
||||
let (started, captured) = oneshot::channel();
|
||||
let (resume, released) = oneshot::channel();
|
||||
*second_executor.discovery_pause.lock().await = Some(DiscoveryPause {
|
||||
@@ -658,7 +641,7 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
|
||||
tokio::time::timeout(Duration::from_secs(/*secs*/ 10), captured).await??;
|
||||
// A has finished and B is still loading. Both readers may finish with their
|
||||
// captured inventories; repeated refreshes must survive until the next lookup.
|
||||
assert_eq!(calls().await, before + 1);
|
||||
assert_eq!(executor.discovery_count().await, before + 1);
|
||||
*executor.inventory.lock().await = Some(json!({"plugins":[],"skills":[],"warnings":[]}));
|
||||
*second_executor.inventory.lock().await =
|
||||
Some(json!({"plugins":[],"skills":[],"warnings":[]}));
|
||||
@@ -689,62 +672,51 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> {
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
calls().await,
|
||||
executor.discovery_count().await,
|
||||
before + 1,
|
||||
"refresh does not restart the current discovery"
|
||||
);
|
||||
assert_eq!(second_calls().await, second_before + 1);
|
||||
assert_eq!(second_executor.discovery_count().await, second_before + 1);
|
||||
|
||||
let response = responses::mount_sse_once(
|
||||
&model,
|
||||
responses::sse(vec![responses::ev_completed("after-refresh")]),
|
||||
)
|
||||
.await;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "Use the refreshed inventories.".into(),
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
test.submit_text_turn("Use the refreshed inventories.")
|
||||
.await?;
|
||||
core_test_support::wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
response.single_request();
|
||||
let snapshots = test
|
||||
.codex
|
||||
.thread_extension_data()
|
||||
.get::<Vec<ExecutorCapabilityDiscoveryV2Snapshot>>()
|
||||
.context("refreshed inventory")?;
|
||||
let inventory = snapshots
|
||||
.iter()
|
||||
.find(|snapshot| snapshot.environment_id == original_selection.environment_id)
|
||||
.context("executor inventory")?
|
||||
.result
|
||||
.as_ref()
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
assert!(inventory.plugins.is_empty(), "the next step must refresh");
|
||||
let _ = test.codex.current_mcp_config_and_runtime_context().await;
|
||||
assert_eq!(calls().await, before + 2);
|
||||
assert_eq!(second_calls().await, second_before + 2);
|
||||
assert_eq!(executor.discovery_count().await, before + 2);
|
||||
assert_eq!(second_executor.discovery_count().await, second_before + 2);
|
||||
assert_eq!(snapshots.len(), 2);
|
||||
let second = snapshots
|
||||
.iter()
|
||||
.find(|snapshot| snapshot.environment_id == "second")
|
||||
.context("second inventory")?;
|
||||
assert!(
|
||||
second
|
||||
for environment_id in [original_selection.environment_id.as_str(), "second"] {
|
||||
let inventory = snapshots
|
||||
.iter()
|
||||
.find(|snapshot| snapshot.environment_id == environment_id)
|
||||
.context("refreshed environment inventory")?
|
||||
.result
|
||||
.as_ref()
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?
|
||||
.plugins
|
||||
.is_empty()
|
||||
);
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
assert!(inventory.plugins.is_empty(), "the next step must refresh");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let prewarm_failed = match failure {
|
||||
DiscoveryFailure::None => false,
|
||||
DiscoveryFailure::Server => true,
|
||||
DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => {
|
||||
unreachable!("transport recovery and cancellation have separate coverage")
|
||||
}
|
||||
};
|
||||
let executor = Executor::start(/*supports_v2*/ true, failure).await?;
|
||||
let (discovery_started, discovery_captured) = oneshot::channel();
|
||||
let (release_discovery, discovery_released) = oneshot::channel();
|
||||
@@ -814,7 +786,7 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
|
||||
selection.config = codex_protocol::protocol::EnvironmentConfigState::Ready(
|
||||
environment_config_for_selection(&test.config, &selection),
|
||||
);
|
||||
// Freeze the prewarm deadline until sampling reaches the readiness barrier.
|
||||
// Control the prewarm deadline and retry cooldown without wall-clock races.
|
||||
tokio::time::pause();
|
||||
let clock_guard = AbortOnDropHandle::new(tokio::spawn(async {
|
||||
// Keep paused time from automatically advancing while mock HTTP/socket I/O is pending.
|
||||
@@ -881,8 +853,6 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
|
||||
release_discovery
|
||||
.send(())
|
||||
.map_err(|_| anyhow::anyhow!("preparation stopped before release"))?;
|
||||
tokio::time::resume();
|
||||
drop(clock_guard);
|
||||
core_test_support::wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
@@ -901,26 +871,13 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
|
||||
.message_input_texts("developer")
|
||||
.join("\n")
|
||||
.contains("inert-sentinel"),
|
||||
!matches!(failure, DiscoveryFailure::Server),
|
||||
"the next step retries transient failures and retains permanent errors"
|
||||
!prewarm_failed,
|
||||
"failed loads stay in cooldown"
|
||||
);
|
||||
let expected_calls = match failure {
|
||||
DiscoveryFailure::None | DiscoveryFailure::Server => 1,
|
||||
DiscoveryFailure::NoReply => 2,
|
||||
// The two failed transport attempts belong to prewarm; the next step recovers.
|
||||
DiscoveryFailure::Disconnect => 3,
|
||||
DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"),
|
||||
};
|
||||
assert_eq!(
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count(),
|
||||
expected_calls,
|
||||
"only retryable prewarm failures should issue another discovery request"
|
||||
executor.discovery_count().await,
|
||||
1,
|
||||
"failed prewarm must not retry before its cooldown expires"
|
||||
);
|
||||
let mut history = starting.requests();
|
||||
history.extend(ready.requests());
|
||||
@@ -940,31 +897,52 @@ pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Resu
|
||||
.message_input_texts("developer")
|
||||
.join("\n")
|
||||
.contains("inert-sentinel"),
|
||||
!matches!(failure, DiscoveryFailure::Server)
|
||||
!prewarm_failed
|
||||
);
|
||||
assert_eq!(
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count(),
|
||||
expected_calls,
|
||||
"unchanged turns reuse the successful inventory or cached permanent error"
|
||||
executor.discovery_count().await,
|
||||
1,
|
||||
"unchanged turns reuse the inventory or the error during cooldown"
|
||||
);
|
||||
history.extend(reused.requests());
|
||||
let snapshot_name = match failure {
|
||||
DiscoveryFailure::None | DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply => {
|
||||
"v2_discovery_readiness"
|
||||
}
|
||||
DiscoveryFailure::Server => "v2_discovery_readiness_permanent_failure",
|
||||
DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"),
|
||||
if prewarm_failed {
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
let recovered = responses::mount_sse_once(
|
||||
&model,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("recovered"),
|
||||
responses::ev_completed("recovered"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
test.submit_text_turn("Inspect the environment after discovery's cooldown.")
|
||||
.await?;
|
||||
assert!(
|
||||
recovered
|
||||
.single_request()
|
||||
.message_input_texts("developer")
|
||||
.join("\n")
|
||||
.contains("inert-sentinel"),
|
||||
"the first turn after cooldown recovers discovery"
|
||||
);
|
||||
assert_eq!(
|
||||
executor.discovery_count().await,
|
||||
2,
|
||||
"recovery needs one discovery request"
|
||||
);
|
||||
history.extend(recovered.requests());
|
||||
}
|
||||
tokio::time::resume();
|
||||
drop(clock_guard);
|
||||
let snapshot_name = if prewarm_failed {
|
||||
"v2_discovery_readiness_failure"
|
||||
} else {
|
||||
"v2_discovery_readiness"
|
||||
};
|
||||
insta::assert_snapshot!(
|
||||
snapshot_name,
|
||||
context_snapshot::format_request_history_snapshot(
|
||||
"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.",
|
||||
"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.",
|
||||
&history,
|
||||
&ContextSnapshotOptions::default(),
|
||||
)
|
||||
@@ -1021,13 +999,7 @@ async fn v2_prewarm_retains_all_selected_environments_until_the_first_step() ->
|
||||
.await?;
|
||||
response.single_request();
|
||||
assert_eq!(
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count(),
|
||||
executor.discovery_count().await,
|
||||
selections.len(),
|
||||
"the first step and later turns must reuse every prewarmed inventory"
|
||||
);
|
||||
@@ -1108,8 +1080,10 @@ async fn v2_prewarm_timeout_does_not_block_session_configured() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<()> {
|
||||
#[test_case::test_case(false; "ordinary_load")]
|
||||
#[test_case::test_case(true; "forced_load")]
|
||||
#[tokio::test]
|
||||
async fn v2_discovery_caches_successes_and_retries_failed_loads(force_reload: bool) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
for failure in [DiscoveryFailure::Server, DiscoveryFailure::Disconnect] {
|
||||
let executor = Executor::start(/*supports_v2*/ true, failure).await?;
|
||||
@@ -1127,45 +1101,42 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<
|
||||
cwd: codex_utils_path_uri::PathUri::parse("file:///repo")?,
|
||||
sandbox: None,
|
||||
};
|
||||
assert!(
|
||||
cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!cache.take_recovered_discovery());
|
||||
let error = cache
|
||||
.discover(
|
||||
"executor",
|
||||
&environment,
|
||||
codex_exec_server::DiscoverV2CapabilitiesRequest {
|
||||
force_reload,
|
||||
..request.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("initial discovery fails");
|
||||
let suppressed = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.expect_err("cooldown suppresses an immediate retry");
|
||||
assert!(Arc::ptr_eq(&error, &suppressed));
|
||||
// Advance only the cooldown; keep the socket fixture on real time.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
let (first, concurrent) = tokio::join!(
|
||||
cache.discover("executor", &environment, request.clone()),
|
||||
cache.discover("executor", &environment, request.clone()),
|
||||
);
|
||||
let first = first.map_err(anyhow::Error::msg)?;
|
||||
let concurrent = concurrent.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &concurrent));
|
||||
let reused = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &reused));
|
||||
let expected_calls = match failure {
|
||||
DiscoveryFailure::Server => {
|
||||
assert!(first.is_err());
|
||||
assert!(concurrent.is_err());
|
||||
let refreshed = codex_exec_server::ExecutorCapabilityDiscoveryV2Cache::default();
|
||||
refreshed
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(!refreshed.take_recovered_discovery());
|
||||
assert!(!cache.take_recovered_discovery());
|
||||
2
|
||||
}
|
||||
DiscoveryFailure::Disconnect => {
|
||||
let first = first.map_err(anyhow::Error::msg)?;
|
||||
let concurrent = concurrent.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &concurrent));
|
||||
assert!(cache.take_recovered_discovery());
|
||||
assert!(!cache.take_recovered_discovery());
|
||||
let reused = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &reused));
|
||||
assert!(!cache.take_recovered_discovery());
|
||||
// Two disconnected attempts, then one shared successful retry in the same cache.
|
||||
3
|
||||
}
|
||||
DiscoveryFailure::Server => 2,
|
||||
// Two disconnected attempts, then one shared successful retry in the same cache.
|
||||
DiscoveryFailure::Disconnect => 3,
|
||||
DiscoveryFailure::None | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => {
|
||||
unreachable!("the test only exercises startup failures")
|
||||
}
|
||||
@@ -1177,13 +1148,85 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count(),
|
||||
expected_calls,
|
||||
.map(|request| request["params"]["forceReload"].as_bool().unwrap_or(false))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![force_reload; expected_calls],
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn v2_discovery_recovery_reprojects_mcp_without_explicit_invalidation() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let executor = Executor::start(/*supports_v2*/ true, DiscoveryFailure::None).await?;
|
||||
// Keep discovery failing until the test repairs the executor's response.
|
||||
*executor.inventory.lock().await = Some(Value::Null);
|
||||
let model = responses::start_mock_server().await;
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let recorder = Arc::new(DiscoveryRecorder {
|
||||
mcp_projections: AtomicUsize::new(0),
|
||||
snapshots: Arc::clone(&observed),
|
||||
selected_plugins: Arc::new(Mutex::new(None)),
|
||||
});
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.mcp_server_contributor(recorder.clone());
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_exec_server_url(executor.url.clone())
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::ExecutorCapabilityDiscoveryV2)
|
||||
.expect("enable discovery V2");
|
||||
})
|
||||
.build_with_auto_env(&model)
|
||||
.await?;
|
||||
let first = responses::mount_sse_once(
|
||||
&model,
|
||||
responses::sse(vec![responses::ev_completed("failed-discovery")]),
|
||||
)
|
||||
.await;
|
||||
test.submit_turn("Hello.").await?;
|
||||
first.single_request();
|
||||
assert!(
|
||||
observed
|
||||
.lock()
|
||||
.await
|
||||
.last()
|
||||
.context("failed MCP inventory")?
|
||||
.result
|
||||
.is_err()
|
||||
);
|
||||
let before = recorder.mcp_projections.load(Ordering::Relaxed);
|
||||
|
||||
*executor.inventory.lock().await = None;
|
||||
// Expire even the maximum cooldown without sleeping or changing executor bindings.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(31)).await;
|
||||
tokio::time::resume();
|
||||
let second = responses::mount_sse_once(
|
||||
&model,
|
||||
responses::sse(vec![responses::ev_completed("recovered-discovery")]),
|
||||
)
|
||||
.await;
|
||||
test.submit_turn("Hello again.").await?;
|
||||
second.single_request();
|
||||
assert!(recorder.mcp_projections.load(Ordering::Relaxed) > before);
|
||||
let observed = observed.lock().await;
|
||||
let recovered = observed
|
||||
.last()
|
||||
.context("recovered MCP inventory")?
|
||||
.result
|
||||
.as_ref()
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
assert_eq!(
|
||||
recovered.plugins.first().context("recovered plugin")?.id,
|
||||
"inert-sentinel@market"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn v2_old_executor_retains_v1_selected_roots() -> Result<()> {
|
||||
check_acquisition(/*enabled*/ true, "0.163.0").await
|
||||
@@ -1247,18 +1290,9 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
})
|
||||
.build_with_auto_env(&model)
|
||||
.await?;
|
||||
let calls = || async {
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count()
|
||||
};
|
||||
if acquires_v2 {
|
||||
let _ = test.codex.current_mcp_config_and_runtime_context().await;
|
||||
let before = calls().await;
|
||||
let before = executor.discovery_count().await;
|
||||
assert!(before > 0);
|
||||
{
|
||||
let observed = observed.lock().await;
|
||||
@@ -1274,9 +1308,13 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
|
||||
*executor.inventory.lock().await = None; // Serve the sentinel plugin and skill.
|
||||
test.codex.invalidate_executor_discovery();
|
||||
assert_eq!(calls().await, before, "invalidation is lazy");
|
||||
assert_eq!(
|
||||
executor.discovery_count().await,
|
||||
before,
|
||||
"invalidation is lazy"
|
||||
);
|
||||
let _ = test.codex.current_mcp_config_and_runtime_context().await;
|
||||
assert_eq!(calls().await, before + 1);
|
||||
assert_eq!(executor.discovery_count().await, before + 1);
|
||||
assert_eq!(
|
||||
executor
|
||||
.requests
|
||||
@@ -1303,7 +1341,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
"inert-sentinel"
|
||||
);
|
||||
}
|
||||
let initial_calls = calls().await;
|
||||
let initial_calls = executor.discovery_count().await;
|
||||
let mut selection = test.executor_environment().selection().clone();
|
||||
selection.environment_id = codex_exec_server::REMOTE_ENVIRONMENT_ID.to_string();
|
||||
let cwd = selection.cwd.clone();
|
||||
@@ -1392,7 +1430,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
"both discovery modes must render the skill catalog",
|
||||
);
|
||||
if acquires_v2 {
|
||||
let discovery_calls = calls().await;
|
||||
let discovery_calls = executor.discovery_count().await;
|
||||
let mcp_projections = recorder.mcp_projections.load(Ordering::Relaxed);
|
||||
let second_response = responses::mount_sse_once(
|
||||
&model,
|
||||
@@ -1409,7 +1447,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
.await?;
|
||||
second_response.single_request();
|
||||
assert_eq!(
|
||||
calls().await,
|
||||
executor.discovery_count().await,
|
||||
discovery_calls,
|
||||
"an unchanged turn must reuse the discovered inventory"
|
||||
);
|
||||
@@ -1429,7 +1467,7 @@ pub(super) async fn check_acquisition(enabled: bool, version: &'static str) -> R
|
||||
)
|
||||
);
|
||||
} else {
|
||||
assert_eq!(calls().await, 0);
|
||||
assert_eq!(executor.discovery_count().await, 0);
|
||||
assert!(observed.lock().await.is_empty());
|
||||
}
|
||||
let requests = executor.requests.lock().await.clone();
|
||||
@@ -1606,17 +1644,8 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
cwd: codex_utils_path_uri::PathUri::from_host_native_path(root.path())?,
|
||||
sandbox: None,
|
||||
};
|
||||
let calls = || async {
|
||||
executor
|
||||
.requests
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|request| request["method"] == "capabilities/discoverV2")
|
||||
.count()
|
||||
};
|
||||
assert_eq!(
|
||||
calls().await,
|
||||
executor.discovery_count().await,
|
||||
0,
|
||||
"creating a cache must not request discovery"
|
||||
);
|
||||
@@ -1629,7 +1658,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
&first,
|
||||
&concurrent.map_err(anyhow::Error::msg)?
|
||||
));
|
||||
assert_eq!(calls().await, 1);
|
||||
assert_eq!(executor.discovery_count().await, 1);
|
||||
let restricted = codex_exec_server::DiscoverV2CapabilitiesRequest {
|
||||
sandbox: Some(
|
||||
codex_exec_server::FileSystemSandboxContext::from_permission_profile(
|
||||
@@ -1648,7 +1677,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &original));
|
||||
assert_eq!(calls().await, 2);
|
||||
assert_eq!(executor.discovery_count().await, 2);
|
||||
cache
|
||||
.discover(
|
||||
"executor",
|
||||
@@ -1661,7 +1690,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert_eq!(
|
||||
calls().await,
|
||||
executor.discovery_count().await,
|
||||
3,
|
||||
"another cwd must not reuse the same inventory"
|
||||
);
|
||||
@@ -1670,7 +1699,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&first, &other));
|
||||
assert_eq!(calls().await, 3);
|
||||
assert_eq!(executor.discovery_count().await, 3);
|
||||
let replacement_manager = codex_exec_server::EnvironmentManager::create_for_tests(
|
||||
Some(executor.url.clone()),
|
||||
/*local_runtime_paths*/ None,
|
||||
@@ -1683,14 +1712,14 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.discover("executor", &replacement, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert_eq!(calls().await, 4);
|
||||
assert_eq!(executor.discovery_count().await, 4);
|
||||
let other_reused = cache
|
||||
.discover("other-executor", &environment, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&other, &other_reused));
|
||||
assert_eq!(
|
||||
calls().await,
|
||||
executor.discovery_count().await,
|
||||
4,
|
||||
"a new environment instance must not invalidate the original instance"
|
||||
);
|
||||
@@ -1715,7 +1744,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&other, &oldest));
|
||||
assert_eq!(calls().await, 17);
|
||||
assert_eq!(executor.discovery_count().await, 17);
|
||||
|
||||
cache
|
||||
.discover(
|
||||
@@ -1733,16 +1762,16 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.discover("other-executor", &environment, request.clone())
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert_eq!(calls().await, 19);
|
||||
assert_eq!(executor.discovery_count().await, 19);
|
||||
assert!(!Arc::ptr_eq(&other, &reloaded));
|
||||
assert_eq!(other.as_ref(), reloaded.as_ref());
|
||||
|
||||
// Explicit requests and dirty entries reload once; ordinary reads reuse the result.
|
||||
// Explicit requests and invalidated entries reload once; ordinary reads reuse the result.
|
||||
let mut previous = reloaded;
|
||||
for (expected_calls, force_reload) in [(20, true), (21, false)] {
|
||||
if !force_reload {
|
||||
cache.invalidate();
|
||||
assert_eq!(calls().await, expected_calls - 1);
|
||||
assert_eq!(executor.discovery_count().await, expected_calls - 1);
|
||||
}
|
||||
let refreshed = cache
|
||||
.discover(
|
||||
@@ -1761,7 +1790,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
.await
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
assert!(Arc::ptr_eq(&refreshed, &ordinary));
|
||||
assert_eq!(calls().await, expected_calls);
|
||||
assert_eq!(executor.discovery_count().await, expected_calls);
|
||||
assert_eq!(
|
||||
executor
|
||||
.requests
|
||||
@@ -1775,7 +1804,7 @@ async fn v2_discovery_cache_reuses_matching_requests_and_isolates_execution_cont
|
||||
previous = refreshed;
|
||||
}
|
||||
|
||||
// Reloading one dirty entry leaves the others dirty; new contexts load normally.
|
||||
// Reloading one invalidated entry preserves the others' reloads; new contexts load normally.
|
||||
for (cwd, force_reload) in [("overflow", true), ("new-context", false)] {
|
||||
cache
|
||||
.discover(
|
||||
|
||||
@@ -10,9 +10,7 @@ async fn executor_discovery_v2_refreshes_inventory_and_reuses_it_across_turns()
|
||||
}
|
||||
|
||||
#[test_case::test_case(DiscoveryFailure::None; "success")]
|
||||
#[test_case::test_case(DiscoveryFailure::Server; "permanent_failure")]
|
||||
#[test_case::test_case(DiscoveryFailure::Disconnect; "retryable_failure")]
|
||||
#[test_case::test_case(DiscoveryFailure::NoReply; "timeout")]
|
||||
#[test_case::test_case(DiscoveryFailure::Server; "server_failure")]
|
||||
#[tokio::test]
|
||||
async fn executor_discovery_v2_prepares_inventory_while_sampling(
|
||||
failure: DiscoveryFailure,
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
---
|
||||
source: core/tests/suite/executor_discovery_v2.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)"
|
||||
---
|
||||
Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.
|
||||
Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.
|
||||
|
||||
## Window 1
|
||||
-- request 1 (turn) --
|
||||
|
||||
+23
-2
@@ -1,8 +1,8 @@
|
||||
---
|
||||
source: core/tests/suite/executor_discovery_v2.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.\",\n&history, &ContextSnapshotOptions::default(),)"
|
||||
---
|
||||
Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.
|
||||
Scenario: Readiness waits for prewarm; unchanged turns reuse inventory or errors, and failures recover after cooldown.
|
||||
|
||||
## Window 1
|
||||
-- request 1 (turn) --
|
||||
@@ -81,3 +81,24 @@ Scenario: Readiness waits for prewarm to finish; steps reuse successes and perma
|
||||
-- request 3 (turn) --
|
||||
07:message/user:
|
||||
Inspect the same environment again.
|
||||
-- request 4 (turn) --
|
||||
08:message/developer:
|
||||
<skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...urce` locators use their provider's access mechanism. [hash=b6782e3abdba03b2]
|
||||
Read a skill package directly with `skills.read({"package":"<package>"})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647]
|
||||
### Available skills
|
||||
- inert-sentinel: Executor skill used by the acquisition test (executor package: skill://codex-home-skills/inert-sentinel)
|
||||
### How to use skills
|
||||
- Discovery: The list above is the skills available in this session (name + description + source locator...ource` entries use their provider's access mechanism. [hash=a9c099703b9c0121]
|
||||
<OMITTED 9 LINES; ~479 TOKENS; hash=c5994539994fb066>
|
||||
- If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
|
||||
- Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
|
||||
- Context hygiene:
|
||||
- Progressive disclosure applies to selecting relevant files, not partially reading a selected instruc...Do not load unrelated references, scripts, or assets. [hash=d6eeb13522175402]
|
||||
- Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
|
||||
- When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
|
||||
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
|
||||
</skills_instructions>
|
||||
09:message/user:
|
||||
Inspect the environment after discovery's cooldown.
|
||||
@@ -1,16 +1,19 @@
|
||||
//! Lazy V2 responses, cached per environment, executor session, cwd, and sandbox.
|
||||
//! Reloads replace the matching entry; in-flight readers may finish with older results.
|
||||
//! Reports recovery from retryable failures so Core can refresh dependent MCP projections.
|
||||
//! Invalidation replaces entries with empty forced loads; in-flight readers may finish with older results.
|
||||
//! Consumers detect recovery and replacement by comparing discovery snapshots.
|
||||
//! Failed loads have a per-context cooldown; lookups never sleep or schedule background retries.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::Weak;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::DiscoverV2CapabilitiesRequest;
|
||||
use crate::DiscoverV2CapabilitiesResponse;
|
||||
@@ -19,16 +22,18 @@ use crate::ExecServerError;
|
||||
use crate::FileSystemSandboxContext;
|
||||
|
||||
const MIN_CACHED_DISCOVERY_CONTEXTS: usize = 16;
|
||||
const INITIAL_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1);
|
||||
const MAX_DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Shared by turn and idle consumers until Core explicitly invalidates discovery.
|
||||
#[derive(Default)]
|
||||
pub struct ExecutorCapabilityDiscoveryV2Cache {
|
||||
entries: Mutex<Vec<CachedEnvironmentDiscovery>>,
|
||||
entries: Mutex<Vec<Arc<DiscoveryV2CacheEntry>>>,
|
||||
capacity: AtomicUsize,
|
||||
recovered_discovery: AtomicBool,
|
||||
}
|
||||
|
||||
// Reload controls how this entry is populated, not which inventory it identifies.
|
||||
#[derive(Clone)]
|
||||
struct DiscoveryCacheKey {
|
||||
environment: Weak<Environment>,
|
||||
session_id: Option<String>,
|
||||
@@ -45,12 +50,81 @@ impl PartialEq for DiscoveryCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
struct CachedEnvironmentDiscovery {
|
||||
struct DiscoveryV2CacheEntry {
|
||||
key: DiscoveryCacheKey,
|
||||
response: Arc<OnceCell<Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>>>,
|
||||
retryable: Arc<AtomicBool>,
|
||||
force_reload: bool,
|
||||
dirty: bool,
|
||||
response: OnceCell<Arc<DiscoverV2CapabilitiesResponse>>,
|
||||
failure: Mutex<Option<DiscoveryV2Failure>>,
|
||||
}
|
||||
|
||||
struct DiscoveryV2Failure {
|
||||
error: Arc<ExecServerError>,
|
||||
attempts: u32,
|
||||
next_retry_at: Instant,
|
||||
}
|
||||
|
||||
impl DiscoveryV2CacheEntry {
|
||||
fn force_reload(&self) -> Self {
|
||||
// Discard the old snapshot and cooldown, but retain the forced reload through retries.
|
||||
Self {
|
||||
key: self.key.clone(),
|
||||
force_reload: true,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_or_try_init<F>(
|
||||
&self,
|
||||
discover: impl FnOnce() -> F,
|
||||
) -> Result<Arc<DiscoverV2CapabilitiesResponse>, Arc<ExecServerError>>
|
||||
where
|
||||
F: Future<Output = Result<DiscoverV2CapabilitiesResponse, ExecServerError>>,
|
||||
{
|
||||
self.response
|
||||
.get_or_try_init(|| async {
|
||||
// Check here so waiting callers don't immediately retry a failed request.
|
||||
{
|
||||
let failure_state = self
|
||||
.failure
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(failure) = failure_state.as_ref()
|
||||
&& Instant::now() < failure.next_retry_at
|
||||
{
|
||||
return Err(Arc::clone(&failure.error));
|
||||
}
|
||||
}
|
||||
let result = discover().await;
|
||||
let mut failure_state = self
|
||||
.failure
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
match result {
|
||||
Ok(response) => {
|
||||
*failure_state = None;
|
||||
Ok(Arc::new(response))
|
||||
}
|
||||
Err(error) => {
|
||||
let attempts = failure_state
|
||||
.as_ref()
|
||||
.map_or(1, |failure| failure.attempts.saturating_add(1));
|
||||
let delay = INITIAL_DISCOVERY_RETRY_DELAY
|
||||
.saturating_mul(2_u32.saturating_pow(attempts - 1))
|
||||
.min(MAX_DISCOVERY_RETRY_DELAY);
|
||||
let error = Arc::new(error);
|
||||
*failure_state = Some(DiscoveryV2Failure {
|
||||
error: Arc::clone(&error),
|
||||
attempts,
|
||||
next_retry_at: Instant::now() + delay,
|
||||
});
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
@@ -59,19 +133,14 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
self.capacity.fetch_max(minimum, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Reports whether a failed discovery recovered since the last observation.
|
||||
pub fn take_recovered_discovery(&self) -> bool {
|
||||
self.recovered_discovery.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
/// Dirties cached entries so their next lookup reloads without interrupting existing loads.
|
||||
/// Makes future lookups reload every context without interrupting existing loads.
|
||||
pub fn invalidate(&self) {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for entry in entries.iter_mut() {
|
||||
entry.dirty = true;
|
||||
*entry = Arc::new(entry.force_reload());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,12 +155,13 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
.iter_mut()
|
||||
.filter(|entry| entry.key.environment.ptr_eq(&environment))
|
||||
{
|
||||
entry.dirty = true;
|
||||
*entry = Arc::new(entry.force_reload());
|
||||
}
|
||||
}
|
||||
|
||||
/// Reuses successes and non-retryable failures unless the request forces a reload.
|
||||
/// Retryable failures and cancelled loads leave the cell available for retry.
|
||||
/// Reuses successful inventories; failed loads retry on the first lookup after their cooldown.
|
||||
/// Cancellation adds no cooldown; explicit invalidation and new sessions start fresh loads.
|
||||
/// A connection error classification does not determine discovery's lifetime.
|
||||
/// Invalidation during a load applies to the next lookup, not its current readers.
|
||||
pub async fn discover(
|
||||
&self,
|
||||
@@ -110,7 +180,7 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
cwd: request.cwd.clone(),
|
||||
sandbox: request.sandbox.clone(),
|
||||
};
|
||||
let (response, retryable, force_reload) = {
|
||||
let entry = {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
@@ -118,77 +188,45 @@ impl ExecutorCapabilityDiscoveryV2Cache {
|
||||
let matching = entries.iter().position(|entry| entry.key == key);
|
||||
if let Some(entry) = matching
|
||||
.map(|index| &entries[index])
|
||||
.filter(|entry| !entry.dirty && !request.force_reload)
|
||||
.filter(|_| !request.force_reload)
|
||||
{
|
||||
(
|
||||
Arc::clone(&entry.response),
|
||||
Arc::clone(&entry.retryable),
|
||||
entry.force_reload,
|
||||
)
|
||||
Arc::clone(entry)
|
||||
} else {
|
||||
let force_reload = request.force_reload || matching.is_some();
|
||||
if let Some(index) = matching {
|
||||
entries.remove(index);
|
||||
}
|
||||
// Recovery belongs to the environment/request, not one executor session.
|
||||
let retryable = entries
|
||||
.iter()
|
||||
.find(|entry| {
|
||||
!entry.dirty
|
||||
&& entry.key.environment.ptr_eq(&key.environment)
|
||||
&& entry.key.cwd == key.cwd
|
||||
&& entry.key.sandbox == key.sandbox
|
||||
})
|
||||
.map(|entry| Arc::clone(&entry.retryable))
|
||||
.unwrap_or_default();
|
||||
let entry = CachedEnvironmentDiscovery {
|
||||
let entry = Arc::new(DiscoveryV2CacheEntry {
|
||||
key,
|
||||
response: Arc::new(OnceCell::new()),
|
||||
retryable,
|
||||
force_reload,
|
||||
dirty: false,
|
||||
};
|
||||
force_reload: request.force_reload,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
});
|
||||
let capacity = self
|
||||
.capacity
|
||||
.load(Ordering::Relaxed)
|
||||
.max(MIN_CACHED_DISCOVERY_CONTEXTS);
|
||||
// Existing readers retain their response cells after replacement or eviction.
|
||||
// Existing readers retain their entries after replacement or eviction.
|
||||
if entries.len() >= capacity {
|
||||
entries.remove(0);
|
||||
}
|
||||
let load = (
|
||||
Arc::clone(&entry.response),
|
||||
Arc::clone(&entry.retryable),
|
||||
force_reload,
|
||||
);
|
||||
entries.push(entry);
|
||||
load
|
||||
entries.push(Arc::clone(&entry));
|
||||
entry
|
||||
}
|
||||
};
|
||||
let response = response
|
||||
entry
|
||||
.get_or_try_init(|| async {
|
||||
request.force_reload = force_reload;
|
||||
match environment.discover_capabilities_v2(request).await {
|
||||
Ok(response) => Ok(Ok(Arc::new(response))),
|
||||
Err(error) => {
|
||||
let can_retry = crate::client::can_retry_connection_attempt(&error);
|
||||
request.force_reload = entry.force_reload;
|
||||
environment
|
||||
.discover_capabilities_v2(request)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
tracing::warn!(environment_id, "executor discovery V2 failed");
|
||||
let error = Arc::new(error);
|
||||
if can_retry {
|
||||
retryable.store(true, Ordering::Release);
|
||||
Err(error)
|
||||
} else {
|
||||
Ok(Err(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.cloned()
|
||||
.unwrap_or_else(Err);
|
||||
if response.is_ok() && retryable.swap(false, Ordering::AcqRel) {
|
||||
self.recovered_discovery.store(true, Ordering::Release);
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "capability_discovery_v2_cache_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::DiscoveryCacheKey;
|
||||
use super::DiscoveryV2CacheEntry;
|
||||
use super::ExecutorCapabilityDiscoveryV2Cache;
|
||||
use crate::DiscoverV2CapabilitiesRequest;
|
||||
use crate::DiscoverV2CapabilitiesResponse;
|
||||
use crate::Environment;
|
||||
use crate::ExecServerError;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::cell::Cell;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::Weak;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::advance;
|
||||
|
||||
fn empty_cache_entry() -> DiscoveryV2CacheEntry {
|
||||
DiscoveryV2CacheEntry {
|
||||
key: DiscoveryCacheKey {
|
||||
environment: Weak::new(),
|
||||
session_id: None,
|
||||
cwd: PathUri::parse("file:///project").expect("test cwd"),
|
||||
sandbox: None,
|
||||
},
|
||||
force_reload: false,
|
||||
response: OnceCell::new(),
|
||||
failure: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_failures_back_off_without_extending_cooldown_on_reads() {
|
||||
let entry = empty_cache_entry();
|
||||
let calls = Cell::new(0);
|
||||
let fail = || async {
|
||||
calls.set(calls.get() + 1);
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
};
|
||||
for (attempt, delay) in [1, 2, 4, 8, 16, 30, 30].into_iter().enumerate() {
|
||||
let error = entry.get_or_try_init(fail).await.expect_err("failed load");
|
||||
assert_eq!(calls.get(), attempt + 1);
|
||||
advance(Duration::from_secs(delay) - Duration::from_millis(1)).await;
|
||||
let suppressed = entry.get_or_try_init(fail).await.expect_err("cooldown");
|
||||
assert!(Arc::ptr_eq(&error, &suppressed));
|
||||
assert_eq!(calls.get(), attempt + 1);
|
||||
// Suppressed lookups must not move the deadline or advance the failure count.
|
||||
advance(Duration::from_millis(1)).await;
|
||||
}
|
||||
|
||||
let response = entry
|
||||
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
|
||||
.await
|
||||
.expect("recovery after cooldown");
|
||||
advance(Duration::from_secs(300)).await;
|
||||
let reused = entry.get_or_try_init(fail).await.expect("cached success");
|
||||
assert!(Arc::ptr_eq(&response, &reused));
|
||||
assert_eq!(calls.get(), 7);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn queued_callers_share_failure_and_the_next_successful_attempt() {
|
||||
let entry = empty_cache_entry();
|
||||
let (release, wait) = oneshot::channel();
|
||||
let first = entry.get_or_try_init(|| async {
|
||||
wait.await.expect("release failed load");
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
});
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
let calls = Cell::new(0);
|
||||
let succeed = || async {
|
||||
calls.set(calls.get() + 1);
|
||||
Ok(DiscoverV2CapabilitiesResponse::default())
|
||||
};
|
||||
let queued = entry.get_or_try_init(succeed);
|
||||
tokio::pin!(queued);
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
release.send(()).expect("release failed load");
|
||||
let (first, queued) = tokio::join!(first, queued);
|
||||
assert!(Arc::ptr_eq(
|
||||
&first.expect_err("failed loader"),
|
||||
&queued.expect_err("queued caller observes cooldown"),
|
||||
));
|
||||
assert_eq!(calls.get(), 0);
|
||||
|
||||
advance(Duration::from_secs(1)).await;
|
||||
let (first, concurrent) = tokio::join!(
|
||||
entry.get_or_try_init(succeed),
|
||||
entry.get_or_try_init(succeed),
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
&first.expect("recovered loader"),
|
||||
&concurrent.expect("shared success"),
|
||||
));
|
||||
assert_eq!(calls.get(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_does_not_add_a_failure_or_cooldown() {
|
||||
let entry = empty_cache_entry();
|
||||
// Exercise cancellation both before any failure and after an expired failure.
|
||||
for delay in [1, 2] {
|
||||
let mut cancelled = Box::pin(entry.get_or_try_init(std::future::pending));
|
||||
assert!(futures::poll!(cancelled.as_mut()).is_pending());
|
||||
drop(cancelled);
|
||||
let attempted = Cell::new(false);
|
||||
entry
|
||||
.get_or_try_init(|| async {
|
||||
attempted.set(true);
|
||||
Err(ExecServerError::Protocol("discovery failed".to_string()))
|
||||
})
|
||||
.await
|
||||
.expect_err("fresh attempt after cancellation");
|
||||
assert!(attempted.get(), "cancellation must not start a cooldown");
|
||||
advance(Duration::from_secs(delay)).await;
|
||||
}
|
||||
entry
|
||||
.get_or_try_init(|| async { Ok(DiscoverV2CapabilitiesResponse::default()) })
|
||||
.await
|
||||
.expect("cancellation did not advance the backoff");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_contexts_are_isolated_and_invalidation_bypasses_cooldown() -> anyhow::Result<()> {
|
||||
// Local environments reject V2; their errors exercise the cache without a network fixture.
|
||||
let environment = Arc::new(Environment::create_for_tests(
|
||||
/*exec_server_url*/ None,
|
||||
)?);
|
||||
let replacement = Arc::new(Environment::create_for_tests(
|
||||
/*exec_server_url*/ None,
|
||||
)?);
|
||||
let cache = ExecutorCapabilityDiscoveryV2Cache::default();
|
||||
let request = DiscoverV2CapabilitiesRequest {
|
||||
force_reload: false,
|
||||
cwd: PathUri::parse("file:///project")?,
|
||||
sandbox: None,
|
||||
};
|
||||
let error = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.expect_err("local environment rejects V2");
|
||||
let other_request = DiscoverV2CapabilitiesRequest {
|
||||
cwd: request.cwd.join("other")?,
|
||||
..request.clone()
|
||||
};
|
||||
let other_error = cache
|
||||
.discover("executor", &environment, other_request.clone())
|
||||
.await
|
||||
.expect_err("independent cwd");
|
||||
let replacement_error = cache
|
||||
.discover("executor", &replacement, request.clone())
|
||||
.await
|
||||
.expect_err("independent environment");
|
||||
assert!(!Arc::ptr_eq(&error, &other_error));
|
||||
assert!(!Arc::ptr_eq(&error, &replacement_error));
|
||||
|
||||
cache.invalidate_environment(&replacement);
|
||||
let suppressed = cache
|
||||
.discover("executor", &environment, request.clone())
|
||||
.await
|
||||
.expect_err("unrelated invalidation preserves cooldown");
|
||||
assert!(Arc::ptr_eq(&error, &suppressed));
|
||||
cache.invalidate_environment(&environment);
|
||||
let refreshed = cache
|
||||
.discover("executor", &environment, request)
|
||||
.await
|
||||
.expect_err("explicit refresh retries immediately");
|
||||
assert!(!Arc::ptr_eq(&error, &refreshed));
|
||||
cache.invalidate();
|
||||
let refreshed_other = cache
|
||||
.discover("executor", &environment, other_request)
|
||||
.await
|
||||
.expect_err("global invalidation retries immediately");
|
||||
assert!(!Arc::ptr_eq(&other_error, &refreshed_other));
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user