Retry transient environment startup failures on the next turn (#57866)

## Why

A transient environment startup failure persisted across turns when the selected environment and workspace were unchanged, preventing commands from recovering after the executor became available.

## What changed

Retry failed environment resolution at the start of a new turn when the error is retryable and the workspace remains selected. Reuse healthy or pending resolutions and preserve provisioning failures until the selection is updated.

## Testing

Add coverage for command execution recovering on an unchanged next turn after endpoint availability is restored. Extend provisioning failure tests to verify that subsequent turns receive the original error without restarting resolution.

GitOrigin-RevId: b80af60990f30d277804160c9c39c286024c7667
This commit is contained in:
pansul47-oai
2026-10-29 00:42:21 +00:00
committed by copyberry
parent f276b82185
commit 4e2dfd2230
3 changed files with 252 additions and 14 deletions
+65 -6
View File
@@ -308,6 +308,39 @@ struct ThreadEnvironmentsState {
environments: Vec<SelectedTurnEnvironment>,
}
#[derive(Clone, Copy)]
enum StartupRetry<'a> {
AfterSelectionUpdate,
OnNewTurn(&'a [TurnEnvironmentSelection]),
}
impl StartupRetry<'_> {
fn should_restart(self, environment: &SelectedTurnEnvironment) -> bool {
if matches!(
environment.selection.config,
EnvironmentConfigState::Failed(_)
) {
return matches!(self, Self::AfterSelectionUpdate);
}
// Avoid cloning a resolved environment on every healthy turn.
if environment.resolution.peek().is_some_and(Result::is_ok) {
return false;
}
match environment.resolution.clone().now_or_never() {
Some(Err(error)) => match self {
Self::AfterSelectionUpdate => true,
Self::OnNewTurn(selected) => {
error.is_retryable_preparation_error()
&& selected
.iter()
.any(|s| s.has_same_workspace(&environment.selection))
}
},
Some(Ok(_)) | None => false,
}
}
}
impl ThreadEnvironments {
pub(crate) fn new(
environment_manager: Arc<EnvironmentManager>,
@@ -449,6 +482,27 @@ impl ThreadEnvironments {
/// Updates the selected list before waking work that was waiting on the previous selection.
pub(crate) fn update_selections(&self, environments: &[TurnEnvironmentSelection]) {
self.apply_selections(environments, StartupRetry::AfterSelectionUpdate);
}
/// A new turn can retry a failed connection even when it keeps the same workspace.
pub(crate) fn activate_selections(&self, environments: &[TurnEnvironmentSelection]) {
let retry = StartupRetry::OnNewTurn(environments);
if environments != self.selections() {
self.update_selections(environments);
} else if self
.state
.lock()
.unwrap_or_else(PoisonError::into_inner)
.environments
.iter()
.any(|environment| retry.should_restart(environment))
{
self.apply_selections(environments, retry);
}
}
fn apply_selections(&self, environments: &[TurnEnvironmentSelection], retry: StartupRetry<'_>) {
let bindings = self
.configured
.lock()
@@ -460,9 +514,16 @@ impl ThreadEnvironments {
environments,
&bindings,
self.connection_event_tx.get(),
retry,
);
// Retry configured bindings with their own settings, keeping turn-only changes temporary.
self.update_configured_selections(&self.configured_snapshot_now().all_selections());
self.update_selections_with_bindings(
&self.configured,
&self.configured_snapshot_now().all_selections(),
&[],
/*tx_event*/ None,
retry,
);
}
/// Retains configured bindings without changing the active turn's attachments.
@@ -472,6 +533,7 @@ impl ThreadEnvironments {
environments,
&[],
/*tx_event*/ None,
StartupRetry::AfterSelectionUpdate,
);
}
@@ -481,6 +543,7 @@ impl ThreadEnvironments {
environments: &[TurnEnvironmentSelection],
bindings: &[SelectedTurnEnvironment],
tx_event: Option<&Sender<Event>>,
retry: StartupRetry<'_>,
) {
let mut state = state.lock().unwrap_or_else(PoisonError::into_inner);
let ThreadEnvironmentsState {
@@ -526,11 +589,7 @@ impl ThreadEnvironments {
let (selected_environment, config_origin) =
resolve_selection_config(selected_environment.clone(), thread_defaults);
if let Some(environment) = previous {
let failed =
matches!(
environment.selection.config,
EnvironmentConfigState::Failed(_)
) || matches!(environment.resolution.clone().now_or_never(), Some(Err(_)));
let failed = retry.should_restart(environment);
let mut environment = if current_environment.is_some() {
environment.clone()
} else {
+3 -5
View File
@@ -214,11 +214,9 @@ impl Session {
self.services
.turn_environments
.set_active_thread_defaults(configuration.inferred_environment_config());
if environments != self.services.turn_environments.selections() {
self.services
.turn_environments
.update_selections(&environments);
}
self.services
.turn_environments
.activate_selections(&environments);
}
self.services.turn_environments.snapshot()
};