Prewarm executor capability discovery during remote startup (#62674)

## Why

Step-time discovery can make the first step that needs a remote executor
inventory wait for a discovery round trip. Starting discovery during
environment setup lets deferred turns begin sampling while the inventory
is being fetched.

## What changed

When executor capability discovery V2 is enabled, prewarm the thread's
discovery cache for supported remote executors before marking their
environments ready. Reserve cache capacity for all selected environments
so the first step can reuse their inventories.

Give each prewarm up to five seconds and let environment startup proceed
after discovery errors or a timeout. Step-time discovery reuses cached
successes and permanent errors and can retry transient failures and
timed-out loads.

## Testing

Add integration coverage for deferred readiness and cache reuse after
success, permanent errors, transient failures, and timeouts. Also cover
cache retention across multiple environments and session startup when
discovery receives no reply.

GitOrigin-RevId: d07fc78bd42953a4caf9e4b39528db8a254eb2b8
This commit is contained in:
Anthony Tafoya
2026-11-20 05:17:34 +00:00
committed by copyberry
parent a0a30c40e8
commit ffa05c8bfb
12 changed files with 739 additions and 56 deletions
+97 -39
View File
@@ -40,6 +40,7 @@ use tokio_util::task::AbortOnDropHandle;
use tracing::Instrument;
use tracing::instrument::WithSubscriber;
use crate::session::ExecutorDiscovery;
use crate::session::ThreadEnvironmentDefaults;
use crate::session::turn_context::ShellSnapshotCache;
use crate::session::turn_context::ShellSnapshotTask;
@@ -200,6 +201,32 @@ struct ResolvedEnvironment {
installed_config: Option<EnvironmentConfig>,
}
impl ResolvedEnvironment {
fn into_turn_environment(
self,
mut selection: TurnEnvironmentSelection,
config_origin: EnvironmentConfigOrigin,
attachment: Arc<EnvironmentAttachment>,
) -> Result<TurnEnvironment, String> {
if matches!(selection.config, EnvironmentConfigState::Pending) {
let config = self
.installed_config
.ok_or_else(|| "Environment configuration was not supplied.".to_string())?;
selection.config = EnvironmentConfigState::Ready(config);
}
let mut environment =
TurnEnvironment::from_attachment(selection, config_origin, attachment, self.shell);
environment.executor_platform_os = self.executor_platform_os;
environment.shell_snapshot = self.shell_snapshot;
environment.shell_snapshot_builder = Some(Box::new(self.shell_snapshot_builder));
environment.shell_snapshot_cache = self.shell_snapshot_cache;
environment.shell_snapshot_v2_supported = self.shell_snapshot_v2_supported;
environment.user_home_dir = self.user_home_dir;
environment.temporary_directories = self.temporary_directories;
Ok(environment)
}
}
#[derive(Clone)]
struct SelectedTurnEnvironment {
selection: TurnEnvironmentSelection,
@@ -249,6 +276,7 @@ impl SelectedTurnEnvironment {
detach_on_drop: Arc<DropGuard>,
local_shell: &Shell,
shell_snapshot: &ShellSnapshot,
executor_discovery: Option<&Arc<ExecutorDiscovery>>,
) -> Self {
if matches!(self.selection.config, EnvironmentConfigState::Pending) {
return ThreadEnvironments::start_environment(
@@ -260,6 +288,7 @@ impl SelectedTurnEnvironment {
/*pending_completion*/ None,
local_shell,
shell_snapshot,
executor_discovery,
);
}
Self {
@@ -311,6 +340,7 @@ pub(crate) struct ThreadEnvironments {
local_shell: Shell,
shell_snapshot: ShellSnapshot,
non_blocking_snapshots: bool,
executor_discovery: Option<Arc<ExecutorDiscovery>>,
state: Mutex<ThreadEnvironmentsState>,
// Retain configured bindings independently of active-turn membership and cancellation.
configured: Mutex<ThreadEnvironmentsState>,
@@ -398,7 +428,11 @@ impl ThreadEnvironments {
shell_snapshot: ShellSnapshot,
current: TurnEnvironmentSnapshot,
non_blocking_snapshots: bool,
executor_discovery: Option<Arc<ExecutorDiscovery>>,
) -> Self {
if let Some(discovery) = &executor_discovery {
discovery.ensure_capacity(current.environments.len());
}
let environments: Vec<_> = current
.environments
.into_iter()
@@ -434,6 +468,7 @@ impl ThreadEnvironments {
/*pending_completion*/ None,
&local_shell,
&shell_snapshot,
executor_discovery.as_ref(),
));
}
TurnEnvironmentState::Failed { .. } => return None,
@@ -487,6 +522,7 @@ impl ThreadEnvironments {
local_shell,
shell_snapshot,
non_blocking_snapshots,
executor_discovery,
state: Mutex::new(ThreadEnvironmentsState {
thread_defaults: thread_defaults.clone(),
environments: Vec::new(),
@@ -523,6 +559,7 @@ impl ThreadEnvironments {
Arc::new(detach_on_drop),
&self.local_shell,
&self.shell_snapshot,
self.executor_discovery.as_ref(),
)
})
.collect();
@@ -627,6 +664,9 @@ impl ThreadEnvironments {
bindings: &[SelectedTurnEnvironment],
retry: StartupRetry<'_>,
) {
if let Some(discovery) = &self.executor_discovery {
discovery.ensure_capacity(environments.len());
}
let ThreadEnvironmentsState {
thread_defaults,
environments: current,
@@ -702,6 +742,7 @@ impl ThreadEnvironments {
detach_on_drop,
&self.local_shell,
&self.shell_snapshot,
self.executor_discovery.as_ref(),
)
};
if let Some(completion) = environment.pending_completion.take() {
@@ -758,6 +799,7 @@ impl ThreadEnvironments {
current_environment.and_then(|previous| previous.pending_completion.clone()),
&self.local_shell,
&self.shell_snapshot,
self.executor_discovery.as_ref(),
);
next.push(selected);
}
@@ -786,6 +828,7 @@ impl ThreadEnvironments {
pending_completion: Option<watch::Sender<Option<PendingConfigurationResult>>>,
local_shell: &Shell,
shell_snapshot: &ShellSnapshot,
executor_discovery: Option<&Arc<ExecutorDiscovery>>,
) -> SelectedTurnEnvironment {
let (pending_completion, configuration_ready) =
if matches!(selection.config, EnvironmentConfigState::Pending) {
@@ -796,24 +839,44 @@ impl ThreadEnvironments {
} else {
(None, None)
};
// The resolver waits for this Session; descendants can follow the original owner.
let (task, resolution) = Self::resolve_environment(
selection.clone(),
Arc::clone(&attachment.environment),
local_shell.clone(),
shell_snapshot.clone(),
configuration_ready,
)
.remote_handle();
let executor_discovery = executor_discovery.filter(|_| attachment.environment.is_remote());
// Deferred turns can sample while this attachment is preparing its inventory.
let (task, resolution) = {
let selection = selection.clone();
let attachment = Arc::clone(&attachment);
let local_shell = local_shell.clone();
let shell_snapshot = shell_snapshot.clone();
let executor_discovery = executor_discovery.cloned();
async move {
let resolved = Self::resolve_environment(
selection.clone(),
Arc::clone(&attachment.environment),
local_shell,
shell_snapshot,
configuration_ready,
)
.await?;
if let Some(discovery) = executor_discovery {
let selected = resolved
.clone()
.into_turn_environment(selection, config_origin, Arc::clone(&attachment))
.map_err(|error| Arc::new(ExecServerError::Protocol(error)))?;
discovery.prewarm(&selected).await;
}
Ok(resolved)
}
.remote_handle()
};
drop(tokio::spawn(
task.in_current_span().with_current_subscriber(),
));
let resolution = resolution.boxed().shared();
SelectedTurnEnvironment {
selection,
config_origin,
attachment,
detach_on_drop,
resolution: resolution.boxed().shared(),
resolution,
pending_completion,
owner_config_result,
}
@@ -1204,35 +1267,20 @@ impl TurnEnvironmentState {
};
}
match resolved {
Some(Ok(environment)) => {
let mut selection = starting.selection;
if matches!(selection.config, EnvironmentConfigState::Pending) {
let Some(config) = environment.installed_config else {
return Self::Failed {
required_skills,
selection: starting.config_origin.into_input_selection(selection),
error: "Environment configuration was not supplied.".to_string(),
};
};
selection.config = EnvironmentConfigState::Ready(config);
}
let mut turn_environment = TurnEnvironment::from_attachment(
selection,
starting.config_origin,
starting.attachment,
environment.shell,
);
turn_environment.executor_platform_os = environment.executor_platform_os;
turn_environment.shell_snapshot = environment.shell_snapshot;
turn_environment.shell_snapshot_builder =
Some(Box::new(environment.shell_snapshot_builder));
turn_environment.shell_snapshot_cache = environment.shell_snapshot_cache;
turn_environment.shell_snapshot_v2_supported =
environment.shell_snapshot_v2_supported;
turn_environment.user_home_dir = environment.user_home_dir;
turn_environment.temporary_directories = environment.temporary_directories;
Self::Ready(turn_environment)
}
Some(Ok(environment)) => match environment.into_turn_environment(
starting.selection.clone(),
starting.config_origin,
starting.attachment,
) {
Ok(environment) => Self::Ready(environment),
Err(error) => Self::Failed {
required_skills,
selection: starting
.config_origin
.into_input_selection(starting.selection),
error,
},
},
Some(Err(err)) => {
tracing::debug!(
environment_id = %starting.selection.environment_id,
@@ -1581,6 +1629,7 @@ mod tests {
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
));
turn_environments.update_selections(selections);
turn_environments.snapshot().await;
@@ -1764,6 +1813,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
turn_environments.update_selections(&[TurnEnvironmentSelection {
selected_capability_roots: Default::default(),
@@ -1943,6 +1993,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
));
environments.update_selections(std::slice::from_ref(&selection));
// Exercise first-use callsite registration from a thread without a subscriber.
@@ -2058,6 +2109,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
);
turn_environments.update_selections(std::slice::from_ref(&local));
let resolution = turn_environments.state.lock().unwrap().environments[0]
@@ -2148,6 +2200,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
attached.clone(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
);
child.activate_configured(async_channel::unbounded().0);
assert_eq!(
@@ -2240,6 +2293,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
);
environments.update_selections(std::slice::from_ref(&selection));
let starting = environments.snapshot().await;
@@ -2345,6 +2399,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
));
environments.activate_configured(tx_event);
environments.update_selections(std::slice::from_ref(&selection));
@@ -2479,6 +2534,7 @@ url = "ws://127.0.0.1:8765"
environments: vec![TurnEnvironmentState::Ready(inherited)],
},
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
environments.update_selections(std::slice::from_ref(&selection));
@@ -2529,6 +2585,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
parent.update_selections(std::slice::from_ref(&selection));
parent.snapshot().await;
@@ -2574,6 +2631,7 @@ url = "ws://127.0.0.1:8765"
ShellSnapshot::disabled(),
parent.snapshot().await,
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
child.activate_configured(async_channel::unbounded().0);
let child_snapshot = child.snapshot().await;
@@ -57,6 +57,7 @@ async fn capability_roots_follow_attachments_through_configuration_changes() {
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
parent.update_selections(&[local.clone(), other.clone()]);
let snapshot = parent.snapshot().await;
@@ -78,6 +79,7 @@ async fn capability_roots_follow_attachments_through_configuration_changes() {
ShellSnapshot::disabled(),
snapshot,
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
child.activate_configured(async_channel::unbounded().0);
assert_eq!(
@@ -146,6 +148,7 @@ async fn unpolled_snapshot_does_not_delay_canceling_a_removed_environment(shared
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
if shared_startup {
environments.update_configured_selections(std::slice::from_ref(&selection));
@@ -222,6 +225,7 @@ async fn updating_another_environment_retries_the_executor_without_canceling_pen
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
);
let current = || {
let state = environments.state.lock().unwrap();
@@ -294,6 +298,7 @@ async fn credential_refresh_does_not_restore_a_removed_environment() {
shell_snapshot,
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
));
environments.update_selections(&[TurnEnvironmentSelection {
selected_capability_roots: Default::default(),
+1
View File
@@ -416,6 +416,7 @@ mod tests {
crate::shell_snapshot::ShellSnapshot::disabled(),
Default::default(),
/*non_blocking_snapshots*/ true,
/*executor_discovery*/ None,
);
environments.update_selections(std::slice::from_ref(&selection));
turn_context.initial_environments = environments.snapshot().await;
@@ -1,8 +1,10 @@
//! Core owns a thread discovery cache shared across turns. Explicit refresh dirties every entry.
//! In-flight readers may keep their inventories; invalidation applies to the next lookup.
//! Prewarming bounds discovery before readiness without publishing MCP state or failing startup.
use super::Session;
use super::TurnContext;
use super::turn_context::TurnEnvironment;
use crate::McpRefreshReason;
use crate::config::Config;
use crate::environment_selection::TurnEnvironmentSnapshot;
@@ -13,6 +15,9 @@ use codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot;
use codex_exec_server::supports_capability_discovery_v2;
use codex_features::Feature;
use std::sync::Arc;
use std::time::Duration;
const DISCOVERY_PREWARM_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Default)]
pub(crate) struct ExecutorDiscovery {
@@ -20,10 +25,52 @@ pub(crate) struct ExecutorDiscovery {
}
impl ExecutorDiscovery {
pub(crate) fn ensure_capacity(&self, minimum: usize) {
self.cache.ensure_capacity(minimum);
}
#[expect(dead_code, reason = "Reserved for future environment-scoped refresh")]
pub(crate) fn invalidate_environment(&self, environment: &Arc<Environment>) {
self.cache.invalidate_environment(environment);
}
pub(crate) async fn prewarm(&self, selected: &TurnEnvironment) {
let prewarm = async {
let Ok(info) = selected.environment().info().await else {
tracing::warn!(
environment_id = selected.selection.environment_id,
"could not read executor discovery V2 support during prewarm"
);
return;
};
if !supports_capability_discovery_v2(&info) {
return;
}
// Discovery logs failures and retains them according to its existing retry policy.
let _ = self
.cache
.discover(
&selected.selection.environment_id,
selected.environment(),
DiscoverV2CapabilitiesRequest {
cwd: selected.cwd().clone(),
sandbox: Some(selected.sandbox_context(/*additional_permissions*/ None)),
},
)
.await;
};
if tokio::time::timeout(DISCOVERY_PREWARM_TIMEOUT, prewarm)
.await
.is_err()
{
// Dropping the load releases the cache's OnceCell for a later step to retry.
tracing::warn!(
environment_id = selected.selection.environment_id,
timeout_ms = DISCOVERY_PREWARM_TIMEOUT.as_millis(),
"executor discovery V2 prewarm timed out; continuing environment startup"
);
}
}
}
impl Session {
+1
View File
@@ -261,6 +261,7 @@ pub(crate) mod context_window;
mod daemon_recovery;
mod environment;
mod executor_discovery;
pub(crate) use executor_discovery::ExecutorDiscovery;
mod extension_interruption;
pub(crate) mod extension_metrics;
mod guardian_checkpoint;
+3
View File
@@ -1548,6 +1548,9 @@ impl Session {
shell_snapshot,
inherited_environments.clone(),
config.features.enabled(Feature::DeferredExecutor),
config.features.enabled(Feature::ExecutorCapabilityDiscoveryV2).then(|| {
thread_extension_data.get_or_init(super::ExecutorDiscovery::default)
}),
));
turn_environments.update_configured_selections(environment_selections);
let configured_environments = turn_environments.configured_snapshot_now();
+3
View File
@@ -5311,6 +5311,7 @@ async fn resolved_environments_for_configuration(
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
turn_environments.update_selections(environment_selections);
(environment_manager, turn_environments.snapshot().await)
@@ -6771,6 +6772,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
ShellSnapshot::disabled(),
resolved_environments,
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
));
turn_environments.activate_configured(tx_event.clone());
let environment = Arc::clone(
@@ -9131,6 +9133,7 @@ where
ShellSnapshot::disabled(),
resolved_turn_environments.clone(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
));
turn_environments.activate_configured(tx_event.clone());
let environment = Arc::clone(
@@ -399,6 +399,7 @@ async fn inactive_profiles_keep_snapshots_but_active_brokers_require_sandbox() -
snapshot_builder.clone(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
environments.update_selections(&[TurnEnvironmentSelection {
selected_capability_roots: Default::default(),
@@ -450,6 +451,7 @@ async fn inactive_profiles_keep_snapshots_but_active_brokers_require_sandbox() -
snapshot_builder.clone(),
updated.clone(),
/*non_blocking_snapshots*/ false,
/*executor_discovery*/ None,
);
child.activate_configured(async_channel::unbounded().0);
let child = child.snapshot().await;