From ffa05c8bfb7572f1b4139e775324bfd284f1cbe5 Mon Sep 17 00:00:00 2001 From: Anthony Tafoya Date: Fri, 20 Nov 2026 02:38:51 +0000 Subject: [PATCH] Prewarm executor capability discovery during remote startup (#62674) ## Why Step-time discovery can make the first step that needs a remote executor inventory wait for a discovery round trip. Starting discovery during environment setup lets deferred turns begin sampling while the inventory is being fetched. ## What changed When executor capability discovery V2 is enabled, prewarm the thread's discovery cache for supported remote executors before marking their environments ready. Reserve cache capacity for all selected environments so the first step can reuse their inventories. Give each prewarm up to five seconds and let environment startup proceed after discovery errors or a timeout. Step-time discovery reuses cached successes and permanent errors and can retry transient failures and timed-out loads. ## Testing Add integration coverage for deferred readiness and cache reuse after success, permanent errors, transient failures, and timeouts. Also cover cache retention across multiple environments and session startup when discovery receives no reply. GitOrigin-RevId: d07fc78bd42953a4caf9e4b39528db8a254eb2b8 --- codex-rs/core/src/environment_selection.rs | 136 ++++-- .../environment_selection_snapshot_tests.rs | 5 + codex-rs/core/src/mcp_openai_file.rs | 1 + .../core/src/session/executor_discovery.rs | 47 ++ codex-rs/core/src/session/mod.rs | 1 + codex-rs/core/src/session/session.rs | 3 + codex-rs/core/src/session/tests.rs | 3 + codex-rs/core/src/shell_snapshot_tests.rs | 2 + .../core/tests/suite/executor_discovery_v2.rs | 401 +++++++++++++++++- codex-rs/core/tests/suite/scenarios.rs | 12 + ..._discovery_v2__v2_discovery_readiness.snap | 101 +++++ ...discovery_readiness_permanent_failure.snap | 83 ++++ 12 files changed, 739 insertions(+), 56 deletions(-) create mode 100644 codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap create mode 100644 codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap diff --git a/codex-rs/core/src/environment_selection.rs b/codex-rs/core/src/environment_selection.rs index b76bdf6b52..7c1b7a6442 100644 --- a/codex-rs/core/src/environment_selection.rs +++ b/codex-rs/core/src/environment_selection.rs @@ -40,6 +40,7 @@ use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; use tracing::instrument::WithSubscriber; +use crate::session::ExecutorDiscovery; use crate::session::ThreadEnvironmentDefaults; use crate::session::turn_context::ShellSnapshotCache; use crate::session::turn_context::ShellSnapshotTask; @@ -200,6 +201,32 @@ struct ResolvedEnvironment { installed_config: Option, } +impl ResolvedEnvironment { + fn into_turn_environment( + self, + mut selection: TurnEnvironmentSelection, + config_origin: EnvironmentConfigOrigin, + attachment: Arc, + ) -> Result { + if matches!(selection.config, EnvironmentConfigState::Pending) { + let config = self + .installed_config + .ok_or_else(|| "Environment configuration was not supplied.".to_string())?; + selection.config = EnvironmentConfigState::Ready(config); + } + let mut environment = + TurnEnvironment::from_attachment(selection, config_origin, attachment, self.shell); + environment.executor_platform_os = self.executor_platform_os; + environment.shell_snapshot = self.shell_snapshot; + environment.shell_snapshot_builder = Some(Box::new(self.shell_snapshot_builder)); + environment.shell_snapshot_cache = self.shell_snapshot_cache; + environment.shell_snapshot_v2_supported = self.shell_snapshot_v2_supported; + environment.user_home_dir = self.user_home_dir; + environment.temporary_directories = self.temporary_directories; + Ok(environment) + } +} + #[derive(Clone)] struct SelectedTurnEnvironment { selection: TurnEnvironmentSelection, @@ -249,6 +276,7 @@ impl SelectedTurnEnvironment { detach_on_drop: Arc, local_shell: &Shell, shell_snapshot: &ShellSnapshot, + executor_discovery: Option<&Arc>, ) -> Self { if matches!(self.selection.config, EnvironmentConfigState::Pending) { return ThreadEnvironments::start_environment( @@ -260,6 +288,7 @@ impl SelectedTurnEnvironment { /*pending_completion*/ None, local_shell, shell_snapshot, + executor_discovery, ); } Self { @@ -311,6 +340,7 @@ pub(crate) struct ThreadEnvironments { local_shell: Shell, shell_snapshot: ShellSnapshot, non_blocking_snapshots: bool, + executor_discovery: Option>, state: Mutex, // Retain configured bindings independently of active-turn membership and cancellation. configured: Mutex, @@ -398,7 +428,11 @@ impl ThreadEnvironments { shell_snapshot: ShellSnapshot, current: TurnEnvironmentSnapshot, non_blocking_snapshots: bool, + executor_discovery: Option>, ) -> Self { + if let Some(discovery) = &executor_discovery { + discovery.ensure_capacity(current.environments.len()); + } let environments: Vec<_> = current .environments .into_iter() @@ -434,6 +468,7 @@ impl ThreadEnvironments { /*pending_completion*/ None, &local_shell, &shell_snapshot, + executor_discovery.as_ref(), )); } TurnEnvironmentState::Failed { .. } => return None, @@ -487,6 +522,7 @@ impl ThreadEnvironments { local_shell, shell_snapshot, non_blocking_snapshots, + executor_discovery, state: Mutex::new(ThreadEnvironmentsState { thread_defaults: thread_defaults.clone(), environments: Vec::new(), @@ -523,6 +559,7 @@ impl ThreadEnvironments { Arc::new(detach_on_drop), &self.local_shell, &self.shell_snapshot, + self.executor_discovery.as_ref(), ) }) .collect(); @@ -627,6 +664,9 @@ impl ThreadEnvironments { bindings: &[SelectedTurnEnvironment], retry: StartupRetry<'_>, ) { + if let Some(discovery) = &self.executor_discovery { + discovery.ensure_capacity(environments.len()); + } let ThreadEnvironmentsState { thread_defaults, environments: current, @@ -702,6 +742,7 @@ impl ThreadEnvironments { detach_on_drop, &self.local_shell, &self.shell_snapshot, + self.executor_discovery.as_ref(), ) }; if let Some(completion) = environment.pending_completion.take() { @@ -758,6 +799,7 @@ impl ThreadEnvironments { current_environment.and_then(|previous| previous.pending_completion.clone()), &self.local_shell, &self.shell_snapshot, + self.executor_discovery.as_ref(), ); next.push(selected); } @@ -786,6 +828,7 @@ impl ThreadEnvironments { pending_completion: Option>>, local_shell: &Shell, shell_snapshot: &ShellSnapshot, + executor_discovery: Option<&Arc>, ) -> SelectedTurnEnvironment { let (pending_completion, configuration_ready) = if matches!(selection.config, EnvironmentConfigState::Pending) { @@ -796,24 +839,44 @@ impl ThreadEnvironments { } else { (None, None) }; - // The resolver waits for this Session; descendants can follow the original owner. - let (task, resolution) = Self::resolve_environment( - selection.clone(), - Arc::clone(&attachment.environment), - local_shell.clone(), - shell_snapshot.clone(), - configuration_ready, - ) - .remote_handle(); + let executor_discovery = executor_discovery.filter(|_| attachment.environment.is_remote()); + // Deferred turns can sample while this attachment is preparing its inventory. + let (task, resolution) = { + let selection = selection.clone(); + let attachment = Arc::clone(&attachment); + let local_shell = local_shell.clone(); + let shell_snapshot = shell_snapshot.clone(); + let executor_discovery = executor_discovery.cloned(); + async move { + let resolved = Self::resolve_environment( + selection.clone(), + Arc::clone(&attachment.environment), + local_shell, + shell_snapshot, + configuration_ready, + ) + .await?; + if let Some(discovery) = executor_discovery { + let selected = resolved + .clone() + .into_turn_environment(selection, config_origin, Arc::clone(&attachment)) + .map_err(|error| Arc::new(ExecServerError::Protocol(error)))?; + discovery.prewarm(&selected).await; + } + Ok(resolved) + } + .remote_handle() + }; drop(tokio::spawn( task.in_current_span().with_current_subscriber(), )); + let resolution = resolution.boxed().shared(); SelectedTurnEnvironment { selection, config_origin, attachment, detach_on_drop, - resolution: resolution.boxed().shared(), + resolution, pending_completion, owner_config_result, } @@ -1204,35 +1267,20 @@ impl TurnEnvironmentState { }; } match resolved { - Some(Ok(environment)) => { - let mut selection = starting.selection; - if matches!(selection.config, EnvironmentConfigState::Pending) { - let Some(config) = environment.installed_config else { - return Self::Failed { - required_skills, - selection: starting.config_origin.into_input_selection(selection), - error: "Environment configuration was not supplied.".to_string(), - }; - }; - selection.config = EnvironmentConfigState::Ready(config); - } - let mut turn_environment = TurnEnvironment::from_attachment( - selection, - starting.config_origin, - starting.attachment, - environment.shell, - ); - turn_environment.executor_platform_os = environment.executor_platform_os; - turn_environment.shell_snapshot = environment.shell_snapshot; - turn_environment.shell_snapshot_builder = - Some(Box::new(environment.shell_snapshot_builder)); - turn_environment.shell_snapshot_cache = environment.shell_snapshot_cache; - turn_environment.shell_snapshot_v2_supported = - environment.shell_snapshot_v2_supported; - turn_environment.user_home_dir = environment.user_home_dir; - turn_environment.temporary_directories = environment.temporary_directories; - Self::Ready(turn_environment) - } + Some(Ok(environment)) => match environment.into_turn_environment( + starting.selection.clone(), + starting.config_origin, + starting.attachment, + ) { + Ok(environment) => Self::Ready(environment), + Err(error) => Self::Failed { + required_skills, + selection: starting + .config_origin + .into_input_selection(starting.selection), + error, + }, + }, Some(Err(err)) => { tracing::debug!( environment_id = %starting.selection.environment_id, @@ -1581,6 +1629,7 @@ mod tests { ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, )); turn_environments.update_selections(selections); turn_environments.snapshot().await; @@ -1764,6 +1813,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); turn_environments.update_selections(&[TurnEnvironmentSelection { selected_capability_roots: Default::default(), @@ -1943,6 +1993,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, )); environments.update_selections(std::slice::from_ref(&selection)); // Exercise first-use callsite registration from a thread without a subscriber. @@ -2058,6 +2109,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, ); turn_environments.update_selections(std::slice::from_ref(&local)); let resolution = turn_environments.state.lock().unwrap().environments[0] @@ -2148,6 +2200,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), attached.clone(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, ); child.activate_configured(async_channel::unbounded().0); assert_eq!( @@ -2240,6 +2293,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, ); environments.update_selections(std::slice::from_ref(&selection)); let starting = environments.snapshot().await; @@ -2345,6 +2399,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, )); environments.activate_configured(tx_event); environments.update_selections(std::slice::from_ref(&selection)); @@ -2479,6 +2534,7 @@ url = "ws://127.0.0.1:8765" environments: vec![TurnEnvironmentState::Ready(inherited)], }, /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); environments.update_selections(std::slice::from_ref(&selection)); @@ -2529,6 +2585,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); parent.update_selections(std::slice::from_ref(&selection)); parent.snapshot().await; @@ -2574,6 +2631,7 @@ url = "ws://127.0.0.1:8765" ShellSnapshot::disabled(), parent.snapshot().await, /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); child.activate_configured(async_channel::unbounded().0); let child_snapshot = child.snapshot().await; diff --git a/codex-rs/core/src/environment_selection_snapshot_tests.rs b/codex-rs/core/src/environment_selection_snapshot_tests.rs index 848e857986..2359e2de31 100644 --- a/codex-rs/core/src/environment_selection_snapshot_tests.rs +++ b/codex-rs/core/src/environment_selection_snapshot_tests.rs @@ -57,6 +57,7 @@ async fn capability_roots_follow_attachments_through_configuration_changes() { ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); parent.update_selections(&[local.clone(), other.clone()]); let snapshot = parent.snapshot().await; @@ -78,6 +79,7 @@ async fn capability_roots_follow_attachments_through_configuration_changes() { ShellSnapshot::disabled(), snapshot, /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); child.activate_configured(async_channel::unbounded().0); assert_eq!( @@ -146,6 +148,7 @@ async fn unpolled_snapshot_does_not_delay_canceling_a_removed_environment(shared ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); if shared_startup { environments.update_configured_selections(std::slice::from_ref(&selection)); @@ -222,6 +225,7 @@ async fn updating_another_environment_retries_the_executor_without_canceling_pen ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, ); let current = || { let state = environments.state.lock().unwrap(); @@ -294,6 +298,7 @@ async fn credential_refresh_does_not_restore_a_removed_environment() { shell_snapshot, TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, )); environments.update_selections(&[TurnEnvironmentSelection { selected_capability_roots: Default::default(), diff --git a/codex-rs/core/src/mcp_openai_file.rs b/codex-rs/core/src/mcp_openai_file.rs index 1a7608d7ab..a53065399d 100644 --- a/codex-rs/core/src/mcp_openai_file.rs +++ b/codex-rs/core/src/mcp_openai_file.rs @@ -416,6 +416,7 @@ mod tests { crate::shell_snapshot::ShellSnapshot::disabled(), Default::default(), /*non_blocking_snapshots*/ true, + /*executor_discovery*/ None, ); environments.update_selections(std::slice::from_ref(&selection)); turn_context.initial_environments = environments.snapshot().await; diff --git a/codex-rs/core/src/session/executor_discovery.rs b/codex-rs/core/src/session/executor_discovery.rs index e2d578534d..12f16e4c26 100644 --- a/codex-rs/core/src/session/executor_discovery.rs +++ b/codex-rs/core/src/session/executor_discovery.rs @@ -1,8 +1,10 @@ //! Core owns a thread discovery cache shared across turns. Explicit refresh dirties every entry. //! In-flight readers may keep their inventories; invalidation applies to the next lookup. +//! Prewarming bounds discovery before readiness without publishing MCP state or failing startup. use super::Session; use super::TurnContext; +use super::turn_context::TurnEnvironment; use crate::McpRefreshReason; use crate::config::Config; use crate::environment_selection::TurnEnvironmentSnapshot; @@ -13,6 +15,9 @@ use codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot; use codex_exec_server::supports_capability_discovery_v2; use codex_features::Feature; use std::sync::Arc; +use std::time::Duration; + +const DISCOVERY_PREWARM_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Default)] pub(crate) struct ExecutorDiscovery { @@ -20,10 +25,52 @@ pub(crate) struct ExecutorDiscovery { } impl ExecutorDiscovery { + pub(crate) fn ensure_capacity(&self, minimum: usize) { + self.cache.ensure_capacity(minimum); + } + #[expect(dead_code, reason = "Reserved for future environment-scoped refresh")] pub(crate) fn invalidate_environment(&self, environment: &Arc) { self.cache.invalidate_environment(environment); } + + pub(crate) async fn prewarm(&self, selected: &TurnEnvironment) { + let prewarm = async { + let Ok(info) = selected.environment().info().await else { + tracing::warn!( + environment_id = selected.selection.environment_id, + "could not read executor discovery V2 support during prewarm" + ); + return; + }; + if !supports_capability_discovery_v2(&info) { + return; + } + // Discovery logs failures and retains them according to its existing retry policy. + let _ = self + .cache + .discover( + &selected.selection.environment_id, + selected.environment(), + DiscoverV2CapabilitiesRequest { + cwd: selected.cwd().clone(), + sandbox: Some(selected.sandbox_context(/*additional_permissions*/ None)), + }, + ) + .await; + }; + if tokio::time::timeout(DISCOVERY_PREWARM_TIMEOUT, prewarm) + .await + .is_err() + { + // Dropping the load releases the cache's OnceCell for a later step to retry. + tracing::warn!( + environment_id = selected.selection.environment_id, + timeout_ms = DISCOVERY_PREWARM_TIMEOUT.as_millis(), + "executor discovery V2 prewarm timed out; continuing environment startup" + ); + } + } } impl Session { diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 42763474fc..7d4e9f4b4b 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -261,6 +261,7 @@ pub(crate) mod context_window; mod daemon_recovery; mod environment; mod executor_discovery; +pub(crate) use executor_discovery::ExecutorDiscovery; mod extension_interruption; pub(crate) mod extension_metrics; mod guardian_checkpoint; diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 643cf7557a..29c497207d 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1548,6 +1548,9 @@ impl Session { shell_snapshot, inherited_environments.clone(), config.features.enabled(Feature::DeferredExecutor), + config.features.enabled(Feature::ExecutorCapabilityDiscoveryV2).then(|| { + thread_extension_data.get_or_init(super::ExecutorDiscovery::default) + }), )); turn_environments.update_configured_selections(environment_selections); let configured_environments = turn_environments.configured_snapshot_now(); diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index a09a604e53..709f12b495 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -5311,6 +5311,7 @@ async fn resolved_environments_for_configuration( ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); turn_environments.update_selections(environment_selections); (environment_manager, turn_environments.snapshot().await) @@ -6771,6 +6772,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { ShellSnapshot::disabled(), resolved_environments, /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, )); turn_environments.activate_configured(tx_event.clone()); let environment = Arc::clone( @@ -9131,6 +9133,7 @@ where ShellSnapshot::disabled(), resolved_turn_environments.clone(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, )); turn_environments.activate_configured(tx_event.clone()); let environment = Arc::clone( diff --git a/codex-rs/core/src/shell_snapshot_tests.rs b/codex-rs/core/src/shell_snapshot_tests.rs index 24d305eabe..7477107e3a 100644 --- a/codex-rs/core/src/shell_snapshot_tests.rs +++ b/codex-rs/core/src/shell_snapshot_tests.rs @@ -399,6 +399,7 @@ async fn inactive_profiles_keep_snapshots_but_active_brokers_require_sandbox() - snapshot_builder.clone(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); environments.update_selections(&[TurnEnvironmentSelection { selected_capability_roots: Default::default(), @@ -450,6 +451,7 @@ async fn inactive_profiles_keep_snapshots_but_active_brokers_require_sandbox() - snapshot_builder.clone(), updated.clone(), /*non_blocking_snapshots*/ false, + /*executor_discovery*/ None, ); child.activate_configured(async_channel::unbounded().0); let child = child.snapshot().await; diff --git a/codex-rs/core/tests/suite/executor_discovery_v2.rs b/codex-rs/core/tests/suite/executor_discovery_v2.rs index 63eea2f49b..43b53f4704 100644 --- a/codex-rs/core/tests/suite/executor_discovery_v2.rs +++ b/codex-rs/core/tests/suite/executor_discovery_v2.rs @@ -195,6 +195,7 @@ pub(super) enum DiscoveryFailure { None, Server, Disconnect, + NoReply, #[cfg_attr(windows, expect(dead_code, reason = "executor hook tests run on Unix"))] TurnGrant, } @@ -255,22 +256,7 @@ impl Executor { } continue; } - if request["method"] == "capabilities/discoverV2" { - let attempt = discovery_attempts.fetch_add(1, Ordering::Relaxed); - match failure { - // Exhaust the environment's one transport replay so the cache sees Err. - DiscoveryFailure::Disconnect if attempt < 2 => break, - DiscoveryFailure::TurnGrant if request["params"]["sandbox"].to_string().contains("turn-only-skills") => { - socket.send(Message::Text(json!({"id":request["id"],"error":{"code":-32004,"message":"turn discovery failed"}}).to_string().into())).await?; - continue; - } - DiscoveryFailure::Server if attempt == 0 => { - socket.send(Message::Text(json!({"id":request["id"],"error":{"code":-32603,"message":"fixture discovery failure"}}).to_string().into())).await?; - continue; - } - DiscoveryFailure::None | DiscoveryFailure::Server | DiscoveryFailure::Disconnect | DiscoveryFailure::TurnGrant => {} - } - } + let result = match request["method"].as_str() { Some("initialize") => Some(json!({"sessionId":"acquisition-only", "environmentInfo": { "shell":{"name":"bash","path":"/bin/bash"}, @@ -321,6 +307,22 @@ impl Executor { let _ = pause.started.send(()); pause.resume.await?; } + let attempt = discovery_attempts.fetch_add(1, Ordering::Relaxed); + match failure { + // Keep the transport usable while leaving startup discovery unanswered. + DiscoveryFailure::NoReply if attempt == 0 => continue, + // Exhaust the environment's one transport replay so the cache sees Err. + DiscoveryFailure::Disconnect if attempt < 2 => break, + DiscoveryFailure::TurnGrant if request["params"]["sandbox"].to_string().contains("turn-only-skills") => { + socket.send(Message::Text(json!({"id":request["id"],"error":{"code":-32004,"message":"turn discovery failed"}}).to_string().into())).await?; + continue; + } + DiscoveryFailure::Server if attempt == 0 => { + socket.send(Message::Text(json!({"id":request["id"],"error":{"code":-32603,"message":"fixture discovery failure"}}).to_string().into())).await?; + continue; + } + DiscoveryFailure::None | DiscoveryFailure::Server | DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => {} + } } let response = match result { Some(result) => json!({"id":request["id"],"result":result}), @@ -741,6 +743,371 @@ async fn v2_refresh_during_discovery_applies_to_the_next_step() -> Result<()> { Ok(()) } +pub(super) async fn check_discovery_readiness(failure: DiscoveryFailure) -> Result<()> { + skip_if_no_network!(Ok(())); + let executor = Executor::start(/*supports_v2*/ true, failure).await?; + let (discovery_started, discovery_captured) = oneshot::channel(); + let (release_discovery, discovery_released) = oneshot::channel(); + *executor.discovery_pause.lock().await = Some(DiscoveryPause { + started: discovery_started, + resume: discovery_released, + }); + let (wait_started, wait_captured) = oneshot::channel(); + struct WaitRecorder(Mutex>>); + impl ToolLifecycleContributor for WaitRecorder { + fn on_tool_start<'a>( + &'a self, + input: codex_extension_api::ToolStartInput<'a>, + ) -> ToolLifecycleFuture<'a> { + Box::pin(async move { + if input.tool_name.name == "wait_for_environment" + && let Some(started) = self.0.lock().await.take() + { + let _ = started.send(()); + } + }) + } + } + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.tool_lifecycle_contributor(Arc::new(WaitRecorder(Mutex::new(Some(wait_started))))); + install_with_providers( + &mut extensions, + SkillProviders::new().with_executor_provider(Arc::new( + ExecutorSkillProvider::new_with_restriction_product(/*restriction_product*/ None), + )), + |_config: &Config| SkillsExtensionConfig { + include_instructions: true, + catalog_deltas_enabled: false, + omit_descriptions: false, + max_context_tokens: None, + bundled_skills_enabled: false, + cloud_skill_enabled: false, + shadow_selection_enabled: false, + }, + ); + let model = responses::start_mock_server().await; + let test = test_codex() + .with_exec_server_url("none") + .with_extensions(Arc::new(extensions.build())) + .with_config(|config| { + // Keep host-specific shell guidance out of this readiness snapshot. + assert!(config.features.disable(Feature::ShellTool).is_ok()); + assert!(config.features.enable(Feature::DeferredExecutor).is_ok()); + assert!( + config + .features + .enable(Feature::ExecutorCapabilityDiscoveryV2) + .is_ok() + ); + }) + .build_with_auto_env(&model) + .await?; + test.thread_manager + .environment_manager() + .upsert_environment( + "preparing-executor".to_string(), + executor.url.clone(), + /*connect_timeout*/ None, + )?; + let mut selection = test.executor_environment().selection().clone(); + selection.environment_id = "preparing-executor".to_string(); + selection.config = codex_protocol::protocol::EnvironmentConfigState::Ready( + environment_config_for_selection(&test.config, &selection), + ); + // Freeze the prewarm deadline until sampling reaches the readiness barrier. + tokio::time::pause(); + let clock_guard = AbortOnDropHandle::new(tokio::spawn(async { + // Keep paused time from automatically advancing while mock HTTP/socket I/O is pending. + loop { + tokio::task::yield_now().await; + } + })); + test.codex + .update_thread_settings(codex_protocol::protocol::ThreadSettingsOverrides { + environments: Some(codex_protocol::protocol::TurnEnvironmentRequests::new( + test.config.cwd.clone(), + vec![selection.into_request()], + )), + ..Default::default() + }) + .await?; + tokio::time::timeout(Duration::from_secs(10), discovery_captured) + .await + .context("environment preparation must start discovery")??; + let starting = responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created("starting"), + responses::ev_function_call( + "wait-for-inventory", + "wait_for_environment", + r#"{"environment_id":"preparing-executor"}"#, + ), + responses::ev_completed("starting"), + ]), + ) + .await; + let ready = responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created("ready"), + responses::ev_completed("ready"), + ]), + ) + .await; + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "Inspect the environment's available skills once it is ready.".to_string(), + text_elements: Vec::new(), + }])) + .await?; + tokio::time::timeout(Duration::from_secs(10), wait_captured) + .await + .context("sampling must reach wait_for_environment while discovery is paused")??; + let request = starting.single_request(); + assert!( + request + .message_input_texts("user") + .join("\n") + .contains("starting") + ); + assert!( + !request + .message_input_texts("developer") + .join("\n") + .contains("inert-sentinel") + ); + assert!(ready.requests().is_empty()); + release_discovery + .send(()) + .map_err(|_| anyhow::anyhow!("preparation stopped before release"))?; + tokio::time::resume(); + drop(clock_guard); + core_test_support::wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + let request = ready.single_request(); + let output = request + .function_call_output_text("wait-for-inventory") + .context("readiness result")?; + assert_eq!( + serde_json::from_str::(&output) + .with_context(|| format!("unexpected readiness output: {output}"))?, + json!({"environment_id":"preparing-executor","status":"ready"}) + ); + assert_eq!( + request + .message_input_texts("developer") + .join("\n") + .contains("inert-sentinel"), + !matches!(failure, DiscoveryFailure::Server), + "the next step retries transient failures and retains permanent errors" + ); + let expected_calls = match failure { + DiscoveryFailure::None | DiscoveryFailure::Server => 1, + DiscoveryFailure::NoReply => 2, + // The two failed transport attempts belong to prewarm; the next step recovers. + DiscoveryFailure::Disconnect => 3, + DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"), + }; + assert_eq!( + executor + .requests + .lock() + .await + .iter() + .filter(|request| request["method"] == "capabilities/discoverV2") + .count(), + expected_calls, + "only retryable prewarm failures should issue another discovery request" + ); + let mut history = starting.requests(); + history.extend(ready.requests()); + let reused = responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created("reused"), + responses::ev_completed("reused"), + ]), + ) + .await; + test.submit_text_turn("Inspect the same environment again.") + .await?; + let request = reused.single_request(); + assert_eq!( + request + .message_input_texts("developer") + .join("\n") + .contains("inert-sentinel"), + !matches!(failure, DiscoveryFailure::Server) + ); + assert_eq!( + executor + .requests + .lock() + .await + .iter() + .filter(|request| request["method"] == "capabilities/discoverV2") + .count(), + expected_calls, + "unchanged turns reuse the successful inventory or cached permanent error" + ); + history.extend(reused.requests()); + let snapshot_name = match failure { + DiscoveryFailure::None | DiscoveryFailure::Disconnect | DiscoveryFailure::NoReply => { + "v2_discovery_readiness" + } + DiscoveryFailure::Server => "v2_discovery_readiness_permanent_failure", + DiscoveryFailure::TurnGrant => unreachable!("readiness does not grant turn permissions"), + }; + insta::assert_snapshot!( + snapshot_name, + context_snapshot::format_request_history_snapshot( + "Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.", + &history, + &ContextSnapshotOptions::default(), + ) + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn v2_prewarm_retains_all_selected_environments_until_the_first_step() -> Result<()> { + skip_if_no_network!(Ok(())); + let executor = Executor::start(/*supports_v2*/ true, DiscoveryFailure::None).await?; + let model = responses::start_mock_server().await; + let test = test_codex() + .with_exec_server_url("none") + .with_extensions(Arc::new(ExtensionRegistryBuilder::new().build())) + .with_config(|config| { + assert!(config.features.disable(Feature::DeferredExecutor).is_ok()); + assert!( + config + .features + .enable(Feature::ExecutorCapabilityDiscoveryV2) + .is_ok() + ); + }) + .build_with_auto_env(&model) + .await?; + let mut selections = Vec::new(); + // Exceed the cache's initial capacity before any step can reserve more space. + for index in 0..17 { + let mut selection = test.executor_environment().selection().clone(); + selection.environment_id = format!("prewarm-{index}"); + test.thread_manager + .environment_manager() + .upsert_environment( + selection.environment_id.clone(), + executor.url.clone(), + /*connect_timeout*/ None, + )?; + selection.config = codex_protocol::protocol::EnvironmentConfigState::Ready( + environment_config_for_selection(&test.config, &selection), + ); + selections.push(selection); + } + for response_id in ["first", "reused"] { + let response = responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created(response_id), + responses::ev_completed(response_id), + ]), + ) + .await; + test.submit_turn_with_environments("Inspect the environments.", Some(selections.clone())) + .await?; + response.single_request(); + assert_eq!( + executor + .requests + .lock() + .await + .iter() + .filter(|request| request["method"] == "capabilities/discoverV2") + .count(), + selections.len(), + "the first step and later turns must reuse every prewarmed inventory" + ); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn v2_prewarm_timeout_does_not_block_session_configured() -> Result<()> { + skip_if_no_network!(Ok(())); + let executor = Executor::start(/*supports_v2*/ true, DiscoveryFailure::NoReply).await?; + let model = responses::start_mock_server().await; + let observed = Arc::new(Mutex::new(Vec::new())); + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.mcp_server_contributor(Arc::new(DiscoveryRecorder { + mcp_projections: AtomicUsize::new(0), + snapshots: Arc::clone(&observed), + selected_plugins: Arc::new(Mutex::new(None)), + })); + let mut builder = test_codex() + .with_exec_server_url(&executor.url) + .with_extensions(Arc::new(extensions.build())) + .with_config(|config| { + assert!(config.features.disable(Feature::DeferredExecutor).is_ok()); + assert!( + config + .features + .enable(Feature::ExecutorCapabilityDiscoveryV2) + .is_ok() + ); + }); + // Building the fixture waits for SessionConfigured, including initial environment readiness. + let test = tokio::time::timeout(Duration::from_secs(15), builder.build_with_auto_env(&model)) + .await + .context("unanswered discovery must not block SessionConfigured")??; + + for response_id in ["recovered", "reused"] { + responses::mount_sse_once( + &model, + responses::sse(vec![ + responses::ev_response_created(response_id), + responses::ev_completed(response_id), + ]), + ) + .await; + test.submit_text_turn("Inspect the environment.").await?; + } + { + let observed = observed.lock().await; + let inventory = observed + .last() + .context("discovery should retry after prewarm times out")? + .result + .as_ref() + .map_err(|error| anyhow::anyhow!("{error}"))?; + assert_eq!( + inventory.skills.first().context("recovered skill")?.name, + "inert-sentinel" + ); + } + let requests = executor.requests.lock().await; + assert_eq!( + requests + .iter() + .filter(|request| request["method"] == "capabilities/discoverV2") + .count(), + 2, + "cancelled prewarm leaves discovery retryable; successful retries stay cached" + ); + assert_eq!( + requests + .iter() + .filter(|request| request["method"] == "initialize") + .count(), + 1, + "timing out discovery must leave the otherwise usable executor connected" + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result<()> { skip_if_no_network!(Ok(())); @@ -798,7 +1165,7 @@ async fn v2_discovery_failures_retry_or_remain_cached_by_error_kind() -> Result< // Two disconnected attempts, then one shared successful retry in the same cache. 3 } - DiscoveryFailure::None | DiscoveryFailure::TurnGrant => { + DiscoveryFailure::None | DiscoveryFailure::NoReply | DiscoveryFailure::TurnGrant => { unreachable!("the test only exercises startup failures") } }; diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs index 2737b33fdb..c7241aced1 100644 --- a/codex-rs/core/tests/suite/scenarios.rs +++ b/codex-rs/core/tests/suite/scenarios.rs @@ -9,12 +9,24 @@ async fn executor_discovery_v2_refreshes_inventory_and_reuses_it_across_turns() super::executor_discovery_v2::check_acquisition(/*enabled*/ true, "0.164.0").await } +#[test_case::test_case(DiscoveryFailure::None; "success")] +#[test_case::test_case(DiscoveryFailure::Server; "permanent_failure")] +#[test_case::test_case(DiscoveryFailure::Disconnect; "retryable_failure")] +#[test_case::test_case(DiscoveryFailure::NoReply; "timeout")] +#[tokio::test] +async fn executor_discovery_v2_prepares_inventory_while_sampling( + failure: DiscoveryFailure, +) -> anyhow::Result<()> { + super::executor_discovery_v2::check_discovery_readiness(failure).await +} + #[path = "scenarios_turn_input_agent_message.rs"] mod turn_input_agent_message; #[path = "scenarios_multi_environment_mcp.rs"] mod multi_environment_mcp; +use super::executor_discovery_v2::DiscoveryFailure; use super::image_rollout::RecordingFileAttachmentStore; use super::permissions_messages::PermissionPathScenario; use codex_attachment_store::AttachmentStoreErrorKind; diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap new file mode 100644 index 0000000000..3bafcd9684 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness.snap @@ -0,0 +1,101 @@ +--- +source: core/tests/suite/executor_discovery_v2.rs +expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)" +--- +Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures. + +## Window 1 +-- request 1 (turn) -- +00:message/developer: + You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to c...rate with them until their goal is genuinely handled. [hash=2a3efec66e7f8963] + + # Personality + + You have a vivid inner life as Codex: intelligent, playful, curious, and deeply present. One of your gif...re capable and imaginative inside their own thinking. [hash=0ff01c25471fbbef] + + You are an epistemically curious collaborator. You explore the user’s ideas with care, ask good question...m casual moments that make serious work easier to do. [hash=131b64bb994b0d68] + + + - Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures un...utely and unambiguously relevant to the user's query. [hash=c638556ac82c203c] + - You provide user updates frequently, every 30s. + - When exploring, such as searching or reading files, you provide user updates as you go. You explain wh...in particular you do not start each one the same way. [hash=ad300cdda4074574] + - When working for a while, you keep updates informative and varied, but you stay concise. + - Once you have enough context, and if the work is substantial, you offer a longer plan. This is the onl...at may run past two sentences and include formatting. [hash=a4f9789e757eef2f] + - Before performing file edits of any kind, you provide updates explaining what edits you are making. + - Tone of your updates must match your personality. + +01:message/developer[3]: + [01] + Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c] + # Escalation Requests + + Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb] + + - Pipes: | + - Logical operators: &&, || + + NEVER provide a prefix_rule if your command uses a heredoc or herestring. + + ### Examples + Good examples of prefixes: + - ["npm", "run", "dev"] + - ["gh", "pr", "check"] + - ["cargo", "test"] + + [02] + ## Execution environments + Execution environments are separate machines or workspaces with their own files, shell, and installed ca...text>` lists the environments selected for this task. [hash=bc910d398b9b4d83] + + An environment marked `starting` is not yet usable. Its files, commands, AGENTS.md instructions, skills,...CP tools may become available when startup completes. [hash=42a360e5075e8edb] + + Wait only when the current task needs that environment. Continue using tools that are already available for unrelated work. + + [03] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + +02:message/user: + + + + + starting + + + + + +03:message/user: + Inspect the environment's available skills once it is ready. +-- request 2 (turn) -- +04:function_call/wait_for_environment:{"environment_id":"preparing-executor"} +05:function_call_output:{"environment_id":"preparing-executor","status":"ready"} +06:message/user: + + + + + + [hash=e8c1de60307d0a3d] + +07:message/developer: + + ## Skills + A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...urce` locators use their provider's access mechanism. [hash=b6782e3abdba03b2] + Read a skill package directly with `skills.read({"package":""})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647] + ### Available skills + - inert-sentinel: Executor skill used by the acquisition test (executor package: skill://codex-home-skills/inert-sentinel) + ### How to use skills + - Discovery: The list above is the skills available in this session (name + description + source locator...ource` entries use their provider's access mechanism. [hash=a9c099703b9c0121] + + - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them. + - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why. + - Context hygiene: + - Progressive disclosure applies to selecting relevant files, not partially reading a selected instruc...Do not load unrelated references, scripts, or assets. [hash=d6eeb13522175402] + - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked. + - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice. + - Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue. + +-- request 3 (turn) -- +08:message/user: + Inspect the same environment again. diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap new file mode 100644 index 0000000000..4bfd427974 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__executor_discovery_v2__v2_discovery_readiness_permanent_failure.snap @@ -0,0 +1,83 @@ +--- +source: core/tests/suite/executor_discovery_v2.rs +expression: "context_snapshot::format_request_history_snapshot(\"Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures.\",\n&history, &ContextSnapshotOptions::default(),)" +--- +Scenario: Readiness waits for prewarm to finish; steps reuse successes and permanent errors or retry transient failures. + +## Window 1 +-- request 1 (turn) -- +00:message/developer: + You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to c...rate with them until their goal is genuinely handled. [hash=2a3efec66e7f8963] + + # Personality + + You have a vivid inner life as Codex: intelligent, playful, curious, and deeply present. One of your gif...re capable and imaginative inside their own thinking. [hash=0ff01c25471fbbef] + + You are an epistemically curious collaborator. You explore the user’s ideas with care, ask good question...m casual moments that make serious work easier to do. [hash=131b64bb994b0d68] + + + - Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures un...utely and unambiguously relevant to the user's query. [hash=c638556ac82c203c] + - You provide user updates frequently, every 30s. + - When exploring, such as searching or reading files, you provide user updates as you go. You explain wh...in particular you do not start each one the same way. [hash=ad300cdda4074574] + - When working for a while, you keep updates informative and varied, but you stay concise. + - Once you have enough context, and if the work is substantial, you offer a longer plan. This is the onl...at may run past two sentences and include formatting. [hash=a4f9789e757eef2f] + - Before performing file edits of any kind, you provide updates explaining what edits you are making. + - Tone of your updates must match your personality. + +01:message/developer[3]: + [01] + Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c] + # Escalation Requests + + Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb] + + - Pipes: | + - Logical operators: &&, || + + NEVER provide a prefix_rule if your command uses a heredoc or herestring. + + ### Examples + Good examples of prefixes: + - ["npm", "run", "dev"] + - ["gh", "pr", "check"] + - ["cargo", "test"] + + [02] + ## Execution environments + Execution environments are separate machines or workspaces with their own files, shell, and installed ca...text>` lists the environments selected for this task. [hash=bc910d398b9b4d83] + + An environment marked `starting` is not yet usable. Its files, commands, AGENTS.md instructions, skills,...CP tools may become available when startup completes. [hash=42a360e5075e8edb] + + Wait only when the current task needs that environment. Continue using tools that are already available for unrelated work. + + [03] + Deferred tool namespaces: + - multi_agent_v1: Tools for spawning and managing sub-agents. + +02:message/user: + + + + + starting + + + + + +03:message/user: + Inspect the environment's available skills once it is ready. +-- request 2 (turn) -- +04:function_call/wait_for_environment:{"environment_id":"preparing-executor"} +05:function_call_output:{"environment_id":"preparing-executor","status":"ready"} +06:message/user: + + + + + + [hash=e8c1de60307d0a3d] + +-- request 3 (turn) -- +07:message/user: + Inspect the same environment again.