From 66618f0512ccd43ebd9784f3f0af3f9e93ca6d0c Mon Sep 17 00:00:00 2001 From: Jay Chen <153026722+jaychen-openai@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:39:04 +0000 Subject: [PATCH] Prewarm MCP servers during cloud catalog refresh (#62876) ## Why A slow cloud skill lookup can delay MCP startup even after the plugin catalog is ready. If the refresh is canceled while waiting for skills, it can miss the chance to prewarm those servers before the first turn. ## What changed Queue MCP prewarm as soon as plugin discovery changes or clears the catalog. Skill discovery alone leaves the MCP server set unchanged. At the end of the refresh, queue a retry for eligible MCP startup failures. If discovery is canceled, check for those failures the next time the catalog lock is acquired. Reuse pending handshakes and leave authentication failures and Apps-managed reconnects to their existing recovery paths. ## Testing Add regression tests for first-turn tool availability after cancellation, startup recovery with cloud plugins disabled, and a single retry for a persistent startup failure. GitOrigin-RevId: f92162d1629358953d8810d20244e55f07416a01 --- .../src/connection_manager/status.rs | 14 + codex-rs/codex-mcp/src/runtime.rs | 8 + codex-rs/core/src/codex_thread.rs | 2 +- codex-rs/core/src/session/cloud_plugins.rs | 42 ++- .../core/src/session/turn_start_mcp_tests.rs | 32 +- codex-rs/core/src/thread_manager.rs | 1 + .../core/tests/common/apps_test_server.rs | 29 +- .../core/tests/suite/mcp_tool_exposure.rs | 353 ++++++++++++++++++ .../ext/extension-api/src/contributors.rs | 9 +- .../src/contributors/turn_lifecycle.rs | 15 + codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/mcp/src/cloud_plugin.rs | 20 +- codex-rs/ext/mcp/src/cloud_plugin_tests.rs | 43 ++- codex-rs/ext/skills/src/cloud_skill.rs | 13 +- codex-rs/ext/skills/tests/skills_extension.rs | 31 +- .../rmcp-client/src/bin/test_stdio_server.rs | 8 + 16 files changed, 564 insertions(+), 57 deletions(-) diff --git a/codex-rs/codex-mcp/src/connection_manager/status.rs b/codex-rs/codex-mcp/src/connection_manager/status.rs index b96f5e9e24..e3e95c2e46 100644 --- a/codex-rs/codex-mcp/src/connection_manager/status.rs +++ b/codex-rs/codex-mcp/src/connection_manager/status.rs @@ -9,6 +9,20 @@ use crate::McpServerKey; use crate::server::McpServerConnectionIdentity; impl McpConnectionSet { + pub(crate) fn has_failed_startup_for_retry(&self) -> bool { + self.servers.values().any(|view| { + let client = &view.connection.client; + // Apps manages its own reconnects. Leave pending handshakes alone. + client.startup_reconnect.is_none() + && !client.cancel_token.is_cancelled() + && matches!( + client.client.peek(), + Some(Err(error @ super::StartupOutcomeError::Failed { .. })) + if !error.is_authentication_required() + ) + }) + } + /// Reads readiness without starting deferred clients or consulting the OAuth store. pub(crate) async fn has_reusable_status_catalog(&self, key: &McpServerKey) -> bool { let Some(view) = self.servers.get(key) else { diff --git a/codex-rs/codex-mcp/src/runtime.rs b/codex-rs/codex-mcp/src/runtime.rs index 240c6b2ebe..096d94f6e1 100644 --- a/codex-rs/codex-mcp/src/runtime.rs +++ b/codex-rs/codex-mcp/src/runtime.rs @@ -691,6 +691,14 @@ impl McpRuntime { .await } + /// Checks for completed startup failures eligible for retry without polling any client. + pub fn has_failed_startup_for_retry(&self) -> bool { + self.current + .load() + .connections + .has_failed_startup_for_retry() + } + /// Checks the current generation before retrying servers detected outside the refresh gate. pub async fn has_authentication_failed_servers(&self, server_names: &[McpServerKey]) -> bool { !self diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 0ac99f599c..f4cc8767e8 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -1622,7 +1622,7 @@ impl CodexThread { } /// Refreshes cloud plugin and skill catalogs before input admission, without starting a turn. - /// Serializes publication with step preparation and marks MCP dirty for later reconciliation. + /// Serializes publication with step preparation and queues MCP prewarm as plugins are published. /// Accepts complete or partial catalogs; failed discovery remains retryable. pub async fn refresh_cloud_plugins(&self) -> anyhow::Result<()> { self.session.refresh_cloud_plugins().await diff --git a/codex-rs/core/src/session/cloud_plugins.rs b/codex-rs/core/src/session/cloud_plugins.rs index 5fa2e4f8f0..636ff530dd 100644 --- a/codex-rs/core/src/session/cloud_plugins.rs +++ b/codex-rs/core/src/session/cloud_plugins.rs @@ -11,15 +11,32 @@ use tokio::sync::Mutex; #[derive(Default)] pub(super) struct CloudCatalogPreparation { /// Serializes catalog refreshes and snapshot capture. - refresh_lock: Mutex<()>, + refresh_lock: Mutex, /// Requests resource invalidation on the next lock acquisition. /// Survives cancellation while waiting. refresh_pending: AtomicBool, } +#[derive(Default)] +pub(crate) struct CatalogRefreshState { + // Leave the check for the next catalog lock holder if discovery is cancelled. + startup_retry_pending: bool, +} + impl Session { - pub(crate) async fn lock_cloud_catalogs(&self) -> tokio::sync::MutexGuard<'_, ()> { - let guard = self.cloud_catalog_preparation.refresh_lock.lock().await; + fn finish_catalog_startup(&self, catalogs: &mut CatalogRefreshState) { + if std::mem::take(&mut catalogs.startup_retry_pending) + && self.services.mcp_runtime.has_failed_startup_for_retry() + { + self.request_mcp_runtime_reprojection(McpRefreshReason::ServerStartupFailed); + } + } + + pub(crate) async fn lock_cloud_catalogs( + &self, + ) -> tokio::sync::MutexGuard<'_, CatalogRefreshState> { + let mut guard = self.cloud_catalog_preparation.refresh_lock.lock().await; + self.finish_catalog_startup(&mut guard); if self .cloud_catalog_preparation .refresh_pending @@ -38,7 +55,8 @@ impl Session { self.cloud_catalog_preparation .refresh_pending .store(true, Ordering::Release); - let _catalogs = self.lock_cloud_catalogs().await; + let mut catalogs = self.lock_cloud_catalogs().await; + catalogs.startup_retry_pending = true; self.refresh_mcp_if_dirty().await; let results = futures::future::join_all( self.services @@ -46,20 +64,24 @@ impl Session { .turn_lifecycle_contributors() .iter() .map(|contributor| async move { - let result = contributor + let outcome = contributor .refresh_cloud_catalog(CloudCatalogRefreshInput { session_store: &self.services.session_extension_data, thread_store: &self.services.thread_extension_data, }) .await; - // Retain publication even if another provider is still pending at cancellation. - self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged); - result + // Queue the work before a slower sibling can time out the whole refresh. + if outcome.needs_mcp_refresh { + self.request_mcp_runtime_reprojection( + McpRefreshReason::ExtensionCatalogChanged, + ); + } + outcome.result }), ) .await; - drop(_catalogs); - self.schedule_mcp_prewarm(); + self.finish_catalog_startup(&mut catalogs); + drop(catalogs); let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); anyhow::ensure!(errors.is_empty(), "{}", errors.join("; ")); Ok(()) diff --git a/codex-rs/core/src/session/turn_start_mcp_tests.rs b/codex-rs/core/src/session/turn_start_mcp_tests.rs index d29604019f..30ce3cb35d 100644 --- a/codex-rs/core/src/session/turn_start_mcp_tests.rs +++ b/codex-rs/core/src/session/turn_start_mcp_tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::McpRefreshReason; +use codex_extension_api::CloudCatalogRefreshOutcome; use codex_mcp::McpResourceClient; use codex_mcp::McpResourceClientCacheKey; use codex_protocol::models::ResponseInputItem; @@ -140,24 +141,24 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_releases_lock_on_cancel session.abort_all_tasks(TurnAbortReason::Interrupted).await; } -/// Explicit refresh overlaps providers under one lock and retains publication while a sibling waits. +/// A published catalog queues MCP prewarm while a sibling is still holding up the refresh. #[tokio::test] async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() { struct CatalogGate { entered: Arc, finish: tokio::sync::Notify, - result: Result<(), &'static str>, + outcome: CloudCatalogRefreshOutcome, } impl codex_extension_api::TurnLifecycleContributor for CatalogGate { fn refresh_cloud_catalog<'a>( &'a self, _input: codex_extension_api::CloudCatalogRefreshInput<'a>, - ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { + ) -> codex_extension_api::ExtensionFuture<'a, CloudCatalogRefreshOutcome> { Box::pin(async move { self.entered.wait().await; self.finish.notified().await; - self.result.map_err(str::to_string) + self.outcome.clone() }) } } @@ -167,17 +168,25 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() { let failing = Arc::new(CatalogGate { entered: Arc::clone(&entered), finish: Default::default(), - result: Err("plugin discovery failed"), + outcome: CloudCatalogRefreshOutcome { + result: Err("skill discovery failed".to_string()), + needs_mcp_refresh: false, + }, }); let succeeding = Arc::new(CatalogGate { entered: Arc::clone(&entered), finish: Default::default(), - result: Ok(()), + outcome: CloudCatalogRefreshOutcome { + result: Ok(()), + needs_mcp_refresh: true, + }, }); let mut builder = codex_extension_api::ExtensionRegistryBuilder::new(); builder.turn_lifecycle_contributor(failing.clone()); builder.turn_lifecycle_contributor(succeeding.clone()); session.services.extensions = Arc::new(builder.build()); + let (tx, prewarm_requests) = async_channel::bounded(1); + session.mcp_prewarm_tx = tx; let session = Arc::new(session); let mut refresh = Box::pin(session.refresh_cloud_plugins()); tokio::time::timeout(std::time::Duration::from_secs(5), async { @@ -194,14 +203,19 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() { assert!(futures::poll!(&mut lock).is_pending()); drop(lock); - failing.finish.notify_one(); + assert!(prewarm_requests.is_empty()); + succeeding.finish.notify_one(); assert!(futures::poll!(&mut refresh).is_pending()); assert!(session.mcp_refresh.is_pending()); - succeeding.finish.notify_one(); + prewarm_requests + .try_recv() + .expect("the worker must be notified before the remaining provider finishes"); + failing.finish.notify_one(); assert_eq!( refresh.await.expect_err("one provider failed").to_string(), - "plugin discovery failed" + "skill discovery failed" ); + assert!(prewarm_requests.is_empty()); drop( tokio::time::timeout( std::time::Duration::from_secs(5), diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 23d9588888..5dc89d3ff0 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -315,6 +315,7 @@ pub enum McpRefreshReason { RemotePluginUninstalled, RuntimeAuthChanged, SelectedCapabilityRootsChanged, + ServerStartupFailed, SessionMcpInputsChanged, SessionPermissionsGranted, SkillMcpDependenciesAdded, diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index 6db166edb3..51b488eefa 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -64,7 +64,8 @@ const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/cale pub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_extract_text"; -type AppsStartupInitializeGate = Arc>>>; +type AppsStartupInitializeGate = + Arc, Option)>>>; pub struct HostedMessagingServer(pub String); @@ -118,11 +119,20 @@ impl AppsTestServerStartupControl { /// Holds one successful startup without turning recovery into a fresh runtime refresh. /// Multiple holds are consumed in order, so tests can gate startup and recovery in advance. pub fn hold_next_successful_initialize(&self) -> mpsc::Sender<()> { + self.hold_initialize(/*response_override*/ None) + } + + /// Hold the next successful handshake and return a non-retryable HTTP error on release. + pub fn hold_initialize_until_failure(&self) -> mpsc::Sender<()> { + self.hold_initialize(Some(ResponseTemplate::new(400))) + } + + fn hold_initialize(&self, response_override: Option) -> mpsc::Sender<()> { let (release, gate) = mpsc::channel(); self.successful_initialize_gate .lock() .expect("Apps initialization gate lock should not be poisoned") - .push_back(gate); + .push_back((gate, response_override)); release } } @@ -584,12 +594,15 @@ impl Respond for CodexAppsJsonRpcResponder { }); // This responder's dedicated mock server stays paused while the test controls // when this existing client's background retry may finish. - if let Some(gate) = gate - && gate.recv().is_err() - { - return ResponseTemplate::new(500).set_body_json(json!({ - "error": "Apps initialization gate was dropped before release", - })); + if let Some((gate, response_override)) = gate { + if gate.recv().is_err() { + return ResponseTemplate::new(500).set_body_json(json!({ + "error": "Apps initialization gate was dropped before release", + })); + } + if let Some(response) = response_override { + return response; + } } let id = body.get("id").cloned().unwrap_or(Value::Null); let protocol_version = body diff --git a/codex-rs/core/tests/suite/mcp_tool_exposure.rs b/codex-rs/core/tests/suite/mcp_tool_exposure.rs index 6b55988fd4..eff8c6cf82 100644 --- a/codex-rs/core/tests/suite/mcp_tool_exposure.rs +++ b/codex-rs/core/tests/suite/mcp_tool_exposure.rs @@ -6,6 +6,7 @@ use codex_core::TurnInputRequest; use codex_core::TurnStartOptions; use codex_core::config::Config; use codex_core::windows_sandbox::WindowsSandboxLevelExt; +use codex_extension_api::CloudCatalogRefreshOutcome; use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::McpServerContribution; @@ -422,6 +423,358 @@ async fn root_and_spawned_subagent_receive_distinct_mcp_session_sources() -> Res Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn first_turn_reuses_local_and_cloud_servers_after_cancelled_refresh() -> Result<()> { + skip_if_no_network!(Ok(())); + core_test_support::skip_if_wine_exec!(Ok(()), "requires a Windows test_stdio_server"); + + let fixture = CloudRefreshFixture::new(/*other_server*/ None).await?; + let release_startup = fixture.startup.hold_next_successful_initialize(); + // Drop the refresh at the caller's deadline with the cloud handshake still in flight. + tokio::select! { + result = fixture.test.codex.refresh_cloud_plugins() => panic!("expected pending catalog: {result:?}"), + started = tokio::time::timeout(Duration::from_secs(5), async { + while fixture.startup.initialize_attempts() == 0 { + tokio::task::yield_now().await; + } + }) => started.expect("cloud startup should begin while the other lookup is pending"), + } + release_startup.send(())?; + fixture.assert_first_turn_uses_both_tools().await?; + fixture.assert_start_counts(/*cloud_attempts*/ 1).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn first_turn_recovers_server_that_failed_during_cloud_refresh() -> Result<()> { + skip_if_no_network!(Ok(())); + core_test_support::skip_if_wine_exec!(Ok(()), "requires a Windows test_stdio_server"); + + // A configured server has already failed when the plugin adds the calendar. + let other = responses::start_mock_server().await; + let (apps, broken) = AppsTestServer::mount_with_startup_control(&other).await?; + broken.fail_next_initialize_attempts(usize::MAX); + let fixture = CloudRefreshFixture::new(Some(apps)).await?; + assert_eq!(broken.initialize_attempts(), 1); + fixture + .refresh_with_startup_failures(/*failures*/ 1) + .await?; + fixture.assert_first_turn_uses_both_tools().await?; + fixture.assert_start_counts(/*cloud_attempts*/ 2).await?; + // Each publication may retry the configured server: initial, plugin update, recovery. + assert!((2..=3).contains(&broken.initialize_attempts())); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cloud_refresh_retries_persistent_startup_failure_only_once() -> Result<()> { + skip_if_no_network!(Ok(())); + core_test_support::skip_if_wine_exec!(Ok(()), "requires a Windows test_stdio_server"); + + let fixture = CloudRefreshFixture::new(/*other_server*/ None).await?; + fixture.refresh_with_startup_failures(usize::MAX).await?; + wait_for_calendar_failure(&fixture.test.codex).await; + for prompt in ["first message", "follow-up"] { + let response = responses::mount_sse_once( + &fixture.server, + sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-1"), + ]), + ) + .await; + fixture.test.submit_text_turn(prompt).await?; + let body = response.single_request().body_json(); + assert!(namespace_child_tool(&body, "mcp__calendar", "calendar_list_events").is_none()); + assert!(namespace_child_tool(&body, "mcp__local", "echo").is_some()); + } + fixture.assert_start_counts(/*cloud_attempts*/ 2).await +} + +#[derive(Default)] +struct PendingCatalog { + entered: tokio::sync::Notify, + finish: tokio::sync::Notify, +} + +impl codex_extension_api::TurnLifecycleContributor for PendingCatalog { + fn refresh_cloud_catalog<'a>( + &'a self, + _input: codex_extension_api::CloudCatalogRefreshInput<'a>, + ) -> ExtensionFuture<'a, CloudCatalogRefreshOutcome> { + Box::pin(async move { + self.entered.notify_one(); + self.finish.notified().await; + CloudCatalogRefreshOutcome::UNCHANGED + }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn first_turn_recovers_configured_server_with_plugins_disabled() -> Result<()> { + check_configured_server_recovery(/*cancel*/ false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn first_turn_recovers_configured_server_after_cancelled_discovery() -> Result<()> { + check_configured_server_recovery(/*cancel*/ true).await +} + +async fn check_configured_server_recovery(cancel: bool) -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (apps, startup) = AppsTestServer::mount_with_startup_control(&server).await?; + let fail_startup = startup.hold_initialize_until_failure(); + let calendar = serde_json::from_value(json!({ + "url": format!("{}/api/codex/ps/mcp", apps.chatgpt_base_url), + }))?; + let skills = Arc::new(PendingCatalog::default()); + let mut extensions = ExtensionRegistryBuilder::::new(); + extensions.turn_lifecycle_contributor(skills.clone()); + let test = core_test_support::test_codex::test_codex() + .with_extensions(Arc::new(extensions.build())) + .with_config(move |config| { + config + .features + .disable(Feature::Plugins) + .expect("test config should allow disabling plugins"); + config + .mcp_servers + .set(HashMap::from([("calendar".to_string(), calendar)])) + .expect("test config should accept MCP servers"); + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow code mode"); + config.code_mode.direct_only_tool_namespaces = vec!["mcp__calendar".to_string()]; + }) + .build(&server) + .await?; + let mut refresh = Box::pin(test.codex.refresh_cloud_plugins()); + tokio::select! { + result = &mut refresh => panic!("expected pending skills: {result:?}"), + _ = skills.entered.notified() => {} + } + fail_startup.send(())?; + wait_for_calendar_failure(&test.codex).await; + assert_eq!(startup.initialize_attempts(), 1); + if cancel { + drop(refresh); + } else { + skills.finish.notify_one(); + refresh.await?; + } + + let response = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-1"), + ]), + ) + .await; + test.submit_text_turn("list calendar events").await?; + let body = response.single_request().body_json(); + assert_eq!(startup.initialize_attempts(), 2); + assert!(namespace_child_tool(&body, "mcp__calendar", "calendar_list_events").is_some()); + test.codex.shutdown_and_wait().await?; + Ok(()) +} + +async fn wait_for_calendar_failure(codex: &CodexThread) { + wait_for_event(codex, |event| matches!(event, + EventMsg::McpStartupUpdate(update) if update.server == "calendar" + && matches!(update.status, codex_protocol::protocol::McpStartupStatus::Failed { .. }) + )).await; +} + +struct CloudRefreshFixture { + test: core_test_support::test_codex::TestCodex, + server: wiremock::MockServer, + startup: core_test_support::apps_test_server::AppsTestServerStartupControl, + skills: Arc, + dir: TempDir, +} + +impl CloudRefreshFixture { + async fn new(other_server: Option) -> Result { + let dir = TempDir::new()?; + let starts = dir.path().join("local-process-starts"); + let local_config = serde_json::from_value(json!({ + "command": core_test_support::stdio_server_bin()?, + "env": {"MCP_TEST_PROCESS_START_LOG": starts}, + "enabled_tools": ["echo"], + "default_tools_approval_mode": "approve", + }))?; + + struct PublishServer { + contributor: Arc, + server: McpServerContribution, + } + impl codex_extension_api::TurnLifecycleContributor for PublishServer { + fn refresh_cloud_catalog<'a>( + &'a self, + _input: codex_extension_api::CloudCatalogRefreshInput<'a>, + ) -> ExtensionFuture<'a, CloudCatalogRefreshOutcome> { + Box::pin(async move { + *self + .contributor + .contributions + .lock() + .expect("test contributions lock should not be poisoned") = + vec![self.server.clone()]; + CloudCatalogRefreshOutcome { + result: Ok(()), + needs_mcp_refresh: true, + } + }) + } + } + let server = responses::start_mock_server().await; + let (apps, startup) = AppsTestServer::mount_with_startup_control(&server).await?; + let contributor = Arc::new(CoalescingMcpContributor::new()); + let mut extensions = ExtensionRegistryBuilder::::new(); + extensions.mcp_server_contributor(contributor.clone()); + extensions.turn_lifecycle_contributor(Arc::new(PublishServer { + contributor, + server: McpServerContribution::Set { + name: "calendar".to_string(), + config: Box::new(serde_json::from_value(json!({ + "url": format!("{}/api/codex/ps/mcp", apps.chatgpt_base_url), + "default_tools_approval_mode": "approve", + }))?), + }, + })); + let skills = Arc::new(PendingCatalog::default()); + extensions.turn_lifecycle_contributor(skills.clone()); + let mut configured = HashMap::from([("local".to_string(), local_config)]); + if let Some(other) = other_server { + configured.insert( + "broken".to_string(), + serde_json::from_value(json!({ + "url": format!("{}/api/codex/ps/mcp", other.chatgpt_base_url), + }))?, + ); + } + let test = core_test_support::test_codex::test_codex() + .with_extensions(Arc::new(extensions.build())) + .with_config(move |config| { + config + .mcp_servers + .set(configured) + .expect("test config should accept MCP servers"); + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow code mode"); + config.code_mode.direct_only_tool_namespaces = + vec!["mcp__calendar".to_string(), "mcp__local".to_string()]; + }) + .build(&server) + .await?; + wait_for_mcp_server(&test.codex, "local").await?; + assert_eq!(std::fs::read_to_string(starts)?.lines().count(), 1); + Ok(Self { + test, + server, + startup, + skills, + dir, + }) + } + + async fn refresh_with_startup_failures(&self, failures: usize) -> Result<()> { + self.startup.fail_next_initialize_attempts(failures); + let refresh = self.test.codex.refresh_cloud_plugins(); + tokio::pin!(refresh); + tokio::select! { + result = &mut refresh => panic!("expected pending catalog: {result:?}"), + _ = wait_for_calendar_failure(&self.test.codex) => {} + } + assert_eq!(self.startup.initialize_attempts(), 1); + // Let every server in the early publication finish before completing skills. + wait_for_event(&self.test.codex, |event| { + matches!(event, EventMsg::McpStartupComplete(_)) + }) + .await; + self.skills.finish.notify_one(); + refresh.await?; + Ok(()) + } + + async fn assert_start_counts(&self, cloud_attempts: usize) -> Result<()> { + self.test.codex.shutdown_and_wait().await?; + let starts = std::fs::read_to_string(self.dir.path().join("local-process-starts"))?; + assert_eq!(starts.lines().count(), 1); + assert_eq!(self.startup.initialize_attempts(), cloud_attempts); + Ok(()) + } + + async fn assert_first_turn_uses_both_tools(&self) -> Result<()> { + wait_for_event(&self.test.codex, |event| { + matches!(event, + EventMsg::McpStartupUpdate(update) if update.server == "calendar" + && matches!(update.status, codex_protocol::protocol::McpStartupStatus::Ready) + ) + }) + .await; + + let response = mount_sse_sequence( + &self.server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call_with_namespace( + "local-call", + "mcp__local", + "echo", + r#"{"message":"hello"}"#, + ), + ev_function_call_with_namespace( + "cloud-call", + "mcp__calendar", + "calendar_list_events", + "{}", + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + self.test + .submit_text_turn("Echo hello locally and list calendar events") + .await?; + let requests = response.requests(); + let body = requests[0].body_json(); + assert!(namespace_child_tool(&body, "mcp__local", "echo").is_some()); + assert!(namespace_child_tool(&body, "mcp__calendar", "calendar_list_events").is_some()); + let calls = + core_test_support::apps_test_server::recorded_apps_tool_calls(&self.server).await; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0]["params"]["name"], "calendar_list_events"); + for (id, expected) in [ + ("local-call", json!({"echo": "ECHOING: hello", "env": null})), + ("cloud-call", json!({"_codex_apps": null})), + ] { + let output = requests[1] + .function_call_output_text(id) + .expect("tool result"); + let (_, payload) = output + .split_once("\nOutput:\n") + .expect("wrapped MCP result"); + assert_eq!(serde_json::from_str::(payload)?, expected, "{id}"); + } + Ok(()) + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn rapid_mcp_refreshes_coalesce_to_the_latest_config() -> Result<()> { let server = responses::start_mock_server().await; diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 18bdf69bb4..5ab1539b90 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -64,6 +64,7 @@ pub use turn_input::TurnInputContext; pub use turn_input::TurnInputEnvironment; pub use turn_lifecycle::CatalogRefreshInput; pub use turn_lifecycle::CloudCatalogRefreshInput; +pub use turn_lifecycle::CloudCatalogRefreshOutcome; pub use turn_lifecycle::TurnAbortInput; pub use turn_lifecycle::TurnErrorInput; pub use turn_lifecycle::TurnStartInput; @@ -249,16 +250,16 @@ pub trait TurnLifecycleContributor: Send + Sync { } /// Publishes cloud catalogs after explicit invalidation, without requiring a turn. - /// Core marks MCP dirty after publication. Accepted complete or partial catalogs - /// replace the previous result; failed or superseded discovery returns an error. + /// Report whether MCP needs updating so Core can start clients while other + /// providers are still running. /// Core invalidates resource keys before calling. MCP-backed catalogs record /// the new key only on acceptance, keeping failed or cancelled refreshes stale. /// Called under the same per-thread catalog lock as step preparation. fn refresh_cloud_catalog<'a>( &'a self, _input: CloudCatalogRefreshInput<'a>, - ) -> ExtensionFuture<'a, Result<(), String>> { - Box::pin(async { Ok(()) }) + ) -> ExtensionFuture<'a, CloudCatalogRefreshOutcome> { + Box::pin(async { CloudCatalogRefreshOutcome::UNCHANGED }) } /// Observes a completed item without changing it or delaying streamed deltas. diff --git a/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs b/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs index a9f4622e4f..95f1d8b4c5 100644 --- a/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs +++ b/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs @@ -58,6 +58,21 @@ impl<'a> From> for CloudCatalogRefreshInput<'a> { } } +/// Result of explicit discovery and its effect on the thread's MCP servers. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CloudCatalogRefreshOutcome { + pub result: Result<(), String>, + /// Also true when discovery failed after clearing a stale plugin catalog. + pub needs_mcp_refresh: bool, +} + +impl CloudCatalogRefreshOutcome { + pub const UNCHANGED: Self = Self { + result: Ok(()), + needs_mcp_refresh: false, + }; +} + /// Input supplied when the host completes a turn. pub struct TurnStopInput<'a> { /// Store scoped to the host session runtime. diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 5d87b363e9..5e9443f882 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -59,6 +59,7 @@ pub use contributors::ApprovalDecisionInput; pub use contributors::ApprovalReviewContributor; pub use contributors::CatalogRefreshInput; pub use contributors::CloudCatalogRefreshInput; +pub use contributors::CloudCatalogRefreshOutcome; pub use contributors::CodeModeCellTiming; pub use contributors::CommandStartInput; pub use contributors::ConfigContributor; diff --git a/codex-rs/ext/mcp/src/cloud_plugin.rs b/codex-rs/ext/mcp/src/cloud_plugin.rs index 3b12c8df28..20431aa59e 100644 --- a/codex-rs/ext/mcp/src/cloud_plugin.rs +++ b/codex-rs/ext/mcp/src/cloud_plugin.rs @@ -12,6 +12,7 @@ use codex_core_plugins::PluginIdentity; use codex_extension_api::CatalogRefreshBackoff; use codex_extension_api::CatalogRefreshInput as StepCatalogRefreshInput; use codex_extension_api::CloudCatalogRefreshInput; +use codex_extension_api::CloudCatalogRefreshOutcome; use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionFuture; @@ -232,17 +233,26 @@ impl TurnLifecycleContributor for PluginContributor { fn refresh_cloud_catalog<'a>( &'a self, input: CloudCatalogRefreshInput<'a>, - ) -> ExtensionFuture<'a, Result<(), String>> { + ) -> ExtensionFuture<'a, CloudCatalogRefreshOutcome> { Box::pin(async move { let Some(state) = input.thread_store.get::() else { - return Ok(()); + return CloudCatalogRefreshOutcome::UNCHANGED; }; // A failed explicit attempt gets an immediate retry on the next step. state.contributor_state().cloud_retry = None; - self.discover_cloud_catalog(&state, cloud_catalog_query(input, /*turn_id*/ None)) + match self + .discover_cloud_catalog(&state, cloud_catalog_query(input, /*turn_id*/ None)) .await - .map(|_| ()) - .map_err(|failure| failure.error) + { + Ok(needs_mcp_refresh) => CloudCatalogRefreshOutcome { + result: Ok(()), + needs_mcp_refresh, + }, + Err(failure) => CloudCatalogRefreshOutcome { + result: Err(failure.error), + needs_mcp_refresh: failure.catalog_cleared, + }, + } }) } } diff --git a/codex-rs/ext/mcp/src/cloud_plugin_tests.rs b/codex-rs/ext/mcp/src/cloud_plugin_tests.rs index 722faf8e2c..ce2cb3e269 100644 --- a/codex-rs/ext/mcp/src/cloud_plugin_tests.rs +++ b/codex-rs/ext/mcp/src/cloud_plugin_tests.rs @@ -79,13 +79,13 @@ async fn cloud_source_policy_skips_startup_discovery() -> anyhow::Result<()> { thread_store: &thread_store, }) .await; - contributor + let outcome = contributor .refresh_cloud_catalog(CloudCatalogRefreshInput { session_store: &session_store, thread_store: &thread_store, }) - .await - .expect("disabled discovery succeeds"); + .await; + assert_eq!(outcome, CloudCatalogRefreshOutcome::UNCHANGED); assert_eq!(provider.calls.load(Ordering::SeqCst), 0); Ok(()) } @@ -431,13 +431,40 @@ async fn explicit_cloud_refresh_replaces_cached_catalog() { }; provider.set_reply(Ok(refreshed_catalog.clone())); assert!(!contributor.refresh_catalog_for_step(input).await); - contributor - .refresh_cloud_catalog(input.into()) - .await - .unwrap(); - assert_eq!(state.cloud_catalog(), Some(refreshed_catalog)); + let outcome = contributor.refresh_cloud_catalog(input.into()).await; + assert_eq!( + outcome, + CloudCatalogRefreshOutcome { + result: Ok(()), + needs_mcp_refresh: true + } + ); + assert_eq!(state.cloud_catalog(), Some(refreshed_catalog.clone())); assert!(!contributor.refresh_catalog_for_step(input).await); assert_eq!(provider.calls.load(Ordering::SeqCst), 2); + + provider.set_reply(Err("discovery failed".to_string())); + assert_eq!( + contributor.refresh_cloud_catalog(input.into()).await, + CloudCatalogRefreshOutcome { + result: Err("discovery failed".to_string()), + needs_mcp_refresh: false, + } + ); + assert_eq!(state.cloud_catalog(), Some(refreshed_catalog)); + + // Switching auth scopes clears the old catalog before the failed lookup. + session.insert(McpResourceClient::new(Arc::new( + codex_mcp::McpRuntime::empty(/*prefix_mcp_tool_names*/ false), + ))); + assert_eq!( + contributor.refresh_cloud_catalog(input.into()).await, + CloudCatalogRefreshOutcome { + result: Err("discovery failed".to_string()), + needs_mcp_refresh: true, + } + ); + assert_eq!(state.cloud_catalog(), None); } struct CloudProvider { diff --git a/codex-rs/ext/skills/src/cloud_skill.rs b/codex-rs/ext/skills/src/cloud_skill.rs index 563c5c13e2..19effdd93d 100644 --- a/codex-rs/ext/skills/src/cloud_skill.rs +++ b/codex-rs/ext/skills/src/cloud_skill.rs @@ -2,6 +2,7 @@ use codex_extension_api::CatalogRefreshInput as StepCatalogRefreshInput; use codex_extension_api::CloudCatalogRefreshInput; +use codex_extension_api::CloudCatalogRefreshOutcome; use codex_extension_api::ExtensionFuture; use codex_extension_api::TurnLifecycleContributor; use std::sync::Arc; @@ -68,15 +69,19 @@ impl TurnLifecycleContributor for SkillsExtension { fn refresh_cloud_catalog<'a>( &'a self, input: CloudCatalogRefreshInput<'a>, - ) -> ExtensionFuture<'a, Result<(), String>> { + ) -> ExtensionFuture<'a, CloudCatalogRefreshOutcome> { Box::pin(async move { let Some((state, query)) = cloud_catalog_query(input, /*turn_id*/ None) else { - return Ok(()); + return CloudCatalogRefreshOutcome::UNCHANGED; }; - state + let result = state .refresh_cloud_catalog(&self.providers, query) .await - .map_err(|error| error.to_string()) + .map_err(|error| error.to_string()); + CloudCatalogRefreshOutcome { + result, + needs_mcp_refresh: false, + } }) } } diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index 3cc7c5287a..9e02aaac54 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -2713,18 +2713,33 @@ async fn cloud_discovery_is_gated_and_reports_failures() -> TestResult { refresh_registered_catalogs_for_step(®istry, &session_store, &thread_store, "turn-3") .await; assert_eq!(list_calls.load(Ordering::Relaxed), 2); - for contributor in registry.turn_lifecycle_contributors() { - contributor - .refresh_cloud_catalog(codex_extension_api::CloudCatalogRefreshInput { - session_store: &session_store, - thread_store: &thread_store, - }) - .await - .map_err(|error| format!("partial catalogs should be accepted: {error}"))?; + let input = codex_extension_api::CloudCatalogRefreshInput { + session_store: &session_store, + thread_store: &thread_store, + }; + let contributors = registry.turn_lifecycle_contributors(); + for contributor in contributors { + assert_eq!( + contributor.refresh_cloud_catalog(input).await, + codex_extension_api::CloudCatalogRefreshOutcome::UNCHANGED + ); } refresh_registered_catalogs_for_step(®istry, &session_store, &thread_store, "turn-4") .await; assert_eq!(list_calls.load(Ordering::Relaxed), 3); + + // Re-arm the provider's first-call failure for an explicit refresh. + list_calls.store(0, Ordering::Relaxed); + let mut errors = Vec::new(); + for contributor in contributors { + let outcome = contributor.refresh_cloud_catalog(input).await; + assert!(!outcome.needs_mcp_refresh); + errors.extend(outcome.result.err()); + } + assert_eq!( + errors, + ["cloud skills unavailable: temporary cloud failure"] + ); } Ok(()) } diff --git a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs index 3688df894e..c731e4c8af 100644 --- a/codex-rs/rmcp-client/src/bin/test_stdio_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_stdio_server.rs @@ -1168,6 +1168,14 @@ async fn main() -> Result<(), Box> { } eprintln!("starting rmcp test server"); + if let Some(path) = std::env::var_os("MCP_TEST_PROCESS_START_LOG") { + use std::io::Write; + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + writeln!(log, "{}", std::process::id())?; + } if let Ok(pid_file) = std::env::var("MCP_TEST_PID_FILE") { std::fs::write(pid_file, std::process::id().to_string())?; }