Prewarm MCP servers during cloud catalog refresh (#62876)

## Why

A slow cloud skill lookup can delay MCP startup even after the plugin
catalog is ready. If the refresh is canceled while waiting for skills,
it can miss the chance to prewarm those servers before the first turn.

## What changed

Queue MCP prewarm as soon as plugin discovery changes or clears the
catalog. Skill discovery alone leaves the MCP server set unchanged.

At the end of the refresh, queue a retry for eligible MCP startup
failures. If discovery is canceled, check for those failures the next
time the catalog lock is acquired. Reuse pending handshakes and leave
authentication failures and Apps-managed reconnects to their existing
recovery paths.

## Testing

Add regression tests for first-turn tool availability after cancellation,
startup recovery with cloud plugins disabled, and a single retry for a
persistent startup failure.

GitOrigin-RevId: f92162d1629358953d8810d20244e55f07416a01
This commit is contained in:
Jay Chen
2026-11-21 00:54:05 +00:00
committed by copyberry
parent 0bfb8467da
commit 66618f0512
16 changed files with 564 additions and 57 deletions
+1 -1
View File
@@ -1622,7 +1622,7 @@ impl CodexThread {
}
/// Refreshes cloud plugin and skill catalogs before input admission, without starting a turn.
/// Serializes publication with step preparation and marks MCP dirty for later reconciliation.
/// Serializes publication with step preparation and queues MCP prewarm as plugins are published.
/// Accepts complete or partial catalogs; failed discovery remains retryable.
pub async fn refresh_cloud_plugins(&self) -> anyhow::Result<()> {
self.session.refresh_cloud_plugins().await
+32 -10
View File
@@ -11,15 +11,32 @@ use tokio::sync::Mutex;
#[derive(Default)]
pub(super) struct CloudCatalogPreparation {
/// Serializes catalog refreshes and snapshot capture.
refresh_lock: Mutex<()>,
refresh_lock: Mutex<CatalogRefreshState>,
/// Requests resource invalidation on the next lock acquisition.
/// Survives cancellation while waiting.
refresh_pending: AtomicBool,
}
#[derive(Default)]
pub(crate) struct CatalogRefreshState {
// Leave the check for the next catalog lock holder if discovery is cancelled.
startup_retry_pending: bool,
}
impl Session {
pub(crate) async fn lock_cloud_catalogs(&self) -> tokio::sync::MutexGuard<'_, ()> {
let guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
fn finish_catalog_startup(&self, catalogs: &mut CatalogRefreshState) {
if std::mem::take(&mut catalogs.startup_retry_pending)
&& self.services.mcp_runtime.has_failed_startup_for_retry()
{
self.request_mcp_runtime_reprojection(McpRefreshReason::ServerStartupFailed);
}
}
pub(crate) async fn lock_cloud_catalogs(
&self,
) -> tokio::sync::MutexGuard<'_, CatalogRefreshState> {
let mut guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
self.finish_catalog_startup(&mut guard);
if self
.cloud_catalog_preparation
.refresh_pending
@@ -38,7 +55,8 @@ impl Session {
self.cloud_catalog_preparation
.refresh_pending
.store(true, Ordering::Release);
let _catalogs = self.lock_cloud_catalogs().await;
let mut catalogs = self.lock_cloud_catalogs().await;
catalogs.startup_retry_pending = true;
self.refresh_mcp_if_dirty().await;
let results = futures::future::join_all(
self.services
@@ -46,20 +64,24 @@ impl Session {
.turn_lifecycle_contributors()
.iter()
.map(|contributor| async move {
let result = contributor
let outcome = contributor
.refresh_cloud_catalog(CloudCatalogRefreshInput {
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
})
.await;
// Retain publication even if another provider is still pending at cancellation.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
result
// Queue the work before a slower sibling can time out the whole refresh.
if outcome.needs_mcp_refresh {
self.request_mcp_runtime_reprojection(
McpRefreshReason::ExtensionCatalogChanged,
);
}
outcome.result
}),
)
.await;
drop(_catalogs);
self.schedule_mcp_prewarm();
self.finish_catalog_startup(&mut catalogs);
drop(catalogs);
let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect();
anyhow::ensure!(errors.is_empty(), "{}", errors.join("; "));
Ok(())
@@ -2,6 +2,7 @@
use super::*;
use crate::McpRefreshReason;
use codex_extension_api::CloudCatalogRefreshOutcome;
use codex_mcp::McpResourceClient;
use codex_mcp::McpResourceClientCacheKey;
use codex_protocol::models::ResponseInputItem;
@@ -140,24 +141,24 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_releases_lock_on_cancel
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
}
/// Explicit refresh overlaps providers under one lock and retains publication while a sibling waits.
/// A published catalog queues MCP prewarm while a sibling is still holding up the refresh.
#[tokio::test]
async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
struct CatalogGate {
entered: Arc<tokio::sync::Barrier>,
finish: tokio::sync::Notify,
result: Result<(), &'static str>,
outcome: CloudCatalogRefreshOutcome,
}
impl codex_extension_api::TurnLifecycleContributor for CatalogGate {
fn refresh_cloud_catalog<'a>(
&'a self,
_input: codex_extension_api::CloudCatalogRefreshInput<'a>,
) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> {
) -> codex_extension_api::ExtensionFuture<'a, CloudCatalogRefreshOutcome> {
Box::pin(async move {
self.entered.wait().await;
self.finish.notified().await;
self.result.map_err(str::to_string)
self.outcome.clone()
})
}
}
@@ -167,17 +168,25 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
let failing = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Err("plugin discovery failed"),
outcome: CloudCatalogRefreshOutcome {
result: Err("skill discovery failed".to_string()),
needs_mcp_refresh: false,
},
});
let succeeding = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Ok(()),
outcome: CloudCatalogRefreshOutcome {
result: Ok(()),
needs_mcp_refresh: true,
},
});
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_lifecycle_contributor(failing.clone());
builder.turn_lifecycle_contributor(succeeding.clone());
session.services.extensions = Arc::new(builder.build());
let (tx, prewarm_requests) = async_channel::bounded(1);
session.mcp_prewarm_tx = tx;
let session = Arc::new(session);
let mut refresh = Box::pin(session.refresh_cloud_plugins());
tokio::time::timeout(std::time::Duration::from_secs(5), async {
@@ -194,14 +203,19 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
assert!(futures::poll!(&mut lock).is_pending());
drop(lock);
failing.finish.notify_one();
assert!(prewarm_requests.is_empty());
succeeding.finish.notify_one();
assert!(futures::poll!(&mut refresh).is_pending());
assert!(session.mcp_refresh.is_pending());
succeeding.finish.notify_one();
prewarm_requests
.try_recv()
.expect("the worker must be notified before the remaining provider finishes");
failing.finish.notify_one();
assert_eq!(
refresh.await.expect_err("one provider failed").to_string(),
"plugin discovery failed"
"skill discovery failed"
);
assert!(prewarm_requests.is_empty());
drop(
tokio::time::timeout(
std::time::Duration::from_secs(5),
+1
View File
@@ -315,6 +315,7 @@ pub enum McpRefreshReason {
RemotePluginUninstalled,
RuntimeAuthChanged,
SelectedCapabilityRootsChanged,
ServerStartupFailed,
SessionMcpInputsChanged,
SessionPermissionsGranted,
SkillMcpDependenciesAdded,