mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Prewarm MCP servers during cloud catalog refresh (#62876)
## Why A slow cloud skill lookup can delay MCP startup even after the plugin catalog is ready. If the refresh is canceled while waiting for skills, it can miss the chance to prewarm those servers before the first turn. ## What changed Queue MCP prewarm as soon as plugin discovery changes or clears the catalog. Skill discovery alone leaves the MCP server set unchanged. At the end of the refresh, queue a retry for eligible MCP startup failures. If discovery is canceled, check for those failures the next time the catalog lock is acquired. Reuse pending handshakes and leave authentication failures and Apps-managed reconnects to their existing recovery paths. ## Testing Add regression tests for first-turn tool availability after cancellation, startup recovery with cloud plugins disabled, and a single retry for a persistent startup failure. GitOrigin-RevId: f92162d1629358953d8810d20244e55f07416a01
This commit is contained in:
@@ -1622,7 +1622,7 @@ impl CodexThread {
|
||||
}
|
||||
|
||||
/// Refreshes cloud plugin and skill catalogs before input admission, without starting a turn.
|
||||
/// Serializes publication with step preparation and marks MCP dirty for later reconciliation.
|
||||
/// Serializes publication with step preparation and queues MCP prewarm as plugins are published.
|
||||
/// Accepts complete or partial catalogs; failed discovery remains retryable.
|
||||
pub async fn refresh_cloud_plugins(&self) -> anyhow::Result<()> {
|
||||
self.session.refresh_cloud_plugins().await
|
||||
|
||||
@@ -11,15 +11,32 @@ use tokio::sync::Mutex;
|
||||
#[derive(Default)]
|
||||
pub(super) struct CloudCatalogPreparation {
|
||||
/// Serializes catalog refreshes and snapshot capture.
|
||||
refresh_lock: Mutex<()>,
|
||||
refresh_lock: Mutex<CatalogRefreshState>,
|
||||
/// Requests resource invalidation on the next lock acquisition.
|
||||
/// Survives cancellation while waiting.
|
||||
refresh_pending: AtomicBool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct CatalogRefreshState {
|
||||
// Leave the check for the next catalog lock holder if discovery is cancelled.
|
||||
startup_retry_pending: bool,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) async fn lock_cloud_catalogs(&self) -> tokio::sync::MutexGuard<'_, ()> {
|
||||
let guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
|
||||
fn finish_catalog_startup(&self, catalogs: &mut CatalogRefreshState) {
|
||||
if std::mem::take(&mut catalogs.startup_retry_pending)
|
||||
&& self.services.mcp_runtime.has_failed_startup_for_retry()
|
||||
{
|
||||
self.request_mcp_runtime_reprojection(McpRefreshReason::ServerStartupFailed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn lock_cloud_catalogs(
|
||||
&self,
|
||||
) -> tokio::sync::MutexGuard<'_, CatalogRefreshState> {
|
||||
let mut guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
|
||||
self.finish_catalog_startup(&mut guard);
|
||||
if self
|
||||
.cloud_catalog_preparation
|
||||
.refresh_pending
|
||||
@@ -38,7 +55,8 @@ impl Session {
|
||||
self.cloud_catalog_preparation
|
||||
.refresh_pending
|
||||
.store(true, Ordering::Release);
|
||||
let _catalogs = self.lock_cloud_catalogs().await;
|
||||
let mut catalogs = self.lock_cloud_catalogs().await;
|
||||
catalogs.startup_retry_pending = true;
|
||||
self.refresh_mcp_if_dirty().await;
|
||||
let results = futures::future::join_all(
|
||||
self.services
|
||||
@@ -46,20 +64,24 @@ impl Session {
|
||||
.turn_lifecycle_contributors()
|
||||
.iter()
|
||||
.map(|contributor| async move {
|
||||
let result = contributor
|
||||
let outcome = contributor
|
||||
.refresh_cloud_catalog(CloudCatalogRefreshInput {
|
||||
session_store: &self.services.session_extension_data,
|
||||
thread_store: &self.services.thread_extension_data,
|
||||
})
|
||||
.await;
|
||||
// Retain publication even if another provider is still pending at cancellation.
|
||||
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
|
||||
result
|
||||
// Queue the work before a slower sibling can time out the whole refresh.
|
||||
if outcome.needs_mcp_refresh {
|
||||
self.request_mcp_runtime_reprojection(
|
||||
McpRefreshReason::ExtensionCatalogChanged,
|
||||
);
|
||||
}
|
||||
outcome.result
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
drop(_catalogs);
|
||||
self.schedule_mcp_prewarm();
|
||||
self.finish_catalog_startup(&mut catalogs);
|
||||
drop(catalogs);
|
||||
let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect();
|
||||
anyhow::ensure!(errors.is_empty(), "{}", errors.join("; "));
|
||||
Ok(())
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::McpRefreshReason;
|
||||
use codex_extension_api::CloudCatalogRefreshOutcome;
|
||||
use codex_mcp::McpResourceClient;
|
||||
use codex_mcp::McpResourceClientCacheKey;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
@@ -140,24 +141,24 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_releases_lock_on_cancel
|
||||
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
|
||||
}
|
||||
|
||||
/// Explicit refresh overlaps providers under one lock and retains publication while a sibling waits.
|
||||
/// A published catalog queues MCP prewarm while a sibling is still holding up the refresh.
|
||||
#[tokio::test]
|
||||
async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
|
||||
struct CatalogGate {
|
||||
entered: Arc<tokio::sync::Barrier>,
|
||||
finish: tokio::sync::Notify,
|
||||
result: Result<(), &'static str>,
|
||||
outcome: CloudCatalogRefreshOutcome,
|
||||
}
|
||||
|
||||
impl codex_extension_api::TurnLifecycleContributor for CatalogGate {
|
||||
fn refresh_cloud_catalog<'a>(
|
||||
&'a self,
|
||||
_input: codex_extension_api::CloudCatalogRefreshInput<'a>,
|
||||
) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> {
|
||||
) -> codex_extension_api::ExtensionFuture<'a, CloudCatalogRefreshOutcome> {
|
||||
Box::pin(async move {
|
||||
self.entered.wait().await;
|
||||
self.finish.notified().await;
|
||||
self.result.map_err(str::to_string)
|
||||
self.outcome.clone()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -167,17 +168,25 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
|
||||
let failing = Arc::new(CatalogGate {
|
||||
entered: Arc::clone(&entered),
|
||||
finish: Default::default(),
|
||||
result: Err("plugin discovery failed"),
|
||||
outcome: CloudCatalogRefreshOutcome {
|
||||
result: Err("skill discovery failed".to_string()),
|
||||
needs_mcp_refresh: false,
|
||||
},
|
||||
});
|
||||
let succeeding = Arc::new(CatalogGate {
|
||||
entered: Arc::clone(&entered),
|
||||
finish: Default::default(),
|
||||
result: Ok(()),
|
||||
outcome: CloudCatalogRefreshOutcome {
|
||||
result: Ok(()),
|
||||
needs_mcp_refresh: true,
|
||||
},
|
||||
});
|
||||
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
builder.turn_lifecycle_contributor(failing.clone());
|
||||
builder.turn_lifecycle_contributor(succeeding.clone());
|
||||
session.services.extensions = Arc::new(builder.build());
|
||||
let (tx, prewarm_requests) = async_channel::bounded(1);
|
||||
session.mcp_prewarm_tx = tx;
|
||||
let session = Arc::new(session);
|
||||
let mut refresh = Box::pin(session.refresh_cloud_plugins());
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
@@ -194,14 +203,19 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
|
||||
assert!(futures::poll!(&mut lock).is_pending());
|
||||
drop(lock);
|
||||
|
||||
failing.finish.notify_one();
|
||||
assert!(prewarm_requests.is_empty());
|
||||
succeeding.finish.notify_one();
|
||||
assert!(futures::poll!(&mut refresh).is_pending());
|
||||
assert!(session.mcp_refresh.is_pending());
|
||||
succeeding.finish.notify_one();
|
||||
prewarm_requests
|
||||
.try_recv()
|
||||
.expect("the worker must be notified before the remaining provider finishes");
|
||||
failing.finish.notify_one();
|
||||
assert_eq!(
|
||||
refresh.await.expect_err("one provider failed").to_string(),
|
||||
"plugin discovery failed"
|
||||
"skill discovery failed"
|
||||
);
|
||||
assert!(prewarm_requests.is_empty());
|
||||
drop(
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
|
||||
@@ -315,6 +315,7 @@ pub enum McpRefreshReason {
|
||||
RemotePluginUninstalled,
|
||||
RuntimeAuthChanged,
|
||||
SelectedCapabilityRootsChanged,
|
||||
ServerStartupFailed,
|
||||
SessionMcpInputsChanged,
|
||||
SessionPermissionsGranted,
|
||||
SkillMcpDependenciesAdded,
|
||||
|
||||
Reference in New Issue
Block a user