Add explicit cloud catalog refresh before turn admission (#61449)

## Why

Hosts need to refresh cloud plugin and skill catalogs without starting a turn or waiting for step preparation.

## What changed

- Add `CodexThread::refresh_cloud_plugins()` and a turn-independent extension callback that bypass cached catalog reuse and failure cooldowns.
- Run catalog contributors concurrently while serializing publication with step snapshot capture. Invalidate MCP resource caches before explicit discovery and schedule MCP reconciliation afterward.
- Accept complete and partial catalogs, report discovery errors, and keep failed or cancelled refreshes retryable.
- Allow hosts to gate cloud plugin discovery through effective configuration.

## Testing

Add coverage for discovery before turn admission, catalog replacement and reuse, immediate retry after failure, concurrent contributors, and cancellation-safe publication and invalidation.

GitOrigin-RevId: 83e847fcab6562581af2c9e831b1d2a310f2b270
This commit is contained in:
Anthony Tafoya
2026-11-14 02:45:51 +00:00
committed by copyberry
parent 32cec881b7
commit d26406987e
32 changed files with 953 additions and 247 deletions
+7
View File
@@ -1392,6 +1392,13 @@ impl CodexThread {
self.session.refresh_codex_apps_tools().await
}
/// Refreshes cloud plugin and skill catalogs before input admission, without starting a turn.
/// Serializes publication with step preparation and marks MCP dirty for later reconciliation.
/// Accepts complete or partial catalogs; failed discovery remains retryable.
pub async fn refresh_cloud_plugins(&self) -> anyhow::Result<()> {
self.session.refresh_cloud_plugins().await
}
/// Returns the environments configured for future turns.
pub async fn environment_selections(&self) -> Vec<TurnEnvironmentSelection> {
self.session.configured_environment_selections().await
@@ -0,0 +1,67 @@
//! Refreshes independent cloud catalogs concurrently, serialized with step preparation.
use super::Session;
use crate::McpRefreshReason;
use codex_extension_api::CloudCatalogRefreshInput;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use tokio::sync::Mutex;
#[derive(Default)]
pub(super) struct CloudCatalogPreparation {
/// Serializes catalog refreshes and snapshot capture.
refresh_lock: Mutex<()>,
/// Requests resource invalidation on the next lock acquisition.
/// Survives cancellation while waiting.
refresh_pending: AtomicBool,
}
impl Session {
pub(crate) async fn lock_cloud_catalogs(&self) -> tokio::sync::MutexGuard<'_, ()> {
let guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
if self
.cloud_catalog_preparation
.refresh_pending
.swap(false, Ordering::AcqRel)
{
self.services.mcp_runtime.invalidate_resource_caches();
}
guard
}
#[expect(
clippy::await_holding_invalid_type,
reason = "catalog publication must remain serialized with step preparation"
)]
pub(crate) async fn refresh_cloud_plugins(self: &Arc<Self>) -> anyhow::Result<()> {
self.cloud_catalog_preparation
.refresh_pending
.store(true, Ordering::Release);
let _catalogs = self.lock_cloud_catalogs().await;
self.refresh_mcp_if_dirty().await;
let results = futures::future::join_all(
self.services
.extensions
.turn_lifecycle_contributors()
.iter()
.map(|contributor| async move {
let result = contributor
.refresh_cloud_catalog(CloudCatalogRefreshInput {
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
})
.await;
// Retain publication even if another provider is still pending at cancellation.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
result
}),
)
.await;
drop(_catalogs);
self.schedule_mcp_prewarm();
let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect();
anyhow::ensure!(errors.is_empty(), "{}", errors.join("; "));
Ok(())
}
}
@@ -194,9 +194,6 @@ impl Session {
}
state.session_configuration.original_config_do_not_use = Arc::clone(&config);
drop(policy_guard);
if matches!(scope, RuntimeConfigRefresh::Mcp) {
self.services.mcp_runtime.invalidate_resource_caches();
}
self.mark_mcp_runtime_dirty(match scope {
RuntimeConfigRefresh::User => McpRefreshReason::UserConfigReloaded,
RuntimeConfigRefresh::Mcp => McpRefreshReason::McpConfigReloaded,
+21 -14
View File
@@ -482,20 +482,27 @@ impl Session {
};
if is_active_regular_turn {
// Refresh cloud catalogs without holding the MCP permit or active-turn lock.
for contributor in self.services.extensions.turn_lifecycle_contributors() {
if contributor
.refresh_catalog_for_step(codex_extension_api::CatalogRefreshInput {
turn_id: &turn_context.sub_id,
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await
{
// Preserve MCP updates if a later refresh is cancelled.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
}
}
futures::future::join_all(
self.services
.extensions
.turn_lifecycle_contributors()
.iter()
.map(|contributor| async move {
if contributor
.refresh_catalog_for_step(codex_extension_api::CatalogRefreshInput {
turn_id: &turn_context.sub_id,
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await
{
// Preserve each publication if the refresh batch is cancelled.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
}
}),
)
.await;
self.refresh_mcp_if_dirty().await;
}
self.wait_for_apps_tool_refresh().await;
+4 -1
View File
@@ -266,6 +266,7 @@ mod reasoning_effort;
mod submission;
pub(crate) use reasoning_effort::RequestEffortUsage;
pub(crate) use submission::Submission;
mod cloud_plugins;
mod input_queue;
mod mcp;
mod mcp_prewarm;
@@ -4325,7 +4326,7 @@ impl Session {
#[expect(
clippy::await_holding_invalid_type,
reason = "step settings and both environment snapshots must be captured under active_turn then state"
reason = "step settings and environments require active_turn then state; cloud snapshots require the catalog publication lock"
)]
#[tracing::instrument(name = "step_context.capture", level = "info", skip_all)]
async fn capture_step_context_inner(
@@ -4495,6 +4496,7 @@ impl Session {
{
required_servers.push(codex_mcp::CODEX_APPS_MCP_SERVER_NAME.to_owned());
}
let catalogs = self.lock_cloud_catalogs().await;
let (mcp, prepared_recommendations) = tokio::join!(
// MCP refresh can be large; keep it off the sampling request's stack.
Box::pin(self.mcp_runtime_for_step(
@@ -4615,6 +4617,7 @@ impl Session {
.or_cancel(cancellation_token)
.await?;
}
drop(catalogs);
extension_data.insert(selected_plugins);
let tool_router = turn::built_tools(
self.as_ref(),
+3
View File
@@ -99,6 +99,8 @@ pub(crate) struct Session {
pub(super) multi_agent_version: OnceLock<MultiAgentVersion>,
/// Owns invalidation and serializes refreshes without blocking captured calls.
pub(super) mcp_refresh: McpRefresh,
/// Serializes cloud publication with capture of a step's plugin and skill snapshots.
pub(super) cloud_catalog_preparation: super::cloud_plugins::CloudCatalogPreparation,
/// Non-owning lookup for approval data retained by running MCP invocations.
pub(crate) mcp_tool_approval_metadata: std::sync::Mutex<McpToolApprovalMetadataMap>,
pub(super) mcp_elicitation_reviewer_handle: OnceLock<codex_mcp::ElicitationReviewerHandle>,
@@ -1981,6 +1983,7 @@ impl Session {
windows_sandbox_proxy_settings_mode,
multi_agent_version,
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
+2
View File
@@ -7044,6 +7044,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
@@ -9391,6 +9392,7 @@ where
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
@@ -1,4 +1,4 @@
//! Step discovery observes ready MCP connections; prewarm skips discovery and cancellation releases preparation.
//! Concurrent catalog discovery retains publication and releases preparation on cancellation.
use super::*;
use crate::McpRefreshReason;
@@ -8,6 +8,7 @@ use pretty_assertions::assert_eq;
struct Recorder {
client: McpResourceClient,
ready: Arc<tokio::sync::Barrier>,
pause: std::sync::atomic::AtomicBool,
keys: std::sync::Mutex<Vec<McpResourceClientCacheKey>>,
discovered: tokio::sync::Notify,
@@ -23,6 +24,7 @@ impl codex_extension_api::TurnLifecycleContributor for Recorder {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(self.client.cache_key());
self.ready.wait().await;
self.discovered.notify_one();
if self.pause.swap(false, std::sync::atomic::Ordering::Relaxed) {
std::future::pending::<()>().await;
@@ -32,17 +34,27 @@ impl codex_extension_api::TurnLifecycleContributor for Recorder {
}
}
/// Step discovery overlaps providers on ready connections; cancellation releases the catalog lock.
#[tokio::test]
async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
async fn step_discovery_reconciles_mcp_skips_prewarm_and_releases_lock_on_cancellation() {
let (mut session, turn_context) = make_session_and_context().await;
let ready = Arc::new(tokio::sync::Barrier::new(2));
let recorder = Arc::new(Recorder {
client: McpResourceClient::new(Arc::clone(&session.services.mcp_runtime)),
ready: Arc::clone(&ready),
pause: std::sync::atomic::AtomicBool::new(true),
keys: Default::default(),
discovered: Default::default(),
});
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_lifecycle_contributor(recorder.clone());
builder.turn_lifecycle_contributor(Arc::new(Recorder {
client: recorder.client.clone(),
ready,
pause: std::sync::atomic::AtomicBool::new(false),
keys: Default::default(),
discovered: Default::default(),
}));
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
@@ -95,7 +107,7 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
session.capture_step_context(Arc::clone(&turn_context), &CancellationToken::new()),
)
.await
.expect("cancelled discovery must allow the next step")
.expect("cancelled discovery must release the catalog lock")
.expect("next step should prepare");
assert!(recorder.client.cache_key() == prepared_key);
@@ -112,3 +124,75 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
assert!(keys[2] == recorder.client.cache_key());
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
}
/// Explicit refresh overlaps providers under one lock and retains publication while a sibling waits.
#[tokio::test]
async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
struct CatalogGate {
entered: Arc<tokio::sync::Barrier>,
finish: tokio::sync::Notify,
result: Result<(), &'static str>,
}
impl codex_extension_api::TurnLifecycleContributor for CatalogGate {
fn refresh_cloud_catalog<'a>(
&'a self,
_input: codex_extension_api::CloudCatalogRefreshInput<'a>,
) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> {
Box::pin(async move {
self.entered.wait().await;
self.finish.notified().await;
self.result.map_err(str::to_string)
})
}
}
let (mut session, _) = make_session_and_context().await;
let entered = Arc::new(tokio::sync::Barrier::new(3));
let failing = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Err("plugin discovery failed"),
});
let succeeding = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Ok(()),
});
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_lifecycle_contributor(failing.clone());
builder.turn_lifecycle_contributor(succeeding.clone());
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let mut refresh = Box::pin(session.refresh_cloud_plugins());
tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::select! {
result = &mut refresh => panic!("refresh completed before both providers started: {result:?}"),
_ = entered.wait() => {}
}
})
.await
.expect("both providers must start before either finishes");
assert!(!session.mcp_refresh.is_pending());
let mut lock = Box::pin(session.lock_cloud_catalogs());
assert!(futures::poll!(&mut lock).is_pending());
drop(lock);
failing.finish.notify_one();
assert!(futures::poll!(&mut refresh).is_pending());
assert!(session.mcp_refresh.is_pending());
succeeding.finish.notify_one();
assert_eq!(
refresh.await.expect_err("one provider failed").to_string(),
"plugin discovery failed"
);
drop(
tokio::time::timeout(
std::time::Duration::from_secs(5),
session.lock_cloud_catalogs(),
)
.await
.expect("completed refresh must release the catalog lock"),
);
}