Add explicit cloud catalog refresh before turn admission (#61449)

## Why

Hosts need to refresh cloud plugin and skill catalogs without starting a turn or waiting for step preparation.

## What changed

- Add `CodexThread::refresh_cloud_plugins()` and a turn-independent extension callback that bypass cached catalog reuse and failure cooldowns.
- Run catalog contributors concurrently while serializing publication with step snapshot capture. Invalidate MCP resource caches before explicit discovery and schedule MCP reconciliation afterward.
- Accept complete and partial catalogs, report discovery errors, and keep failed or cancelled refreshes retryable.
- Allow hosts to gate cloud plugin discovery through effective configuration.

## Testing

Add coverage for discovery before turn admission, catalog replacement and reuse, immediate retry after failure, concurrent contributors, and cancellation-safe publication and invalidation.

GitOrigin-RevId: 83e847fcab6562581af2c9e831b1d2a310f2b270
This commit is contained in:
Anthony Tafoya
2026-11-14 02:45:51 +00:00
committed by copyberry
parent 32cec881b7
commit d26406987e
32 changed files with 953 additions and 247 deletions
+7
View File
@@ -1392,6 +1392,13 @@ impl CodexThread {
self.session.refresh_codex_apps_tools().await
}
/// Refreshes cloud plugin and skill catalogs before input admission, without starting a turn.
/// Serializes publication with step preparation and marks MCP dirty for later reconciliation.
/// Accepts complete or partial catalogs; failed discovery remains retryable.
pub async fn refresh_cloud_plugins(&self) -> anyhow::Result<()> {
self.session.refresh_cloud_plugins().await
}
/// Returns the environments configured for future turns.
pub async fn environment_selections(&self) -> Vec<TurnEnvironmentSelection> {
self.session.configured_environment_selections().await
@@ -0,0 +1,67 @@
//! Refreshes independent cloud catalogs concurrently, serialized with step preparation.
use super::Session;
use crate::McpRefreshReason;
use codex_extension_api::CloudCatalogRefreshInput;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use tokio::sync::Mutex;
#[derive(Default)]
pub(super) struct CloudCatalogPreparation {
/// Serializes catalog refreshes and snapshot capture.
refresh_lock: Mutex<()>,
/// Requests resource invalidation on the next lock acquisition.
/// Survives cancellation while waiting.
refresh_pending: AtomicBool,
}
impl Session {
pub(crate) async fn lock_cloud_catalogs(&self) -> tokio::sync::MutexGuard<'_, ()> {
let guard = self.cloud_catalog_preparation.refresh_lock.lock().await;
if self
.cloud_catalog_preparation
.refresh_pending
.swap(false, Ordering::AcqRel)
{
self.services.mcp_runtime.invalidate_resource_caches();
}
guard
}
#[expect(
clippy::await_holding_invalid_type,
reason = "catalog publication must remain serialized with step preparation"
)]
pub(crate) async fn refresh_cloud_plugins(self: &Arc<Self>) -> anyhow::Result<()> {
self.cloud_catalog_preparation
.refresh_pending
.store(true, Ordering::Release);
let _catalogs = self.lock_cloud_catalogs().await;
self.refresh_mcp_if_dirty().await;
let results = futures::future::join_all(
self.services
.extensions
.turn_lifecycle_contributors()
.iter()
.map(|contributor| async move {
let result = contributor
.refresh_cloud_catalog(CloudCatalogRefreshInput {
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
})
.await;
// Retain publication even if another provider is still pending at cancellation.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
result
}),
)
.await;
drop(_catalogs);
self.schedule_mcp_prewarm();
let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect();
anyhow::ensure!(errors.is_empty(), "{}", errors.join("; "));
Ok(())
}
}
@@ -194,9 +194,6 @@ impl Session {
}
state.session_configuration.original_config_do_not_use = Arc::clone(&config);
drop(policy_guard);
if matches!(scope, RuntimeConfigRefresh::Mcp) {
self.services.mcp_runtime.invalidate_resource_caches();
}
self.mark_mcp_runtime_dirty(match scope {
RuntimeConfigRefresh::User => McpRefreshReason::UserConfigReloaded,
RuntimeConfigRefresh::Mcp => McpRefreshReason::McpConfigReloaded,
+21 -14
View File
@@ -482,20 +482,27 @@ impl Session {
};
if is_active_regular_turn {
// Refresh cloud catalogs without holding the MCP permit or active-turn lock.
for contributor in self.services.extensions.turn_lifecycle_contributors() {
if contributor
.refresh_catalog_for_step(codex_extension_api::CatalogRefreshInput {
turn_id: &turn_context.sub_id,
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await
{
// Preserve MCP updates if a later refresh is cancelled.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
}
}
futures::future::join_all(
self.services
.extensions
.turn_lifecycle_contributors()
.iter()
.map(|contributor| async move {
if contributor
.refresh_catalog_for_step(codex_extension_api::CatalogRefreshInput {
turn_id: &turn_context.sub_id,
session_store: &self.services.session_extension_data,
thread_store: &self.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await
{
// Preserve each publication if the refresh batch is cancelled.
self.mark_mcp_runtime_dirty(McpRefreshReason::ExtensionCatalogChanged);
}
}),
)
.await;
self.refresh_mcp_if_dirty().await;
}
self.wait_for_apps_tool_refresh().await;
+4 -1
View File
@@ -266,6 +266,7 @@ mod reasoning_effort;
mod submission;
pub(crate) use reasoning_effort::RequestEffortUsage;
pub(crate) use submission::Submission;
mod cloud_plugins;
mod input_queue;
mod mcp;
mod mcp_prewarm;
@@ -4325,7 +4326,7 @@ impl Session {
#[expect(
clippy::await_holding_invalid_type,
reason = "step settings and both environment snapshots must be captured under active_turn then state"
reason = "step settings and environments require active_turn then state; cloud snapshots require the catalog publication lock"
)]
#[tracing::instrument(name = "step_context.capture", level = "info", skip_all)]
async fn capture_step_context_inner(
@@ -4495,6 +4496,7 @@ impl Session {
{
required_servers.push(codex_mcp::CODEX_APPS_MCP_SERVER_NAME.to_owned());
}
let catalogs = self.lock_cloud_catalogs().await;
let (mcp, prepared_recommendations) = tokio::join!(
// MCP refresh can be large; keep it off the sampling request's stack.
Box::pin(self.mcp_runtime_for_step(
@@ -4615,6 +4617,7 @@ impl Session {
.or_cancel(cancellation_token)
.await?;
}
drop(catalogs);
extension_data.insert(selected_plugins);
let tool_router = turn::built_tools(
self.as_ref(),
+3
View File
@@ -99,6 +99,8 @@ pub(crate) struct Session {
pub(super) multi_agent_version: OnceLock<MultiAgentVersion>,
/// Owns invalidation and serializes refreshes without blocking captured calls.
pub(super) mcp_refresh: McpRefresh,
/// Serializes cloud publication with capture of a step's plugin and skill snapshots.
pub(super) cloud_catalog_preparation: super::cloud_plugins::CloudCatalogPreparation,
/// Non-owning lookup for approval data retained by running MCP invocations.
pub(crate) mcp_tool_approval_metadata: std::sync::Mutex<McpToolApprovalMetadataMap>,
pub(super) mcp_elicitation_reviewer_handle: OnceLock<codex_mcp::ElicitationReviewerHandle>,
@@ -1981,6 +1983,7 @@ impl Session {
windows_sandbox_proxy_settings_mode,
multi_agent_version,
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
+2
View File
@@ -7044,6 +7044,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
@@ -9391,6 +9392,7 @@ where
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
mcp_refresh: McpRefresh::new(),
cloud_catalog_preparation: Default::default(),
mcp_tool_approval_metadata: Default::default(),
mcp_elicitation_reviewer_handle: OnceLock::new(),
mcp_elicitation_lifecycle_handle: OnceLock::new(),
@@ -1,4 +1,4 @@
//! Step discovery observes ready MCP connections; prewarm skips discovery and cancellation releases preparation.
//! Concurrent catalog discovery retains publication and releases preparation on cancellation.
use super::*;
use crate::McpRefreshReason;
@@ -8,6 +8,7 @@ use pretty_assertions::assert_eq;
struct Recorder {
client: McpResourceClient,
ready: Arc<tokio::sync::Barrier>,
pause: std::sync::atomic::AtomicBool,
keys: std::sync::Mutex<Vec<McpResourceClientCacheKey>>,
discovered: tokio::sync::Notify,
@@ -23,6 +24,7 @@ impl codex_extension_api::TurnLifecycleContributor for Recorder {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(self.client.cache_key());
self.ready.wait().await;
self.discovered.notify_one();
if self.pause.swap(false, std::sync::atomic::Ordering::Relaxed) {
std::future::pending::<()>().await;
@@ -32,17 +34,27 @@ impl codex_extension_api::TurnLifecycleContributor for Recorder {
}
}
/// Step discovery overlaps providers on ready connections; cancellation releases the catalog lock.
#[tokio::test]
async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
async fn step_discovery_reconciles_mcp_skips_prewarm_and_releases_lock_on_cancellation() {
let (mut session, turn_context) = make_session_and_context().await;
let ready = Arc::new(tokio::sync::Barrier::new(2));
let recorder = Arc::new(Recorder {
client: McpResourceClient::new(Arc::clone(&session.services.mcp_runtime)),
ready: Arc::clone(&ready),
pause: std::sync::atomic::AtomicBool::new(true),
keys: Default::default(),
discovered: Default::default(),
});
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_lifecycle_contributor(recorder.clone());
builder.turn_lifecycle_contributor(Arc::new(Recorder {
client: recorder.client.clone(),
ready,
pause: std::sync::atomic::AtomicBool::new(false),
keys: Default::default(),
discovered: Default::default(),
}));
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
@@ -95,7 +107,7 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
session.capture_step_context(Arc::clone(&turn_context), &CancellationToken::new()),
)
.await
.expect("cancelled discovery must allow the next step")
.expect("cancelled discovery must release the catalog lock")
.expect("next step should prepare");
assert!(recorder.client.cache_key() == prepared_key);
@@ -112,3 +124,75 @@ async fn step_discovery_reconciles_mcp_skips_prewarm_and_cancels_preparation() {
assert!(keys[2] == recorder.client.cache_key());
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
}
/// Explicit refresh overlaps providers under one lock and retains publication while a sibling waits.
#[tokio::test]
async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
struct CatalogGate {
entered: Arc<tokio::sync::Barrier>,
finish: tokio::sync::Notify,
result: Result<(), &'static str>,
}
impl codex_extension_api::TurnLifecycleContributor for CatalogGate {
fn refresh_cloud_catalog<'a>(
&'a self,
_input: codex_extension_api::CloudCatalogRefreshInput<'a>,
) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> {
Box::pin(async move {
self.entered.wait().await;
self.finish.notified().await;
self.result.map_err(str::to_string)
})
}
}
let (mut session, _) = make_session_and_context().await;
let entered = Arc::new(tokio::sync::Barrier::new(3));
let failing = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Err("plugin discovery failed"),
});
let succeeding = Arc::new(CatalogGate {
entered: Arc::clone(&entered),
finish: Default::default(),
result: Ok(()),
});
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_lifecycle_contributor(failing.clone());
builder.turn_lifecycle_contributor(succeeding.clone());
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let mut refresh = Box::pin(session.refresh_cloud_plugins());
tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::select! {
result = &mut refresh => panic!("refresh completed before both providers started: {result:?}"),
_ = entered.wait() => {}
}
})
.await
.expect("both providers must start before either finishes");
assert!(!session.mcp_refresh.is_pending());
let mut lock = Box::pin(session.lock_cloud_catalogs());
assert!(futures::poll!(&mut lock).is_pending());
drop(lock);
failing.finish.notify_one();
assert!(futures::poll!(&mut refresh).is_pending());
assert!(session.mcp_refresh.is_pending());
succeeding.finish.notify_one();
assert_eq!(
refresh.await.expect_err("one provider failed").to_string(),
"plugin discovery failed"
);
drop(
tokio::time::timeout(
std::time::Duration::from_secs(5),
session.lock_cloud_catalogs(),
)
.await
.expect("completed refresh must release the catalog lock"),
);
}
@@ -87,8 +87,13 @@ async fn frozen_cloud_skills_with_named_executor_request_history() -> Result<()>
})
.build(&server)
.await?;
let executor =
core_test_support::exec_server::ExecServerProcess::start_with_version("0.164.0").await?;
test.codex
.refresh_cloud_plugins()
.await
.expect("explicit startup discovery");
let executor = core_test_support::exec_server::ExecServerProcess::start_with_version("0.164.0")
.await
.expect("executor fixture");
test.thread_manager
.environment_manager()
.upsert_environment(
@@ -161,7 +161,7 @@ impl SkillProvider for PausedCatalogSkillProvider {
}
if self.first_discovery.swap(/*val*/ false, Ordering::SeqCst) {
self.started
.send(query.turn_id.clone())
.send(query.turn_id.clone().expect("turn discovery"))
.expect("discovery receiver");
self.resume
.acquire()
@@ -90,6 +90,197 @@ impl ThreadLifecycleContributor<Config> for SessionResources {
}
}
async fn mount_cloud_skill_list(
server: &wiremock::MockServer,
call_id: &str,
) -> responses::ResponseMock {
responses::mount_sse_sequence(
server,
vec![
sse(vec![
ev_response_created(call_id),
responses::ev_function_call_with_namespace(
call_id,
"skills",
"list",
&json!({"authority":{"kind":"cloud"}}).to_string(),
),
ev_completed(call_id),
]),
sse(vec![ev_response_created("done"), ev_completed("done")]),
],
)
.await
}
/// Hosts refresh before admission; publication is serialized with step preparation.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn explicit_cloud_refresh_publishes_before_admission_and_reuses_catalogs() -> Result<()> {
#[derive(Default)]
struct CatalogGate {
pause: AtomicBool,
entered: tokio::sync::Notify,
resume: tokio::sync::Notify,
}
impl codex_extension_api::TurnLifecycleContributor for CatalogGate {
fn refresh_catalog_for_step<'a>(
&'a self,
_: codex_extension_api::CatalogRefreshInput<'a>,
) -> ExtensionFuture<'a, bool> {
Box::pin(async move {
if self.pause.swap(false, Ordering::SeqCst) {
self.entered.notify_one();
self.resume.notified().await;
}
false
})
}
}
core_test_support::skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let apps = AppsTestServer::mount(&server).await?;
let provider = Arc::new(CloudTurnProvider {
reply: Mutex::new(Ok(Some(("calendar", 1)))),
lists: AtomicUsize::new(0),
partial: AtomicBool::new(false),
reads: AtomicUsize::new(0),
resource_key: Mutex::new(None),
});
let mut extensions = ExtensionRegistryBuilder::new();
let resources = Arc::new(SessionResources::default());
extensions.thread_lifecycle_contributor(resources.clone());
install_with_providers(
&mut extensions,
SkillProviders::new().with_cloud_provider(provider.clone()),
|_config: &Config| SkillsExtensionConfig {
include_instructions: true,
catalog_deltas_enabled: false,
omit_descriptions: false,
max_context_tokens: None,
bundled_skills_enabled: false,
cloud_skill_enabled: true,
shadow_selection_enabled: false,
},
);
let gate = Arc::new(CatalogGate::default());
extensions.turn_lifecycle_contributor(gate.clone());
let mut builder = apps_enabled_builder(apps.chatgpt_base_url)
.with_exec_server_url("none")
.with_extensions(Arc::new(extensions.build()))
.with_auth(CodexAuth::from_external_chatgpt_tokens(
"header.e30.initial",
"account-a",
/*chatgpt_plan_type*/ None,
)?)
.with_config(|config| {
config.include_skill_instructions = true;
config.cloud_skill_enabled = true;
});
let test = builder.build_with_auto_env(&server).await?;
for (index, plugin) in ["calendar", "gmail"].into_iter().enumerate() {
*provider.reply.lock().expect("reply") = Ok(Some((plugin, 1)));
provider.partial.store(plugin == "gmail", Ordering::SeqCst);
test.codex.refresh_cloud_plugins().await?;
assert_eq!(provider.lists.load(Ordering::SeqCst), index + 1);
test.codex.refresh_codex_apps_tools().await?;
let current = test.codex.config().await;
let mut next = (*current).clone();
next.config_layer_stack = next
.config_layer_stack
.with_session_plugins(toml::toml! { _default = { enabled = false } }.into())?;
assert_eq!(
test.codex.refresh_mcp_config(current, next).await,
codex_core::ConfigRefreshOutcome::Published,
);
let list_id = format!("list-{index}");
let response = mount_cloud_skill_list(&server, &list_id).await;
test.submit_turn("List available cloud skills.").await?;
assert_eq!(
provider.lists.load(Ordering::SeqCst),
index + 1,
"tool/config refresh and steps must reuse explicitly published catalogs"
);
let requests = response.requests();
let listed = requests[1]
.function_call_output_text(&list_id)
.expect("skill list");
assert!(listed.contains(plugin), "expected {plugin}: {listed}");
}
provider.partial.store(false, Ordering::SeqCst);
let accepted_key = provider
.resource_key
.lock()
.expect("resource key lock")
.clone()
.expect("accepted catalog should have an Apps resource key");
*provider.reply.lock().expect("reply") = Err(());
assert!(test.codex.refresh_cloud_plugins().await.is_err());
assert!(
provider
.resource_key
.lock()
.expect("resource key lock")
.as_ref()
.is_some_and(|key| key != &accepted_key),
"explicit refresh must invalidate resources before discovery"
);
*provider.reply.lock().expect("reply") = Ok(Some(("gmail", 2)));
let response = mount_cloud_skill_list(&server, "retried-list").await;
test.submit_turn("List skills after retrying discovery.")
.await?;
assert_eq!(provider.lists.load(Ordering::SeqCst), 4);
assert!(
response.requests()[1]
.function_call_output_text("retried-list")
.expect("skill list")
.contains("Cloud gmail revision 2")
);
// Cancelling an explicit refresh while it waits must preserve pending invalidation.
let previous_calls = provider.lists.load(Ordering::SeqCst);
*provider.reply.lock().expect("reply") = Ok(Some(("gmail", 3)));
gate.pause.store(true, Ordering::SeqCst);
let response = mount_cloud_skill_list(&server, "cancelled-refresh-list").await;
tokio::time::timeout(Duration::from_secs(10), async {
tokio::try_join!(
test.submit_turn("List skills after a cancelled refresh request."),
async {
gate.entered.notified().await;
let client = resources.0.lock().expect("resources").clone().unwrap();
let key = client.server_cache_key(CODEX_APPS_MCP_SERVER_NAME);
let mut refresh = Box::pin(test.codex.refresh_cloud_plugins());
assert!(futures::poll!(&mut refresh).is_pending());
drop(refresh);
assert!(client.server_cache_key(CODEX_APPS_MCP_SERVER_NAME) == key);
assert_eq!(provider.lists.load(Ordering::SeqCst), previous_calls);
gate.resume.notify_one();
anyhow::Ok(())
}
)
})
.await??;
assert_eq!(provider.lists.load(Ordering::SeqCst), previous_calls + 1);
assert!(
response.requests()[1]
.function_call_output_text("cancelled-refresh-list")
.expect("skill list")
.contains("Cloud gmail revision 2")
);
let response = mount_cloud_skill_list(&server, "after-cancellation-list").await;
test.submit_turn("List the catalog recovered after cancellation.")
.await?;
assert_eq!(provider.lists.load(Ordering::SeqCst), previous_calls + 1);
assert!(
response.requests()[1]
.function_call_output_text("after-cancellation-list")
.expect("skill list")
.contains("Cloud gmail revision 3")
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change() -> Result<()> {
core_test_support::skip_if_no_network!(Ok(()));