Refresh Apps tools outside the MCP runtime gate (#60639)

## Why

Holding the session MCP gate during `tools/list` blocks other runtime access. Releasing it requires preventing stale refresh results from publishing after configuration or authentication changes.

## What changed

- Capture the Apps client under the session gate, then release the gate before fetching tools.
- Validate runtime and authentication state before publishing the shared cache and client catalog; retry a superseded refresh once.
- Wait for in-flight Apps refreshes before capturing model tools, preparing Apps calls, or shutting down the runtime.

## Testing

Add regression coverage for stale refreshes after runtime invalidation or logout, publication validation after call preparation drains, and a model step that waits for refreshed Apps tools while MCP status queries remain available.

GitOrigin-RevId: 42d4be8c4a9218933f8e3d07fc7f32f2da9cbd9d
This commit is contained in:
Dmitriy Kovalenko
2026-11-10 18:33:49 +00:00
committed by copyberry
parent 057d9140a8
commit 3f797555cd
13 changed files with 405 additions and 42 deletions
+12
View File
@@ -356,6 +356,7 @@ async fn prepared_call_is_rejected_after_catalog_refresh() {
step.tool_catalog
.refresh(
|| async { Ok((step.step.tools().to_vec(), None, ())) },
|| Some(()),
|_, _, ()| {},
)
.await
@@ -390,6 +391,7 @@ async fn stale_prepared_call_does_not_run_preparation() {
step.tool_catalog
.refresh(
|| async { Ok((step.step.tools().to_vec(), None, ())) },
|| Some(()),
|_, _, ()| {},
)
.await
@@ -435,8 +437,13 @@ async fn preparation_holds_catalog_authority_until_it_finishes() {
});
preparation_started.notified().await;
let validation_reached = AtomicBool::new(false);
let refresh = step.tool_catalog.refresh(
|| async { Ok((step.step.tools().to_vec(), None, ())) },
|| {
validation_reached.store(true, Ordering::SeqCst);
Some(())
},
|_, _, ()| {},
);
tokio::pin!(refresh);
@@ -444,6 +451,10 @@ async fn preparation_holds_catalog_authority_until_it_finishes() {
futures::poll!(&mut refresh).is_pending(),
"catalog replacement must wait for irreversible call preparation"
);
assert!(
!validation_reached.load(Ordering::SeqCst),
"publication validation must not hold its guard while calls drain"
);
finish_preparation.notify_one();
call.await
.expect("call task should finish")
@@ -451,6 +462,7 @@ async fn preparation_holds_catalog_authority_until_it_finishes() {
refresh
.await
.expect("catalog refresh should finish after preparation");
assert!(validation_reached.load(Ordering::SeqCst));
}
#[tokio::test]
+12 -6
View File
@@ -232,22 +232,28 @@ impl ClientToolCatalog {
result
}
/// Serialize fetching and publication, leaving the current catalog usable during the fetch.
/// The publication callback runs alongside the exact-client update under the write lock.
/// Validate under the caller's publication guard after fetching and draining calls.
/// The guard covers both shared-provider publication and this client's revision update.
#[expect(
clippy::await_holding_invalid_type,
reason = "refreshes must remain serialized through fetching and catalog publication"
)]
pub(crate) async fn refresh<C, R, F, Fut, P>(&self, fetch: F, publish: P) -> Result<R>
pub(crate) async fn refresh<C, R, Fut, G>(
&self,
fetch: impl FnOnce() -> Fut,
validate: impl FnOnce() -> Option<G>,
publish: impl FnOnce(&[ToolInfo], Option<Arc<serde_json::Value>>, C) -> R,
) -> Result<Option<R>>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<(Vec<ToolInfo>, Option<Arc<serde_json::Value>>, C)>>,
P: FnOnce(&[ToolInfo], Option<Arc<serde_json::Value>>, C) -> R,
{
let _refresh = self.refresh_lock.lock().await;
let (tools, meta, context) = fetch().await?;
let _calls = self.calls.write().await;
let mut current = self.current.write().await;
let Some(_validity) = validate() else {
return Ok(None);
};
let result = publish(&tools, meta.clone(), context);
match &mut current.source {
CatalogSource::Local {
@@ -269,7 +275,7 @@ impl ClientToolCatalog {
}
current.revision += 1;
current.last_refresh_revision = current.revision;
Ok(result)
Ok(Some(result))
}
/// Reject stale calls before preparation. Explicit refreshes wait for execution;
@@ -660,16 +660,19 @@ impl McpConnectionSet {
}
/// Refreshes one exact Apps catalog, preserving the raw inventory for app policy.
pub(crate) async fn refresh_codex_apps_client_catalog(
pub(crate) async fn refresh_codex_apps_client_catalog<G>(
&self,
config: &crate::McpConfig,
) -> Result<CodexAppsToolSnapshot> {
validate: impl FnOnce() -> Option<G>,
) -> Result<Option<CodexAppsToolSnapshot>> {
let refresh_start = Instant::now();
let view = self
.servers
.get(CODEX_APPS_MCP_SERVER_NAME)
.ok_or_else(|| anyhow!("unknown MCP server '{CODEX_APPS_MCP_SERVER_NAME}'"))?;
let (tools, _) = self.refresh_codex_apps_tool_catalog().await?;
let Some((tools, _)) = self.refresh_codex_apps_tool_catalog(validate).await? else {
return Ok(None);
};
let server_has_permission = config
.permission_profile_for_server(CODEX_APPS_MCP_SERVER_NAME)
.is_some();
@@ -690,10 +693,10 @@ impl McpConnectionSet {
refresh_start.elapsed(),
&[("path", "legacy"), ("trigger", "explicit")],
);
Ok(CodexAppsToolSnapshot {
Ok(Some(CodexAppsToolSnapshot {
tools,
model_visible_tool_names,
})
}))
}
/// Refreshes Apps tools and returns the prepared shared-cache winner for discovery.
@@ -703,7 +706,10 @@ impl McpConnectionSet {
.servers
.get(CODEX_APPS_MCP_SERVER_NAME)
.ok_or_else(|| anyhow!("unknown MCP server '{CODEX_APPS_MCP_SERVER_NAME}'"))?;
let (_, tools) = self.refresh_codex_apps_tool_catalog().await?;
let (_, tools) = self
.refresh_codex_apps_tool_catalog(|| Some(()))
.await?
.context("Apps discovery refresh skipped publication")?;
let tools = prepare_codex_apps_tools_for_model(
filter_tools(tools, &view.tool_filter),
&self.tool_plugin_context,
@@ -744,7 +750,10 @@ impl McpConnectionSet {
}
/// Publishes the exact client catalog and returns both raw inventories.
async fn refresh_codex_apps_tool_catalog(&self) -> Result<(Vec<ToolInfo>, Vec<ToolInfo>)> {
async fn refresh_codex_apps_tool_catalog<G>(
&self,
validate: impl FnOnce() -> Option<G>,
) -> Result<Option<(Vec<ToolInfo>, Vec<ToolInfo>)>> {
let view = self
.servers
.get(CODEX_APPS_MCP_SERVER_NAME)
@@ -754,7 +763,7 @@ impl McpConnectionSet {
.client()
.await
.context("failed to get client")?;
let (tools, list_start) = managed_client
let Some((tools, list_start)) = managed_client
.tool_catalog
.refresh(
|| async {
@@ -781,6 +790,7 @@ impl McpConnectionSet {
})?;
Ok((client_tools, meta, (fetch_ticket, list_start)))
},
validate,
|client_tools, meta, (fetch_ticket, list_start)| {
// Discovery can accept another scope's winner; executable catalogs
// receive only the latest successful fetch from their own scope.
@@ -803,7 +813,10 @@ impl McpConnectionSet {
(tools, list_start)
},
)
.await?;
.await?
else {
return Ok(None);
};
let client_tools = managed_client
.tool_catalog
.read(|catalog| catalog.tools.to_vec())
@@ -813,7 +826,7 @@ impl McpConnectionSet {
list_start.elapsed(),
&[("cache", "miss")],
);
Ok((client_tools, tools))
Ok(Some((client_tools, tools)))
}
fn with_server_metadata(mut tool: ToolInfo, metadata: &McpServerMetadata) -> ToolInfo {
@@ -2701,13 +2701,13 @@ async fn hard_refresh_keeps_client_catalog_local_when_shared_cache_loses_race()
let config_for_refresh = config.clone();
let refresh_a = tokio::spawn(async move {
manager_a_for_refresh
.refresh_codex_apps_client_catalog(&config_for_refresh)
.refresh_codex_apps_client_catalog(&config_for_refresh, || Some(()))
.await
});
list_started.notified().await;
manager_b.refresh_codex_apps_tools_for_discovery().await?;
release_list.notify_one();
let snapshot_a = refresh_a.await??;
let snapshot_a = refresh_a.await??.expect("Apps catalog was published");
assert_eq!(
snapshot_a
.tools
@@ -2737,7 +2737,10 @@ async fn hard_refresh_keeps_client_catalog_local_when_shared_cache_loses_race()
.tool_filter
.disabled
.insert("b_only".to_string());
let snapshot_b = manager_b.refresh_codex_apps_client_catalog(&config).await?;
let snapshot_b = manager_b
.refresh_codex_apps_client_catalog(&config, || Some(()))
.await?
.expect("Apps catalog was published");
assert_eq!(
(
snapshot_b
@@ -5743,6 +5746,7 @@ async fn apps_catalog_broadcast_restores_equivalent_calls() -> anyhow::Result<()
catalog
.refresh(
|| async { Ok((snapshot.tools.to_vec(), None, ())) },
|| Some(()),
|tools, _, ()| store_current_tools(&context, tools.to_vec()),
)
.await?;
@@ -5801,6 +5805,7 @@ async fn apps_catalog_broadcast_survives_an_older_local_refresh() -> anyhow::Res
catalog
.refresh(
|| async { Ok((Vec::new(), None, older_ticket)) },
|| Some(()),
|tools, _, ticket| {
context
.publish_runtime_if_newest_accepted(
@@ -5859,7 +5864,11 @@ async fn refreshed_catalog_follows_reused_client_without_mutating_old_bindings()
let startup_tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "startup")];
client
.tool_catalog
.refresh(|| async { Ok((startup_tools, None, ())) }, |_, _, ()| {})
.refresh(
|| async { Ok((startup_tools, None, ())) },
|| Some(()),
|_, _, ()| {},
)
.await?;
let old_binding = capture_binding(&previous).await;
let old_tools = serde_json::to_value(old_binding.tools())?;
@@ -767,6 +767,7 @@ async fn new_chat_relists_tools_without_invalidating_or_waiting_for_existing_cal
Duration::from_secs(1),
second.tool_catalog.refresh(
|| async { Ok((vec![create_test_tool("test_server", "refreshed")], None, ())) },
|| Some(()),
|_, _, ()| {},
),
)
+67 -11
View File
@@ -6,6 +6,9 @@
//! [`crate::connection_manager`]. Registry removal revokes EMA authority before
//! the thread becomes untracked.
#[cfg(test)]
#[path = "runtime/apps_tool_refresh_tests.rs"]
mod apps_tool_refresh_tests;
mod status;
#[cfg(test)]
@@ -124,6 +127,8 @@ pub struct McpRuntimeInput {
/// their exact connections and configuration for as long as they are needed.
pub struct McpRuntime {
current: ArcSwap<PublishedMcpRuntime>,
// This mutex is held only through synchronous publication, never network I/O or call drain.
catalog_publication: Mutex<u64>,
// Old bindings keep leases alive after publication; cancellation alone does not release them.
superseded_connections: Mutex<Vec<Weak<McpConnectionSet>>>,
event_stream_cancellation: Mutex<EventStreamCancellation>,
@@ -263,6 +268,15 @@ impl McpPublicationGate {
}
impl McpRuntime {
/// Supersede pending fetches as soon as Core requests a configuration change.
pub fn invalidate_apps_tool_refresh(&self) {
let mut generation = self
.catalog_publication
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*generation = generation.wrapping_add(1);
}
/// Creates a runtime with no configured servers.
///
/// This is useful while constructing a thread that must publish a stable
@@ -283,6 +297,7 @@ impl McpRuntime {
ready_environments: HashMap::new(),
cached_binding: Mutex::new(None),
}),
catalog_publication: Mutex::new(0),
superseded_connections: Mutex::default(),
event_stream_cancellation: Mutex::new(EventStreamCancellation {
event_server_available: false,
@@ -389,6 +404,8 @@ impl McpRuntime {
}
async fn publish(&self, input: McpRuntimeInput, reuse_previous: bool) {
// Also cover callers which replace the runtime without Core's dirty-state machinery.
self.invalidate_apps_tool_refresh();
let current = self.current.load_full();
let retain_ema = current.config.is_some();
let ema_auth = (input.config.xaa_enabled
@@ -475,6 +492,10 @@ impl McpRuntime {
superseded.retain(|connections| connections.strong_count() > 0);
superseded.push(Arc::downgrade(&current.connections));
}
let publication = self
.catalog_publication
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
self.current.store(Arc::new(PublishedMcpRuntime {
connections,
config: Some(config),
@@ -489,6 +510,7 @@ impl McpRuntime {
ready_environments,
cached_binding: Mutex::new(None),
}));
drop(publication);
let _ = publish.send(true);
cancellation.event_server_available = hosted_event_server_retained;
if !hosted_event_server_retained {
@@ -732,17 +754,51 @@ impl McpRuntime {
.await
}
/// Refreshes the published Apps client and returns its exact inventory and MCP eligibility.
pub async fn refresh_codex_apps_tools(&self) -> anyhow::Result<CodexAppsToolSnapshot> {
let current = self.current.load_full();
let config = current
.config
.as_deref()
.ok_or_else(|| anyhow::anyhow!("MCP runtime is not configured"))?;
current
.connections
.refresh_codex_apps_client_catalog(config)
.await
/// Captures the current Apps client under Core's session permit.
/// Await the returned fetch after releasing the permit. A stale result is skipped.
pub fn capture_apps_tool_refresh(
&self,
) -> impl Future<Output = anyhow::Result<Option<CodexAppsToolSnapshot>>> + Send {
let (generation, current) = {
let generation = self
.catalog_publication
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
(*generation, self.current.load_full())
};
let auth_changes = current.auth_manager.as_ref().map(|manager| {
let mut changes = manager.auth_change_state_receiver();
// Subscribe before reading auth to catch changes during capture.
if !current.auth_matches(manager.auth_cached().as_ref()) {
changes.mark_changed();
}
changes
});
async move {
let validate = || {
let publication = self
.catalog_publication
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
// Hold the watch read guard through publication. Auth writers take cache then watch.
let auth = auth_changes.as_ref().map(watch::Receiver::borrow);
(*publication == generation
&& Arc::ptr_eq(&self.current.load(), &current)
&& auth_changes
.as_ref()
.is_none_or(|changes| matches!(changes.has_changed(), Ok(false))))
.then_some((publication, auth))
};
let config = current
.config
.as_deref()
.ok_or_else(|| anyhow::anyhow!("MCP runtime is not configured"))?;
let snapshot = current
.connections
.refresh_codex_apps_client_catalog(config, validate)
.await?;
Ok(snapshot.filter(|_| validate().is_some()))
}
}
/// Returns tools from this thread’s ready, unshared Apps MCP client.
@@ -0,0 +1,91 @@
#![cfg(test)]
//! Verifies that an in-flight Apps refresh cannot publish after invalidation.
use super::*;
use tokio::sync::Notify;
#[tokio::test]
async fn stale_apps_refresh_does_not_publish() -> anyhow::Result<()> {
let home = tempfile::tempdir()?;
let auth = CodexAuth::from_external_chatgpt_tokens(
"header.e30.token",
"account",
/*chatgpt_plan_type*/ None,
)?;
let auth_manager =
AuthManager::from_auth_for_testing_with_home(auth.clone(), home.path().to_path_buf());
let cache = ConnectorRuntimeManager::<ToolInfo>::default().context(
home.path().to_path_buf(),
ConnectorRuntimeContextKey::personal(
/*account_id*/ None, /*chatgpt_user_id*/ None,
),
);
let started = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let connections = crate::connection_manager::tests::create_test_manager_with_ready_apps_client(
cache.clone(),
"search",
Some(started.clone()),
Some(release.clone()),
)
.await?;
connections.list_all_tools().await;
let revision = connections
.stable_catalog_revisions(/*required_servers*/ &[], &HashSet::new())
.await
.expect("Apps catalog is ready");
let runtime = McpRuntime::empty(/*prefix_mcp_tool_names*/ true);
runtime.current.store(Arc::new(PublishedMcpRuntime {
connections: connections.clone(),
config: Some(Arc::new(crate::mcp::tests::test_mcp_config(
home.path().to_path_buf(),
))),
auth_token: Some(auth.get_token()?),
auth: Some(auth),
auth_manager: Some(auth_manager.clone()),
auth_owner: None,
auth_generation: Arc::new(()),
plugins_available: false,
ready_selected_capability_roots: Vec::new(),
environment_selections: Arc::default(),
ready_environments: HashMap::new(),
cached_binding: Mutex::new(None),
}));
let refresh = runtime.capture_apps_tool_refresh();
tokio::pin!(refresh);
tokio::select! {
_ = started.notified() => {}
_ = &mut refresh => panic!("refresh completed before tools/list was released"),
}
runtime.invalidate_apps_tool_refresh();
release.notify_one();
assert!(
refresh.await?.is_none(),
"runtime invalidation supersedes the fetch"
);
let refresh = runtime.capture_apps_tool_refresh();
tokio::pin!(refresh);
tokio::select! {
_ = started.notified() => {}
_ = &mut refresh => panic!("refresh completed before tools/list was released"),
}
auth_manager.logout().await?;
release.notify_one();
assert!(refresh.await?.is_none(), "logout supersedes the fetch");
assert!(
!cache.has_current_tools(),
"stale tools reached the shared cache"
);
assert!(
connections
.stable_catalog_revisions(/*required_servers*/ &[], &HashSet::new())
.await
.as_ref()
== Some(&revision),
"a stale fetch invalidated the existing client catalog"
);
Ok(())
}