Trace MCP binding capture and tool catalog refresh (#62320)

## Why

When MCP binding capture or a catalog refresh is slow, separate spans
help identify whether the time went to startup, lock waits, or tool
catalog processing.

## What changed

- Trace server startup waits, catalog revision checks and snapshot
  reads, and tool preparation and normalization.
- Trace catalog refresh lock waits, fetching, and publication, including
  time spent waiting for active tool calls to finish.
- Record `cache_hit` on the binding capture span and trace the cache lock
  wait. Box the catalog capture future on cache misses to keep it out of
  callers' future layouts.

GitOrigin-RevId: e5a5c037455c755fe74ead9f0efabfcd7a9529bf
This commit is contained in:
bkotsopoulos
2026-11-18 17:37:57 +00:00
committed by copyberry
parent 6257fbc4bd
commit f2d3086682
5 changed files with 46 additions and 17 deletions
+21 -4
View File
@@ -25,6 +25,7 @@ use tokio::sync::RwLock;
use tokio::sync::RwLockReadGuard;
use tokio::sync::watch;
use tokio::time::Instant;
use tracing::Instrument;
use crate::tools::ToolInfo;
@@ -117,6 +118,7 @@ impl ClientToolCatalog {
}
/// Captures tools and their client-local revision together, without retaining them in the client.
#[tracing::instrument(name = "mcp.catalog.read_snapshot", skip_all)]
async fn read_current(&self) -> (RwLockReadGuard<'_, CatalogState>, ToolCatalogSnapshot) {
loop {
{
@@ -247,13 +249,28 @@ impl ClientToolCatalog {
where
Fut: Future<Output = Result<(Vec<ToolInfo>, Option<Arc<serde_json::Value>>, C)>>,
{
let _refresh = self.refresh_lock.lock().await;
let (tools, meta, context) = fetch().await?;
let _calls = self.calls.write().await;
let mut current = self.current.write().await;
let _refresh = self
.refresh_lock
.lock()
.instrument(tracing::info_span!("mcp.catalog.refresh_lock_wait"))
.await;
let (tools, meta, context) = fetch()
.instrument(tracing::info_span!("mcp.catalog.fetch"))
.await?;
let _calls = self
.calls
.write()
.instrument(tracing::info_span!("mcp.catalog.active_calls_wait"))
.await;
let mut current = self
.current
.write()
.instrument(tracing::info_span!("mcp.catalog.publication_lock_wait"))
.await;
let Some(_validity) = validate() else {
return Ok(None);
};
let _publication = tracing::info_span!("mcp.catalog.publish").entered();
let result = publish(&tools, meta.clone(), context);
match &mut current.source {
CatalogSource::Local {
@@ -335,6 +335,7 @@ impl McpServerView {
/// Publishes raw client tools with this view's policy, owner, and plugin attribution.
/// Shared client catalogs stay unannotated so later views can reuse them safely.
#[tracing::instrument(name = "mcp.tools.prepare_for_model", skip_all)]
fn prepare_tools_for_model(
&self,
tools: Vec<ToolInfo>,
@@ -119,6 +119,7 @@ impl McpConnectionSet {
}
}
#[instrument(name = "mcp.binding.catalog_revision_wait", skip_all)]
pub(crate) async fn stable_catalog_revisions(
&self,
required_servers: &[McpServerKey],
@@ -380,6 +381,7 @@ impl McpConnectionSet {
}
(server_name, view, None)
}))
.instrument(tracing::info_span!("mcp.binding.startup_wait"))
.await;
let catalog = &config.mcp_server_catalog;
let server_results = join_all(server_snapshots.into_iter().map(|(server_name, view, cached_tools)| async move {
@@ -469,6 +471,7 @@ impl McpConnectionSet {
raw_definition_json_bytes,
))
}))
.instrument(tracing::info_span!("mcp.binding.catalog_capture"))
.await;
let mut raw_definition_json_bytes = 0usize;
for (server_name, client, server_tools, server_definition_json_bytes) in
@@ -791,6 +794,7 @@ impl McpConnectionSet {
let managed_client = view
.connection
.client()
.instrument(tracing::info_span!("mcp.apps.client_ready_wait"))
.await
.context("failed to get client")?;
let Some((tools, list_start)) = managed_client
+19 -13
View File
@@ -596,6 +596,7 @@ impl McpRuntime {
Self::binding_from_published_runtime(current, &required_servers, required_plugins).await
}
#[tracing::instrument(name = "mcp.binding.capture", skip_all, fields(cache_hit))]
async fn binding_from_published_runtime(
current: Arc<PublishedMcpRuntime>,
required_servers: &[McpServerKey],
@@ -607,28 +608,33 @@ impl McpRuntime {
.stable_catalog_revisions(required_servers, required_plugins)
.await;
if let Some(catalog_revisions) = &stable_catalog_revisions {
let cached = current
.cached_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let cached = tracing::info_span!("mcp.binding.cache_lock_wait").in_scope(|| {
current
.cached_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
});
if let Some(cached) = cached.as_ref()
&& &cached.catalog_revisions == catalog_revisions
&& let Some(binding) = cached.binding.upgrade()
{
tracing::Span::current().record("cache_hit", true);
return Some(binding);
}
}
// A binding built below is a miss even if a concurrent capture wins publication.
tracing::Span::current().record("cache_hit", false);
// Keep the instrumented catalog fan-out out of its callers' future layouts.
// The cache-hit path above does not need this allocation.
let binding = Arc::new(
current
.connections
.capture_binding_with_metadata(
config,
current.plugins_available,
required_servers,
required_plugins,
)
.await,
Box::pin(current.connections.capture_binding_with_metadata(
config,
current.plugins_available,
required_servers,
required_plugins,
))
.await,
);
if let Some(catalog_revisions) = stable_catalog_revisions
&& current
+1
View File
@@ -131,6 +131,7 @@ pub(crate) fn filter_tools(tools: Vec<ToolInfo>, filter: &ToolFilter) -> Vec<Too
///
/// When `prefix_mcp_tool_names` is true, the historical `mcp__` namespace
/// prefix is added except for tools from `non_prefixed_mcp_tool_servers`.
#[tracing::instrument(name = "mcp.binding.normalize_tools", skip_all)]
pub(crate) fn normalize_tools_for_model_with_prefix<I>(
tools: I,
prefix_mcp_tool_names: bool,