From a811c72969ccf3edf6be205f19bc535e4fb191a7 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 6 Oct 2026 01:07:15 +0000 Subject: [PATCH] Measure raw MCP tool catalog sizes in telemetry (#51215) ## What changed Record the total serialized JSON size of raw MCP tool definitions in the `codex.mcp.binding_catalog.raw_definition_json_bytes` histogram, before filtering or model preparation. Emit an `mcp.binding_catalog` trace event for each server with its tool count, definition size, server kind, plugin ID, and cached or live catalog source. Reuse bounded product SKU attribution for both metrics and events, and skip size measurement when neither telemetry output is enabled. GitOrigin-RevId: abda7cbf46d5470eb321455f047dd7429084740e --- codex-rs/codex-mcp/src/connection_manager.rs | 2 + .../connection_manager/catalog_telemetry.rs | 98 +++++++++++++++++++ .../src/connection_manager/tool_catalog.rs | 65 +++++++++++- codex-rs/codex-mcp/src/mcp/mod.rs | 2 +- codex-rs/otel/src/events/session_telemetry.rs | 29 +++--- codex-rs/otel/src/lib.rs | 1 + 6 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 codex-rs/codex-mcp/src/connection_manager/catalog_telemetry.rs diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index 65f7c5b445..7aaba94fbc 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -5,6 +5,8 @@ //! events, keeps server metadata, and aggregates tools and resources across //! running RMCP clients. +#[path = "connection_manager/catalog_telemetry.rs"] +mod catalog_telemetry; #[path = "connection_manager/required.rs"] mod required; #[path = "connection_manager/resources.rs"] diff --git a/codex-rs/codex-mcp/src/connection_manager/catalog_telemetry.rs b/codex-rs/codex-mcp/src/connection_manager/catalog_telemetry.rs new file mode 100644 index 0000000000..8ea65bb691 --- /dev/null +++ b/codex-rs/codex-mcp/src/connection_manager/catalog_telemetry.rs @@ -0,0 +1,98 @@ +//! Privacy-safe size telemetry for raw MCP tool catalogs. + +use rmcp::model::Tool; +use serde::Serialize; +use std::io::Write; + +const RAW_DEFINITION_JSON_BYTES_METRIC: &str = + "codex.mcp.binding_catalog.raw_definition_json_bytes"; +const KIB: f64 = 1024.0; +const MIB: f64 = 1024.0 * KIB; +const RAW_DEFINITION_JSON_BYTES_BUCKETS: &[f64] = &[ + 64.0 * KIB, + 128.0 * KIB, + 256.0 * KIB, + 512.0 * KIB, + 768.0 * KIB, + MIB, + 1.5 * MIB, + 2.0 * MIB, + 3.0 * MIB, + 4.0 * MIB, + 6.0 * MIB, + 8.0 * MIB, + 12.0 * MIB, + 16.0 * MIB, + 24.0 * MIB, + 32.0 * MIB, + 48.0 * MIB, + 64.0 * MIB, + 96.0 * MIB, + 128.0 * MIB, + 192.0 * MIB, + 256.0 * MIB, + 384.0 * MIB, + 512.0 * MIB, +]; + +pub(super) fn tool_definition_json_bytes<'a>(tools: impl Iterator) -> usize { + tools.map(serialized_json_bytes).sum() +} + +pub(super) fn record_binding_catalog_size( + metrics: &codex_otel::MetricsClient, + product_sku: &'static str, + raw_definition_json_bytes: usize, +) { + let _ = metrics.histogram_with_boundaries( + RAW_DEFINITION_JSON_BYTES_METRIC, + i64::try_from(raw_definition_json_bytes).unwrap_or(i64::MAX), + RAW_DEFINITION_JSON_BYTES_BUCKETS, + &[("product_sku", product_sku)], + ); +} + +pub(super) fn emit_binding_catalog( + product_sku: &'static str, + server_kind: &'static str, + plugin_id: Option<&str>, + catalog_source: &'static str, + tool_count: usize, + tool_definition_json_bytes: usize, +) { + tracing::event!( + target: "codex_otel.trace_safe", + tracing::Level::INFO, + event.name = "mcp.binding_catalog", + product_sku, + server_kind, + plugin_id = plugin_id.unwrap_or(""), + catalog_source, + tool_count, + tool_definition_json_bytes, + "MCP binding catalog materialized" + ); +} + +#[derive(Default)] +struct CountingWriter(usize); + +impl Write for CountingWriter { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.0 += buffer.len(); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn serialized_json_bytes(value: &T) -> usize { + let mut writer = CountingWriter::default(); + if serde_json::to_writer(&mut writer, value).is_ok() { + writer.0 + } else { + 0 + } +} diff --git a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs index d6040dcdf5..54c6de2e8c 100644 --- a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs +++ b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs @@ -17,6 +17,9 @@ use tracing::trace_span; use super::McpConnectionSet; use super::McpServerMetadata; +use super::catalog_telemetry::emit_binding_catalog; +use super::catalog_telemetry::record_binding_catalog_size; +use super::catalog_telemetry::tool_definition_json_bytes; use crate::binding::McpBinding; use crate::binding::PreparedMcpCall; use crate::binding_clients::McpBindingClients; @@ -267,6 +270,19 @@ impl McpConnectionSet { ) -> McpBinding { let mut listed_tools = Vec::new(); let mut clients = HashMap::new(); + let catalog_log_enabled = tracing::enabled!( + target: "codex_otel.trace_safe", + tracing::Level::INFO + ); + let catalog_metrics = codex_otel::global(); + let catalog_measurement_enabled = catalog_log_enabled || catalog_metrics.is_some(); + let product_sku = codex_otel::bounded_product_sku(Some( + config + .apps_mcp_product_sku + .as_deref() + .unwrap_or(crate::mcp::DEFAULT_CODEX_APPS_MCP_PRODUCT_SKU), + )) + .unwrap_or("unknown"); let optional_mcp_startup_grace = config.optional_mcp_startup_grace; let server_snapshots = join_all(self.servers.iter().map(|(server_name, view)| async move { if !view @@ -350,8 +366,8 @@ impl McpConnectionSet { } else { None }; - let (client, server_tools) = if let Some(cached_tools) = cached_tools { - (None, cached_tools) + let (client, server_tools, catalog_source) = if let Some(cached_tools) = cached_tools { + (None, cached_tools, "cached") } else { // A server may opt out of caching after the first pass. Required catalog // readiness must then wait for discovery rather than silently omit its tools. @@ -369,8 +385,34 @@ impl McpConnectionSet { client.tool_timeout = view.tool_timeout; let snapshot = client.tool_catalog.read(Arc::new).await; let server_tools = snapshot.tools.to_vec(); - (Some((Arc::new(client), snapshot)), server_tools) + ( + Some((Arc::new(client), snapshot)), + server_tools, + "live", + ) }; + let raw_definition_json_bytes = if catalog_measurement_enabled { + tool_definition_json_bytes(server_tools.iter().map(|tool| &tool.tool)) + } else { + 0 + }; + if catalog_log_enabled { + let plugin_id = self.plugin_id_for_mcp_server_name(server_name); + emit_binding_catalog( + product_sku, + if server_name == CODEX_APPS_MCP_SERVER_NAME { + "codex_apps" + } else if plugin_id.is_some() { + "plugin" + } else { + "configured" + }, + plugin_id, + catalog_source, + server_tools.len(), + raw_definition_json_bytes, + ); + } let server_tools = filter_tools(server_tools, &view.tool_filter); let server_tools = if server_name == CODEX_APPS_MCP_SERVER_NAME { prepare_codex_apps_tools_for_model(server_tools, &self.tool_plugin_context) @@ -391,15 +433,28 @@ impl McpConnectionSet { Self::with_server_metadata(tool, &view.metadata) }) .collect::>(); - Some((server_name.clone(), client, server_tools)) + Some(( + server_name.clone(), + client, + server_tools, + raw_definition_json_bytes, + )) })) .await; - for (server_name, client, server_tools) in server_results.into_iter().flatten() { + let mut raw_definition_json_bytes = 0usize; + for (server_name, client, server_tools, server_definition_json_bytes) in + server_results.into_iter().flatten() + { + raw_definition_json_bytes = + raw_definition_json_bytes.saturating_add(server_definition_json_bytes); if let Some((client, snapshot)) = client { clients.insert(server_name, (client, snapshot)); } listed_tools.extend(server_tools); } + if let Some(metrics) = catalog_metrics.as_ref() { + record_binding_catalog_size(metrics, product_sku, raw_definition_json_bytes); + } let listed_tools = normalize_tools_for_model_with_prefix( listed_tools, self.prefix_mcp_tool_names, diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index 7f6be9376d..c690f08838 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -62,7 +62,7 @@ use crate::server::EffectiveMcpServer; use crate::tools::ToolInfo; pub const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps"; -const DEFAULT_CODEX_APPS_MCP_PRODUCT_SKU: &str = "codex"; +pub(crate) const DEFAULT_CODEX_APPS_MCP_PRODUCT_SKU: &str = "codex"; const MCP_TOOL_NAME_PREFIX: &str = "mcp"; const MCP_TOOL_NAME_DELIMITER: &str = "__"; const CODEX_CONNECTORS_TOKEN_ENV_VAR: &str = "CODEX_CONNECTORS_TOKEN"; diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index 352265d9ea..c5fb87bf5f 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -123,6 +123,22 @@ pub struct SessionTelemetry { pub(crate) metrics_use_metadata_tags: bool, } +/// Bounds product SKU attribution before it is used in telemetry dimensions. +pub fn bounded_product_sku(product_sku: Option<&str>) -> Option<&'static str> { + const KNOWN_PRODUCT_SKUS: &[&str] = &["codex"]; + + match product_sku { + None | Some("") => None, + Some(sku) => Some( + KNOWN_PRODUCT_SKUS + .iter() + .copied() + .find(|known| *known == sku) + .unwrap_or("other"), + ), + } +} + impl SessionTelemetry { pub fn with_tool_result_log_config(mut self, config: ToolResultLogConfig) -> Self { self.tool_result_log_config = config; @@ -157,18 +173,7 @@ impl SessionTelemetry { /// Attributes bounded telemetry without turning arbitrary configuration into metric labels. pub fn with_product_sku(mut self, product_sku: Option<&str>) -> Self { - const KNOWN_PRODUCT_SKUS: &[&str] = &["codex"]; - - self.metadata.product_sku = match product_sku { - None | Some("") => None, - Some(sku) => Some( - KNOWN_PRODUCT_SKUS - .iter() - .copied() - .find(|known| *known == sku) - .unwrap_or("other"), - ), - }; + self.metadata.product_sku = bounded_product_sku(product_sku); self } diff --git a/codex-rs/otel/src/lib.rs b/codex-rs/otel/src/lib.rs index 690fdd9094..b719515e2f 100644 --- a/codex-rs/otel/src/lib.rs +++ b/codex-rs/otel/src/lib.rs @@ -29,6 +29,7 @@ pub use crate::config::validate_span_attributes; pub use crate::events::session_telemetry::AuthEnvTelemetryMetadata; pub use crate::events::session_telemetry::SessionTelemetry; pub use crate::events::session_telemetry::SessionTelemetryMetadata; +pub use crate::events::session_telemetry::bounded_product_sku; pub use crate::metrics::runtime_metrics::RuntimeMetricTotals; pub use crate::metrics::runtime_metrics::RuntimeMetricsSummary; pub use crate::metrics::timer::Timer;