From d4e03acb6147dc83f5e013fd91f729da72712a03 Mon Sep 17 00:00:00 2001 From: Alex Kira Date: Tue, 20 Oct 2026 22:24:06 +0000 Subject: [PATCH] Tag tool registration metrics with effective incremental tools mode (#55692) ## What changed Add an `incremental_tools` attribute to `codex.tools.registered` using the effective mode for the current sampling request. Compute the mode once and reuse it for registration metrics and prompt construction across transport retries. ## Testing Extend integration tests to verify the attribute for incremental and non-incremental requests, including when the feature flag is enabled but the Responses API does not support incremental tools. Assert registration sample counts and retain coverage that stream retries do not duplicate registration metrics. GitOrigin-RevId: e5b7a1075fe722ef6bdf900ae07e2b1b06aecfb5 --- codex-rs/core/src/session/turn.rs | 12 +++++++--- codex-rs/core/src/tools/router.rs | 13 +++++++++-- .../suite/scenarios_incremental_tools.rs | 22 +++++++++++++++++++ .../tests/suite/tool_registration_metrics.rs | 7 ++++++ codex-rs/otel/src/metrics/names.rs | 3 ++- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 7333c9060c..f30130e641 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -1742,7 +1742,14 @@ async fn run_sampling_request( Arc::clone(&turn_diff_tracker), ); let max_retries = turn_context.provider.info().stream_max_retries(); - record_registration_metrics(&step_context.tool_router, &step_context.session_telemetry); + let incremental_tools = sess + .current_window_uses_incremental_tools(&step_context) + .await; + record_registration_metrics( + &step_context.tool_router, + &step_context.session_telemetry, + incremental_tools, + ); let mut retry_state = ResponsesStreamRetryState::default(); let mut initial_input = Some(input); let mut original_input = None; @@ -1765,8 +1772,7 @@ async fn run_sampling_request( prompt_input, step_context.as_ref(), base_instructions.clone(), - sess.current_window_uses_incremental_tools(&step_context) - .await, + incremental_tools, ); let responses_metadata = sess .responses_metadata(step_context.as_ref(), CodexResponsesRequestKind::Turn) diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index a7a192815b..3c01ab9818 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -395,7 +395,11 @@ impl ToolRouter { /// Records the registered inventory once per sampling request, before transport retries. /// Registrations do not imply model visibility or reachability; hosted tools are excluded. -pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &SessionTelemetry) { +pub(crate) fn record_registration_metrics( + router: &ToolRouter, + telemetry: &SessionTelemetry, + incremental_tools: bool, +) { let mut direct = 0usize; let mut direct_model_only = 0usize; let mut deferred = 0usize; @@ -418,6 +422,7 @@ pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &Sessi ToolMode::CodeMode => "code_mode", ToolMode::CodeModeOnly => "code_mode_only", }; + let incremental_tools = if incremental_tools { "true" } else { "false" }; for (exposure, count) in [ (EXPOSURE_DIRECT, direct), (EXPOSURE_DIRECT_MODEL_ONLY, direct_model_only), @@ -430,7 +435,11 @@ pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &Sessi TOOL_REGISTRATIONS_METRIC, i64::try_from(count).unwrap_or(i64::MAX), TOOL_REGISTRATIONS_METRIC_BUCKETS, - &[("exposure", exposure), ("tool_mode", tool_mode)], + &[ + ("exposure", exposure), + ("tool_mode", tool_mode), + ("incremental_tools", incremental_tools), + ], ); } } diff --git a/codex-rs/core/tests/suite/scenarios_incremental_tools.rs b/codex-rs/core/tests/suite/scenarios_incremental_tools.rs index a07fe7129e..27173e088b 100644 --- a/codex-rs/core/tests/suite/scenarios_incremental_tools.rs +++ b/codex-rs/core/tests/suite/scenarios_incremental_tools.rs @@ -8,6 +8,7 @@ use codex_features::Feature; use codex_otel::MetricsClient; use codex_otel::MetricsConfig; use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC; +use codex_otel::TOOL_REGISTRATIONS_METRIC; use codex_otel::TOOL_SPEC_DESCRIPTION_BYTES_METRIC; use codex_protocol::openai_models::CodeModeToolMessages; use codex_protocol::openai_models::IncrementalToolMessages; @@ -255,10 +256,26 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history( let requests = mock.requests(); assert_eq!(requests.len(), if use_responses_lite { 6 } else { 4 }); let snapshot = metrics.snapshot()?; + let mut registration_samples = 0; let mut counts = BTreeMap::<_, u64>::new(); let mut bytes = BTreeMap::new(); for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) { match metric.name() { + TOOL_REGISTRATIONS_METRIC => { + let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else { + panic!("tool registrations must be a histogram"); + }; + for point in histogram.data_points() { + registration_samples += point.count(); + assert_eq!( + point + .attributes() + .find(|tag| tag.key.as_str() == "incremental_tools") + .map(|tag| tag.value.to_string()), + Some(use_responses_lite.to_string()) + ); + } + } TOOL_INCREMENTAL_UPDATES_METRIC => { let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { panic!("tool updates must be a counter"); @@ -292,6 +309,11 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history( _ => {} } } + // Six exposure samples per sampling request, excluding compaction. + assert_eq!( + registration_samples, + 6 * if use_responses_lite { 5 } else { 4 } + ); if use_responses_lite { let body = requests[2].body_json(); let exec = body["input"] diff --git a/codex-rs/core/tests/suite/tool_registration_metrics.rs b/codex-rs/core/tests/suite/tool_registration_metrics.rs index a6751df289..b37a4076dc 100644 --- a/codex-rs/core/tests/suite/tool_registration_metrics.rs +++ b/codex-rs/core/tests/suite/tool_registration_metrics.rs @@ -64,6 +64,11 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul model.experimental_supported_tools.clear(); }) .with_config(|config| { + // The feature flag alone cannot enable incremental tools on Responses API. + config + .features + .enable(Feature::IncrementalTools) + .expect("enable incremental tools"); // Keep request_user_input as the only model-only registration. for feature in [ Feature::Collab, @@ -166,6 +171,7 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul .collect::>(); assert_eq!(attributes.get("model").map(String::as_str), Some(MODEL)); assert_eq!(attributes["tool_mode"], "direct"); + assert_eq!(attributes["incremental_tools"], "false"); assert!( attributes.keys().all(|key| matches!( *key, @@ -177,6 +183,7 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul | "app.version" | "exposure" | "tool_mode" + | "incremental_tools" )), "registration telemetry must not label tool names or descriptions" ); diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs index ba35b516f1..b003365dd7 100644 --- a/codex-rs/otel/src/metrics/names.rs +++ b/codex-rs/otel/src/metrics/names.rs @@ -79,7 +79,8 @@ pub const THREAD_SKILLS_TRUNCATED_METRIC: &str = "codex.thread.skills.truncated" pub const THREAD_TOOLS_NAMESPACES_TOTAL_METRIC: &str = "codex.thread.tools.namespaces_total"; pub const THREAD_TOOLS_FRAGMENT_BYTES_METRIC: &str = "codex.thread.tools.fragment_bytes"; -/// Finalized tool registrations per sampling request, tagged by exposure and tool_mode. +/// Finalized tool registrations per sampling request, tagged by exposure, tool_mode, +/// and effective incremental_tools mode. pub const TOOL_REGISTRATIONS_METRIC: &str = "codex.tools.registered"; pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[ 0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 50.0, 75.0, 100.0, 150.0, 200.0, 300.0, 500.0, 750.0,