diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 7333c9060c..f30130e641 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -1742,7 +1742,14 @@ async fn run_sampling_request( Arc::clone(&turn_diff_tracker), ); let max_retries = turn_context.provider.info().stream_max_retries(); - record_registration_metrics(&step_context.tool_router, &step_context.session_telemetry); + let incremental_tools = sess + .current_window_uses_incremental_tools(&step_context) + .await; + record_registration_metrics( + &step_context.tool_router, + &step_context.session_telemetry, + incremental_tools, + ); let mut retry_state = ResponsesStreamRetryState::default(); let mut initial_input = Some(input); let mut original_input = None; @@ -1765,8 +1772,7 @@ async fn run_sampling_request( prompt_input, step_context.as_ref(), base_instructions.clone(), - sess.current_window_uses_incremental_tools(&step_context) - .await, + incremental_tools, ); let responses_metadata = sess .responses_metadata(step_context.as_ref(), CodexResponsesRequestKind::Turn) diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index a7a192815b..3c01ab9818 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -395,7 +395,11 @@ impl ToolRouter { /// Records the registered inventory once per sampling request, before transport retries. /// Registrations do not imply model visibility or reachability; hosted tools are excluded. -pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &SessionTelemetry) { +pub(crate) fn record_registration_metrics( + router: &ToolRouter, + telemetry: &SessionTelemetry, + incremental_tools: bool, +) { let mut direct = 0usize; let mut direct_model_only = 0usize; let mut deferred = 0usize; @@ -418,6 +422,7 @@ pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &Sessi ToolMode::CodeMode => "code_mode", ToolMode::CodeModeOnly => "code_mode_only", }; + let incremental_tools = if incremental_tools { "true" } else { "false" }; for (exposure, count) in [ (EXPOSURE_DIRECT, direct), (EXPOSURE_DIRECT_MODEL_ONLY, direct_model_only), @@ -430,7 +435,11 @@ pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &Sessi TOOL_REGISTRATIONS_METRIC, i64::try_from(count).unwrap_or(i64::MAX), TOOL_REGISTRATIONS_METRIC_BUCKETS, - &[("exposure", exposure), ("tool_mode", tool_mode)], + &[ + ("exposure", exposure), + ("tool_mode", tool_mode), + ("incremental_tools", incremental_tools), + ], ); } } diff --git a/codex-rs/core/tests/suite/scenarios_incremental_tools.rs b/codex-rs/core/tests/suite/scenarios_incremental_tools.rs index a07fe7129e..27173e088b 100644 --- a/codex-rs/core/tests/suite/scenarios_incremental_tools.rs +++ b/codex-rs/core/tests/suite/scenarios_incremental_tools.rs @@ -8,6 +8,7 @@ use codex_features::Feature; use codex_otel::MetricsClient; use codex_otel::MetricsConfig; use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC; +use codex_otel::TOOL_REGISTRATIONS_METRIC; use codex_otel::TOOL_SPEC_DESCRIPTION_BYTES_METRIC; use codex_protocol::openai_models::CodeModeToolMessages; use codex_protocol::openai_models::IncrementalToolMessages; @@ -255,10 +256,26 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history( let requests = mock.requests(); assert_eq!(requests.len(), if use_responses_lite { 6 } else { 4 }); let snapshot = metrics.snapshot()?; + let mut registration_samples = 0; let mut counts = BTreeMap::<_, u64>::new(); let mut bytes = BTreeMap::new(); for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) { match metric.name() { + TOOL_REGISTRATIONS_METRIC => { + let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else { + panic!("tool registrations must be a histogram"); + }; + for point in histogram.data_points() { + registration_samples += point.count(); + assert_eq!( + point + .attributes() + .find(|tag| tag.key.as_str() == "incremental_tools") + .map(|tag| tag.value.to_string()), + Some(use_responses_lite.to_string()) + ); + } + } TOOL_INCREMENTAL_UPDATES_METRIC => { let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { panic!("tool updates must be a counter"); @@ -292,6 +309,11 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history( _ => {} } } + // Six exposure samples per sampling request, excluding compaction. + assert_eq!( + registration_samples, + 6 * if use_responses_lite { 5 } else { 4 } + ); if use_responses_lite { let body = requests[2].body_json(); let exec = body["input"] diff --git a/codex-rs/core/tests/suite/tool_registration_metrics.rs b/codex-rs/core/tests/suite/tool_registration_metrics.rs index a6751df289..b37a4076dc 100644 --- a/codex-rs/core/tests/suite/tool_registration_metrics.rs +++ b/codex-rs/core/tests/suite/tool_registration_metrics.rs @@ -64,6 +64,11 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul model.experimental_supported_tools.clear(); }) .with_config(|config| { + // The feature flag alone cannot enable incremental tools on Responses API. + config + .features + .enable(Feature::IncrementalTools) + .expect("enable incremental tools"); // Keep request_user_input as the only model-only registration. for feature in [ Feature::Collab, @@ -166,6 +171,7 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul .collect::>(); assert_eq!(attributes.get("model").map(String::as_str), Some(MODEL)); assert_eq!(attributes["tool_mode"], "direct"); + assert_eq!(attributes["incremental_tools"], "false"); assert!( attributes.keys().all(|key| matches!( *key, @@ -177,6 +183,7 @@ async fn sampling_records_tool_registrations_once_across_stream_retry() -> Resul | "app.version" | "exposure" | "tool_mode" + | "incremental_tools" )), "registration telemetry must not label tool names or descriptions" ); diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs index ba35b516f1..b003365dd7 100644 --- a/codex-rs/otel/src/metrics/names.rs +++ b/codex-rs/otel/src/metrics/names.rs @@ -79,7 +79,8 @@ pub const THREAD_SKILLS_TRUNCATED_METRIC: &str = "codex.thread.skills.truncated" pub const THREAD_TOOLS_NAMESPACES_TOTAL_METRIC: &str = "codex.thread.tools.namespaces_total"; pub const THREAD_TOOLS_FRAGMENT_BYTES_METRIC: &str = "codex.thread.tools.fragment_bytes"; -/// Finalized tool registrations per sampling request, tagged by exposure and tool_mode. +/// Finalized tool registrations per sampling request, tagged by exposure, tool_mode, +/// and effective incremental_tools mode. pub const TOOL_REGISTRATIONS_METRIC: &str = "codex.tools.registered"; pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[ 0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 50.0, 75.0, 100.0, 150.0, 200.0, 300.0, 500.0, 750.0,