From 7457b7268fe109e5387bf0eac80ef4acd7c31365 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 17 Nov 2026 19:06:23 +0000 Subject: [PATCH] Preserve trace context during WebSocket startup prewarm (#62103) ## What changed - Inject the current span's W3C trace headers into WebSocket handshakes. - Promote `startup_prewarm` to an info span and fall back to the current span when it is disabled, preserving trace context in the spawned task. ## Testing Add a regression test for `traceparent` and `tracestate` propagation through the handshake, prewarm request, and subsequent turn on the same connection. Verify that prewarm uses the startup trace while the turn uses its own trace, with the prewarm span both enabled and filtered out. GitOrigin-RevId: 1e196e132a1991206469060ab80a474857fa7909 --- codex-rs/core/src/client.rs | 2 + codex-rs/core/src/session/startup_prewarm.rs | 16 ++- .../websocket_startup_prewarm_tracing.rs | 123 ++++++++++++++++++ 3 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 codex-rs/core/tests/websocket_startup_prewarm_tracing.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 3874e4bba8..21060f212d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -83,6 +83,7 @@ use codex_login::default_client::add_originator_header; use codex_otel::SessionTelemetry; use codex_otel::WEBSOCKET_CONTINUATION_COUNT_METRIC; use codex_otel::current_span_w3c_trace_context; +use codex_otel::inject_span_w3c_trace_headers; use codex_protocol::ItemIdGenerator; use codex_protocol::ResponseItemId; use codex_protocol::auth::AuthMode; @@ -1504,6 +1505,7 @@ impl ModelClient { ) -> std::result::Result { let mut headers = self.build_websocket_headers(responses_metadata).await; headers.extend(responses_headers.clone()); + inject_span_w3c_trace_headers(&tracing::Span::current(), &mut headers); if let Some(affinity) = parent_routing_affinity { let uri = api_provider .websocket_url_for_path(RESPONSES_PATH) diff --git a/codex-rs/core/src/session/startup_prewarm.rs b/codex-rs/core/src/session/startup_prewarm.rs index 3014ccde96..c1ed35e5b4 100644 --- a/codex-rs/core/src/session/startup_prewarm.rs +++ b/codex-rs/core/src/session/startup_prewarm.rs @@ -14,8 +14,8 @@ use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; use tracing::info; +use tracing::info_span; use tracing::instrument; -use tracing::trace_span; use tracing::warn; use crate::client::ModelClientSession; @@ -262,6 +262,13 @@ impl Session { let started_at = Instant::now(); let startup_prewarm_session = Arc::clone(self); let input_label = input.as_str(); + let prewarm_span = info_span!( + "startup_prewarm", + otel.name = "startup_prewarm", + thread.id = %self.thread_id(), + prewarm.input = input_label, + ) + .or_current(); let startup_prewarm = tokio::spawn( async move { let result = schedule_startup_prewarm_inner(startup_prewarm_session, input).await; @@ -278,12 +285,7 @@ impl Session { ); result } - .instrument(trace_span!( - "startup_prewarm", - otel.name = "startup_prewarm", - thread.id = %self.thread_id(), - prewarm.input = input_label, - )), + .instrument(prewarm_span), ); state.set_session_startup_prewarm(SessionStartupPrewarmHandle::new( startup_prewarm, diff --git a/codex-rs/core/tests/websocket_startup_prewarm_tracing.rs b/codex-rs/core/tests/websocket_startup_prewarm_tracing.rs new file mode 100644 index 0000000000..08436aaff7 --- /dev/null +++ b/codex-rs/core/tests/websocket_startup_prewarm_tracing.rs @@ -0,0 +1,123 @@ +//! Use a process-wide subscriber so another test cannot disable shared core callsites +//! while the startup task and regular turn are running. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; + +use codex_api::WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY; +use codex_api::WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY; +use codex_protocol::protocol::W3cTraceContext; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::start_websocket_server; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::SdkTracerProvider; +use pretty_assertions::assert_eq; +use tracing::Instrument; +use tracing_subscriber::filter::dynamic_filter_fn; +use tracing_subscriber::layer::SubscriberExt; + +#[tokio::test(flavor = "current_thread")] +async fn websocket_startup_prewarm_propagates_trace() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let include_prewarm_span = Arc::new(AtomicBool::new(false)); + let filter_setting = Arc::clone(&include_prewarm_span); + let provider = SdkTracerProvider::builder().build(); + tracing::subscriber::set_global_default( + tracing_subscriber::registry() + .with( + tracing_opentelemetry::layer().with_tracer(provider.tracer("startup-prewarm-test")), + ) + .with(dynamic_filter_fn(move |meta, _| { + *meta.level() <= tracing::Level::INFO + && (meta.name() != "startup_prewarm" || filter_setting.load(Ordering::Relaxed)) + })), + )?; + + for enabled in [false, true] { + include_prewarm_span.store(enabled, Ordering::Relaxed); + assert_prewarm_and_turn_trace().await?; + } + Ok(()) +} + +async fn assert_prewarm_and_turn_trace() -> anyhow::Result<()> { + let request_span = |trace_id: &str, tracestate: &str| { + let span = tracing::info_span!(parent: None, "request"); + assert!(codex_otel::set_parent_from_w3c_trace_context( + &span, + &W3cTraceContext { + traceparent: Some(format!("00-{trace_id}-0000000000000001-01")), + tracestate: Some(tracestate.to_string()), + } + )); + span + }; + let start_trace_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let turn_trace_id = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let server = start_websocket_server(vec![vec![ + vec![ev_response_created("warm-1"), ev_completed("warm-1")], + vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-1"), + ], + ]]) + .await; + let test = test_codex() + .build_with_websocket_server(&server) + .instrument(request_span(start_trace_id, "vendor=start")) + .await?; + let warmup = server + .wait_for_request(/*connection_index*/ 0, /*request_index*/ 0) + .await + .body_json(); + assert_eq!(warmup["type"], "response.create"); + assert_eq!(warmup["generate"], false); + + test.submit_text_turn("hello") + .instrument(request_span(turn_trace_id, "vendor=turn")) + .await?; + let handshake = server.single_handshake(); + let connection = server.single_connection(); + assert_eq!(connection.len(), 2); + let turn = connection[1].body_json(); + assert_eq!(turn["type"], "response.create"); + assert_eq!(turn.get("generate"), None); + + let assert_trace = + |parent: Option<&str>, state: Option<&str>, expected_id: &str, expected_state: &str| { + // Each carrier can have its own span ID within the originating trace. + assert_eq!( + parent.and_then(|value| value.split('-').nth(1)), + Some(expected_id) + ); + assert_eq!(state, Some(expected_state)); + }; + assert_trace( + handshake.header("traceparent").as_deref(), + handshake.header("tracestate").as_deref(), + start_trace_id, + "vendor=start", + ); + for (frame, expected_id, expected_state) in [ + (&warmup, start_trace_id, "vendor=start"), + (&turn, turn_trace_id, "vendor=turn"), + ] { + let metadata = &frame["client_metadata"]; + assert_trace( + metadata[WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY].as_str(), + metadata[WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY].as_str(), + expected_id, + expected_state, + ); + } + test.codex.shutdown_and_wait().await?; + server.shutdown().await; + Ok(()) +}