Preserve trace context during WebSocket startup prewarm (#62103)

## What changed

- Inject the current span's W3C trace headers into WebSocket handshakes.
- Promote `startup_prewarm` to an info span and fall back to the current span when it is disabled, preserving trace context in the spawned task.

## Testing

Add a regression test for `traceparent` and `tracestate` propagation through the handshake, prewarm request, and subsequent turn on the same connection. Verify that prewarm uses the startup trace while the turn uses its own trace, with the prewarm span both enabled and filtered out.

GitOrigin-RevId: 1e196e132a1991206469060ab80a474857fa7909
This commit is contained in:
Tom
2026-11-17 19:24:17 +00:00
committed by copyberry
parent 7fb2cd20c8
commit 7457b7268f
3 changed files with 134 additions and 7 deletions
+2
View File
@@ -83,6 +83,7 @@ use codex_login::default_client::add_originator_header;
use codex_otel::SessionTelemetry;
use codex_otel::WEBSOCKET_CONTINUATION_COUNT_METRIC;
use codex_otel::current_span_w3c_trace_context;
use codex_otel::inject_span_w3c_trace_headers;
use codex_protocol::ItemIdGenerator;
use codex_protocol::ResponseItemId;
use codex_protocol::auth::AuthMode;
@@ -1504,6 +1505,7 @@ impl ModelClient {
) -> std::result::Result<ApiWebSocketConnection, ApiError> {
let mut headers = self.build_websocket_headers(responses_metadata).await;
headers.extend(responses_headers.clone());
inject_span_w3c_trace_headers(&tracing::Span::current(), &mut headers);
if let Some(affinity) = parent_routing_affinity {
let uri = api_provider
.websocket_url_for_path(RESPONSES_PATH)
+9 -7
View File
@@ -14,8 +14,8 @@ use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::Instrument;
use tracing::info;
use tracing::info_span;
use tracing::instrument;
use tracing::trace_span;
use tracing::warn;
use crate::client::ModelClientSession;
@@ -262,6 +262,13 @@ impl Session {
let started_at = Instant::now();
let startup_prewarm_session = Arc::clone(self);
let input_label = input.as_str();
let prewarm_span = info_span!(
"startup_prewarm",
otel.name = "startup_prewarm",
thread.id = %self.thread_id(),
prewarm.input = input_label,
)
.or_current();
let startup_prewarm = tokio::spawn(
async move {
let result = schedule_startup_prewarm_inner(startup_prewarm_session, input).await;
@@ -278,12 +285,7 @@ impl Session {
);
result
}
.instrument(trace_span!(
"startup_prewarm",
otel.name = "startup_prewarm",
thread.id = %self.thread_id(),
prewarm.input = input_label,
)),
.instrument(prewarm_span),
);
state.set_session_startup_prewarm(SessionStartupPrewarmHandle::new(
startup_prewarm,
@@ -0,0 +1,123 @@
//! Use a process-wide subscriber so another test cannot disable shared core callsites
//! while the startup task and regular turn are running.
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use codex_api::WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY;
use codex_api::WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY;
use codex_protocol::protocol::W3cTraceContext;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::start_websocket_server;
use core_test_support::skip_if_no_network;
use core_test_support::test_codex::test_codex;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::SdkTracerProvider;
use pretty_assertions::assert_eq;
use tracing::Instrument;
use tracing_subscriber::filter::dynamic_filter_fn;
use tracing_subscriber::layer::SubscriberExt;
#[tokio::test(flavor = "current_thread")]
async fn websocket_startup_prewarm_propagates_trace() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let include_prewarm_span = Arc::new(AtomicBool::new(false));
let filter_setting = Arc::clone(&include_prewarm_span);
let provider = SdkTracerProvider::builder().build();
tracing::subscriber::set_global_default(
tracing_subscriber::registry()
.with(
tracing_opentelemetry::layer().with_tracer(provider.tracer("startup-prewarm-test")),
)
.with(dynamic_filter_fn(move |meta, _| {
*meta.level() <= tracing::Level::INFO
&& (meta.name() != "startup_prewarm" || filter_setting.load(Ordering::Relaxed))
})),
)?;
for enabled in [false, true] {
include_prewarm_span.store(enabled, Ordering::Relaxed);
assert_prewarm_and_turn_trace().await?;
}
Ok(())
}
async fn assert_prewarm_and_turn_trace() -> anyhow::Result<()> {
let request_span = |trace_id: &str, tracestate: &str| {
let span = tracing::info_span!(parent: None, "request");
assert!(codex_otel::set_parent_from_w3c_trace_context(
&span,
&W3cTraceContext {
traceparent: Some(format!("00-{trace_id}-0000000000000001-01")),
tracestate: Some(tracestate.to_string()),
}
));
span
};
let start_trace_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
let turn_trace_id = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
let server = start_websocket_server(vec![vec![
vec![ev_response_created("warm-1"), ev_completed("warm-1")],
vec![
ev_response_created("resp-1"),
ev_assistant_message("msg-1", "done"),
ev_completed("resp-1"),
],
]])
.await;
let test = test_codex()
.build_with_websocket_server(&server)
.instrument(request_span(start_trace_id, "vendor=start"))
.await?;
let warmup = server
.wait_for_request(/*connection_index*/ 0, /*request_index*/ 0)
.await
.body_json();
assert_eq!(warmup["type"], "response.create");
assert_eq!(warmup["generate"], false);
test.submit_text_turn("hello")
.instrument(request_span(turn_trace_id, "vendor=turn"))
.await?;
let handshake = server.single_handshake();
let connection = server.single_connection();
assert_eq!(connection.len(), 2);
let turn = connection[1].body_json();
assert_eq!(turn["type"], "response.create");
assert_eq!(turn.get("generate"), None);
let assert_trace =
|parent: Option<&str>, state: Option<&str>, expected_id: &str, expected_state: &str| {
// Each carrier can have its own span ID within the originating trace.
assert_eq!(
parent.and_then(|value| value.split('-').nth(1)),
Some(expected_id)
);
assert_eq!(state, Some(expected_state));
};
assert_trace(
handshake.header("traceparent").as_deref(),
handshake.header("tracestate").as_deref(),
start_trace_id,
"vendor=start",
);
for (frame, expected_id, expected_state) in [
(&warmup, start_trace_id, "vendor=start"),
(&turn, turn_trace_id, "vendor=turn"),
] {
let metadata = &frame["client_metadata"];
assert_trace(
metadata[WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY].as_str(),
metadata[WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY].as_str(),
expected_id,
expected_state,
);
}
test.codex.shutdown_and_wait().await?;
server.shutdown().await;
Ok(())
}