From 19c516242c38e68cb94a0a7d3586300791459811 Mon Sep 17 00:00:00 2001 From: Anton Panasenko Date: Mon, 29 Jun 2026 22:43:43 -0700 Subject: [PATCH] telemetry(exec-server): trace Noise virtual streams --- .../src/noise_relay/executor_stream.rs | 322 +++++++++++++++--- .../src/noise_relay/executor_stream_tests.rs | 78 ++++- codex-rs/exec-server/src/noise_relay/mod.rs | 1 + .../src/noise_relay/trace_context.rs | 63 ++++ .../src/noise_relay/trace_context_tests.rs | 48 +++ codex-rs/exec-server/src/relay.rs | 61 +++- 6 files changed, 501 insertions(+), 72 deletions(-) create mode 100644 codex-rs/exec-server/src/noise_relay/trace_context.rs create mode 100644 codex-rs/exec-server/src/noise_relay/trace_context_tests.rs diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream.rs b/codex-rs/exec-server/src/noise_relay/executor_stream.rs index 89f33d938a..258f152317 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream.rs @@ -6,9 +6,16 @@ use std::sync::Arc; use std::sync::Mutex; +use std::time::Duration; +use std::time::Instant; +use codex_exec_server_protocol::JSONRPCMessage; +use codex_protocol::protocol::W3cTraceContext; use tokio::sync::mpsc; use tokio::sync::watch; +use tracing::Instrument; +use tracing::Span; +use tracing::field; use tracing::warn; use crate::ExecServerError; @@ -23,6 +30,7 @@ use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN; use crate::noise_relay::message_framing::frame_jsonrpc_message; use crate::noise_relay::ordered_ciphertext::OrderedCiphertextFrames; use crate::noise_relay::take_next_sequence; +use crate::noise_relay::trace_context::NoiseTraceContext; use crate::relay::encode_relay_message_frame; use crate::relay_proto::RelayData; use crate::relay_proto::RelayMessageFrame; @@ -39,6 +47,35 @@ pub(crate) struct ClosedNoiseVirtualStream { pub(crate) instance_id: u64, } +/// One frame queued for the executor's shared physical rendezvous websocket. +/// +/// Data frames carry the current trace context across the channel so the +/// physical websocket send remains attributable to the originating RPC or +/// process notification. Relay control frames intentionally carry no context. +pub(crate) struct NoisePhysicalFrame { + pub(crate) encoded: Vec, + pub(crate) queued_at: Instant, + pub(crate) trace: Option>, +} + +impl NoisePhysicalFrame { + pub(crate) fn data(encoded: Vec, trace: Option>) -> Self { + Self { + encoded, + queued_at: Instant::now(), + trace, + } + } + + pub(crate) fn control(encoded: Vec) -> Self { + Self { + encoded, + queued_at: Instant::now(), + trace: None, + } + } +} + /// One authenticated JSON-RPC stream carried by the executor's physical relay. /// /// Inbound delivery is intentionally nonblocking. An overloaded or abandoned @@ -50,9 +87,19 @@ pub(crate) struct NoiseVirtualStream { transport: Arc>, inbound_ciphertexts: OrderedCiphertextFrames, inbound_decoder: JsonRpcMessageDecoder, + trace_context: Arc>, pub(crate) instance_id: u64, } +struct ExecutorInboundTimings { + websocket_ingress_to_decode: Duration, + relay_dispatch: Duration, + reorder: Duration, + decrypt: Duration, + decode: Duration, + trace_context: Duration, +} + impl NoiseVirtualStream { pub(crate) fn disconnect(self, reason: Option) { let _ = self.disconnected_tx.send(true); @@ -63,8 +110,19 @@ impl NoiseVirtualStream { /// Reorder and decrypt one inbound record, then queue complete JSON-RPC messages. /// This must stay nonblocking because all virtual streams share the read loop. - pub(crate) fn receive_data(&mut self, data: RelayData) -> Result<(), ExecServerError> { - for ciphertext in self.inbound_ciphertexts.push(data.seq, data.payload)? { + pub(crate) fn receive_data( + &mut self, + data: RelayData, + physical_received_at: Instant, + ) -> Result<(), ExecServerError> { + let relay_seq = data.seq; + let ciphertext_bytes = data.payload.len(); + let relay_dispatch_elapsed = physical_received_at.elapsed(); + let reorder_started_at = Instant::now(); + let ciphertexts = self.inbound_ciphertexts.push(data.seq, data.payload)?; + let reorder_elapsed = reorder_started_at.elapsed(); + for ciphertext in ciphertexts { + let decrypt_started_at = Instant::now(); let plaintext = { let mut transport = self .transport @@ -74,20 +132,113 @@ impl NoiseVirtualStream { ExecServerError::Protocol(format!("Noise relay decryption failed: {error}")) })? }; - for message in self.inbound_decoder.push(&plaintext)? { - self.incoming_tx - .try_send(JsonRpcConnectionEvent::Message(message)) - .map_err(|_| { - ExecServerError::Protocol( - "Noise virtual stream inbound queue is full or closed".to_string(), - ) - })?; + let decrypt_elapsed = decrypt_started_at.elapsed(); + let decode_started_at = Instant::now(); + let messages = self.inbound_decoder.push(&plaintext)?; + let decode_elapsed = decode_started_at.elapsed(); + for message in messages { + let trace_context_started_at = Instant::now(); + self.trace_context + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .observe_request(&message); + let trace_context_elapsed = trace_context_started_at.elapsed(); + let decoded_span = decoded_message_span( + &message, + relay_seq, + ciphertext_bytes, + ExecutorInboundTimings { + websocket_ingress_to_decode: physical_received_at.elapsed(), + relay_dispatch: relay_dispatch_elapsed, + reorder: reorder_elapsed, + decrypt: decrypt_elapsed, + decode: decode_elapsed, + trace_context: trace_context_elapsed, + }, + ); + let enqueued_span = enqueued_message_span(&message, relay_seq); + let enqueue_started_at = Instant::now(); + let enqueue_result = decoded_span.in_scope(|| { + self.incoming_tx + .try_send(JsonRpcConnectionEvent::Message(message)) + .map_err(|_| { + ExecServerError::Protocol( + "Noise virtual stream inbound queue is full or closed".to_string(), + ) + }) + }); + enqueued_span.record( + "enqueue_ms", + enqueue_started_at.elapsed().as_secs_f64() * 1_000.0, + ); + enqueue_result?; + enqueued_span.in_scope(|| {}); } } Ok(()) } } +fn decoded_message_span( + message: &JSONRPCMessage, + relay_seq: u32, + ciphertext_bytes: usize, + timings: ExecutorInboundTimings, +) -> Span { + let (message_kind, method, trace) = match message { + JSONRPCMessage::Request(request) => { + ("request", request.method.as_str(), request.trace.as_ref()) + } + JSONRPCMessage::Notification(notification) => { + ("notification", notification.method.as_str(), None) + } + JSONRPCMessage::Response(_) => ("response", "", None), + JSONRPCMessage::Error(_) => ("error", "", None), + }; + let span = tracing::info_span!( + "exec_server.noise.executor_message_decoded", + message_kind, + method, + relay_seq, + ciphertext_bytes, + executor_websocket_ingress_to_decode_ms = + timings.websocket_ingress_to_decode.as_secs_f64() * 1_000.0, + relay_dispatch_ms = timings.relay_dispatch.as_secs_f64() * 1_000.0, + reorder_ms = timings.reorder.as_secs_f64() * 1_000.0, + decrypt_ms = timings.decrypt.as_secs_f64() * 1_000.0, + decode_ms = timings.decode.as_secs_f64() * 1_000.0, + trace_context_ms = timings.trace_context.as_secs_f64() * 1_000.0, + ); + if let Some(trace) = trace { + let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace); + } + span +} + +fn enqueued_message_span(message: &JSONRPCMessage, relay_seq: u32) -> Span { + let (message_kind, method, trace) = match message { + JSONRPCMessage::Request(request) => { + ("request", request.method.as_str(), request.trace.as_ref()) + } + JSONRPCMessage::Notification(notification) => { + ("notification", notification.method.as_str(), None) + } + JSONRPCMessage::Response(_) => ("response", "", None), + JSONRPCMessage::Error(_) => ("error", "", None), + }; + let span = tracing::info_span!( + "exec_server.noise.executor_message_enqueued", + message_kind, + method, + relay_seq, + enqueue_ms = field::Empty, + ); + if let Some(trace) = trace { + let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace); + } + span +} + /// Start JSON-RPC processing for a completed handshake. /// /// The returned value is the read half; the spawned task owns outbound framing @@ -96,7 +247,7 @@ pub(crate) fn spawn_noise_virtual_stream( stream_id: String, instance_id: u64, processor: ConnectionProcessor, - physical_outgoing_tx: mpsc::Sender>, + physical_outgoing_tx: mpsc::Sender, closed_stream_tx: mpsc::Sender, transport: NoiseTransport, ) -> NoiseVirtualStream { @@ -105,49 +256,27 @@ pub(crate) fn spawn_noise_virtual_stream( let (disconnected_tx, disconnected_rx) = watch::channel(false); let transport = Arc::new(Mutex::new(transport)); let writer_transport = Arc::clone(&transport); + let trace_context = Arc::new(Mutex::new(NoiseTraceContext::default())); + let writer_trace_context = Arc::clone(&trace_context); let processor_stream_id = stream_id.clone(); let processor_closed_stream_tx = closed_stream_tx.clone(); let writer_stream_id = stream_id; let writer_task = tokio::spawn(async move { let mut next_seq = 0u32; 'writer: while let Some(message) = json_outgoing_rx.recv().await { - // Each chunk becomes one Noise record and consumes one nonce. - let framed = match frame_jsonrpc_message(&message) { - Ok(framed) => framed, - Err(error) => { - warn!("failed to frame Noise virtual stream JSON-RPC payload: {error}"); - break; - } - }; - for plaintext_record in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN) { - let seq = match take_next_sequence(&mut next_seq) { - Ok(seq) => seq, - Err(error) => { - warn!("Noise virtual stream sequence exhausted: {error}"); - break 'writer; - } - }; - let ciphertext = { - let mut transport = writer_transport - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - transport.encrypt(plaintext_record) - }; - let ciphertext = match ciphertext { - Ok(ciphertext) => ciphertext, - Err(error) => { - warn!("failed to encrypt Noise virtual stream payload: {error}"); - break 'writer; - } - }; - let frame = RelayMessageFrame::data(writer_stream_id.clone(), seq, ciphertext); - if physical_outgoing_tx - .send(encode_relay_message_frame(&frame)) - .await - .is_err() - { - break 'writer; - } + let outbound_span = outbound_message_span(&message, &writer_trace_context); + if let Err(error) = send_outbound_message( + &physical_outgoing_tx, + &writer_transport, + &writer_stream_id, + &mut next_seq, + &message, + ) + .instrument(outbound_span) + .await + { + warn!("failed to send Noise virtual stream JSON-RPC payload: {error}"); + break 'writer; } } @@ -158,7 +287,9 @@ pub(crate) fn spawn_noise_virtual_stream( }; let reset = RelayMessageFrame::reset(writer_stream_id, NOISE_RELAY_RESET_REASON.to_string()); - let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset)); + let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control( + encode_relay_message_frame(&reset), + )); let _ = closed_stream_tx.send(closed_stream).await; }); @@ -187,10 +318,103 @@ pub(crate) fn spawn_noise_virtual_stream( transport, inbound_ciphertexts: OrderedCiphertextFrames::default(), inbound_decoder: JsonRpcMessageDecoder::default(), + trace_context, instance_id, } } +fn outbound_message_span( + message: &JSONRPCMessage, + trace_context: &Mutex, +) -> Span { + let (message_kind, method) = match message { + JSONRPCMessage::Request(request) => ("request", request.method.as_str()), + JSONRPCMessage::Notification(notification) => { + ("notification", notification.method.as_str()) + } + JSONRPCMessage::Response(_) => ("response", ""), + JSONRPCMessage::Error(_) => ("error", ""), + }; + let trace = trace_context + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .return_trace(message); + let span = tracing::info_span!( + "exec_server.noise.executor_outbound", + message_kind, + method, + framed_bytes = field::Empty, + records = field::Empty, + frame_complete_ms = field::Empty, + encrypt_complete_ms = field::Empty, + physical_queue_complete_ms = field::Empty, + ); + if let Some(trace) = trace.as_ref() { + let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace); + } + span +} + +async fn send_outbound_message( + physical_outgoing_tx: &mpsc::Sender, + transport: &Mutex, + stream_id: &str, + next_seq: &mut u32, + message: &JSONRPCMessage, +) -> Result<(), ExecServerError> { + let started_at = Instant::now(); + let framed = tracing::info_span!("exec_server.noise.executor_frame_message") + .in_scope(|| frame_jsonrpc_message(message))?; + let record_count = framed.len().div_ceil(NOISE_RECORD_PLAINTEXT_LEN); + let span = Span::current(); + span.record("framed_bytes", framed.len()); + span.record("records", record_count); + span.record( + "frame_complete_ms", + started_at.elapsed().as_secs_f64() * 1_000.0, + ); + // Extracting W3C context allocates its string carrier. Do that once for the + // logical JSON-RPC message, then share it across all physical Noise records. + let trace = codex_otel::current_span_w3c_trace_context().map(Arc::new); + + for (record_index, plaintext_record) in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN).enumerate() { + let seq = take_next_sequence(next_seq)?; + let ciphertext = + tracing::info_span!("exec_server.noise.executor_encrypt_record", record_index,) + .in_scope(|| { + transport + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .encrypt(plaintext_record) + }) + .map_err(|error| { + ExecServerError::Protocol(format!( + "failed to encrypt Noise virtual stream payload: {error}" + )) + })?; + span.record( + "encrypt_complete_ms", + started_at.elapsed().as_secs_f64() * 1_000.0, + ); + let frame = RelayMessageFrame::data(stream_id.to_string(), seq, ciphertext); + let encoded = encode_relay_message_frame(&frame); + let permit = physical_outgoing_tx + .reserve() + .instrument(tracing::info_span!( + "exec_server.noise.executor_physical_queue", + record_index, + )) + .await + .map_err(|_| ExecServerError::Closed)?; + permit.send(NoisePhysicalFrame::data(encoded, trace.clone())); + span.record( + "physical_queue_complete_ms", + started_at.elapsed().as_secs_f64() * 1_000.0, + ); + } + Ok(()) +} + #[cfg(test)] #[path = "executor_stream_tests.rs"] mod tests; diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs index 232dfa479e..fd310f9a9c 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream_tests.rs @@ -1,23 +1,86 @@ +use std::sync::Arc; use std::time::Duration; use anyhow::Result; use codex_exec_server_protocol::JSONRPCMessage; use codex_exec_server_protocol::JSONRPCResponse; use codex_exec_server_protocol::RequestId; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::SdkTracerProvider; use tokio::sync::mpsc; use tokio::time::timeout; +use tracing::Instrument; +use tracing::instrument::WithSubscriber; +use tracing_subscriber::prelude::*; use super::ClosedNoiseVirtualStream; +use super::send_outbound_message; use super::spawn_noise_virtual_stream; use crate::ExecServerRuntimePaths; use crate::connection::CHANNEL_CAPACITY; use crate::noise_channel::InitiatorHandshake; use crate::noise_channel::NoiseChannelIdentity; use crate::noise_channel::PendingResponderHandshake; +use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN; use crate::noise_relay::message_framing::frame_jsonrpc_message; use crate::relay_proto::RelayData; use crate::server::ConnectionProcessor; +#[tokio::test(flavor = "current_thread")] +async fn outbound_records_share_one_message_trace_context() -> Result<()> { + let executor_identity = NoiseChannelIdentity::generate()?; + let harness_identity = NoiseChannelIdentity::generate()?; + let (initiator, request) = InitiatorHandshake::start( + &harness_identity, + &executor_identity.public_key(), + b"test-prologue", + b"authorization", + )?; + let pending = + PendingResponderHandshake::read_request(&executor_identity, b"test-prologue", &request)?; + let (executor_transport, response) = pending.complete()?; + let _harness_transport = initiator.finish(&response)?; + let executor_transport = std::sync::Mutex::new(executor_transport); + let (physical_outgoing_tx, mut physical_outgoing_rx) = mpsc::channel(CHANNEL_CAPACITY); + let message = JSONRPCMessage::Response(JSONRPCResponse { + id: RequestId::Integer(1), + result: serde_json::Value::String("x".repeat(NOISE_RECORD_PLAINTEXT_LEN * 2)), + }); + + let provider = SdkTracerProvider::builder().build(); + let subscriber = tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("exec-server-test"))); + let mut next_seq = 0; + async { + tracing::callsite::rebuild_interest_cache(); + send_outbound_message( + &physical_outgoing_tx, + &executor_transport, + "stream-1", + &mut next_seq, + &message, + ) + .instrument(tracing::info_span!("outbound-message")) + .await + } + .with_subscriber(subscriber) + .await?; + + let mut frames = Vec::new(); + while let Ok(frame) = physical_outgoing_rx.try_recv() { + frames.push(frame); + } + assert!(frames.len() > 1, "expected multiple physical records"); + let first_trace = frames[0].trace.as_ref().expect("first record trace"); + assert!(frames.iter().skip(1).all(|frame| { + Arc::ptr_eq( + first_trace, + frame.trace.as_ref().expect("subsequent record trace"), + ) + })); + Ok(()) +} + #[tokio::test] async fn processor_exit_reports_closed_virtual_stream() -> Result<()> { let executor_identity = NoiseChannelIdentity::generate()?; @@ -52,12 +115,15 @@ async fn processor_exit_reports_closed_virtual_stream() -> Result<()> { result: serde_json::Value::Null, }); let ciphertext = harness_transport.encrypt(&frame_jsonrpc_message(&message)?)?; - stream.receive_data(RelayData { - seq: 0, - segment_index: 0, - segment_count: 1, - payload: ciphertext, - })?; + stream.receive_data( + RelayData { + seq: 0, + segment_index: 0, + segment_count: 1, + payload: ciphertext, + }, + std::time::Instant::now(), + )?; assert!(matches!( timeout(Duration::from_secs(1), closed_stream_rx.recv()).await?, diff --git a/codex-rs/exec-server/src/noise_relay/mod.rs b/codex-rs/exec-server/src/noise_relay/mod.rs index 56fabd5cb5..d1775d73f0 100644 --- a/codex-rs/exec-server/src/noise_relay/mod.rs +++ b/codex-rs/exec-server/src/noise_relay/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod executor_stream; mod harness; mod message_framing; mod ordered_ciphertext; +mod trace_context; use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; diff --git a/codex-rs/exec-server/src/noise_relay/trace_context.rs b/codex-rs/exec-server/src/noise_relay/trace_context.rs new file mode 100644 index 0000000000..a6b42b63d2 --- /dev/null +++ b/codex-rs/exec-server/src/noise_relay/trace_context.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use codex_exec_server_protocol::EXEC_CLOSED_METHOD; +use codex_exec_server_protocol::JSONRPCMessage; +use codex_exec_server_protocol::RequestId; +use codex_protocol::protocol::W3cTraceContext; + +/// Correlates return traffic with the trace carried by its originating request. +/// +/// Responses do not repeat the request's W3C carrier, and process notifications +/// are correlated by `processId`. Keeping this state per virtual stream avoids +/// adding telemetry-only fields to the JSON-RPC wire protocol. +#[derive(Default)] +pub(super) struct NoiseTraceContext { + requests: HashMap, + processes: HashMap, +} + +impl NoiseTraceContext { + pub(super) fn observe_request(&mut self, message: &JSONRPCMessage) { + let JSONRPCMessage::Request(request) = message else { + return; + }; + let Some(trace) = request.trace.as_ref() else { + return; + }; + self.requests.insert(request.id.clone(), trace.clone()); + if let Some(process_id) = message_process_id(message) { + self.processes + .entry(process_id.to_string()) + .or_insert_with(|| trace.clone()); + } + } + + pub(super) fn return_trace(&mut self, message: &JSONRPCMessage) -> Option { + match message { + JSONRPCMessage::Response(response) => self.requests.remove(&response.id), + JSONRPCMessage::Error(error) => self.requests.remove(&error.id), + JSONRPCMessage::Notification(notification) => { + let process_id = message_process_id(message)?; + let trace = self.processes.get(process_id).cloned(); + if notification.method == EXEC_CLOSED_METHOD { + self.processes.remove(process_id); + } + trace + } + JSONRPCMessage::Request(request) => request.trace.clone(), + } + } +} + +fn message_process_id(message: &JSONRPCMessage) -> Option<&str> { + let params = match message { + JSONRPCMessage::Request(request) => request.params.as_ref(), + JSONRPCMessage::Notification(notification) => notification.params.as_ref(), + JSONRPCMessage::Response(_) | JSONRPCMessage::Error(_) => None, + }?; + params.get("processId")?.as_str() +} + +#[cfg(test)] +#[path = "trace_context_tests.rs"] +mod tests; diff --git a/codex-rs/exec-server/src/noise_relay/trace_context_tests.rs b/codex-rs/exec-server/src/noise_relay/trace_context_tests.rs new file mode 100644 index 0000000000..0c293e552d --- /dev/null +++ b/codex-rs/exec-server/src/noise_relay/trace_context_tests.rs @@ -0,0 +1,48 @@ +use codex_exec_server_protocol::EXEC_CLOSED_METHOD; +use codex_exec_server_protocol::EXEC_METHOD; +use codex_exec_server_protocol::JSONRPCMessage; +use codex_exec_server_protocol::JSONRPCNotification; +use codex_exec_server_protocol::JSONRPCRequest; +use codex_exec_server_protocol::JSONRPCResponse; +use codex_exec_server_protocol::RequestId; +use codex_protocol::protocol::W3cTraceContext; +use pretty_assertions::assert_eq; + +use super::NoiseTraceContext; + +fn trace_context() -> W3cTraceContext { + W3cTraceContext { + traceparent: Some("00-00000000000000000000000000000001-0000000000000002-01".to_string()), + tracestate: None, + } +} + +fn process_start_request(trace: W3cTraceContext) -> JSONRPCMessage { + JSONRPCMessage::Request(JSONRPCRequest { + id: RequestId::Integer(7), + method: EXEC_METHOD.to_string(), + params: Some(serde_json::json!({"processId": "process-1"})), + trace: Some(trace), + }) +} + +#[test] +fn correlates_response_and_terminal_notification_with_request_trace() { + let trace = trace_context(); + let mut context = NoiseTraceContext::default(); + context.observe_request(&process_start_request(trace.clone())); + + let response = JSONRPCMessage::Response(JSONRPCResponse { + id: RequestId::Integer(7), + result: serde_json::Value::Null, + }); + assert_eq!(context.return_trace(&response), Some(trace.clone())); + assert_eq!(context.return_trace(&response), None); + + let closed = JSONRPCMessage::Notification(JSONRPCNotification { + method: EXEC_CLOSED_METHOD.to_string(), + params: Some(serde_json::json!({"processId": "process-1", "seq": 1})), + }); + assert_eq!(context.return_trace(&closed), Some(trace)); + assert_eq!(context.return_trace(&closed), None); +} diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rs index b8d477f5db..e0073e9151 100644 --- a/codex-rs/exec-server/src/relay.rs +++ b/codex-rs/exec-server/src/relay.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::time::Duration; +use std::time::Instant; use codex_exec_server_protocol::JSONRPCMessage; use futures::Sink; @@ -15,6 +16,7 @@ use tokio::task::JoinSet; use tokio::time::timeout; use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::tungstenite::Message; +use tracing::Instrument; use tracing::debug; use tracing::info; use tracing::warn; @@ -32,6 +34,7 @@ use crate::noise_channel::PendingResponderHandshake; use crate::noise_channel::noise_channel_prologue; use crate::noise_relay::NOISE_RELAY_RESET_REASON; use crate::noise_relay::executor_stream::ClosedNoiseVirtualStream; +use crate::noise_relay::executor_stream::NoisePhysicalFrame; use crate::noise_relay::executor_stream::NoiseVirtualStream; use crate::noise_relay::executor_stream::spawn_noise_virtual_stream; use crate::relay_proto::RelayData; @@ -483,7 +486,7 @@ where ); let (mut websocket_sink, mut websocket_stream) = stream.split(); let (physical_outgoing_tx, mut physical_outgoing_rx) = - mpsc::channel::>(CHANNEL_CAPACITY); + mpsc::channel::(CHANNEL_CAPACITY); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel::(MAX_ACTIVE_NOISE_RELAY_STREAMS); let (pong_tx, mut pong_rx) = mpsc::channel(1); @@ -498,7 +501,7 @@ where let pong_deadline = tokio::time::sleep(WEBSOCKET_PONG_TIMEOUT); tokio::pin!(pong_deadline); loop { - let message = tokio::select! { + let (message, queued_frame) = tokio::select! { pong = pong_rx.recv() => { let Some(()) = pong else { break RendezvousDisconnectReason::LocalShutdown; @@ -521,18 +524,35 @@ where } } _ = keepalive.tick(), if pong_watchdog.deadline().is_none() => { - Message::Ping(Vec::new().into()) + (Message::Ping(Vec::new().into()), None) } - encoded = physical_outgoing_rx.recv() => { - let Some(encoded) = encoded else { + frame = physical_outgoing_rx.recv() => { + let Some(frame) = frame else { break RendezvousDisconnectReason::LocalShutdown; }; - Message::Binary(encoded.into()) + let bytes = frame.encoded.len(); + let message = Message::Binary(frame.encoded.into()); + (message, Some((frame.queued_at, frame.trace, bytes))) } }; let is_keepalive_ping = matches!(message, Message::Ping(_)); let write_deadline = pong_watchdog.write_deadline(tokio::time::Instant::now()); - match tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await { + let send_result = if let Some((queued_at, Some(trace), bytes)) = queued_frame { + let span = tracing::info_span!( + "exec_server.noise.executor_physical_send", + bytes, + queue_wait_ms = queued_at.elapsed().as_secs_f64() * 1_000.0, + ); + let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace.as_ref()); + tokio::time::timeout_at( + write_deadline, + websocket_sink.send(message).instrument(span), + ) + .await + } else { + tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await + }; + match send_result { Ok(Ok(())) => { if is_keepalive_ping { pong_watchdog.ping_sent(tokio::time::Instant::now()); @@ -561,7 +581,7 @@ where loop { // Registry calls run separately so a slow check does not block the relay. - let frame = tokio::select! { + let (frame, physical_received_at) = tokio::select! { writer_result = &mut physical_writer_task => { match writer_result { Ok(reason) => disconnect_reason = reason, @@ -647,7 +667,9 @@ where // Do not leave a half-open stream if the handshake reply // cannot be queued immediately. if physical_outgoing_tx - .try_send(encode_relay_message_frame(&response)) + .try_send(NoisePhysicalFrame::control(encode_relay_message_frame( + &response, + ))) .is_err() { break; @@ -687,11 +709,14 @@ where continue; } incoming_message = websocket_stream.next() => match incoming_message { - Some(Ok(Message::Binary(payload))) => match decode_relay_message_frame(payload.as_ref()) { - Ok(frame) => frame, - Err(error) => { - warn!("dropping malformed Noise relay frame from harness: {error}"); - continue; + Some(Ok(Message::Binary(payload))) => { + let physical_received_at = Instant::now(); + match decode_relay_message_frame(payload.as_ref()) { + Ok(frame) => (frame, physical_received_at), + Err(error) => { + warn!("dropping malformed Noise relay frame from harness: {error}"); + continue; + } } }, Some(Ok(Message::Close(_))) | None => { @@ -856,7 +881,7 @@ where continue; } }; - if let Err(error) = stream.receive_data(data) { + if let Err(error) = stream.receive_data(data, physical_received_at) { warn!("failed to process Noise relay payload: {error}"); streams.remove(&stream_id); send_reset(&physical_outgoing_tx, stream_id); @@ -910,9 +935,11 @@ struct HarnessKeyValidationResult { /// Queue a best-effort reset without blocking the shared websocket loop. /// Reset reasons are relay control data and are not treated as trusted text. -fn send_reset(physical_outgoing_tx: &mpsc::Sender>, stream_id: String) { +fn send_reset(physical_outgoing_tx: &mpsc::Sender, stream_id: String) { let reset = RelayMessageFrame::reset(stream_id, NOISE_RELAY_RESET_REASON.to_string()); - let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset)); + let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control(encode_relay_message_frame( + &reset, + ))); } #[cfg(test)]