telemetry(exec-server): trace Noise virtual streams

This commit is contained in:
Anton Panasenko
2026-07-07 23:47:41 -07:00
parent f87140dc75
commit 19c516242c
6 changed files with 501 additions and 72 deletions
@@ -6,9 +6,16 @@
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use std::time::Instant;
use codex_exec_server_protocol::JSONRPCMessage;
use codex_protocol::protocol::W3cTraceContext;
use tokio::sync::mpsc;
use tokio::sync::watch;
use tracing::Instrument;
use tracing::Span;
use tracing::field;
use tracing::warn;
use crate::ExecServerError;
@@ -23,6 +30,7 @@ use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN;
use crate::noise_relay::message_framing::frame_jsonrpc_message;
use crate::noise_relay::ordered_ciphertext::OrderedCiphertextFrames;
use crate::noise_relay::take_next_sequence;
use crate::noise_relay::trace_context::NoiseTraceContext;
use crate::relay::encode_relay_message_frame;
use crate::relay_proto::RelayData;
use crate::relay_proto::RelayMessageFrame;
@@ -39,6 +47,35 @@ pub(crate) struct ClosedNoiseVirtualStream {
pub(crate) instance_id: u64,
}
/// One frame queued for the executor's shared physical rendezvous websocket.
///
/// Data frames carry the current trace context across the channel so the
/// physical websocket send remains attributable to the originating RPC or
/// process notification. Relay control frames intentionally carry no context.
pub(crate) struct NoisePhysicalFrame {
pub(crate) encoded: Vec<u8>,
pub(crate) queued_at: Instant,
pub(crate) trace: Option<Arc<W3cTraceContext>>,
}
impl NoisePhysicalFrame {
pub(crate) fn data(encoded: Vec<u8>, trace: Option<Arc<W3cTraceContext>>) -> Self {
Self {
encoded,
queued_at: Instant::now(),
trace,
}
}
pub(crate) fn control(encoded: Vec<u8>) -> Self {
Self {
encoded,
queued_at: Instant::now(),
trace: None,
}
}
}
/// One authenticated JSON-RPC stream carried by the executor's physical relay.
///
/// Inbound delivery is intentionally nonblocking. An overloaded or abandoned
@@ -50,9 +87,19 @@ pub(crate) struct NoiseVirtualStream {
transport: Arc<Mutex<NoiseTransport>>,
inbound_ciphertexts: OrderedCiphertextFrames,
inbound_decoder: JsonRpcMessageDecoder,
trace_context: Arc<Mutex<NoiseTraceContext>>,
pub(crate) instance_id: u64,
}
struct ExecutorInboundTimings {
websocket_ingress_to_decode: Duration,
relay_dispatch: Duration,
reorder: Duration,
decrypt: Duration,
decode: Duration,
trace_context: Duration,
}
impl NoiseVirtualStream {
pub(crate) fn disconnect(self, reason: Option<String>) {
let _ = self.disconnected_tx.send(true);
@@ -63,8 +110,19 @@ impl NoiseVirtualStream {
/// Reorder and decrypt one inbound record, then queue complete JSON-RPC messages.
/// This must stay nonblocking because all virtual streams share the read loop.
pub(crate) fn receive_data(&mut self, data: RelayData) -> Result<(), ExecServerError> {
for ciphertext in self.inbound_ciphertexts.push(data.seq, data.payload)? {
pub(crate) fn receive_data(
&mut self,
data: RelayData,
physical_received_at: Instant,
) -> Result<(), ExecServerError> {
let relay_seq = data.seq;
let ciphertext_bytes = data.payload.len();
let relay_dispatch_elapsed = physical_received_at.elapsed();
let reorder_started_at = Instant::now();
let ciphertexts = self.inbound_ciphertexts.push(data.seq, data.payload)?;
let reorder_elapsed = reorder_started_at.elapsed();
for ciphertext in ciphertexts {
let decrypt_started_at = Instant::now();
let plaintext = {
let mut transport = self
.transport
@@ -74,20 +132,113 @@ impl NoiseVirtualStream {
ExecServerError::Protocol(format!("Noise relay decryption failed: {error}"))
})?
};
for message in self.inbound_decoder.push(&plaintext)? {
self.incoming_tx
.try_send(JsonRpcConnectionEvent::Message(message))
.map_err(|_| {
ExecServerError::Protocol(
"Noise virtual stream inbound queue is full or closed".to_string(),
)
})?;
let decrypt_elapsed = decrypt_started_at.elapsed();
let decode_started_at = Instant::now();
let messages = self.inbound_decoder.push(&plaintext)?;
let decode_elapsed = decode_started_at.elapsed();
for message in messages {
let trace_context_started_at = Instant::now();
self.trace_context
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe_request(&message);
let trace_context_elapsed = trace_context_started_at.elapsed();
let decoded_span = decoded_message_span(
&message,
relay_seq,
ciphertext_bytes,
ExecutorInboundTimings {
websocket_ingress_to_decode: physical_received_at.elapsed(),
relay_dispatch: relay_dispatch_elapsed,
reorder: reorder_elapsed,
decrypt: decrypt_elapsed,
decode: decode_elapsed,
trace_context: trace_context_elapsed,
},
);
let enqueued_span = enqueued_message_span(&message, relay_seq);
let enqueue_started_at = Instant::now();
let enqueue_result = decoded_span.in_scope(|| {
self.incoming_tx
.try_send(JsonRpcConnectionEvent::Message(message))
.map_err(|_| {
ExecServerError::Protocol(
"Noise virtual stream inbound queue is full or closed".to_string(),
)
})
});
enqueued_span.record(
"enqueue_ms",
enqueue_started_at.elapsed().as_secs_f64() * 1_000.0,
);
enqueue_result?;
enqueued_span.in_scope(|| {});
}
}
Ok(())
}
}
fn decoded_message_span(
message: &JSONRPCMessage,
relay_seq: u32,
ciphertext_bytes: usize,
timings: ExecutorInboundTimings,
) -> Span {
let (message_kind, method, trace) = match message {
JSONRPCMessage::Request(request) => {
("request", request.method.as_str(), request.trace.as_ref())
}
JSONRPCMessage::Notification(notification) => {
("notification", notification.method.as_str(), None)
}
JSONRPCMessage::Response(_) => ("response", "", None),
JSONRPCMessage::Error(_) => ("error", "", None),
};
let span = tracing::info_span!(
"exec_server.noise.executor_message_decoded",
message_kind,
method,
relay_seq,
ciphertext_bytes,
executor_websocket_ingress_to_decode_ms =
timings.websocket_ingress_to_decode.as_secs_f64() * 1_000.0,
relay_dispatch_ms = timings.relay_dispatch.as_secs_f64() * 1_000.0,
reorder_ms = timings.reorder.as_secs_f64() * 1_000.0,
decrypt_ms = timings.decrypt.as_secs_f64() * 1_000.0,
decode_ms = timings.decode.as_secs_f64() * 1_000.0,
trace_context_ms = timings.trace_context.as_secs_f64() * 1_000.0,
);
if let Some(trace) = trace {
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
}
span
}
fn enqueued_message_span(message: &JSONRPCMessage, relay_seq: u32) -> Span {
let (message_kind, method, trace) = match message {
JSONRPCMessage::Request(request) => {
("request", request.method.as_str(), request.trace.as_ref())
}
JSONRPCMessage::Notification(notification) => {
("notification", notification.method.as_str(), None)
}
JSONRPCMessage::Response(_) => ("response", "", None),
JSONRPCMessage::Error(_) => ("error", "", None),
};
let span = tracing::info_span!(
"exec_server.noise.executor_message_enqueued",
message_kind,
method,
relay_seq,
enqueue_ms = field::Empty,
);
if let Some(trace) = trace {
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
}
span
}
/// Start JSON-RPC processing for a completed handshake.
///
/// The returned value is the read half; the spawned task owns outbound framing
@@ -96,7 +247,7 @@ pub(crate) fn spawn_noise_virtual_stream(
stream_id: String,
instance_id: u64,
processor: ConnectionProcessor,
physical_outgoing_tx: mpsc::Sender<Vec<u8>>,
physical_outgoing_tx: mpsc::Sender<NoisePhysicalFrame>,
closed_stream_tx: mpsc::Sender<ClosedNoiseVirtualStream>,
transport: NoiseTransport,
) -> NoiseVirtualStream {
@@ -105,49 +256,27 @@ pub(crate) fn spawn_noise_virtual_stream(
let (disconnected_tx, disconnected_rx) = watch::channel(false);
let transport = Arc::new(Mutex::new(transport));
let writer_transport = Arc::clone(&transport);
let trace_context = Arc::new(Mutex::new(NoiseTraceContext::default()));
let writer_trace_context = Arc::clone(&trace_context);
let processor_stream_id = stream_id.clone();
let processor_closed_stream_tx = closed_stream_tx.clone();
let writer_stream_id = stream_id;
let writer_task = tokio::spawn(async move {
let mut next_seq = 0u32;
'writer: while let Some(message) = json_outgoing_rx.recv().await {
// Each chunk becomes one Noise record and consumes one nonce.
let framed = match frame_jsonrpc_message(&message) {
Ok(framed) => framed,
Err(error) => {
warn!("failed to frame Noise virtual stream JSON-RPC payload: {error}");
break;
}
};
for plaintext_record in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN) {
let seq = match take_next_sequence(&mut next_seq) {
Ok(seq) => seq,
Err(error) => {
warn!("Noise virtual stream sequence exhausted: {error}");
break 'writer;
}
};
let ciphertext = {
let mut transport = writer_transport
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
transport.encrypt(plaintext_record)
};
let ciphertext = match ciphertext {
Ok(ciphertext) => ciphertext,
Err(error) => {
warn!("failed to encrypt Noise virtual stream payload: {error}");
break 'writer;
}
};
let frame = RelayMessageFrame::data(writer_stream_id.clone(), seq, ciphertext);
if physical_outgoing_tx
.send(encode_relay_message_frame(&frame))
.await
.is_err()
{
break 'writer;
}
let outbound_span = outbound_message_span(&message, &writer_trace_context);
if let Err(error) = send_outbound_message(
&physical_outgoing_tx,
&writer_transport,
&writer_stream_id,
&mut next_seq,
&message,
)
.instrument(outbound_span)
.await
{
warn!("failed to send Noise virtual stream JSON-RPC payload: {error}");
break 'writer;
}
}
@@ -158,7 +287,9 @@ pub(crate) fn spawn_noise_virtual_stream(
};
let reset =
RelayMessageFrame::reset(writer_stream_id, NOISE_RELAY_RESET_REASON.to_string());
let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset));
let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control(
encode_relay_message_frame(&reset),
));
let _ = closed_stream_tx.send(closed_stream).await;
});
@@ -187,10 +318,103 @@ pub(crate) fn spawn_noise_virtual_stream(
transport,
inbound_ciphertexts: OrderedCiphertextFrames::default(),
inbound_decoder: JsonRpcMessageDecoder::default(),
trace_context,
instance_id,
}
}
fn outbound_message_span(
message: &JSONRPCMessage,
trace_context: &Mutex<NoiseTraceContext>,
) -> Span {
let (message_kind, method) = match message {
JSONRPCMessage::Request(request) => ("request", request.method.as_str()),
JSONRPCMessage::Notification(notification) => {
("notification", notification.method.as_str())
}
JSONRPCMessage::Response(_) => ("response", ""),
JSONRPCMessage::Error(_) => ("error", ""),
};
let trace = trace_context
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.return_trace(message);
let span = tracing::info_span!(
"exec_server.noise.executor_outbound",
message_kind,
method,
framed_bytes = field::Empty,
records = field::Empty,
frame_complete_ms = field::Empty,
encrypt_complete_ms = field::Empty,
physical_queue_complete_ms = field::Empty,
);
if let Some(trace) = trace.as_ref() {
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
}
span
}
async fn send_outbound_message(
physical_outgoing_tx: &mpsc::Sender<NoisePhysicalFrame>,
transport: &Mutex<NoiseTransport>,
stream_id: &str,
next_seq: &mut u32,
message: &JSONRPCMessage,
) -> Result<(), ExecServerError> {
let started_at = Instant::now();
let framed = tracing::info_span!("exec_server.noise.executor_frame_message")
.in_scope(|| frame_jsonrpc_message(message))?;
let record_count = framed.len().div_ceil(NOISE_RECORD_PLAINTEXT_LEN);
let span = Span::current();
span.record("framed_bytes", framed.len());
span.record("records", record_count);
span.record(
"frame_complete_ms",
started_at.elapsed().as_secs_f64() * 1_000.0,
);
// Extracting W3C context allocates its string carrier. Do that once for the
// logical JSON-RPC message, then share it across all physical Noise records.
let trace = codex_otel::current_span_w3c_trace_context().map(Arc::new);
for (record_index, plaintext_record) in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN).enumerate() {
let seq = take_next_sequence(next_seq)?;
let ciphertext =
tracing::info_span!("exec_server.noise.executor_encrypt_record", record_index,)
.in_scope(|| {
transport
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.encrypt(plaintext_record)
})
.map_err(|error| {
ExecServerError::Protocol(format!(
"failed to encrypt Noise virtual stream payload: {error}"
))
})?;
span.record(
"encrypt_complete_ms",
started_at.elapsed().as_secs_f64() * 1_000.0,
);
let frame = RelayMessageFrame::data(stream_id.to_string(), seq, ciphertext);
let encoded = encode_relay_message_frame(&frame);
let permit = physical_outgoing_tx
.reserve()
.instrument(tracing::info_span!(
"exec_server.noise.executor_physical_queue",
record_index,
))
.await
.map_err(|_| ExecServerError::Closed)?;
permit.send(NoisePhysicalFrame::data(encoded, trace.clone()));
span.record(
"physical_queue_complete_ms",
started_at.elapsed().as_secs_f64() * 1_000.0,
);
}
Ok(())
}
#[cfg(test)]
#[path = "executor_stream_tests.rs"]
mod tests;
@@ -1,23 +1,86 @@
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use codex_exec_server_protocol::JSONRPCMessage;
use codex_exec_server_protocol::JSONRPCResponse;
use codex_exec_server_protocol::RequestId;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::SdkTracerProvider;
use tokio::sync::mpsc;
use tokio::time::timeout;
use tracing::Instrument;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::prelude::*;
use super::ClosedNoiseVirtualStream;
use super::send_outbound_message;
use super::spawn_noise_virtual_stream;
use crate::ExecServerRuntimePaths;
use crate::connection::CHANNEL_CAPACITY;
use crate::noise_channel::InitiatorHandshake;
use crate::noise_channel::NoiseChannelIdentity;
use crate::noise_channel::PendingResponderHandshake;
use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN;
use crate::noise_relay::message_framing::frame_jsonrpc_message;
use crate::relay_proto::RelayData;
use crate::server::ConnectionProcessor;
#[tokio::test(flavor = "current_thread")]
async fn outbound_records_share_one_message_trace_context() -> Result<()> {
let executor_identity = NoiseChannelIdentity::generate()?;
let harness_identity = NoiseChannelIdentity::generate()?;
let (initiator, request) = InitiatorHandshake::start(
&harness_identity,
&executor_identity.public_key(),
b"test-prologue",
b"authorization",
)?;
let pending =
PendingResponderHandshake::read_request(&executor_identity, b"test-prologue", &request)?;
let (executor_transport, response) = pending.complete()?;
let _harness_transport = initiator.finish(&response)?;
let executor_transport = std::sync::Mutex::new(executor_transport);
let (physical_outgoing_tx, mut physical_outgoing_rx) = mpsc::channel(CHANNEL_CAPACITY);
let message = JSONRPCMessage::Response(JSONRPCResponse {
id: RequestId::Integer(1),
result: serde_json::Value::String("x".repeat(NOISE_RECORD_PLAINTEXT_LEN * 2)),
});
let provider = SdkTracerProvider::builder().build();
let subscriber = tracing_subscriber::registry()
.with(tracing_opentelemetry::layer().with_tracer(provider.tracer("exec-server-test")));
let mut next_seq = 0;
async {
tracing::callsite::rebuild_interest_cache();
send_outbound_message(
&physical_outgoing_tx,
&executor_transport,
"stream-1",
&mut next_seq,
&message,
)
.instrument(tracing::info_span!("outbound-message"))
.await
}
.with_subscriber(subscriber)
.await?;
let mut frames = Vec::new();
while let Ok(frame) = physical_outgoing_rx.try_recv() {
frames.push(frame);
}
assert!(frames.len() > 1, "expected multiple physical records");
let first_trace = frames[0].trace.as_ref().expect("first record trace");
assert!(frames.iter().skip(1).all(|frame| {
Arc::ptr_eq(
first_trace,
frame.trace.as_ref().expect("subsequent record trace"),
)
}));
Ok(())
}
#[tokio::test]
async fn processor_exit_reports_closed_virtual_stream() -> Result<()> {
let executor_identity = NoiseChannelIdentity::generate()?;
@@ -52,12 +115,15 @@ async fn processor_exit_reports_closed_virtual_stream() -> Result<()> {
result: serde_json::Value::Null,
});
let ciphertext = harness_transport.encrypt(&frame_jsonrpc_message(&message)?)?;
stream.receive_data(RelayData {
seq: 0,
segment_index: 0,
segment_count: 1,
payload: ciphertext,
})?;
stream.receive_data(
RelayData {
seq: 0,
segment_index: 0,
segment_count: 1,
payload: ciphertext,
},
std::time::Instant::now(),
)?;
assert!(matches!(
timeout(Duration::from_secs(1), closed_stream_rx.recv()).await?,
@@ -2,6 +2,7 @@ pub(crate) mod executor_stream;
mod harness;
mod message_framing;
mod ordered_ciphertext;
mod trace_context;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
@@ -0,0 +1,63 @@
use std::collections::HashMap;
use codex_exec_server_protocol::EXEC_CLOSED_METHOD;
use codex_exec_server_protocol::JSONRPCMessage;
use codex_exec_server_protocol::RequestId;
use codex_protocol::protocol::W3cTraceContext;
/// Correlates return traffic with the trace carried by its originating request.
///
/// Responses do not repeat the request's W3C carrier, and process notifications
/// are correlated by `processId`. Keeping this state per virtual stream avoids
/// adding telemetry-only fields to the JSON-RPC wire protocol.
#[derive(Default)]
pub(super) struct NoiseTraceContext {
requests: HashMap<RequestId, W3cTraceContext>,
processes: HashMap<String, W3cTraceContext>,
}
impl NoiseTraceContext {
pub(super) fn observe_request(&mut self, message: &JSONRPCMessage) {
let JSONRPCMessage::Request(request) = message else {
return;
};
let Some(trace) = request.trace.as_ref() else {
return;
};
self.requests.insert(request.id.clone(), trace.clone());
if let Some(process_id) = message_process_id(message) {
self.processes
.entry(process_id.to_string())
.or_insert_with(|| trace.clone());
}
}
pub(super) fn return_trace(&mut self, message: &JSONRPCMessage) -> Option<W3cTraceContext> {
match message {
JSONRPCMessage::Response(response) => self.requests.remove(&response.id),
JSONRPCMessage::Error(error) => self.requests.remove(&error.id),
JSONRPCMessage::Notification(notification) => {
let process_id = message_process_id(message)?;
let trace = self.processes.get(process_id).cloned();
if notification.method == EXEC_CLOSED_METHOD {
self.processes.remove(process_id);
}
trace
}
JSONRPCMessage::Request(request) => request.trace.clone(),
}
}
}
fn message_process_id(message: &JSONRPCMessage) -> Option<&str> {
let params = match message {
JSONRPCMessage::Request(request) => request.params.as_ref(),
JSONRPCMessage::Notification(notification) => notification.params.as_ref(),
JSONRPCMessage::Response(_) | JSONRPCMessage::Error(_) => None,
}?;
params.get("processId")?.as_str()
}
#[cfg(test)]
#[path = "trace_context_tests.rs"]
mod tests;
@@ -0,0 +1,48 @@
use codex_exec_server_protocol::EXEC_CLOSED_METHOD;
use codex_exec_server_protocol::EXEC_METHOD;
use codex_exec_server_protocol::JSONRPCMessage;
use codex_exec_server_protocol::JSONRPCNotification;
use codex_exec_server_protocol::JSONRPCRequest;
use codex_exec_server_protocol::JSONRPCResponse;
use codex_exec_server_protocol::RequestId;
use codex_protocol::protocol::W3cTraceContext;
use pretty_assertions::assert_eq;
use super::NoiseTraceContext;
fn trace_context() -> W3cTraceContext {
W3cTraceContext {
traceparent: Some("00-00000000000000000000000000000001-0000000000000002-01".to_string()),
tracestate: None,
}
}
fn process_start_request(trace: W3cTraceContext) -> JSONRPCMessage {
JSONRPCMessage::Request(JSONRPCRequest {
id: RequestId::Integer(7),
method: EXEC_METHOD.to_string(),
params: Some(serde_json::json!({"processId": "process-1"})),
trace: Some(trace),
})
}
#[test]
fn correlates_response_and_terminal_notification_with_request_trace() {
let trace = trace_context();
let mut context = NoiseTraceContext::default();
context.observe_request(&process_start_request(trace.clone()));
let response = JSONRPCMessage::Response(JSONRPCResponse {
id: RequestId::Integer(7),
result: serde_json::Value::Null,
});
assert_eq!(context.return_trace(&response), Some(trace.clone()));
assert_eq!(context.return_trace(&response), None);
let closed = JSONRPCMessage::Notification(JSONRPCNotification {
method: EXEC_CLOSED_METHOD.to_string(),
params: Some(serde_json::json!({"processId": "process-1", "seq": 1})),
});
assert_eq!(context.return_trace(&closed), Some(trace));
assert_eq!(context.return_trace(&closed), None);
}
+44 -17
View File
@@ -1,5 +1,6 @@
use std::collections::HashMap;
use std::time::Duration;
use std::time::Instant;
use codex_exec_server_protocol::JSONRPCMessage;
use futures::Sink;
@@ -15,6 +16,7 @@ use tokio::task::JoinSet;
use tokio::time::timeout;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Message;
use tracing::Instrument;
use tracing::debug;
use tracing::info;
use tracing::warn;
@@ -32,6 +34,7 @@ use crate::noise_channel::PendingResponderHandshake;
use crate::noise_channel::noise_channel_prologue;
use crate::noise_relay::NOISE_RELAY_RESET_REASON;
use crate::noise_relay::executor_stream::ClosedNoiseVirtualStream;
use crate::noise_relay::executor_stream::NoisePhysicalFrame;
use crate::noise_relay::executor_stream::NoiseVirtualStream;
use crate::noise_relay::executor_stream::spawn_noise_virtual_stream;
use crate::relay_proto::RelayData;
@@ -483,7 +486,7 @@ where
);
let (mut websocket_sink, mut websocket_stream) = stream.split();
let (physical_outgoing_tx, mut physical_outgoing_rx) =
mpsc::channel::<Vec<u8>>(CHANNEL_CAPACITY);
mpsc::channel::<NoisePhysicalFrame>(CHANNEL_CAPACITY);
let (closed_stream_tx, mut closed_stream_rx) =
mpsc::channel::<ClosedNoiseVirtualStream>(MAX_ACTIVE_NOISE_RELAY_STREAMS);
let (pong_tx, mut pong_rx) = mpsc::channel(1);
@@ -498,7 +501,7 @@ where
let pong_deadline = tokio::time::sleep(WEBSOCKET_PONG_TIMEOUT);
tokio::pin!(pong_deadline);
loop {
let message = tokio::select! {
let (message, queued_frame) = tokio::select! {
pong = pong_rx.recv() => {
let Some(()) = pong else {
break RendezvousDisconnectReason::LocalShutdown;
@@ -521,18 +524,35 @@ where
}
}
_ = keepalive.tick(), if pong_watchdog.deadline().is_none() => {
Message::Ping(Vec::new().into())
(Message::Ping(Vec::new().into()), None)
}
encoded = physical_outgoing_rx.recv() => {
let Some(encoded) = encoded else {
frame = physical_outgoing_rx.recv() => {
let Some(frame) = frame else {
break RendezvousDisconnectReason::LocalShutdown;
};
Message::Binary(encoded.into())
let bytes = frame.encoded.len();
let message = Message::Binary(frame.encoded.into());
(message, Some((frame.queued_at, frame.trace, bytes)))
}
};
let is_keepalive_ping = matches!(message, Message::Ping(_));
let write_deadline = pong_watchdog.write_deadline(tokio::time::Instant::now());
match tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await {
let send_result = if let Some((queued_at, Some(trace), bytes)) = queued_frame {
let span = tracing::info_span!(
"exec_server.noise.executor_physical_send",
bytes,
queue_wait_ms = queued_at.elapsed().as_secs_f64() * 1_000.0,
);
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace.as_ref());
tokio::time::timeout_at(
write_deadline,
websocket_sink.send(message).instrument(span),
)
.await
} else {
tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await
};
match send_result {
Ok(Ok(())) => {
if is_keepalive_ping {
pong_watchdog.ping_sent(tokio::time::Instant::now());
@@ -561,7 +581,7 @@ where
loop {
// Registry calls run separately so a slow check does not block the relay.
let frame = tokio::select! {
let (frame, physical_received_at) = tokio::select! {
writer_result = &mut physical_writer_task => {
match writer_result {
Ok(reason) => disconnect_reason = reason,
@@ -647,7 +667,9 @@ where
// Do not leave a half-open stream if the handshake reply
// cannot be queued immediately.
if physical_outgoing_tx
.try_send(encode_relay_message_frame(&response))
.try_send(NoisePhysicalFrame::control(encode_relay_message_frame(
&response,
)))
.is_err()
{
break;
@@ -687,11 +709,14 @@ where
continue;
}
incoming_message = websocket_stream.next() => match incoming_message {
Some(Ok(Message::Binary(payload))) => match decode_relay_message_frame(payload.as_ref()) {
Ok(frame) => frame,
Err(error) => {
warn!("dropping malformed Noise relay frame from harness: {error}");
continue;
Some(Ok(Message::Binary(payload))) => {
let physical_received_at = Instant::now();
match decode_relay_message_frame(payload.as_ref()) {
Ok(frame) => (frame, physical_received_at),
Err(error) => {
warn!("dropping malformed Noise relay frame from harness: {error}");
continue;
}
}
},
Some(Ok(Message::Close(_))) | None => {
@@ -856,7 +881,7 @@ where
continue;
}
};
if let Err(error) = stream.receive_data(data) {
if let Err(error) = stream.receive_data(data, physical_received_at) {
warn!("failed to process Noise relay payload: {error}");
streams.remove(&stream_id);
send_reset(&physical_outgoing_tx, stream_id);
@@ -910,9 +935,11 @@ struct HarnessKeyValidationResult {
/// Queue a best-effort reset without blocking the shared websocket loop.
/// Reset reasons are relay control data and are not treated as trusted text.
fn send_reset(physical_outgoing_tx: &mpsc::Sender<Vec<u8>>, stream_id: String) {
fn send_reset(physical_outgoing_tx: &mpsc::Sender<NoisePhysicalFrame>, stream_id: String) {
let reset = RelayMessageFrame::reset(stream_id, NOISE_RELAY_RESET_REASON.to_string());
let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset));
let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control(encode_relay_message_frame(
&reset,
)));
}
#[cfg(test)]