mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
telemetry(exec-server): trace Noise virtual streams
This commit is contained in:
@@ -6,9 +6,16 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::watch;
|
||||
use tracing::Instrument;
|
||||
use tracing::Span;
|
||||
use tracing::field;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::ExecServerError;
|
||||
@@ -23,6 +30,7 @@ use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN;
|
||||
use crate::noise_relay::message_framing::frame_jsonrpc_message;
|
||||
use crate::noise_relay::ordered_ciphertext::OrderedCiphertextFrames;
|
||||
use crate::noise_relay::take_next_sequence;
|
||||
use crate::noise_relay::trace_context::NoiseTraceContext;
|
||||
use crate::relay::encode_relay_message_frame;
|
||||
use crate::relay_proto::RelayData;
|
||||
use crate::relay_proto::RelayMessageFrame;
|
||||
@@ -39,6 +47,35 @@ pub(crate) struct ClosedNoiseVirtualStream {
|
||||
pub(crate) instance_id: u64,
|
||||
}
|
||||
|
||||
/// One frame queued for the executor's shared physical rendezvous websocket.
|
||||
///
|
||||
/// Data frames carry the current trace context across the channel so the
|
||||
/// physical websocket send remains attributable to the originating RPC or
|
||||
/// process notification. Relay control frames intentionally carry no context.
|
||||
pub(crate) struct NoisePhysicalFrame {
|
||||
pub(crate) encoded: Vec<u8>,
|
||||
pub(crate) queued_at: Instant,
|
||||
pub(crate) trace: Option<Arc<W3cTraceContext>>,
|
||||
}
|
||||
|
||||
impl NoisePhysicalFrame {
|
||||
pub(crate) fn data(encoded: Vec<u8>, trace: Option<Arc<W3cTraceContext>>) -> Self {
|
||||
Self {
|
||||
encoded,
|
||||
queued_at: Instant::now(),
|
||||
trace,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn control(encoded: Vec<u8>) -> Self {
|
||||
Self {
|
||||
encoded,
|
||||
queued_at: Instant::now(),
|
||||
trace: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One authenticated JSON-RPC stream carried by the executor's physical relay.
|
||||
///
|
||||
/// Inbound delivery is intentionally nonblocking. An overloaded or abandoned
|
||||
@@ -50,9 +87,19 @@ pub(crate) struct NoiseVirtualStream {
|
||||
transport: Arc<Mutex<NoiseTransport>>,
|
||||
inbound_ciphertexts: OrderedCiphertextFrames,
|
||||
inbound_decoder: JsonRpcMessageDecoder,
|
||||
trace_context: Arc<Mutex<NoiseTraceContext>>,
|
||||
pub(crate) instance_id: u64,
|
||||
}
|
||||
|
||||
struct ExecutorInboundTimings {
|
||||
websocket_ingress_to_decode: Duration,
|
||||
relay_dispatch: Duration,
|
||||
reorder: Duration,
|
||||
decrypt: Duration,
|
||||
decode: Duration,
|
||||
trace_context: Duration,
|
||||
}
|
||||
|
||||
impl NoiseVirtualStream {
|
||||
pub(crate) fn disconnect(self, reason: Option<String>) {
|
||||
let _ = self.disconnected_tx.send(true);
|
||||
@@ -63,8 +110,19 @@ impl NoiseVirtualStream {
|
||||
|
||||
/// Reorder and decrypt one inbound record, then queue complete JSON-RPC messages.
|
||||
/// This must stay nonblocking because all virtual streams share the read loop.
|
||||
pub(crate) fn receive_data(&mut self, data: RelayData) -> Result<(), ExecServerError> {
|
||||
for ciphertext in self.inbound_ciphertexts.push(data.seq, data.payload)? {
|
||||
pub(crate) fn receive_data(
|
||||
&mut self,
|
||||
data: RelayData,
|
||||
physical_received_at: Instant,
|
||||
) -> Result<(), ExecServerError> {
|
||||
let relay_seq = data.seq;
|
||||
let ciphertext_bytes = data.payload.len();
|
||||
let relay_dispatch_elapsed = physical_received_at.elapsed();
|
||||
let reorder_started_at = Instant::now();
|
||||
let ciphertexts = self.inbound_ciphertexts.push(data.seq, data.payload)?;
|
||||
let reorder_elapsed = reorder_started_at.elapsed();
|
||||
for ciphertext in ciphertexts {
|
||||
let decrypt_started_at = Instant::now();
|
||||
let plaintext = {
|
||||
let mut transport = self
|
||||
.transport
|
||||
@@ -74,20 +132,113 @@ impl NoiseVirtualStream {
|
||||
ExecServerError::Protocol(format!("Noise relay decryption failed: {error}"))
|
||||
})?
|
||||
};
|
||||
for message in self.inbound_decoder.push(&plaintext)? {
|
||||
self.incoming_tx
|
||||
.try_send(JsonRpcConnectionEvent::Message(message))
|
||||
.map_err(|_| {
|
||||
ExecServerError::Protocol(
|
||||
"Noise virtual stream inbound queue is full or closed".to_string(),
|
||||
)
|
||||
})?;
|
||||
let decrypt_elapsed = decrypt_started_at.elapsed();
|
||||
let decode_started_at = Instant::now();
|
||||
let messages = self.inbound_decoder.push(&plaintext)?;
|
||||
let decode_elapsed = decode_started_at.elapsed();
|
||||
for message in messages {
|
||||
let trace_context_started_at = Instant::now();
|
||||
self.trace_context
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.observe_request(&message);
|
||||
let trace_context_elapsed = trace_context_started_at.elapsed();
|
||||
let decoded_span = decoded_message_span(
|
||||
&message,
|
||||
relay_seq,
|
||||
ciphertext_bytes,
|
||||
ExecutorInboundTimings {
|
||||
websocket_ingress_to_decode: physical_received_at.elapsed(),
|
||||
relay_dispatch: relay_dispatch_elapsed,
|
||||
reorder: reorder_elapsed,
|
||||
decrypt: decrypt_elapsed,
|
||||
decode: decode_elapsed,
|
||||
trace_context: trace_context_elapsed,
|
||||
},
|
||||
);
|
||||
let enqueued_span = enqueued_message_span(&message, relay_seq);
|
||||
let enqueue_started_at = Instant::now();
|
||||
let enqueue_result = decoded_span.in_scope(|| {
|
||||
self.incoming_tx
|
||||
.try_send(JsonRpcConnectionEvent::Message(message))
|
||||
.map_err(|_| {
|
||||
ExecServerError::Protocol(
|
||||
"Noise virtual stream inbound queue is full or closed".to_string(),
|
||||
)
|
||||
})
|
||||
});
|
||||
enqueued_span.record(
|
||||
"enqueue_ms",
|
||||
enqueue_started_at.elapsed().as_secs_f64() * 1_000.0,
|
||||
);
|
||||
enqueue_result?;
|
||||
enqueued_span.in_scope(|| {});
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn decoded_message_span(
|
||||
message: &JSONRPCMessage,
|
||||
relay_seq: u32,
|
||||
ciphertext_bytes: usize,
|
||||
timings: ExecutorInboundTimings,
|
||||
) -> Span {
|
||||
let (message_kind, method, trace) = match message {
|
||||
JSONRPCMessage::Request(request) => {
|
||||
("request", request.method.as_str(), request.trace.as_ref())
|
||||
}
|
||||
JSONRPCMessage::Notification(notification) => {
|
||||
("notification", notification.method.as_str(), None)
|
||||
}
|
||||
JSONRPCMessage::Response(_) => ("response", "", None),
|
||||
JSONRPCMessage::Error(_) => ("error", "", None),
|
||||
};
|
||||
let span = tracing::info_span!(
|
||||
"exec_server.noise.executor_message_decoded",
|
||||
message_kind,
|
||||
method,
|
||||
relay_seq,
|
||||
ciphertext_bytes,
|
||||
executor_websocket_ingress_to_decode_ms =
|
||||
timings.websocket_ingress_to_decode.as_secs_f64() * 1_000.0,
|
||||
relay_dispatch_ms = timings.relay_dispatch.as_secs_f64() * 1_000.0,
|
||||
reorder_ms = timings.reorder.as_secs_f64() * 1_000.0,
|
||||
decrypt_ms = timings.decrypt.as_secs_f64() * 1_000.0,
|
||||
decode_ms = timings.decode.as_secs_f64() * 1_000.0,
|
||||
trace_context_ms = timings.trace_context.as_secs_f64() * 1_000.0,
|
||||
);
|
||||
if let Some(trace) = trace {
|
||||
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
|
||||
}
|
||||
span
|
||||
}
|
||||
|
||||
fn enqueued_message_span(message: &JSONRPCMessage, relay_seq: u32) -> Span {
|
||||
let (message_kind, method, trace) = match message {
|
||||
JSONRPCMessage::Request(request) => {
|
||||
("request", request.method.as_str(), request.trace.as_ref())
|
||||
}
|
||||
JSONRPCMessage::Notification(notification) => {
|
||||
("notification", notification.method.as_str(), None)
|
||||
}
|
||||
JSONRPCMessage::Response(_) => ("response", "", None),
|
||||
JSONRPCMessage::Error(_) => ("error", "", None),
|
||||
};
|
||||
let span = tracing::info_span!(
|
||||
"exec_server.noise.executor_message_enqueued",
|
||||
message_kind,
|
||||
method,
|
||||
relay_seq,
|
||||
enqueue_ms = field::Empty,
|
||||
);
|
||||
if let Some(trace) = trace {
|
||||
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
|
||||
}
|
||||
span
|
||||
}
|
||||
|
||||
/// Start JSON-RPC processing for a completed handshake.
|
||||
///
|
||||
/// The returned value is the read half; the spawned task owns outbound framing
|
||||
@@ -96,7 +247,7 @@ pub(crate) fn spawn_noise_virtual_stream(
|
||||
stream_id: String,
|
||||
instance_id: u64,
|
||||
processor: ConnectionProcessor,
|
||||
physical_outgoing_tx: mpsc::Sender<Vec<u8>>,
|
||||
physical_outgoing_tx: mpsc::Sender<NoisePhysicalFrame>,
|
||||
closed_stream_tx: mpsc::Sender<ClosedNoiseVirtualStream>,
|
||||
transport: NoiseTransport,
|
||||
) -> NoiseVirtualStream {
|
||||
@@ -105,49 +256,27 @@ pub(crate) fn spawn_noise_virtual_stream(
|
||||
let (disconnected_tx, disconnected_rx) = watch::channel(false);
|
||||
let transport = Arc::new(Mutex::new(transport));
|
||||
let writer_transport = Arc::clone(&transport);
|
||||
let trace_context = Arc::new(Mutex::new(NoiseTraceContext::default()));
|
||||
let writer_trace_context = Arc::clone(&trace_context);
|
||||
let processor_stream_id = stream_id.clone();
|
||||
let processor_closed_stream_tx = closed_stream_tx.clone();
|
||||
let writer_stream_id = stream_id;
|
||||
let writer_task = tokio::spawn(async move {
|
||||
let mut next_seq = 0u32;
|
||||
'writer: while let Some(message) = json_outgoing_rx.recv().await {
|
||||
// Each chunk becomes one Noise record and consumes one nonce.
|
||||
let framed = match frame_jsonrpc_message(&message) {
|
||||
Ok(framed) => framed,
|
||||
Err(error) => {
|
||||
warn!("failed to frame Noise virtual stream JSON-RPC payload: {error}");
|
||||
break;
|
||||
}
|
||||
};
|
||||
for plaintext_record in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN) {
|
||||
let seq = match take_next_sequence(&mut next_seq) {
|
||||
Ok(seq) => seq,
|
||||
Err(error) => {
|
||||
warn!("Noise virtual stream sequence exhausted: {error}");
|
||||
break 'writer;
|
||||
}
|
||||
};
|
||||
let ciphertext = {
|
||||
let mut transport = writer_transport
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
transport.encrypt(plaintext_record)
|
||||
};
|
||||
let ciphertext = match ciphertext {
|
||||
Ok(ciphertext) => ciphertext,
|
||||
Err(error) => {
|
||||
warn!("failed to encrypt Noise virtual stream payload: {error}");
|
||||
break 'writer;
|
||||
}
|
||||
};
|
||||
let frame = RelayMessageFrame::data(writer_stream_id.clone(), seq, ciphertext);
|
||||
if physical_outgoing_tx
|
||||
.send(encode_relay_message_frame(&frame))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break 'writer;
|
||||
}
|
||||
let outbound_span = outbound_message_span(&message, &writer_trace_context);
|
||||
if let Err(error) = send_outbound_message(
|
||||
&physical_outgoing_tx,
|
||||
&writer_transport,
|
||||
&writer_stream_id,
|
||||
&mut next_seq,
|
||||
&message,
|
||||
)
|
||||
.instrument(outbound_span)
|
||||
.await
|
||||
{
|
||||
warn!("failed to send Noise virtual stream JSON-RPC payload: {error}");
|
||||
break 'writer;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +287,9 @@ pub(crate) fn spawn_noise_virtual_stream(
|
||||
};
|
||||
let reset =
|
||||
RelayMessageFrame::reset(writer_stream_id, NOISE_RELAY_RESET_REASON.to_string());
|
||||
let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset));
|
||||
let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control(
|
||||
encode_relay_message_frame(&reset),
|
||||
));
|
||||
let _ = closed_stream_tx.send(closed_stream).await;
|
||||
});
|
||||
|
||||
@@ -187,10 +318,103 @@ pub(crate) fn spawn_noise_virtual_stream(
|
||||
transport,
|
||||
inbound_ciphertexts: OrderedCiphertextFrames::default(),
|
||||
inbound_decoder: JsonRpcMessageDecoder::default(),
|
||||
trace_context,
|
||||
instance_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn outbound_message_span(
|
||||
message: &JSONRPCMessage,
|
||||
trace_context: &Mutex<NoiseTraceContext>,
|
||||
) -> Span {
|
||||
let (message_kind, method) = match message {
|
||||
JSONRPCMessage::Request(request) => ("request", request.method.as_str()),
|
||||
JSONRPCMessage::Notification(notification) => {
|
||||
("notification", notification.method.as_str())
|
||||
}
|
||||
JSONRPCMessage::Response(_) => ("response", ""),
|
||||
JSONRPCMessage::Error(_) => ("error", ""),
|
||||
};
|
||||
let trace = trace_context
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.return_trace(message);
|
||||
let span = tracing::info_span!(
|
||||
"exec_server.noise.executor_outbound",
|
||||
message_kind,
|
||||
method,
|
||||
framed_bytes = field::Empty,
|
||||
records = field::Empty,
|
||||
frame_complete_ms = field::Empty,
|
||||
encrypt_complete_ms = field::Empty,
|
||||
physical_queue_complete_ms = field::Empty,
|
||||
);
|
||||
if let Some(trace) = trace.as_ref() {
|
||||
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace);
|
||||
}
|
||||
span
|
||||
}
|
||||
|
||||
async fn send_outbound_message(
|
||||
physical_outgoing_tx: &mpsc::Sender<NoisePhysicalFrame>,
|
||||
transport: &Mutex<NoiseTransport>,
|
||||
stream_id: &str,
|
||||
next_seq: &mut u32,
|
||||
message: &JSONRPCMessage,
|
||||
) -> Result<(), ExecServerError> {
|
||||
let started_at = Instant::now();
|
||||
let framed = tracing::info_span!("exec_server.noise.executor_frame_message")
|
||||
.in_scope(|| frame_jsonrpc_message(message))?;
|
||||
let record_count = framed.len().div_ceil(NOISE_RECORD_PLAINTEXT_LEN);
|
||||
let span = Span::current();
|
||||
span.record("framed_bytes", framed.len());
|
||||
span.record("records", record_count);
|
||||
span.record(
|
||||
"frame_complete_ms",
|
||||
started_at.elapsed().as_secs_f64() * 1_000.0,
|
||||
);
|
||||
// Extracting W3C context allocates its string carrier. Do that once for the
|
||||
// logical JSON-RPC message, then share it across all physical Noise records.
|
||||
let trace = codex_otel::current_span_w3c_trace_context().map(Arc::new);
|
||||
|
||||
for (record_index, plaintext_record) in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN).enumerate() {
|
||||
let seq = take_next_sequence(next_seq)?;
|
||||
let ciphertext =
|
||||
tracing::info_span!("exec_server.noise.executor_encrypt_record", record_index,)
|
||||
.in_scope(|| {
|
||||
transport
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.encrypt(plaintext_record)
|
||||
})
|
||||
.map_err(|error| {
|
||||
ExecServerError::Protocol(format!(
|
||||
"failed to encrypt Noise virtual stream payload: {error}"
|
||||
))
|
||||
})?;
|
||||
span.record(
|
||||
"encrypt_complete_ms",
|
||||
started_at.elapsed().as_secs_f64() * 1_000.0,
|
||||
);
|
||||
let frame = RelayMessageFrame::data(stream_id.to_string(), seq, ciphertext);
|
||||
let encoded = encode_relay_message_frame(&frame);
|
||||
let permit = physical_outgoing_tx
|
||||
.reserve()
|
||||
.instrument(tracing::info_span!(
|
||||
"exec_server.noise.executor_physical_queue",
|
||||
record_index,
|
||||
))
|
||||
.await
|
||||
.map_err(|_| ExecServerError::Closed)?;
|
||||
permit.send(NoisePhysicalFrame::data(encoded, trace.clone()));
|
||||
span.record(
|
||||
"physical_queue_complete_ms",
|
||||
started_at.elapsed().as_secs_f64() * 1_000.0,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "executor_stream_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -1,23 +1,86 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::JSONRPCResponse;
|
||||
use codex_exec_server_protocol::RequestId;
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::timeout;
|
||||
use tracing::Instrument;
|
||||
use tracing::instrument::WithSubscriber;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
use super::ClosedNoiseVirtualStream;
|
||||
use super::send_outbound_message;
|
||||
use super::spawn_noise_virtual_stream;
|
||||
use crate::ExecServerRuntimePaths;
|
||||
use crate::connection::CHANNEL_CAPACITY;
|
||||
use crate::noise_channel::InitiatorHandshake;
|
||||
use crate::noise_channel::NoiseChannelIdentity;
|
||||
use crate::noise_channel::PendingResponderHandshake;
|
||||
use crate::noise_relay::message_framing::NOISE_RECORD_PLAINTEXT_LEN;
|
||||
use crate::noise_relay::message_framing::frame_jsonrpc_message;
|
||||
use crate::relay_proto::RelayData;
|
||||
use crate::server::ConnectionProcessor;
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn outbound_records_share_one_message_trace_context() -> Result<()> {
|
||||
let executor_identity = NoiseChannelIdentity::generate()?;
|
||||
let harness_identity = NoiseChannelIdentity::generate()?;
|
||||
let (initiator, request) = InitiatorHandshake::start(
|
||||
&harness_identity,
|
||||
&executor_identity.public_key(),
|
||||
b"test-prologue",
|
||||
b"authorization",
|
||||
)?;
|
||||
let pending =
|
||||
PendingResponderHandshake::read_request(&executor_identity, b"test-prologue", &request)?;
|
||||
let (executor_transport, response) = pending.complete()?;
|
||||
let _harness_transport = initiator.finish(&response)?;
|
||||
let executor_transport = std::sync::Mutex::new(executor_transport);
|
||||
let (physical_outgoing_tx, mut physical_outgoing_rx) = mpsc::channel(CHANNEL_CAPACITY);
|
||||
let message = JSONRPCMessage::Response(JSONRPCResponse {
|
||||
id: RequestId::Integer(1),
|
||||
result: serde_json::Value::String("x".repeat(NOISE_RECORD_PLAINTEXT_LEN * 2)),
|
||||
});
|
||||
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(tracing_opentelemetry::layer().with_tracer(provider.tracer("exec-server-test")));
|
||||
let mut next_seq = 0;
|
||||
async {
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
send_outbound_message(
|
||||
&physical_outgoing_tx,
|
||||
&executor_transport,
|
||||
"stream-1",
|
||||
&mut next_seq,
|
||||
&message,
|
||||
)
|
||||
.instrument(tracing::info_span!("outbound-message"))
|
||||
.await
|
||||
}
|
||||
.with_subscriber(subscriber)
|
||||
.await?;
|
||||
|
||||
let mut frames = Vec::new();
|
||||
while let Ok(frame) = physical_outgoing_rx.try_recv() {
|
||||
frames.push(frame);
|
||||
}
|
||||
assert!(frames.len() > 1, "expected multiple physical records");
|
||||
let first_trace = frames[0].trace.as_ref().expect("first record trace");
|
||||
assert!(frames.iter().skip(1).all(|frame| {
|
||||
Arc::ptr_eq(
|
||||
first_trace,
|
||||
frame.trace.as_ref().expect("subsequent record trace"),
|
||||
)
|
||||
}));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn processor_exit_reports_closed_virtual_stream() -> Result<()> {
|
||||
let executor_identity = NoiseChannelIdentity::generate()?;
|
||||
@@ -52,12 +115,15 @@ async fn processor_exit_reports_closed_virtual_stream() -> Result<()> {
|
||||
result: serde_json::Value::Null,
|
||||
});
|
||||
let ciphertext = harness_transport.encrypt(&frame_jsonrpc_message(&message)?)?;
|
||||
stream.receive_data(RelayData {
|
||||
seq: 0,
|
||||
segment_index: 0,
|
||||
segment_count: 1,
|
||||
payload: ciphertext,
|
||||
})?;
|
||||
stream.receive_data(
|
||||
RelayData {
|
||||
seq: 0,
|
||||
segment_index: 0,
|
||||
segment_count: 1,
|
||||
payload: ciphertext,
|
||||
},
|
||||
std::time::Instant::now(),
|
||||
)?;
|
||||
|
||||
assert!(matches!(
|
||||
timeout(Duration::from_secs(1), closed_stream_rx.recv()).await?,
|
||||
|
||||
@@ -2,6 +2,7 @@ pub(crate) mod executor_stream;
|
||||
mod harness;
|
||||
mod message_framing;
|
||||
mod ordered_ciphertext;
|
||||
mod trace_context;
|
||||
|
||||
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use codex_exec_server_protocol::EXEC_CLOSED_METHOD;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::RequestId;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
|
||||
/// Correlates return traffic with the trace carried by its originating request.
|
||||
///
|
||||
/// Responses do not repeat the request's W3C carrier, and process notifications
|
||||
/// are correlated by `processId`. Keeping this state per virtual stream avoids
|
||||
/// adding telemetry-only fields to the JSON-RPC wire protocol.
|
||||
#[derive(Default)]
|
||||
pub(super) struct NoiseTraceContext {
|
||||
requests: HashMap<RequestId, W3cTraceContext>,
|
||||
processes: HashMap<String, W3cTraceContext>,
|
||||
}
|
||||
|
||||
impl NoiseTraceContext {
|
||||
pub(super) fn observe_request(&mut self, message: &JSONRPCMessage) {
|
||||
let JSONRPCMessage::Request(request) = message else {
|
||||
return;
|
||||
};
|
||||
let Some(trace) = request.trace.as_ref() else {
|
||||
return;
|
||||
};
|
||||
self.requests.insert(request.id.clone(), trace.clone());
|
||||
if let Some(process_id) = message_process_id(message) {
|
||||
self.processes
|
||||
.entry(process_id.to_string())
|
||||
.or_insert_with(|| trace.clone());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn return_trace(&mut self, message: &JSONRPCMessage) -> Option<W3cTraceContext> {
|
||||
match message {
|
||||
JSONRPCMessage::Response(response) => self.requests.remove(&response.id),
|
||||
JSONRPCMessage::Error(error) => self.requests.remove(&error.id),
|
||||
JSONRPCMessage::Notification(notification) => {
|
||||
let process_id = message_process_id(message)?;
|
||||
let trace = self.processes.get(process_id).cloned();
|
||||
if notification.method == EXEC_CLOSED_METHOD {
|
||||
self.processes.remove(process_id);
|
||||
}
|
||||
trace
|
||||
}
|
||||
JSONRPCMessage::Request(request) => request.trace.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn message_process_id(message: &JSONRPCMessage) -> Option<&str> {
|
||||
let params = match message {
|
||||
JSONRPCMessage::Request(request) => request.params.as_ref(),
|
||||
JSONRPCMessage::Notification(notification) => notification.params.as_ref(),
|
||||
JSONRPCMessage::Response(_) | JSONRPCMessage::Error(_) => None,
|
||||
}?;
|
||||
params.get("processId")?.as_str()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "trace_context_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,48 @@
|
||||
use codex_exec_server_protocol::EXEC_CLOSED_METHOD;
|
||||
use codex_exec_server_protocol::EXEC_METHOD;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::JSONRPCNotification;
|
||||
use codex_exec_server_protocol::JSONRPCRequest;
|
||||
use codex_exec_server_protocol::JSONRPCResponse;
|
||||
use codex_exec_server_protocol::RequestId;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use super::NoiseTraceContext;
|
||||
|
||||
fn trace_context() -> W3cTraceContext {
|
||||
W3cTraceContext {
|
||||
traceparent: Some("00-00000000000000000000000000000001-0000000000000002-01".to_string()),
|
||||
tracestate: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn process_start_request(trace: W3cTraceContext) -> JSONRPCMessage {
|
||||
JSONRPCMessage::Request(JSONRPCRequest {
|
||||
id: RequestId::Integer(7),
|
||||
method: EXEC_METHOD.to_string(),
|
||||
params: Some(serde_json::json!({"processId": "process-1"})),
|
||||
trace: Some(trace),
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlates_response_and_terminal_notification_with_request_trace() {
|
||||
let trace = trace_context();
|
||||
let mut context = NoiseTraceContext::default();
|
||||
context.observe_request(&process_start_request(trace.clone()));
|
||||
|
||||
let response = JSONRPCMessage::Response(JSONRPCResponse {
|
||||
id: RequestId::Integer(7),
|
||||
result: serde_json::Value::Null,
|
||||
});
|
||||
assert_eq!(context.return_trace(&response), Some(trace.clone()));
|
||||
assert_eq!(context.return_trace(&response), None);
|
||||
|
||||
let closed = JSONRPCMessage::Notification(JSONRPCNotification {
|
||||
method: EXEC_CLOSED_METHOD.to_string(),
|
||||
params: Some(serde_json::json!({"processId": "process-1", "seq": 1})),
|
||||
});
|
||||
assert_eq!(context.return_trace(&closed), Some(trace));
|
||||
assert_eq!(context.return_trace(&closed), None);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use futures::Sink;
|
||||
@@ -15,6 +16,7 @@ use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_tungstenite::WebSocketStream;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::Instrument;
|
||||
use tracing::debug;
|
||||
use tracing::info;
|
||||
use tracing::warn;
|
||||
@@ -32,6 +34,7 @@ use crate::noise_channel::PendingResponderHandshake;
|
||||
use crate::noise_channel::noise_channel_prologue;
|
||||
use crate::noise_relay::NOISE_RELAY_RESET_REASON;
|
||||
use crate::noise_relay::executor_stream::ClosedNoiseVirtualStream;
|
||||
use crate::noise_relay::executor_stream::NoisePhysicalFrame;
|
||||
use crate::noise_relay::executor_stream::NoiseVirtualStream;
|
||||
use crate::noise_relay::executor_stream::spawn_noise_virtual_stream;
|
||||
use crate::relay_proto::RelayData;
|
||||
@@ -483,7 +486,7 @@ where
|
||||
);
|
||||
let (mut websocket_sink, mut websocket_stream) = stream.split();
|
||||
let (physical_outgoing_tx, mut physical_outgoing_rx) =
|
||||
mpsc::channel::<Vec<u8>>(CHANNEL_CAPACITY);
|
||||
mpsc::channel::<NoisePhysicalFrame>(CHANNEL_CAPACITY);
|
||||
let (closed_stream_tx, mut closed_stream_rx) =
|
||||
mpsc::channel::<ClosedNoiseVirtualStream>(MAX_ACTIVE_NOISE_RELAY_STREAMS);
|
||||
let (pong_tx, mut pong_rx) = mpsc::channel(1);
|
||||
@@ -498,7 +501,7 @@ where
|
||||
let pong_deadline = tokio::time::sleep(WEBSOCKET_PONG_TIMEOUT);
|
||||
tokio::pin!(pong_deadline);
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
let (message, queued_frame) = tokio::select! {
|
||||
pong = pong_rx.recv() => {
|
||||
let Some(()) = pong else {
|
||||
break RendezvousDisconnectReason::LocalShutdown;
|
||||
@@ -521,18 +524,35 @@ where
|
||||
}
|
||||
}
|
||||
_ = keepalive.tick(), if pong_watchdog.deadline().is_none() => {
|
||||
Message::Ping(Vec::new().into())
|
||||
(Message::Ping(Vec::new().into()), None)
|
||||
}
|
||||
encoded = physical_outgoing_rx.recv() => {
|
||||
let Some(encoded) = encoded else {
|
||||
frame = physical_outgoing_rx.recv() => {
|
||||
let Some(frame) = frame else {
|
||||
break RendezvousDisconnectReason::LocalShutdown;
|
||||
};
|
||||
Message::Binary(encoded.into())
|
||||
let bytes = frame.encoded.len();
|
||||
let message = Message::Binary(frame.encoded.into());
|
||||
(message, Some((frame.queued_at, frame.trace, bytes)))
|
||||
}
|
||||
};
|
||||
let is_keepalive_ping = matches!(message, Message::Ping(_));
|
||||
let write_deadline = pong_watchdog.write_deadline(tokio::time::Instant::now());
|
||||
match tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await {
|
||||
let send_result = if let Some((queued_at, Some(trace), bytes)) = queued_frame {
|
||||
let span = tracing::info_span!(
|
||||
"exec_server.noise.executor_physical_send",
|
||||
bytes,
|
||||
queue_wait_ms = queued_at.elapsed().as_secs_f64() * 1_000.0,
|
||||
);
|
||||
let _ = codex_otel::set_parent_from_w3c_trace_context(&span, trace.as_ref());
|
||||
tokio::time::timeout_at(
|
||||
write_deadline,
|
||||
websocket_sink.send(message).instrument(span),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await
|
||||
};
|
||||
match send_result {
|
||||
Ok(Ok(())) => {
|
||||
if is_keepalive_ping {
|
||||
pong_watchdog.ping_sent(tokio::time::Instant::now());
|
||||
@@ -561,7 +581,7 @@ where
|
||||
|
||||
loop {
|
||||
// Registry calls run separately so a slow check does not block the relay.
|
||||
let frame = tokio::select! {
|
||||
let (frame, physical_received_at) = tokio::select! {
|
||||
writer_result = &mut physical_writer_task => {
|
||||
match writer_result {
|
||||
Ok(reason) => disconnect_reason = reason,
|
||||
@@ -647,7 +667,9 @@ where
|
||||
// Do not leave a half-open stream if the handshake reply
|
||||
// cannot be queued immediately.
|
||||
if physical_outgoing_tx
|
||||
.try_send(encode_relay_message_frame(&response))
|
||||
.try_send(NoisePhysicalFrame::control(encode_relay_message_frame(
|
||||
&response,
|
||||
)))
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
@@ -687,11 +709,14 @@ where
|
||||
continue;
|
||||
}
|
||||
incoming_message = websocket_stream.next() => match incoming_message {
|
||||
Some(Ok(Message::Binary(payload))) => match decode_relay_message_frame(payload.as_ref()) {
|
||||
Ok(frame) => frame,
|
||||
Err(error) => {
|
||||
warn!("dropping malformed Noise relay frame from harness: {error}");
|
||||
continue;
|
||||
Some(Ok(Message::Binary(payload))) => {
|
||||
let physical_received_at = Instant::now();
|
||||
match decode_relay_message_frame(payload.as_ref()) {
|
||||
Ok(frame) => (frame, physical_received_at),
|
||||
Err(error) => {
|
||||
warn!("dropping malformed Noise relay frame from harness: {error}");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
},
|
||||
Some(Ok(Message::Close(_))) | None => {
|
||||
@@ -856,7 +881,7 @@ where
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Err(error) = stream.receive_data(data) {
|
||||
if let Err(error) = stream.receive_data(data, physical_received_at) {
|
||||
warn!("failed to process Noise relay payload: {error}");
|
||||
streams.remove(&stream_id);
|
||||
send_reset(&physical_outgoing_tx, stream_id);
|
||||
@@ -910,9 +935,11 @@ struct HarnessKeyValidationResult {
|
||||
|
||||
/// Queue a best-effort reset without blocking the shared websocket loop.
|
||||
/// Reset reasons are relay control data and are not treated as trusted text.
|
||||
fn send_reset(physical_outgoing_tx: &mpsc::Sender<Vec<u8>>, stream_id: String) {
|
||||
fn send_reset(physical_outgoing_tx: &mpsc::Sender<NoisePhysicalFrame>, stream_id: String) {
|
||||
let reset = RelayMessageFrame::reset(stream_id, NOISE_RELAY_RESET_REASON.to_string());
|
||||
let _ = physical_outgoing_tx.try_send(encode_relay_message_frame(&reset));
|
||||
let _ = physical_outgoing_tx.try_send(NoisePhysicalFrame::control(encode_relay_message_frame(
|
||||
&reset,
|
||||
)));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user