exec-server: trace JSON-RPC requests

This commit is contained in:
Richard Lee
2026-06-10 11:40:24 -07:00
parent bf4be25a43
commit e06c4e5e1d
5 changed files with 236 additions and 17 deletions
+5
View File
@@ -2840,6 +2840,7 @@ dependencies = [
"codex-app-server-protocol",
"codex-client",
"codex-file-system",
"codex-otel",
"codex-protocol",
"codex-sandboxing",
"codex-shell-command",
@@ -2850,6 +2851,8 @@ dependencies = [
"ctor 0.6.3",
"futures",
"http 1.4.0",
"opentelemetry",
"opentelemetry_sdk",
"pretty_assertions",
"prost 0.14.3",
"reqwest 0.12.28",
@@ -2864,6 +2867,8 @@ dependencies = [
"tokio-util",
"toml 0.9.11+spec-1.1.0",
"tracing",
"tracing-opentelemetry",
"tracing-subscriber",
"uuid",
"wiremock",
]
+2
View File
@@ -4,6 +4,8 @@ codex_rust_crate(
name = "exec-server",
crate_name = "codex_exec_server",
deps_extra = [
"@crates//:opentelemetry",
"@crates//:opentelemetry_sdk",
"@crates//:toml",
],
# Keep the crate's integration tests single-threaded under Bazel because
+5
View File
@@ -20,6 +20,7 @@ codex-app-server-protocol = { workspace = true }
codex-api = { workspace = true }
codex-client = { workspace = true }
codex-file-system = { workspace = true }
codex-otel = { workspace = true }
codex-protocol = { workspace = true }
codex-sandboxing = { workspace = true }
codex-shell-command = { workspace = true }
@@ -54,8 +55,12 @@ anyhow = { workspace = true }
codex-test-binary-support = { workspace = true }
ctor = { workspace = true }
http = { workspace = true }
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true }
pretty_assertions = { workspace = true }
serial_test = { workspace = true }
tempfile = { workspace = true }
test-case = "3.3.1"
tracing-opentelemetry = { workspace = true }
tracing-subscriber = { workspace = true }
wiremock = { workspace = true }
+60 -3
View File
@@ -209,8 +209,10 @@ where
);
}
pub(crate) fn request_route(&self, method: &str) -> Option<&RequestRoute<S>> {
self.request_routes.get(method)
pub(crate) fn request_route(&self, method: &str) -> Option<(&'static str, &RequestRoute<S>)> {
self.request_routes
.get_key_value(method)
.map(|(&method, route)| (method, route))
}
pub(crate) fn notification_route(&self, method: &str) -> Option<&NotificationRoute<S>> {
@@ -347,7 +349,7 @@ impl RpcClient {
id: request_id.clone(),
method: method.to_string(),
params: Some(params),
trace: None,
trace: codex_otel::current_span_w3c_trace_context(),
}))
.await
.is_err()
@@ -531,11 +533,17 @@ mod tests {
use codex_app_server_protocol::JSONRPCMessage;
use codex_app_server_protocol::JSONRPCResponse;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::InMemorySpanExporter;
use opentelemetry_sdk::trace::SdkTracerProvider;
use pretty_assertions::assert_eq;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::time::timeout;
use tracing::Instrument;
use tracing_subscriber::filter::filter_fn;
use tracing_subscriber::prelude::*;
use super::RpcClient;
use crate::connection::JsonRpcConnection;
@@ -641,4 +649,53 @@ mod tests {
panic!("server task failed: {err}");
}
}
#[tokio::test(flavor = "current_thread")]
async fn rpc_client_propagates_current_trace_context() {
let span_exporter = InMemorySpanExporter::default();
let tracer_provider = SdkTracerProvider::builder()
.with_simple_exporter(span_exporter)
.build();
let tracer = tracer_provider.tracer("exec-server-test");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(codex_otel::OtelProvider::trace_export_filter)),
);
let _subscriber_guard = tracing::subscriber::set_default(subscriber);
tracing::callsite::rebuild_interest_cache();
let parent_span = tracing::info_span!("outbound-parent");
let (client_stdin, server_reader) = tokio::io::duplex(4096);
let (mut server_writer, client_stdout) = tokio::io::duplex(4096);
let connection =
JsonRpcConnection::from_stdio(client_stdout, client_stdin, "test-rpc".to_string());
let (client, _events_rx) = RpcClient::new(connection);
let server = tokio::spawn(async move {
let mut lines = BufReader::new(server_reader).lines();
let request = match read_jsonrpc_line(&mut lines).await {
JSONRPCMessage::Request(request) => request,
other => panic!("expected JSON-RPC request, got {other:?}"),
};
write_jsonrpc_line(
&mut server_writer,
JSONRPCMessage::Response(JSONRPCResponse {
id: request.id.clone(),
result: serde_json::json!({}),
}),
)
.await;
request.trace
});
let response = client
.call::<_, serde_json::Value>("traced", &serde_json::json!({}))
.instrument(parent_span)
.await
.expect("RPC response");
assert_eq!(response, serde_json::json!({}));
let trace = server.await.expect("server task").expect("trace context");
assert!(trace.traceparent.is_some());
}
}
+164 -14
View File
@@ -1,6 +1,7 @@
use std::sync::Arc;
use tokio::sync::mpsc;
use tracing::Instrument;
use tracing::debug;
use tracing::warn;
@@ -100,31 +101,43 @@ async fn run_connection(
}
JsonRpcConnectionEvent::Message(message) => match message {
codex_app_server_protocol::JSONRPCMessage::Request(request) => {
if let Some(route) = router.request_route(request.method.as_str()) {
if let Some((method, route)) = router.request_route(request.method.as_str()) {
let request_span = request_span(method, &request);
let message = tokio::select! {
message = route(Arc::clone(&handler), request) => message,
message = route(Arc::clone(&handler), request).instrument(request_span.clone()) => message,
_ = disconnected_rx.changed() => {
request_span.record("result", "disconnected");
drop(request_span);
debug!("exec-server transport disconnected while handling request");
break;
}
};
let result = request_result(&message);
request_span.record("result", result);
drop(request_span);
if let Some(message) = message
&& outgoing_tx.send(message).await.is_err()
{
break;
}
} else if outgoing_tx
.send(RpcServerOutboundMessage::Error {
request_id: request.id,
error: method_not_found(format!(
"exec-server stub does not implement `{}` yet",
request.method
)),
})
.await
.is_err()
{
break;
} else {
let method = "unknown";
let request_span = request_span(method, &request);
request_span.record("result", "error");
drop(request_span);
if outgoing_tx
.send(RpcServerOutboundMessage::Error {
request_id: request.id,
error: method_not_found(format!(
"exec-server stub does not implement `{}` yet",
request.method
)),
})
.await
.is_err()
{
break;
}
}
}
codex_app_server_protocol::JSONRPCMessage::Notification(notification) => {
@@ -184,6 +197,35 @@ async fn run_connection(
let _ = outbound_task.await;
}
fn request_span(
method: &'static str,
request: &codex_app_server_protocol::JSONRPCRequest,
) -> tracing::Span {
let span = tracing::info_span!(
"codex.exec_server.request",
otel.kind = "server",
otel.name = method,
method,
result = tracing::field::Empty,
);
if let Some(trace) = &request.trace
&& !codex_otel::set_parent_from_w3c_trace_context(&span, trace)
{
warn!(method, "ignoring invalid inbound exec-server trace carrier");
}
span
}
fn request_result(message: &Option<RpcServerOutboundMessage>) -> &'static str {
match message {
Some(RpcServerOutboundMessage::Error { .. }) => "error",
Some(
RpcServerOutboundMessage::Response { .. } | RpcServerOutboundMessage::Notification(_),
)
| None => "success",
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
@@ -195,6 +237,12 @@ mod tests {
use codex_app_server_protocol::JSONRPCRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use opentelemetry::trace::SpanId;
use opentelemetry::trace::TraceId;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::InMemorySpanExporter;
use opentelemetry_sdk::trace::SdkTracerProvider;
use pretty_assertions::assert_eq;
use serde::Serialize;
use serde::de::DeserializeOwned;
use tokio::io::AsyncBufReadExt;
@@ -205,7 +253,10 @@ mod tests {
use tokio::io::duplex;
use tokio::task::JoinHandle;
use tokio::time::timeout;
use tracing_subscriber::filter::filter_fn;
use tracing_subscriber::prelude::*;
use super::request_span;
use super::run_connection;
use crate::ExecServerRuntimePaths;
use crate::ProcessId;
@@ -223,6 +274,105 @@ mod tests {
use crate::protocol::TerminateParams;
use crate::protocol::TerminateResponse;
use crate::server::session_registry::SessionRegistry;
use crate::telemetry::runtime_span;
#[test]
fn request_routes_return_bounded_protocol_method_names() {
let router = crate::server::registry::build_router();
assert_eq!(
router
.request_route(EXEC_TERMINATE_METHOD)
.map(|(method, _)| method),
Some(EXEC_TERMINATE_METHOD)
);
assert!(router.request_route("custom/method").is_none());
}
#[test]
fn runtime_and_request_spans_are_exported() {
let span_exporter = InMemorySpanExporter::default();
let tracer_provider = SdkTracerProvider::builder()
.with_simple_exporter(span_exporter.clone())
.build();
let tracer = tracer_provider.tracer("exec-server-test");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(codex_otel::OtelProvider::trace_export_filter)),
);
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let runtime_span = runtime_span();
runtime_span.in_scope(|| {
let request_span = request_span(INITIALIZE_METHOD, &test_request(/*trace*/ None));
request_span.in_scope(|| {});
request_span.record("result", "success");
drop(request_span);
});
drop(runtime_span);
});
tracer_provider.force_flush().expect("flush traces");
let spans = span_exporter.get_finished_spans().expect("span export");
assert!(
spans
.iter()
.any(|span| span.name.as_ref() == "codex.exec_server"),
"expected runtime span, got {spans:?}"
);
assert!(
spans
.iter()
.any(|span| span.name.as_ref() == INITIALIZE_METHOD),
"expected initialize request span, got {spans:?}"
);
}
#[test]
fn request_span_uses_inbound_trace_parent() {
let span_exporter = InMemorySpanExporter::default();
let tracer_provider = SdkTracerProvider::builder()
.with_simple_exporter(span_exporter.clone())
.build();
let tracer = tracer_provider.tracer("exec-server-test");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(codex_otel::OtelProvider::trace_export_filter)),
);
let trace_id = TraceId::from_hex("00000000000000000000000000000001").expect("trace id");
let parent_span_id = SpanId::from_hex("0000000000000002").expect("span id");
let trace = codex_protocol::protocol::W3cTraceContext {
traceparent: Some(format!("00-{trace_id}-{parent_span_id}-01")),
tracestate: None,
};
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let request_span = request_span(INITIALIZE_METHOD, &test_request(Some(trace)));
request_span.in_scope(|| {});
drop(request_span);
});
tracer_provider.force_flush().expect("flush traces");
let spans = span_exporter.get_finished_spans().expect("span export");
let request_span = spans
.iter()
.find(|span| span.name.as_ref() == INITIALIZE_METHOD)
.expect("initialize span");
assert_eq!(request_span.span_context.trace_id(), trace_id);
assert_eq!(request_span.parent_span_id, parent_span_id);
}
fn test_request(trace: Option<codex_protocol::protocol::W3cTraceContext>) -> JSONRPCRequest {
JSONRPCRequest {
id: RequestId::Integer(1),
method: INITIALIZE_METHOD.to_string(),
params: None,
trace,
}
}
#[tokio::test]
async fn transport_disconnect_detaches_session_during_in_flight_read() {