exec-server: observe remote WebSocket lifecycle

This commit is contained in:
Richard Lee
2026-06-10 13:49:50 -07:00
parent cb15fe94bc
commit 9dd39b74a3
5 changed files with 435 additions and 3 deletions
+153
View File
@@ -1,4 +1,10 @@
use std::io::Read as _;
use std::io::Write as _;
use std::net::TcpListener;
use std::path::Path;
use std::thread;
use std::time::Duration;
use std::time::Instant;
use anyhow::Result;
use predicates::prelude::PredicateBooleanExt;
@@ -48,3 +54,150 @@ fn local_exec_server_ignores_invalid_config_without_strict_config() -> Result<()
Ok(())
}
#[test]
fn remote_exec_server_preserves_websocket_error_in_stderr() -> Result<()> {
let failed_websocket_listener = TcpListener::bind("127.0.0.1:0")?;
let failed_websocket_url = format!("ws://{}", failed_websocket_listener.local_addr()?);
drop(failed_websocket_listener);
let registry = TestRegistry::start(&failed_websocket_url)?;
let codex_home = TempDir::new()?;
let stderr_path = codex_home.path().join("remote.stderr");
let stderr_file = std::fs::File::create(&stderr_path)?;
let mut cmd = std::process::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?);
cmd.env("CODEX_HOME", codex_home.path())
.env("CODEX_API_KEY", "test-key")
.env("RUST_LOG", "codex_exec_server=warn")
.args([
"exec-server",
"--remote",
&registry.base_url,
"--environment-id",
"env-test",
])
.stdout(std::process::Stdio::null())
.stderr(stderr_file);
let mut child = cmd.spawn()?;
let deadline = Instant::now() + Duration::from_secs(15);
let stderr = loop {
let stderr = std::fs::read_to_string(&stderr_path)?;
if stderr.contains("failed to connect remote exec-server websocket")
|| Instant::now() >= deadline
{
break stderr;
}
thread::sleep(Duration::from_millis(50));
};
let _ = child.kill();
child.wait()?;
registry.finish()?;
assert!(
stderr.contains("failed to connect remote exec-server websocket"),
"{stderr}"
);
assert!(stderr.contains("environment_id env-test"), "{stderr}");
assert!(stderr.contains("IO error"), "{stderr}");
Ok(())
}
struct CapturedRequest {
path: String,
body: String,
}
struct TestRegistry {
base_url: String,
server: thread::JoinHandle<Result<()>>,
}
impl TestRegistry {
fn start(websocket_url: &str) -> Result<Self> {
let listener = TcpListener::bind("127.0.0.1:0")?;
let addr = listener.local_addr()?;
listener.set_nonblocking(true)?;
let websocket_url = websocket_url.to_string();
let server = thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(20);
let (mut stream, _) = loop {
match listener.accept() {
Ok(connection) => break connection,
Err(err)
if err.kind() == std::io::ErrorKind::WouldBlock
&& Instant::now() < deadline =>
{
thread::sleep(Duration::from_millis(50));
}
Err(err) => return Err(err.into()),
}
};
let request = read_http_request(&mut stream)?;
assert_eq!(request.path, "/cloud/environment/env-test/register");
assert!(request.body.is_empty());
let body = format!(r#"{{"environment_id":"env-test","url":"{websocket_url}"}}"#);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes())?;
Ok(())
});
Ok(Self {
base_url: format!("http://{addr}"),
server,
})
}
fn finish(self) -> Result<()> {
self.server
.join()
.map_err(|_| anyhow::anyhow!("registry thread panicked"))?
}
}
fn read_http_request(stream: &mut std::net::TcpStream) -> std::io::Result<CapturedRequest> {
stream.set_read_timeout(Some(Duration::from_secs(2)))?;
let mut bytes = Vec::new();
let mut scratch = [0_u8; 8192];
let header_end = loop {
let read = stream.read(&mut scratch)?;
if read == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"request closed before headers",
));
}
bytes.extend_from_slice(&scratch[..read]);
if let Some(header_end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
break header_end;
}
};
let headers = String::from_utf8_lossy(&bytes[..header_end]);
let mut lines = headers.split("\r\n");
let path = lines
.next()
.and_then(|line| line.split_whitespace().nth(1))
.unwrap_or_default()
.to_string();
let content_length = lines
.filter_map(|line| line.split_once(':'))
.find(|(key, _)| key.eq_ignore_ascii_case("content-length"))
.and_then(|(_, value)| value.trim().parse::<usize>().ok())
.unwrap_or_default();
let mut body = bytes[header_end + 4..].to_vec();
while body.len() < content_length {
let read = stream.read(&mut scratch)?;
if read == 0 {
break;
}
body.extend_from_slice(&scratch[..read]);
}
body.truncate(content_length);
Ok(CapturedRequest {
path,
body: String::from_utf8_lossy(&body).into_owned(),
})
}
+57 -3
View File
@@ -7,7 +7,6 @@ use serde::Deserialize;
use tokio::time::sleep;
use tokio_tungstenite::connect_async;
use tracing::Instrument;
use tracing::warn;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
@@ -145,6 +144,7 @@ pub async fn run_remote_environment(
EnvironmentRegistryClient::new(config.base_url.clone(), config.auth_provider.clone())?;
let processor = ConnectionProcessor::new(runtime_paths, config.telemetry.clone());
let mut backoff = Duration::from_secs(1);
let mut connection_attempt = 0_u32;
loop {
let registration_started_at = Instant::now();
@@ -172,16 +172,39 @@ pub async fn run_remote_environment(
&response.environment_id,
);
match connect_async(response.url.as_str()).await {
let websocket_connect_started_at = Instant::now();
let websocket_connect_span =
remote_observability::websocket_connect_span(connection_attempt + 1);
match connect_async(response.url.as_str())
.instrument(websocket_connect_span.clone())
.await
{
Ok((websocket, _)) => {
connection_attempt += 1;
let remote_websocket = remote_observability::websocket_connected(
&config.telemetry,
websocket_connect_span,
websocket_connect_started_at,
connection_attempt,
);
backoff = Duration::from_secs(1);
run_multiplexed_environment(websocket, processor.clone()).await;
drop(remote_websocket);
remote_observability::websocket_disconnected(&config.telemetry, connection_attempt);
}
Err(err) => {
warn!("failed to connect remote exec-server websocket: {err}");
connection_attempt += 1;
remote_observability::websocket_connect_failed(
&config.telemetry,
websocket_connect_span,
websocket_connect_started_at,
connection_attempt,
&err,
);
}
}
remote_observability::websocket_retrying(connection_attempt, backoff);
sleep(backoff).await;
backoff = (backoff * 2).min(Duration::from_secs(30));
}
@@ -446,4 +469,35 @@ mod tests {
"raw environment id leaked into exported spans: {spans:?}"
);
}
#[test]
fn remote_websocket_operation_span_finishes_immediately() {
let span_exporter = InMemorySpanExporter::default();
let tracer_provider = SdkTracerProvider::builder()
.with_simple_exporter(span_exporter.clone())
.build();
let tracer = tracer_provider.tracer("exec-server-test");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(codex_otel::OtelProvider::trace_export_filter)),
);
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let websocket_span = remote_observability::websocket_connect_span(/*attempt*/ 2);
websocket_span.in_scope(|| {});
websocket_span.record("result", "success");
drop(websocket_span);
});
tracer_provider.force_flush().expect("flush traces");
let spans = span_exporter.get_finished_spans().expect("span export");
assert!(
spans
.iter()
.any(|span| { span.name.as_ref() == "codex.exec_server.remote.websocket.connect" }),
"expected finished remote websocket connect span, got {spans:?}"
);
}
}
@@ -1,3 +1,4 @@
use std::time::Duration;
use std::time::Instant;
use tracing::Span;
@@ -6,6 +7,7 @@ use tracing::warn;
use crate::ExecServerError;
use crate::ExecServerTelemetry;
use crate::telemetry::RemoteWebSocketMetricGuard;
macro_rules! emit_remote_otel_event {
($level:ident, $event_name:literal, $($fields:tt)*) => {{
@@ -81,3 +83,84 @@ pub(crate) fn registration_failed(
"failed to register remote exec-server environment"
);
}
pub(crate) fn websocket_connect_span(attempt: u32) -> Span {
tracing::info_span!(
"codex.exec_server.remote.websocket.connect",
otel.kind = "client",
otel.name = "codex.exec_server.remote.websocket.connect",
attempt,
result = tracing::field::Empty,
)
}
pub(crate) fn websocket_connected(
telemetry: &ExecServerTelemetry,
span: Span,
started_at: Instant,
attempt: u32,
) -> RemoteWebSocketMetricGuard {
span.record("result", "success");
telemetry.remote_websocket_connect_completed("success", started_at.elapsed());
drop(span);
info!(attempt, "connected remote exec-server websocket");
emit_remote_otel_event!(
INFO,
"codex.exec_server.remote_websocket_connected",
attempt,
"connected remote exec-server websocket"
);
telemetry.remote_websocket_connected()
}
pub(crate) fn websocket_connect_failed(
telemetry: &ExecServerTelemetry,
span: Span,
started_at: Instant,
attempt: u32,
err: &tokio_tungstenite::tungstenite::Error,
) {
span.record("result", "error");
telemetry.remote_websocket_connect_completed("error", started_at.elapsed());
drop(span);
warn!(
attempt,
error = %err,
"failed to connect remote exec-server websocket"
);
emit_remote_otel_event!(
WARN,
"codex.exec_server.remote_websocket_connect_failed",
attempt,
"failed to connect remote exec-server websocket"
);
telemetry.remote_websocket_reconnect("connect_failed");
}
pub(crate) fn websocket_disconnected(telemetry: &ExecServerTelemetry, attempt: u32) {
telemetry.remote_websocket_reconnect("disconnected");
warn!(
attempt,
"remote exec-server websocket disconnected; retrying"
);
emit_remote_otel_event!(
WARN,
"codex.exec_server.remote_websocket_disconnected",
attempt,
"remote exec-server websocket disconnected; retrying"
);
}
pub(crate) fn websocket_retrying(attempt: u32, backoff: Duration) {
let backoff_ms = backoff.as_millis();
info!(attempt, backoff_ms, "retrying remote exec-server websocket");
emit_remote_otel_event!(
INFO,
"codex.exec_server.remote_websocket_retrying",
attempt,
backoff_ms,
"retrying remote exec-server websocket"
);
}
+83
View File
@@ -17,6 +17,19 @@ const REMOTE_REGISTRATION_DURATION_METRIC: &str =
"exec_server_remote_registration_duration_seconds";
const REMOTE_REGISTRATION_DURATION_DESCRIPTION: &str =
"Duration of remote exec-server registration attempts in seconds.";
const REMOTE_WEBSOCKET_ACTIVE_METRIC: &str = "exec_server_remote_websocket_active";
const REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION: &str =
"Number of active remote exec-server WebSocket connections.";
const REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC: &str = "exec_server_remote_websocket_connect_total";
const REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION: &str =
"Total number of remote exec-server WebSocket connection attempts.";
const REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC: &str =
"exec_server_remote_websocket_connect_duration_seconds";
const REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION: &str =
"Duration of remote exec-server WebSocket connection attempts in seconds.";
const REMOTE_WEBSOCKET_RECONNECTS_METRIC: &str = "exec_server_remote_websocket_reconnects_total";
const REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION: &str =
"Total number of remote exec-server WebSocket reconnects.";
const REQUESTS_TOTAL_METRIC: &str = "exec_server_requests_total";
const REQUESTS_TOTAL_DESCRIPTION: &str = "Total number of exec-server requests.";
const REQUEST_DURATION_METRIC: &str = "exec_server_request_duration_seconds";
@@ -60,6 +73,7 @@ struct ExecServerTelemetryInner {
relay_connections: AtomicI64,
stdio_connections: AtomicI64,
websocket_connections: AtomicI64,
remote_websockets: AtomicI64,
active_processes: AtomicI64,
}
@@ -68,6 +82,10 @@ pub(crate) struct ConnectionMetricGuard {
transport: ConnectionTransport,
}
pub(crate) struct RemoteWebSocketMetricGuard {
telemetry: ExecServerTelemetry,
}
impl ExecServerTelemetry {
pub fn new(metrics: Option<MetricsClient>) -> Self {
Self {
@@ -77,6 +95,7 @@ impl ExecServerTelemetry {
relay_connections: AtomicI64::new(0),
stdio_connections: AtomicI64::new(0),
websocket_connections: AtomicI64::new(0),
remote_websockets: AtomicI64::new(0),
active_processes: AtomicI64::new(0),
})
}),
@@ -148,6 +167,52 @@ impl ExecServerTelemetry {
});
}
pub(crate) fn remote_websocket_connected(&self) -> RemoteWebSocketMetricGuard {
self.with_inner(|inner| {
let active = inner.remote_websockets.fetch_add(1, Ordering::AcqRel) + 1;
inner.gauge(
REMOTE_WEBSOCKET_ACTIVE_METRIC,
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
active,
&[],
);
});
RemoteWebSocketMetricGuard {
telemetry: self.clone(),
}
}
pub(crate) fn remote_websocket_connect_completed(
&self,
result: &'static str,
duration: Duration,
) {
self.with_inner(|inner| {
let tags = [("result", result)];
inner.counter(
REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC,
REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION,
&tags,
);
inner.duration(
REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC,
REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION,
duration,
&tags,
);
});
}
pub(crate) fn remote_websocket_reconnect(&self, reason: &'static str) {
self.with_inner(|inner| {
inner.counter(
REMOTE_WEBSOCKET_RECONNECTS_METRIC,
REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION,
&[("reason", reason)],
);
});
}
pub(crate) fn process_started(&self) {
self.with_inner(|inner| {
let active = inner.active_processes.fetch_add(1, Ordering::AcqRel) + 1;
@@ -198,6 +263,18 @@ impl ExecServerTelemetry {
});
}
fn remote_websocket_disconnected(&self) {
self.with_inner(|inner| {
let active = inner.remote_websockets.fetch_sub(1, Ordering::AcqRel) - 1;
inner.gauge(
REMOTE_WEBSOCKET_ACTIVE_METRIC,
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
active,
&[],
);
});
}
fn with_inner(&self, emit: impl FnOnce(&ExecServerTelemetryInner)) {
if let Some(inner) = &self.inner {
emit(inner);
@@ -211,6 +288,12 @@ impl Drop for ConnectionMetricGuard {
}
}
impl Drop for RemoteWebSocketMetricGuard {
fn drop(&mut self) {
self.telemetry.remote_websocket_disconnected();
}
}
impl ExecServerTelemetryInner {
fn connection_counter(&self, transport: ConnectionTransport) -> &AtomicI64 {
match transport {
@@ -160,6 +160,65 @@ fn emits_remote_registration_metrics() {
);
}
#[test]
fn emits_remote_websocket_metrics() {
let (telemetry, metrics, exporter) = test_telemetry();
let websocket = telemetry.remote_websocket_connected();
telemetry.remote_websocket_connect_completed("success", Duration::from_millis(7));
telemetry.remote_websocket_reconnect("connect_failed");
drop(websocket);
metrics.shutdown().expect("shutdown metrics");
let metrics = latest_metrics(&exporter);
assert_eq!(
metric_points(&metrics, REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC),
vec![(
1.0,
BTreeMap::from([("result".to_string(), "success".to_string())]),
)]
);
assert_eq!(
metric_points(&metrics, REMOTE_WEBSOCKET_ACTIVE_METRIC),
vec![(0.0, BTreeMap::new())]
);
assert_eq!(
metric_points(&metrics, REMOTE_WEBSOCKET_RECONNECTS_METRIC),
vec![(
1.0,
BTreeMap::from([("reason".to_string(), "connect_failed".to_string())]),
)]
);
assert_eq!(
histogram_count(&metrics, REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC),
1
);
for (name, description, unit) in [
(
REMOTE_WEBSOCKET_ACTIVE_METRIC,
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
"",
),
(
REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC,
REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION,
"",
),
(
REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC,
REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION,
"s",
),
(
REMOTE_WEBSOCKET_RECONNECTS_METRIC,
REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION,
"",
),
] {
assert_metric_metadata(&metrics, name, description, unit);
}
}
fn test_telemetry() -> (
ExecServerTelemetry,
codex_otel::MetricsClient,