mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
exec-server: observe remote WebSocket lifecycle
This commit is contained in:
@@ -1,4 +1,10 @@
|
||||
use std::io::Read as _;
|
||||
use std::io::Write as _;
|
||||
use std::net::TcpListener;
|
||||
use std::path::Path;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::Result;
|
||||
use predicates::prelude::PredicateBooleanExt;
|
||||
@@ -48,3 +54,150 @@ fn local_exec_server_ignores_invalid_config_without_strict_config() -> Result<()
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_exec_server_preserves_websocket_error_in_stderr() -> Result<()> {
|
||||
let failed_websocket_listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let failed_websocket_url = format!("ws://{}", failed_websocket_listener.local_addr()?);
|
||||
drop(failed_websocket_listener);
|
||||
|
||||
let registry = TestRegistry::start(&failed_websocket_url)?;
|
||||
let codex_home = TempDir::new()?;
|
||||
let stderr_path = codex_home.path().join("remote.stderr");
|
||||
let stderr_file = std::fs::File::create(&stderr_path)?;
|
||||
let mut cmd = std::process::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?);
|
||||
cmd.env("CODEX_HOME", codex_home.path())
|
||||
.env("CODEX_API_KEY", "test-key")
|
||||
.env("RUST_LOG", "codex_exec_server=warn")
|
||||
.args([
|
||||
"exec-server",
|
||||
"--remote",
|
||||
®istry.base_url,
|
||||
"--environment-id",
|
||||
"env-test",
|
||||
])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(stderr_file);
|
||||
|
||||
let mut child = cmd.spawn()?;
|
||||
let deadline = Instant::now() + Duration::from_secs(15);
|
||||
let stderr = loop {
|
||||
let stderr = std::fs::read_to_string(&stderr_path)?;
|
||||
if stderr.contains("failed to connect remote exec-server websocket")
|
||||
|| Instant::now() >= deadline
|
||||
{
|
||||
break stderr;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
};
|
||||
let _ = child.kill();
|
||||
child.wait()?;
|
||||
registry.finish()?;
|
||||
|
||||
assert!(
|
||||
stderr.contains("failed to connect remote exec-server websocket"),
|
||||
"{stderr}"
|
||||
);
|
||||
assert!(stderr.contains("environment_id env-test"), "{stderr}");
|
||||
assert!(stderr.contains("IO error"), "{stderr}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct CapturedRequest {
|
||||
path: String,
|
||||
body: String,
|
||||
}
|
||||
|
||||
struct TestRegistry {
|
||||
base_url: String,
|
||||
server: thread::JoinHandle<Result<()>>,
|
||||
}
|
||||
|
||||
impl TestRegistry {
|
||||
fn start(websocket_url: &str) -> Result<Self> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let addr = listener.local_addr()?;
|
||||
listener.set_nonblocking(true)?;
|
||||
let websocket_url = websocket_url.to_string();
|
||||
let server = thread::spawn(move || {
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
let (mut stream, _) = loop {
|
||||
match listener.accept() {
|
||||
Ok(connection) => break connection,
|
||||
Err(err)
|
||||
if err.kind() == std::io::ErrorKind::WouldBlock
|
||||
&& Instant::now() < deadline =>
|
||||
{
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
};
|
||||
let request = read_http_request(&mut stream)?;
|
||||
assert_eq!(request.path, "/cloud/environment/env-test/register");
|
||||
assert!(request.body.is_empty());
|
||||
let body = format!(r#"{{"environment_id":"env-test","url":"{websocket_url}"}}"#);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes())?;
|
||||
Ok(())
|
||||
});
|
||||
Ok(Self {
|
||||
base_url: format!("http://{addr}"),
|
||||
server,
|
||||
})
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<()> {
|
||||
self.server
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("registry thread panicked"))?
|
||||
}
|
||||
}
|
||||
|
||||
fn read_http_request(stream: &mut std::net::TcpStream) -> std::io::Result<CapturedRequest> {
|
||||
stream.set_read_timeout(Some(Duration::from_secs(2)))?;
|
||||
let mut bytes = Vec::new();
|
||||
let mut scratch = [0_u8; 8192];
|
||||
let header_end = loop {
|
||||
let read = stream.read(&mut scratch)?;
|
||||
if read == 0 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"request closed before headers",
|
||||
));
|
||||
}
|
||||
bytes.extend_from_slice(&scratch[..read]);
|
||||
if let Some(header_end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break header_end;
|
||||
}
|
||||
};
|
||||
let headers = String::from_utf8_lossy(&bytes[..header_end]);
|
||||
let mut lines = headers.split("\r\n");
|
||||
let path = lines
|
||||
.next()
|
||||
.and_then(|line| line.split_whitespace().nth(1))
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let content_length = lines
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case("content-length"))
|
||||
.and_then(|(_, value)| value.trim().parse::<usize>().ok())
|
||||
.unwrap_or_default();
|
||||
let mut body = bytes[header_end + 4..].to_vec();
|
||||
while body.len() < content_length {
|
||||
let read = stream.read(&mut scratch)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
body.extend_from_slice(&scratch[..read]);
|
||||
}
|
||||
body.truncate(content_length);
|
||||
Ok(CapturedRequest {
|
||||
path,
|
||||
body: String::from_utf8_lossy(&body).into_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ use serde::Deserialize;
|
||||
use tokio::time::sleep;
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tracing::Instrument;
|
||||
use tracing::warn;
|
||||
|
||||
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
|
||||
|
||||
@@ -145,6 +144,7 @@ pub async fn run_remote_environment(
|
||||
EnvironmentRegistryClient::new(config.base_url.clone(), config.auth_provider.clone())?;
|
||||
let processor = ConnectionProcessor::new(runtime_paths, config.telemetry.clone());
|
||||
let mut backoff = Duration::from_secs(1);
|
||||
let mut connection_attempt = 0_u32;
|
||||
|
||||
loop {
|
||||
let registration_started_at = Instant::now();
|
||||
@@ -172,16 +172,39 @@ pub async fn run_remote_environment(
|
||||
&response.environment_id,
|
||||
);
|
||||
|
||||
match connect_async(response.url.as_str()).await {
|
||||
let websocket_connect_started_at = Instant::now();
|
||||
let websocket_connect_span =
|
||||
remote_observability::websocket_connect_span(connection_attempt + 1);
|
||||
match connect_async(response.url.as_str())
|
||||
.instrument(websocket_connect_span.clone())
|
||||
.await
|
||||
{
|
||||
Ok((websocket, _)) => {
|
||||
connection_attempt += 1;
|
||||
let remote_websocket = remote_observability::websocket_connected(
|
||||
&config.telemetry,
|
||||
websocket_connect_span,
|
||||
websocket_connect_started_at,
|
||||
connection_attempt,
|
||||
);
|
||||
backoff = Duration::from_secs(1);
|
||||
run_multiplexed_environment(websocket, processor.clone()).await;
|
||||
drop(remote_websocket);
|
||||
remote_observability::websocket_disconnected(&config.telemetry, connection_attempt);
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("failed to connect remote exec-server websocket: {err}");
|
||||
connection_attempt += 1;
|
||||
remote_observability::websocket_connect_failed(
|
||||
&config.telemetry,
|
||||
websocket_connect_span,
|
||||
websocket_connect_started_at,
|
||||
connection_attempt,
|
||||
&err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
remote_observability::websocket_retrying(connection_attempt, backoff);
|
||||
sleep(backoff).await;
|
||||
backoff = (backoff * 2).min(Duration::from_secs(30));
|
||||
}
|
||||
@@ -446,4 +469,35 @@ mod tests {
|
||||
"raw environment id leaked into exported spans: {spans:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_websocket_operation_span_finishes_immediately() {
|
||||
let span_exporter = InMemorySpanExporter::default();
|
||||
let tracer_provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(span_exporter.clone())
|
||||
.build();
|
||||
let tracer = tracer_provider.tracer("exec-server-test");
|
||||
let subscriber = tracing_subscriber::registry().with(
|
||||
tracing_opentelemetry::layer()
|
||||
.with_tracer(tracer)
|
||||
.with_filter(filter_fn(codex_otel::OtelProvider::trace_export_filter)),
|
||||
);
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
let websocket_span = remote_observability::websocket_connect_span(/*attempt*/ 2);
|
||||
websocket_span.in_scope(|| {});
|
||||
websocket_span.record("result", "success");
|
||||
drop(websocket_span);
|
||||
});
|
||||
|
||||
tracer_provider.force_flush().expect("flush traces");
|
||||
let spans = span_exporter.get_finished_spans().expect("span export");
|
||||
assert!(
|
||||
spans
|
||||
.iter()
|
||||
.any(|span| { span.name.as_ref() == "codex.exec_server.remote.websocket.connect" }),
|
||||
"expected finished remote websocket connect span, got {spans:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use tracing::Span;
|
||||
@@ -6,6 +7,7 @@ use tracing::warn;
|
||||
|
||||
use crate::ExecServerError;
|
||||
use crate::ExecServerTelemetry;
|
||||
use crate::telemetry::RemoteWebSocketMetricGuard;
|
||||
|
||||
macro_rules! emit_remote_otel_event {
|
||||
($level:ident, $event_name:literal, $($fields:tt)*) => {{
|
||||
@@ -81,3 +83,84 @@ pub(crate) fn registration_failed(
|
||||
"failed to register remote exec-server environment"
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn websocket_connect_span(attempt: u32) -> Span {
|
||||
tracing::info_span!(
|
||||
"codex.exec_server.remote.websocket.connect",
|
||||
otel.kind = "client",
|
||||
otel.name = "codex.exec_server.remote.websocket.connect",
|
||||
attempt,
|
||||
result = tracing::field::Empty,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn websocket_connected(
|
||||
telemetry: &ExecServerTelemetry,
|
||||
span: Span,
|
||||
started_at: Instant,
|
||||
attempt: u32,
|
||||
) -> RemoteWebSocketMetricGuard {
|
||||
span.record("result", "success");
|
||||
telemetry.remote_websocket_connect_completed("success", started_at.elapsed());
|
||||
drop(span);
|
||||
|
||||
info!(attempt, "connected remote exec-server websocket");
|
||||
emit_remote_otel_event!(
|
||||
INFO,
|
||||
"codex.exec_server.remote_websocket_connected",
|
||||
attempt,
|
||||
"connected remote exec-server websocket"
|
||||
);
|
||||
telemetry.remote_websocket_connected()
|
||||
}
|
||||
|
||||
pub(crate) fn websocket_connect_failed(
|
||||
telemetry: &ExecServerTelemetry,
|
||||
span: Span,
|
||||
started_at: Instant,
|
||||
attempt: u32,
|
||||
err: &tokio_tungstenite::tungstenite::Error,
|
||||
) {
|
||||
span.record("result", "error");
|
||||
telemetry.remote_websocket_connect_completed("error", started_at.elapsed());
|
||||
drop(span);
|
||||
|
||||
warn!(
|
||||
attempt,
|
||||
error = %err,
|
||||
"failed to connect remote exec-server websocket"
|
||||
);
|
||||
emit_remote_otel_event!(
|
||||
WARN,
|
||||
"codex.exec_server.remote_websocket_connect_failed",
|
||||
attempt,
|
||||
"failed to connect remote exec-server websocket"
|
||||
);
|
||||
telemetry.remote_websocket_reconnect("connect_failed");
|
||||
}
|
||||
|
||||
pub(crate) fn websocket_disconnected(telemetry: &ExecServerTelemetry, attempt: u32) {
|
||||
telemetry.remote_websocket_reconnect("disconnected");
|
||||
warn!(
|
||||
attempt,
|
||||
"remote exec-server websocket disconnected; retrying"
|
||||
);
|
||||
emit_remote_otel_event!(
|
||||
WARN,
|
||||
"codex.exec_server.remote_websocket_disconnected",
|
||||
attempt,
|
||||
"remote exec-server websocket disconnected; retrying"
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn websocket_retrying(attempt: u32, backoff: Duration) {
|
||||
let backoff_ms = backoff.as_millis();
|
||||
info!(attempt, backoff_ms, "retrying remote exec-server websocket");
|
||||
emit_remote_otel_event!(
|
||||
INFO,
|
||||
"codex.exec_server.remote_websocket_retrying",
|
||||
attempt,
|
||||
backoff_ms,
|
||||
"retrying remote exec-server websocket"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,19 @@ const REMOTE_REGISTRATION_DURATION_METRIC: &str =
|
||||
"exec_server_remote_registration_duration_seconds";
|
||||
const REMOTE_REGISTRATION_DURATION_DESCRIPTION: &str =
|
||||
"Duration of remote exec-server registration attempts in seconds.";
|
||||
const REMOTE_WEBSOCKET_ACTIVE_METRIC: &str = "exec_server_remote_websocket_active";
|
||||
const REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION: &str =
|
||||
"Number of active remote exec-server WebSocket connections.";
|
||||
const REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC: &str = "exec_server_remote_websocket_connect_total";
|
||||
const REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION: &str =
|
||||
"Total number of remote exec-server WebSocket connection attempts.";
|
||||
const REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC: &str =
|
||||
"exec_server_remote_websocket_connect_duration_seconds";
|
||||
const REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION: &str =
|
||||
"Duration of remote exec-server WebSocket connection attempts in seconds.";
|
||||
const REMOTE_WEBSOCKET_RECONNECTS_METRIC: &str = "exec_server_remote_websocket_reconnects_total";
|
||||
const REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION: &str =
|
||||
"Total number of remote exec-server WebSocket reconnects.";
|
||||
const REQUESTS_TOTAL_METRIC: &str = "exec_server_requests_total";
|
||||
const REQUESTS_TOTAL_DESCRIPTION: &str = "Total number of exec-server requests.";
|
||||
const REQUEST_DURATION_METRIC: &str = "exec_server_request_duration_seconds";
|
||||
@@ -60,6 +73,7 @@ struct ExecServerTelemetryInner {
|
||||
relay_connections: AtomicI64,
|
||||
stdio_connections: AtomicI64,
|
||||
websocket_connections: AtomicI64,
|
||||
remote_websockets: AtomicI64,
|
||||
active_processes: AtomicI64,
|
||||
}
|
||||
|
||||
@@ -68,6 +82,10 @@ pub(crate) struct ConnectionMetricGuard {
|
||||
transport: ConnectionTransport,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteWebSocketMetricGuard {
|
||||
telemetry: ExecServerTelemetry,
|
||||
}
|
||||
|
||||
impl ExecServerTelemetry {
|
||||
pub fn new(metrics: Option<MetricsClient>) -> Self {
|
||||
Self {
|
||||
@@ -77,6 +95,7 @@ impl ExecServerTelemetry {
|
||||
relay_connections: AtomicI64::new(0),
|
||||
stdio_connections: AtomicI64::new(0),
|
||||
websocket_connections: AtomicI64::new(0),
|
||||
remote_websockets: AtomicI64::new(0),
|
||||
active_processes: AtomicI64::new(0),
|
||||
})
|
||||
}),
|
||||
@@ -148,6 +167,52 @@ impl ExecServerTelemetry {
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn remote_websocket_connected(&self) -> RemoteWebSocketMetricGuard {
|
||||
self.with_inner(|inner| {
|
||||
let active = inner.remote_websockets.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
inner.gauge(
|
||||
REMOTE_WEBSOCKET_ACTIVE_METRIC,
|
||||
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
|
||||
active,
|
||||
&[],
|
||||
);
|
||||
});
|
||||
RemoteWebSocketMetricGuard {
|
||||
telemetry: self.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remote_websocket_connect_completed(
|
||||
&self,
|
||||
result: &'static str,
|
||||
duration: Duration,
|
||||
) {
|
||||
self.with_inner(|inner| {
|
||||
let tags = [("result", result)];
|
||||
inner.counter(
|
||||
REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC,
|
||||
REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION,
|
||||
&tags,
|
||||
);
|
||||
inner.duration(
|
||||
REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC,
|
||||
REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION,
|
||||
duration,
|
||||
&tags,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn remote_websocket_reconnect(&self, reason: &'static str) {
|
||||
self.with_inner(|inner| {
|
||||
inner.counter(
|
||||
REMOTE_WEBSOCKET_RECONNECTS_METRIC,
|
||||
REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION,
|
||||
&[("reason", reason)],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn process_started(&self) {
|
||||
self.with_inner(|inner| {
|
||||
let active = inner.active_processes.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
@@ -198,6 +263,18 @@ impl ExecServerTelemetry {
|
||||
});
|
||||
}
|
||||
|
||||
fn remote_websocket_disconnected(&self) {
|
||||
self.with_inner(|inner| {
|
||||
let active = inner.remote_websockets.fetch_sub(1, Ordering::AcqRel) - 1;
|
||||
inner.gauge(
|
||||
REMOTE_WEBSOCKET_ACTIVE_METRIC,
|
||||
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
|
||||
active,
|
||||
&[],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn with_inner(&self, emit: impl FnOnce(&ExecServerTelemetryInner)) {
|
||||
if let Some(inner) = &self.inner {
|
||||
emit(inner);
|
||||
@@ -211,6 +288,12 @@ impl Drop for ConnectionMetricGuard {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteWebSocketMetricGuard {
|
||||
fn drop(&mut self) {
|
||||
self.telemetry.remote_websocket_disconnected();
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecServerTelemetryInner {
|
||||
fn connection_counter(&self, transport: ConnectionTransport) -> &AtomicI64 {
|
||||
match transport {
|
||||
|
||||
@@ -160,6 +160,65 @@ fn emits_remote_registration_metrics() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_remote_websocket_metrics() {
|
||||
let (telemetry, metrics, exporter) = test_telemetry();
|
||||
|
||||
let websocket = telemetry.remote_websocket_connected();
|
||||
telemetry.remote_websocket_connect_completed("success", Duration::from_millis(7));
|
||||
telemetry.remote_websocket_reconnect("connect_failed");
|
||||
drop(websocket);
|
||||
metrics.shutdown().expect("shutdown metrics");
|
||||
|
||||
let metrics = latest_metrics(&exporter);
|
||||
assert_eq!(
|
||||
metric_points(&metrics, REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC),
|
||||
vec![(
|
||||
1.0,
|
||||
BTreeMap::from([("result".to_string(), "success".to_string())]),
|
||||
)]
|
||||
);
|
||||
assert_eq!(
|
||||
metric_points(&metrics, REMOTE_WEBSOCKET_ACTIVE_METRIC),
|
||||
vec![(0.0, BTreeMap::new())]
|
||||
);
|
||||
assert_eq!(
|
||||
metric_points(&metrics, REMOTE_WEBSOCKET_RECONNECTS_METRIC),
|
||||
vec![(
|
||||
1.0,
|
||||
BTreeMap::from([("reason".to_string(), "connect_failed".to_string())]),
|
||||
)]
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_count(&metrics, REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC),
|
||||
1
|
||||
);
|
||||
for (name, description, unit) in [
|
||||
(
|
||||
REMOTE_WEBSOCKET_ACTIVE_METRIC,
|
||||
REMOTE_WEBSOCKET_ACTIVE_DESCRIPTION,
|
||||
"",
|
||||
),
|
||||
(
|
||||
REMOTE_WEBSOCKET_CONNECT_TOTAL_METRIC,
|
||||
REMOTE_WEBSOCKET_CONNECT_TOTAL_DESCRIPTION,
|
||||
"",
|
||||
),
|
||||
(
|
||||
REMOTE_WEBSOCKET_CONNECT_DURATION_METRIC,
|
||||
REMOTE_WEBSOCKET_CONNECT_DURATION_DESCRIPTION,
|
||||
"s",
|
||||
),
|
||||
(
|
||||
REMOTE_WEBSOCKET_RECONNECTS_METRIC,
|
||||
REMOTE_WEBSOCKET_RECONNECTS_DESCRIPTION,
|
||||
"",
|
||||
),
|
||||
] {
|
||||
assert_metric_metadata(&metrics, name, description, unit);
|
||||
}
|
||||
}
|
||||
|
||||
fn test_telemetry() -> (
|
||||
ExecServerTelemetry,
|
||||
codex_otel::MetricsClient,
|
||||
|
||||
Reference in New Issue
Block a user