Route blocking OTLP exporters through shared HTTP policy

This commit is contained in:
celia-oai
2026-08-17 16:13:39 -07:00
parent 96c24890ca
commit 34ffab4d4f
12 changed files with 191 additions and 96 deletions

2
codex-rs/Cargo.lock generated
View File

@@ -3841,6 +3841,7 @@ version = "0.0.0"
dependencies = [
"chrono",
"codex-api",
"codex-http-client",
"codex-protocol",
"codex-utils-absolute-path",
"codex-utils-string",
@@ -4755,6 +4756,7 @@ dependencies = [
"anyhow",
"base64 0.22.1",
"chrono",
"codex-http-client",
"codex-otel",
"codex-protocol",
"codex-utils-absolute-path",

View File

@@ -88,6 +88,7 @@ pub fn build_provider(
exporter,
trace_exporter,
metrics_exporter,
http_client_factory: config.http_client_factory(),
runtime_metrics,
span_attributes: config.otel.span_attributes.clone(),
tracestate: config.otel.tracestate.clone(),

View File

@@ -17,6 +17,7 @@ chrono = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-string = { workspace = true }
codex-api = { workspace = true }
codex-http-client = { workspace = true }
codex-protocol = { workspace = true }
eventsource-stream = { workspace = true }
gethostname = { workspace = true }

View File

@@ -15,6 +15,7 @@ metrics (when enabled), then attach its layers to your `tracing_subscriber`
registry:
```rust
use codex_http_client::{HttpClientFactory, OutboundProxyPolicy};
use codex_otel::config::OtelExporter;
use codex_otel::config::OtelHttpProtocol;
use codex_otel::config::OtelSettings;
@@ -39,6 +40,8 @@ let settings = OtelSettings {
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
runtime_metrics: false,
span_attributes: std::collections::BTreeMap::new(),
tracestate: std::collections::BTreeMap::new(),
};
@@ -109,6 +112,7 @@ to Statsig using Codex-internal defaults.
Statsig ingestion (OTLP/HTTP JSON) example:
```rust
use codex_http_client::{HttpClientFactory, OutboundProxyPolicy};
use codex_otel::config::{OtelExporter, OtelHttpProtocol};
let metrics = MetricsClient::new(MetricsConfig::otlp(
@@ -124,6 +128,7 @@ let metrics = MetricsClient::new(MetricsConfig::otlp(
protocol: OtelHttpProtocol::Json,
tls: None,
},
HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
))?;
metrics.counter("codex.session_started", 1, &[("source", "tui")])?;

View File

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
use std::collections::HashMap;
use std::path::PathBuf;
use codex_http_client::HttpClientFactory;
use codex_utils_absolute_path::AbsolutePathBuf;
use serde::Deserialize;
use serde::Serialize;
@@ -56,6 +57,7 @@ pub struct OtelSettings {
pub exporter: OtelExporter,
pub trace_exporter: OtelExporter,
pub metrics_exporter: OtelExporter,
pub http_client_factory: HttpClientFactory,
pub runtime_metrics: bool,
pub span_attributes: BTreeMap<String, String>,
pub tracestate: BTreeMap<String, BTreeMap<String, String>>,
@@ -116,4 +118,5 @@ mod tests {
OtelExporter::None
));
}
}

View File

@@ -349,8 +349,12 @@ impl MetricsClient {
MetricsExporter::InMemory(exporter) => {
build_provider(resource, exporter, export_interval, runtime_reader.clone())
}
MetricsExporter::Otlp(exporter) => {
let exporter = build_otlp_metric_exporter(exporter, Temporality::Delta)?;
MetricsExporter::Otlp {
exporter,
http_client_factory,
} => {
let exporter =
build_otlp_metric_exporter(exporter, Temporality::Delta, &http_client_factory)?;
build_provider(resource, exporter, export_interval, runtime_reader.clone())
}
};
@@ -555,12 +559,14 @@ where
fn build_otlp_metric_exporter(
exporter: OtelExporter,
temporality: Temporality,
http_client_factory: &codex_http_client::HttpClientFactory,
) -> Result<opentelemetry_otlp::MetricExporter> {
match exporter {
OtelExporter::None => Err(MetricsError::ExporterDisabled),
OtelExporter::Statsig => build_otlp_metric_exporter(
crate::config::resolve_exporter(&OtelExporter::Statsig),
temporality,
http_client_factory,
),
OtelExporter::OtlpGrpc {
endpoint,
@@ -605,21 +611,23 @@ fn build_otlp_metric_exporter(
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = opentelemetry_otlp::MetricExporter::builder()
let client = crate::otlp::build_http_client(
http_client_factory,
&endpoint,
tls.as_ref(),
OTEL_EXPORTER_OTLP_METRICS_TIMEOUT,
)
.map_err(|err| MetricsError::InvalidConfig {
message: err.to_string(),
})?;
let exporter_builder = opentelemetry_otlp::MetricExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_temporality(temporality)
.with_protocol(protocol)
.with_headers(headers);
if let Some(tls) = tls.as_ref() {
let client =
crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT)
.map_err(|err| MetricsError::InvalidConfig {
message: err.to_string(),
})?;
exporter_builder = exporter_builder.with_http_client(client);
}
.with_headers(headers)
.with_http_client(client);
exporter_builder
.build()

View File

@@ -10,6 +10,7 @@ use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
use crate::metrics::validation::validate_tag_key;
use crate::metrics::validation::validate_tag_value;
use codex_http_client::HttpClientFactory;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use std::collections::BTreeMap;
use std::time::Duration;
@@ -32,7 +33,10 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
#[derive(Clone, Debug)]
pub enum MetricsExporter {
Otlp(OtelExporter),
Otlp {
exporter: OtelExporter,
http_client_factory: HttpClientFactory,
},
InMemory(InMemoryMetricExporter),
}
@@ -54,6 +58,7 @@ impl MetricsConfig {
service_name: impl Into<String>,
service_version: impl Into<String>,
exporter: OtelExporter,
http_client_factory: HttpClientFactory,
) -> Self {
let statsig_disabled_metrics = if matches!(exporter, OtelExporter::Statsig) {
STATSIG_DISABLED_METRICS
@@ -64,7 +69,10 @@ impl MetricsConfig {
environment: environment.into(),
service_name: service_name.into(),
service_version: service_version.into(),
exporter: MetricsExporter::Otlp(exporter),
exporter: MetricsExporter::Otlp {
exporter,
http_client_factory,
},
export_interval: None,
runtime_reader: false,
statsig_disabled_metrics,

View File

@@ -1,4 +1,7 @@
use crate::config::OtelTlsConfig;
use codex_http_client::HttpClientFactory;
use codex_http_client::TelemetryClientTlsConfig;
use codex_http_client::TelemetryHttpClient;
use codex_utils_absolute_path::AbsolutePathBuf;
use http::Uri;
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT;
@@ -67,27 +70,39 @@ pub(crate) fn build_grpc_tls_config(
Ok(config)
}
/// Build a blocking HTTP client with TLS configuration for OTLP HTTP exporters.
/// Build a policy-aware blocking HTTP client for OTLP HTTP exporters.
///
/// We use `reqwest::blocking::Client` because OTEL exporters run on dedicated
/// OS threads that are not necessarily backed by tokio.
/// OTEL exporters run on dedicated OS threads that are not necessarily backed
/// by Tokio, so logs and metrics require a blocking-compatible transport.
pub(crate) fn build_http_client(
tls: &OtelTlsConfig,
http_client_factory: &HttpClientFactory,
endpoint: &str,
tls: Option<&OtelTlsConfig>,
timeout_var: &str,
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
) -> Result<impl TelemetryHttpClient + 'static + use<>, Box<dyn Error>> {
let tls = telemetry_tls_config(tls);
let timeout = resolve_otlp_timeout(timeout_var);
if current_tokio_runtime_is_multi_thread() {
tokio::task::block_in_place(|| build_http_client_inner(tls, timeout_var))
tokio::task::block_in_place(|| {
http_client_factory
.build_blocking_telemetry_client(endpoint, timeout, &tls)
.map_err(|error| Box::new(error) as Box<dyn Error>)
})
} else if tokio::runtime::Handle::try_current().is_ok() {
let tls = tls.clone();
let timeout_var = timeout_var.to_string();
let http_client_factory = http_client_factory.clone();
let endpoint = endpoint.to_owned();
std::thread::spawn(move || {
build_http_client_inner(&tls, &timeout_var).map_err(|err| err.to_string())
http_client_factory
.build_blocking_telemetry_client(&endpoint, timeout, &tls)
.map_err(|error| error.to_string())
})
.join()
.map_err(|_| config_error("failed to join OTLP blocking HTTP client builder thread"))?
.map_err(config_error)
} else {
build_http_client_inner(tls, timeout_var)
http_client_factory
.build_blocking_telemetry_client(endpoint, timeout, &tls)
.map_err(|error| Box::new(error) as Box<dyn Error>)
}
}
@@ -98,53 +113,6 @@ pub(crate) fn current_tokio_runtime_is_multi_thread() -> bool {
}
}
fn build_http_client_inner(
tls: &OtelTlsConfig,
timeout_var: &str,
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
let mut builder =
reqwest::blocking::Client::builder().timeout(resolve_otlp_timeout(timeout_var));
if let Some(path) = tls.ca_certificate.as_ref() {
let (pem, location) = read_bytes(path)?;
let certificate = ReqwestCertificate::from_pem(pem.as_slice()).map_err(|error| {
config_error(format!(
"failed to parse certificate {}: {error}",
location.display()
))
})?;
builder = builder
.tls_built_in_root_certs(false)
.add_root_certificate(certificate);
}
match (&tls.client_certificate, &tls.client_private_key) {
(Some(cert_path), Some(key_path)) => {
let (mut cert_pem, cert_location) = read_bytes(cert_path)?;
let (key_pem, key_location) = read_bytes(key_path)?;
cert_pem.extend_from_slice(key_pem.as_slice());
let identity = ReqwestIdentity::from_pem(cert_pem.as_slice()).map_err(|error| {
config_error(format!(
"failed to parse client identity using {} and {}: {error}",
cert_location.display(),
key_location.display()
))
})?;
builder = builder.identity(identity).https_only(true);
}
(Some(_), None) | (None, Some(_)) => {
return Err(config_error(
"client_certificate and client_private_key must both be provided for mTLS",
));
}
(None, None) => {}
}
builder
.build()
.map_err(|error| Box::new(error) as Box<dyn Error>)
}
pub(crate) fn build_async_http_client(
tls: Option<&OtelTlsConfig>,
timeout_var: &str,
@@ -193,6 +161,27 @@ pub(crate) fn build_async_http_client(
.map_err(|error| Box::new(error) as Box<dyn Error>)
}
fn telemetry_tls_config(tls: Option<&OtelTlsConfig>) -> TelemetryClientTlsConfig {
let Some(tls) = tls else {
return TelemetryClientTlsConfig::default();
};
TelemetryClientTlsConfig {
ca_certificate: tls
.ca_certificate
.as_ref()
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
client_certificate: tls
.client_certificate
.as_ref()
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
client_private_key: tls
.client_private_key
.as_ref()
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
}
}
pub(crate) fn resolve_otlp_timeout(signal_var: &str) -> Duration {
if let Some(timeout) = read_timeout_env(signal_var) {
return timeout;
@@ -229,6 +218,7 @@ fn config_error(message: impl Into<String>) -> Box<dyn Error> {
#[cfg(test)]
mod tests {
use super::*;
use codex_http_client::OutboundProxyPolicy;
use pretty_assertions::assert_eq;
use tokio::runtime::Builder;
@@ -264,7 +254,12 @@ mod tests {
.expect("current-thread runtime");
let client = runtime.block_on(async {
build_http_client(&OtelTlsConfig::default(), OTEL_EXPORTER_OTLP_TIMEOUT)
build_http_client(
&HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
"http://localhost:4318/v1/traces",
Some(&OtelTlsConfig::default()),
OTEL_EXPORTER_OTLP_TIMEOUT,
)
});
assert!(client.is_ok());

View File

@@ -6,6 +6,7 @@ use crate::metrics::MetricsClient;
use crate::metrics::MetricsConfig;
use crate::targets::is_log_export_target;
use crate::targets::is_trace_safe_target;
use codex_http_client::HttpClientFactory;
use gethostname::gethostname;
use opentelemetry::Context;
use opentelemetry::KeyValue;
@@ -222,6 +223,7 @@ impl OtelProvider {
settings.service_name.clone(),
settings.service_version.clone(),
settings.metrics_exporter.clone(),
settings.http_client_factory.clone(),
);
if settings.runtime_metrics {
config = config.with_runtime_reader();
@@ -232,7 +234,13 @@ impl OtelProvider {
let log_resource = make_resource(settings, ResourceKind::Logs);
let trace_resource = make_resource(settings, ResourceKind::Traces);
let logger = log_enabled
.then(|| build_logger(&log_resource, &settings.exporter))
.then(|| {
build_logger(
&log_resource,
&settings.exporter,
&settings.http_client_factory,
)
})
.transpose()?;
let tracer_provider = trace_enabled
@@ -241,6 +249,7 @@ impl OtelProvider {
&trace_resource,
&settings.trace_exporter,
settings.span_attributes.clone(),
&settings.http_client_factory,
)
})
.transpose()?;
@@ -427,6 +436,7 @@ impl SpanProcessor for SpanAttributesProcessor {
fn build_logger(
resource: &Resource,
exporter: &OtelExporter,
http_client_factory: &HttpClientFactory,
) -> Result<SdkLoggerProvider, Box<dyn Error>> {
let mut builder = SdkLoggerProvider::builder().with_resource(resource.clone());
@@ -473,16 +483,19 @@ fn build_logger(
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = LogExporter::builder()
let client = crate::otlp::build_http_client(
http_client_factory,
&endpoint,
tls.as_ref(),
OTEL_EXPORTER_OTLP_LOGS_TIMEOUT,
)?;
let exporter_builder = LogExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers);
if let Some(tls) = tls.as_ref() {
let client = crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT)?;
exporter_builder = exporter_builder.with_http_client(client);
}
.with_headers(headers)
.with_http_client(client);
let exporter = exporter_builder.build()?;
@@ -497,6 +510,7 @@ fn build_tracer_provider(
resource: &Resource,
exporter: &OtelExporter,
span_attributes: BTreeMap<String, String>,
http_client_factory: &HttpClientFactory,
) -> Result<SdkTracerProvider, Box<dyn Error>> {
let span_exporter = match crate::config::resolve_exporter(exporter) {
OtelExporter::None => return Ok(tracer_provider_builder(resource, span_attributes).build()),
@@ -540,17 +554,17 @@ fn build_tracer_provider(
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = SpanExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers);
let client = crate::otlp::build_async_http_client(
tls.as_ref(),
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
)?;
exporter_builder = exporter_builder.with_http_client(client);
let exporter_builder = SpanExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers)
.with_http_client(client);
let processor =
TokioBatchSpanProcessor::builder(exporter_builder.build()?, runtime::Tokio)
@@ -566,17 +580,19 @@ fn build_tracer_provider(
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = SpanExporter::builder()
let client = crate::otlp::build_http_client(
http_client_factory,
&endpoint,
tls.as_ref(),
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
)?;
let exporter_builder = SpanExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers);
if let Some(tls) = tls.as_ref() {
let client =
crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT)?;
exporter_builder = exporter_builder.with_http_client(client);
}
.with_headers(headers)
.with_http_client(client);
exporter_builder.build()?
}
@@ -596,6 +612,7 @@ mod shutdown_tests;
#[cfg(test)]
mod tests {
use super::*;
use codex_http_client::OutboundProxyPolicy;
use crate::metrics::API_CALL_COUNT_METRIC;
use crate::metrics::API_CALL_DURATION_METRIC;
use crate::metrics::MetricsExporter;
@@ -717,6 +734,7 @@ mod tests {
"codex-cli",
env!("CARGO_PKG_VERSION"),
OtelExporter::Statsig,
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
);
config.exporter = MetricsExporter::InMemory(exporter.clone());
let metrics = MetricsClient::new(config)?;
@@ -768,6 +786,7 @@ mod tests {
exporter: OtelExporter::None,
trace_exporter: OtelExporter::None,
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),

View File

@@ -1,3 +1,6 @@
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_http_client::cache_system_proxy_route_for_test;
use codex_otel::MetricsClient;
use codex_otel::MetricsConfig;
use codex_otel::OtelExporter;
@@ -139,6 +142,45 @@ fn write_http_response(stream: &mut TcpStream, status: &str) -> std::io::Result<
stream.flush()
}
#[test]
fn otlp_http_metrics_exporter_uses_configured_system_proxy() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").expect("proxy should bind");
let address = listener.local_addr().expect("proxy should have address");
let endpoint = "http://otel-metrics-system-proxy.test/v1/metrics";
cache_system_proxy_route_for_test(endpoint, format!("http://{address}"));
let proxy = thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("proxy should receive export");
let request = read_http_request(&mut stream).expect("proxy should read export");
write_http_response(&mut stream, "202 Accepted").expect("proxy should respond");
request
});
let metrics = MetricsClient::new(MetricsConfig::otlp(
"test",
"codex-cli",
env!("CARGO_PKG_VERSION"),
OtelExporter::OtlpHttp {
endpoint: endpoint.to_string(),
headers: HashMap::new(),
protocol: OtelHttpProtocol::Json,
tls: None,
},
HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
))?;
metrics.counter("codex.proxy_test", /*inc*/ 1, &[])?;
metrics.shutdown()?;
let (path, _, body) = proxy.join().expect("proxy should complete");
assert_eq!(path, endpoint);
assert!(
String::from_utf8_lossy(&body).contains("codex.proxy_test"),
"proxied request should contain the exported metric"
);
Ok(())
}
#[test]
fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
@@ -183,6 +225,7 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
protocol: OtelHttpProtocol::Json,
tls: None,
},
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
))?;
metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?;
@@ -356,6 +399,7 @@ fn otlp_http_exporter_sends_logs_to_collector()
},
trace_exporter: OtelExporter::None,
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),
@@ -415,6 +459,7 @@ fn otel_provider_rejects_header_unsafe_configured_tracestate() {
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::from([(
@@ -480,6 +525,7 @@ fn otlp_http_exporter_sends_traces_to_collector()
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::from([(
"test.configured_attribute".to_string(),
@@ -625,6 +671,7 @@ async fn otlp_http_exporter_sends_traces_to_collector_with_bounded_shutdown_in_t
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),
@@ -718,6 +765,7 @@ fn otlp_http_exporter_times_out_when_collector_stalls_during_bounded_shutdown()
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),
@@ -817,6 +865,7 @@ fn otlp_http_exporter_sends_traces_to_collector_in_current_thread_tokio_runtime(
tls: None,
},
metrics_exporter: OtelExporter::None,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),

View File

@@ -31,6 +31,7 @@ chrono = { version = "0.4.42", default-features = false, features = [
codex-utils-pty = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-string = { workspace = true }
codex-http-client = { workspace = true }
codex-otel = { workspace = true }
dunce = "1.0"
glob = { workspace = true }

View File

@@ -1,6 +1,8 @@
use crate::install_wfp_filters_for_account;
use crate::setup_error::sanitize_setup_metric_tag_value;
use anyhow::Result;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_otel::OtelExporter;
use codex_otel::OtelProvider;
use codex_otel::OtelSettings;
@@ -54,6 +56,7 @@ fn build_wfp_metrics_provider(
exporter: OtelExporter::None,
trace_exporter: OtelExporter::None,
metrics_exporter: OtelExporter::Statsig,
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
runtime_metrics: false,
span_attributes: BTreeMap::new(),
tracestate: BTreeMap::new(),