mirror of
https://github.com/openai/codex.git
synced 2026-08-23 13:09:46 +00:00
Route blocking OTLP exporters through shared HTTP policy
This commit is contained in:
2
codex-rs/Cargo.lock
generated
2
codex-rs/Cargo.lock
generated
@@ -3841,6 +3841,7 @@ version = "0.0.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"codex-api",
|
"codex-api",
|
||||||
|
"codex-http-client",
|
||||||
"codex-protocol",
|
"codex-protocol",
|
||||||
"codex-utils-absolute-path",
|
"codex-utils-absolute-path",
|
||||||
"codex-utils-string",
|
"codex-utils-string",
|
||||||
@@ -4755,6 +4756,7 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"codex-http-client",
|
||||||
"codex-otel",
|
"codex-otel",
|
||||||
"codex-protocol",
|
"codex-protocol",
|
||||||
"codex-utils-absolute-path",
|
"codex-utils-absolute-path",
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ pub fn build_provider(
|
|||||||
exporter,
|
exporter,
|
||||||
trace_exporter,
|
trace_exporter,
|
||||||
metrics_exporter,
|
metrics_exporter,
|
||||||
|
http_client_factory: config.http_client_factory(),
|
||||||
runtime_metrics,
|
runtime_metrics,
|
||||||
span_attributes: config.otel.span_attributes.clone(),
|
span_attributes: config.otel.span_attributes.clone(),
|
||||||
tracestate: config.otel.tracestate.clone(),
|
tracestate: config.otel.tracestate.clone(),
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ chrono = { workspace = true }
|
|||||||
codex-utils-absolute-path = { workspace = true }
|
codex-utils-absolute-path = { workspace = true }
|
||||||
codex-utils-string = { workspace = true }
|
codex-utils-string = { workspace = true }
|
||||||
codex-api = { workspace = true }
|
codex-api = { workspace = true }
|
||||||
|
codex-http-client = { workspace = true }
|
||||||
codex-protocol = { workspace = true }
|
codex-protocol = { workspace = true }
|
||||||
eventsource-stream = { workspace = true }
|
eventsource-stream = { workspace = true }
|
||||||
gethostname = { workspace = true }
|
gethostname = { workspace = true }
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ metrics (when enabled), then attach its layers to your `tracing_subscriber`
|
|||||||
registry:
|
registry:
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
|
use codex_http_client::{HttpClientFactory, OutboundProxyPolicy};
|
||||||
use codex_otel::config::OtelExporter;
|
use codex_otel::config::OtelExporter;
|
||||||
use codex_otel::config::OtelHttpProtocol;
|
use codex_otel::config::OtelHttpProtocol;
|
||||||
use codex_otel::config::OtelSettings;
|
use codex_otel::config::OtelSettings;
|
||||||
@@ -39,6 +40,8 @@ let settings = OtelSettings {
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
|
||||||
|
runtime_metrics: false,
|
||||||
span_attributes: std::collections::BTreeMap::new(),
|
span_attributes: std::collections::BTreeMap::new(),
|
||||||
tracestate: std::collections::BTreeMap::new(),
|
tracestate: std::collections::BTreeMap::new(),
|
||||||
};
|
};
|
||||||
@@ -109,6 +112,7 @@ to Statsig using Codex-internal defaults.
|
|||||||
Statsig ingestion (OTLP/HTTP JSON) example:
|
Statsig ingestion (OTLP/HTTP JSON) example:
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
|
use codex_http_client::{HttpClientFactory, OutboundProxyPolicy};
|
||||||
use codex_otel::config::{OtelExporter, OtelHttpProtocol};
|
use codex_otel::config::{OtelExporter, OtelHttpProtocol};
|
||||||
|
|
||||||
let metrics = MetricsClient::new(MetricsConfig::otlp(
|
let metrics = MetricsClient::new(MetricsConfig::otlp(
|
||||||
@@ -124,6 +128,7 @@ let metrics = MetricsClient::new(MetricsConfig::otlp(
|
|||||||
protocol: OtelHttpProtocol::Json,
|
protocol: OtelHttpProtocol::Json,
|
||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
|
HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
metrics.counter("codex.session_started", 1, &[("source", "tui")])?;
|
metrics.counter("codex.session_started", 1, &[("source", "tui")])?;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
@@ -56,6 +57,7 @@ pub struct OtelSettings {
|
|||||||
pub exporter: OtelExporter,
|
pub exporter: OtelExporter,
|
||||||
pub trace_exporter: OtelExporter,
|
pub trace_exporter: OtelExporter,
|
||||||
pub metrics_exporter: OtelExporter,
|
pub metrics_exporter: OtelExporter,
|
||||||
|
pub http_client_factory: HttpClientFactory,
|
||||||
pub runtime_metrics: bool,
|
pub runtime_metrics: bool,
|
||||||
pub span_attributes: BTreeMap<String, String>,
|
pub span_attributes: BTreeMap<String, String>,
|
||||||
pub tracestate: BTreeMap<String, BTreeMap<String, String>>,
|
pub tracestate: BTreeMap<String, BTreeMap<String, String>>,
|
||||||
@@ -116,4 +118,5 @@ mod tests {
|
|||||||
OtelExporter::None
|
OtelExporter::None
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -349,8 +349,12 @@ impl MetricsClient {
|
|||||||
MetricsExporter::InMemory(exporter) => {
|
MetricsExporter::InMemory(exporter) => {
|
||||||
build_provider(resource, exporter, export_interval, runtime_reader.clone())
|
build_provider(resource, exporter, export_interval, runtime_reader.clone())
|
||||||
}
|
}
|
||||||
MetricsExporter::Otlp(exporter) => {
|
MetricsExporter::Otlp {
|
||||||
let exporter = build_otlp_metric_exporter(exporter, Temporality::Delta)?;
|
exporter,
|
||||||
|
http_client_factory,
|
||||||
|
} => {
|
||||||
|
let exporter =
|
||||||
|
build_otlp_metric_exporter(exporter, Temporality::Delta, &http_client_factory)?;
|
||||||
build_provider(resource, exporter, export_interval, runtime_reader.clone())
|
build_provider(resource, exporter, export_interval, runtime_reader.clone())
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -555,12 +559,14 @@ where
|
|||||||
fn build_otlp_metric_exporter(
|
fn build_otlp_metric_exporter(
|
||||||
exporter: OtelExporter,
|
exporter: OtelExporter,
|
||||||
temporality: Temporality,
|
temporality: Temporality,
|
||||||
|
http_client_factory: &codex_http_client::HttpClientFactory,
|
||||||
) -> Result<opentelemetry_otlp::MetricExporter> {
|
) -> Result<opentelemetry_otlp::MetricExporter> {
|
||||||
match exporter {
|
match exporter {
|
||||||
OtelExporter::None => Err(MetricsError::ExporterDisabled),
|
OtelExporter::None => Err(MetricsError::ExporterDisabled),
|
||||||
OtelExporter::Statsig => build_otlp_metric_exporter(
|
OtelExporter::Statsig => build_otlp_metric_exporter(
|
||||||
crate::config::resolve_exporter(&OtelExporter::Statsig),
|
crate::config::resolve_exporter(&OtelExporter::Statsig),
|
||||||
temporality,
|
temporality,
|
||||||
|
http_client_factory,
|
||||||
),
|
),
|
||||||
OtelExporter::OtlpGrpc {
|
OtelExporter::OtlpGrpc {
|
||||||
endpoint,
|
endpoint,
|
||||||
@@ -605,21 +611,23 @@ fn build_otlp_metric_exporter(
|
|||||||
OtelHttpProtocol::Json => Protocol::HttpJson,
|
OtelHttpProtocol::Json => Protocol::HttpJson,
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut exporter_builder = opentelemetry_otlp::MetricExporter::builder()
|
let client = crate::otlp::build_http_client(
|
||||||
|
http_client_factory,
|
||||||
|
&endpoint,
|
||||||
|
tls.as_ref(),
|
||||||
|
OTEL_EXPORTER_OTLP_METRICS_TIMEOUT,
|
||||||
|
)
|
||||||
|
.map_err(|err| MetricsError::InvalidConfig {
|
||||||
|
message: err.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let exporter_builder = opentelemetry_otlp::MetricExporter::builder()
|
||||||
.with_http()
|
.with_http()
|
||||||
.with_endpoint(endpoint)
|
.with_endpoint(endpoint)
|
||||||
.with_temporality(temporality)
|
.with_temporality(temporality)
|
||||||
.with_protocol(protocol)
|
.with_protocol(protocol)
|
||||||
.with_headers(headers);
|
.with_headers(headers)
|
||||||
|
.with_http_client(client);
|
||||||
if let Some(tls) = tls.as_ref() {
|
|
||||||
let client =
|
|
||||||
crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT)
|
|
||||||
.map_err(|err| MetricsError::InvalidConfig {
|
|
||||||
message: err.to_string(),
|
|
||||||
})?;
|
|
||||||
exporter_builder = exporter_builder.with_http_client(client);
|
|
||||||
}
|
|
||||||
|
|
||||||
exporter_builder
|
exporter_builder
|
||||||
.build()
|
.build()
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
|
|||||||
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
|
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
|
||||||
use crate::metrics::validation::validate_tag_key;
|
use crate::metrics::validation::validate_tag_key;
|
||||||
use crate::metrics::validation::validate_tag_value;
|
use crate::metrics::validation::validate_tag_value;
|
||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -32,7 +33,10 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
|
|||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub enum MetricsExporter {
|
pub enum MetricsExporter {
|
||||||
Otlp(OtelExporter),
|
Otlp {
|
||||||
|
exporter: OtelExporter,
|
||||||
|
http_client_factory: HttpClientFactory,
|
||||||
|
},
|
||||||
InMemory(InMemoryMetricExporter),
|
InMemory(InMemoryMetricExporter),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,6 +58,7 @@ impl MetricsConfig {
|
|||||||
service_name: impl Into<String>,
|
service_name: impl Into<String>,
|
||||||
service_version: impl Into<String>,
|
service_version: impl Into<String>,
|
||||||
exporter: OtelExporter,
|
exporter: OtelExporter,
|
||||||
|
http_client_factory: HttpClientFactory,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let statsig_disabled_metrics = if matches!(exporter, OtelExporter::Statsig) {
|
let statsig_disabled_metrics = if matches!(exporter, OtelExporter::Statsig) {
|
||||||
STATSIG_DISABLED_METRICS
|
STATSIG_DISABLED_METRICS
|
||||||
@@ -64,7 +69,10 @@ impl MetricsConfig {
|
|||||||
environment: environment.into(),
|
environment: environment.into(),
|
||||||
service_name: service_name.into(),
|
service_name: service_name.into(),
|
||||||
service_version: service_version.into(),
|
service_version: service_version.into(),
|
||||||
exporter: MetricsExporter::Otlp(exporter),
|
exporter: MetricsExporter::Otlp {
|
||||||
|
exporter,
|
||||||
|
http_client_factory,
|
||||||
|
},
|
||||||
export_interval: None,
|
export_interval: None,
|
||||||
runtime_reader: false,
|
runtime_reader: false,
|
||||||
statsig_disabled_metrics,
|
statsig_disabled_metrics,
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
use crate::config::OtelTlsConfig;
|
use crate::config::OtelTlsConfig;
|
||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
|
use codex_http_client::TelemetryClientTlsConfig;
|
||||||
|
use codex_http_client::TelemetryHttpClient;
|
||||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||||
use http::Uri;
|
use http::Uri;
|
||||||
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT;
|
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT;
|
||||||
@@ -67,27 +70,39 @@ pub(crate) fn build_grpc_tls_config(
|
|||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a blocking HTTP client with TLS configuration for OTLP HTTP exporters.
|
/// Build a policy-aware blocking HTTP client for OTLP HTTP exporters.
|
||||||
///
|
///
|
||||||
/// We use `reqwest::blocking::Client` because OTEL exporters run on dedicated
|
/// OTEL exporters run on dedicated OS threads that are not necessarily backed
|
||||||
/// OS threads that are not necessarily backed by tokio.
|
/// by Tokio, so logs and metrics require a blocking-compatible transport.
|
||||||
pub(crate) fn build_http_client(
|
pub(crate) fn build_http_client(
|
||||||
tls: &OtelTlsConfig,
|
http_client_factory: &HttpClientFactory,
|
||||||
|
endpoint: &str,
|
||||||
|
tls: Option<&OtelTlsConfig>,
|
||||||
timeout_var: &str,
|
timeout_var: &str,
|
||||||
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
|
) -> Result<impl TelemetryHttpClient + 'static + use<>, Box<dyn Error>> {
|
||||||
|
let tls = telemetry_tls_config(tls);
|
||||||
|
let timeout = resolve_otlp_timeout(timeout_var);
|
||||||
if current_tokio_runtime_is_multi_thread() {
|
if current_tokio_runtime_is_multi_thread() {
|
||||||
tokio::task::block_in_place(|| build_http_client_inner(tls, timeout_var))
|
tokio::task::block_in_place(|| {
|
||||||
|
http_client_factory
|
||||||
|
.build_blocking_telemetry_client(endpoint, timeout, &tls)
|
||||||
|
.map_err(|error| Box::new(error) as Box<dyn Error>)
|
||||||
|
})
|
||||||
} else if tokio::runtime::Handle::try_current().is_ok() {
|
} else if tokio::runtime::Handle::try_current().is_ok() {
|
||||||
let tls = tls.clone();
|
let http_client_factory = http_client_factory.clone();
|
||||||
let timeout_var = timeout_var.to_string();
|
let endpoint = endpoint.to_owned();
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
build_http_client_inner(&tls, &timeout_var).map_err(|err| err.to_string())
|
http_client_factory
|
||||||
|
.build_blocking_telemetry_client(&endpoint, timeout, &tls)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
})
|
})
|
||||||
.join()
|
.join()
|
||||||
.map_err(|_| config_error("failed to join OTLP blocking HTTP client builder thread"))?
|
.map_err(|_| config_error("failed to join OTLP blocking HTTP client builder thread"))?
|
||||||
.map_err(config_error)
|
.map_err(config_error)
|
||||||
} else {
|
} else {
|
||||||
build_http_client_inner(tls, timeout_var)
|
http_client_factory
|
||||||
|
.build_blocking_telemetry_client(endpoint, timeout, &tls)
|
||||||
|
.map_err(|error| Box::new(error) as Box<dyn Error>)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,53 +113,6 @@ pub(crate) fn current_tokio_runtime_is_multi_thread() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_http_client_inner(
|
|
||||||
tls: &OtelTlsConfig,
|
|
||||||
timeout_var: &str,
|
|
||||||
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
|
|
||||||
let mut builder =
|
|
||||||
reqwest::blocking::Client::builder().timeout(resolve_otlp_timeout(timeout_var));
|
|
||||||
|
|
||||||
if let Some(path) = tls.ca_certificate.as_ref() {
|
|
||||||
let (pem, location) = read_bytes(path)?;
|
|
||||||
let certificate = ReqwestCertificate::from_pem(pem.as_slice()).map_err(|error| {
|
|
||||||
config_error(format!(
|
|
||||||
"failed to parse certificate {}: {error}",
|
|
||||||
location.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
builder = builder
|
|
||||||
.tls_built_in_root_certs(false)
|
|
||||||
.add_root_certificate(certificate);
|
|
||||||
}
|
|
||||||
|
|
||||||
match (&tls.client_certificate, &tls.client_private_key) {
|
|
||||||
(Some(cert_path), Some(key_path)) => {
|
|
||||||
let (mut cert_pem, cert_location) = read_bytes(cert_path)?;
|
|
||||||
let (key_pem, key_location) = read_bytes(key_path)?;
|
|
||||||
cert_pem.extend_from_slice(key_pem.as_slice());
|
|
||||||
let identity = ReqwestIdentity::from_pem(cert_pem.as_slice()).map_err(|error| {
|
|
||||||
config_error(format!(
|
|
||||||
"failed to parse client identity using {} and {}: {error}",
|
|
||||||
cert_location.display(),
|
|
||||||
key_location.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
builder = builder.identity(identity).https_only(true);
|
|
||||||
}
|
|
||||||
(Some(_), None) | (None, Some(_)) => {
|
|
||||||
return Err(config_error(
|
|
||||||
"client_certificate and client_private_key must both be provided for mTLS",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
(None, None) => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
builder
|
|
||||||
.build()
|
|
||||||
.map_err(|error| Box::new(error) as Box<dyn Error>)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn build_async_http_client(
|
pub(crate) fn build_async_http_client(
|
||||||
tls: Option<&OtelTlsConfig>,
|
tls: Option<&OtelTlsConfig>,
|
||||||
timeout_var: &str,
|
timeout_var: &str,
|
||||||
@@ -193,6 +161,27 @@ pub(crate) fn build_async_http_client(
|
|||||||
.map_err(|error| Box::new(error) as Box<dyn Error>)
|
.map_err(|error| Box::new(error) as Box<dyn Error>)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn telemetry_tls_config(tls: Option<&OtelTlsConfig>) -> TelemetryClientTlsConfig {
|
||||||
|
let Some(tls) = tls else {
|
||||||
|
return TelemetryClientTlsConfig::default();
|
||||||
|
};
|
||||||
|
|
||||||
|
TelemetryClientTlsConfig {
|
||||||
|
ca_certificate: tls
|
||||||
|
.ca_certificate
|
||||||
|
.as_ref()
|
||||||
|
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
|
||||||
|
client_certificate: tls
|
||||||
|
.client_certificate
|
||||||
|
.as_ref()
|
||||||
|
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
|
||||||
|
client_private_key: tls
|
||||||
|
.client_private_key
|
||||||
|
.as_ref()
|
||||||
|
.map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn resolve_otlp_timeout(signal_var: &str) -> Duration {
|
pub(crate) fn resolve_otlp_timeout(signal_var: &str) -> Duration {
|
||||||
if let Some(timeout) = read_timeout_env(signal_var) {
|
if let Some(timeout) = read_timeout_env(signal_var) {
|
||||||
return timeout;
|
return timeout;
|
||||||
@@ -229,6 +218,7 @@ fn config_error(message: impl Into<String>) -> Box<dyn Error> {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use codex_http_client::OutboundProxyPolicy;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
use tokio::runtime::Builder;
|
use tokio::runtime::Builder;
|
||||||
|
|
||||||
@@ -264,7 +254,12 @@ mod tests {
|
|||||||
.expect("current-thread runtime");
|
.expect("current-thread runtime");
|
||||||
|
|
||||||
let client = runtime.block_on(async {
|
let client = runtime.block_on(async {
|
||||||
build_http_client(&OtelTlsConfig::default(), OTEL_EXPORTER_OTLP_TIMEOUT)
|
build_http_client(
|
||||||
|
&HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
|
"http://localhost:4318/v1/traces",
|
||||||
|
Some(&OtelTlsConfig::default()),
|
||||||
|
OTEL_EXPORTER_OTLP_TIMEOUT,
|
||||||
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
assert!(client.is_ok());
|
assert!(client.is_ok());
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ use crate::metrics::MetricsClient;
|
|||||||
use crate::metrics::MetricsConfig;
|
use crate::metrics::MetricsConfig;
|
||||||
use crate::targets::is_log_export_target;
|
use crate::targets::is_log_export_target;
|
||||||
use crate::targets::is_trace_safe_target;
|
use crate::targets::is_trace_safe_target;
|
||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
use gethostname::gethostname;
|
use gethostname::gethostname;
|
||||||
use opentelemetry::Context;
|
use opentelemetry::Context;
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
@@ -222,6 +223,7 @@ impl OtelProvider {
|
|||||||
settings.service_name.clone(),
|
settings.service_name.clone(),
|
||||||
settings.service_version.clone(),
|
settings.service_version.clone(),
|
||||||
settings.metrics_exporter.clone(),
|
settings.metrics_exporter.clone(),
|
||||||
|
settings.http_client_factory.clone(),
|
||||||
);
|
);
|
||||||
if settings.runtime_metrics {
|
if settings.runtime_metrics {
|
||||||
config = config.with_runtime_reader();
|
config = config.with_runtime_reader();
|
||||||
@@ -232,7 +234,13 @@ impl OtelProvider {
|
|||||||
let log_resource = make_resource(settings, ResourceKind::Logs);
|
let log_resource = make_resource(settings, ResourceKind::Logs);
|
||||||
let trace_resource = make_resource(settings, ResourceKind::Traces);
|
let trace_resource = make_resource(settings, ResourceKind::Traces);
|
||||||
let logger = log_enabled
|
let logger = log_enabled
|
||||||
.then(|| build_logger(&log_resource, &settings.exporter))
|
.then(|| {
|
||||||
|
build_logger(
|
||||||
|
&log_resource,
|
||||||
|
&settings.exporter,
|
||||||
|
&settings.http_client_factory,
|
||||||
|
)
|
||||||
|
})
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
|
|
||||||
let tracer_provider = trace_enabled
|
let tracer_provider = trace_enabled
|
||||||
@@ -241,6 +249,7 @@ impl OtelProvider {
|
|||||||
&trace_resource,
|
&trace_resource,
|
||||||
&settings.trace_exporter,
|
&settings.trace_exporter,
|
||||||
settings.span_attributes.clone(),
|
settings.span_attributes.clone(),
|
||||||
|
&settings.http_client_factory,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
@@ -427,6 +436,7 @@ impl SpanProcessor for SpanAttributesProcessor {
|
|||||||
fn build_logger(
|
fn build_logger(
|
||||||
resource: &Resource,
|
resource: &Resource,
|
||||||
exporter: &OtelExporter,
|
exporter: &OtelExporter,
|
||||||
|
http_client_factory: &HttpClientFactory,
|
||||||
) -> Result<SdkLoggerProvider, Box<dyn Error>> {
|
) -> Result<SdkLoggerProvider, Box<dyn Error>> {
|
||||||
let mut builder = SdkLoggerProvider::builder().with_resource(resource.clone());
|
let mut builder = SdkLoggerProvider::builder().with_resource(resource.clone());
|
||||||
|
|
||||||
@@ -473,16 +483,19 @@ fn build_logger(
|
|||||||
OtelHttpProtocol::Json => Protocol::HttpJson,
|
OtelHttpProtocol::Json => Protocol::HttpJson,
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut exporter_builder = LogExporter::builder()
|
let client = crate::otlp::build_http_client(
|
||||||
|
http_client_factory,
|
||||||
|
&endpoint,
|
||||||
|
tls.as_ref(),
|
||||||
|
OTEL_EXPORTER_OTLP_LOGS_TIMEOUT,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let exporter_builder = LogExporter::builder()
|
||||||
.with_http()
|
.with_http()
|
||||||
.with_endpoint(endpoint)
|
.with_endpoint(endpoint)
|
||||||
.with_protocol(protocol)
|
.with_protocol(protocol)
|
||||||
.with_headers(headers);
|
.with_headers(headers)
|
||||||
|
.with_http_client(client);
|
||||||
if let Some(tls) = tls.as_ref() {
|
|
||||||
let client = crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT)?;
|
|
||||||
exporter_builder = exporter_builder.with_http_client(client);
|
|
||||||
}
|
|
||||||
|
|
||||||
let exporter = exporter_builder.build()?;
|
let exporter = exporter_builder.build()?;
|
||||||
|
|
||||||
@@ -497,6 +510,7 @@ fn build_tracer_provider(
|
|||||||
resource: &Resource,
|
resource: &Resource,
|
||||||
exporter: &OtelExporter,
|
exporter: &OtelExporter,
|
||||||
span_attributes: BTreeMap<String, String>,
|
span_attributes: BTreeMap<String, String>,
|
||||||
|
http_client_factory: &HttpClientFactory,
|
||||||
) -> Result<SdkTracerProvider, Box<dyn Error>> {
|
) -> Result<SdkTracerProvider, Box<dyn Error>> {
|
||||||
let span_exporter = match crate::config::resolve_exporter(exporter) {
|
let span_exporter = match crate::config::resolve_exporter(exporter) {
|
||||||
OtelExporter::None => return Ok(tracer_provider_builder(resource, span_attributes).build()),
|
OtelExporter::None => return Ok(tracer_provider_builder(resource, span_attributes).build()),
|
||||||
@@ -540,17 +554,17 @@ fn build_tracer_provider(
|
|||||||
OtelHttpProtocol::Json => Protocol::HttpJson,
|
OtelHttpProtocol::Json => Protocol::HttpJson,
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut exporter_builder = SpanExporter::builder()
|
|
||||||
.with_http()
|
|
||||||
.with_endpoint(endpoint)
|
|
||||||
.with_protocol(protocol)
|
|
||||||
.with_headers(headers);
|
|
||||||
|
|
||||||
let client = crate::otlp::build_async_http_client(
|
let client = crate::otlp::build_async_http_client(
|
||||||
tls.as_ref(),
|
tls.as_ref(),
|
||||||
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
|
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
|
||||||
)?;
|
)?;
|
||||||
exporter_builder = exporter_builder.with_http_client(client);
|
|
||||||
|
let exporter_builder = SpanExporter::builder()
|
||||||
|
.with_http()
|
||||||
|
.with_endpoint(endpoint)
|
||||||
|
.with_protocol(protocol)
|
||||||
|
.with_headers(headers)
|
||||||
|
.with_http_client(client);
|
||||||
|
|
||||||
let processor =
|
let processor =
|
||||||
TokioBatchSpanProcessor::builder(exporter_builder.build()?, runtime::Tokio)
|
TokioBatchSpanProcessor::builder(exporter_builder.build()?, runtime::Tokio)
|
||||||
@@ -566,17 +580,19 @@ fn build_tracer_provider(
|
|||||||
OtelHttpProtocol::Json => Protocol::HttpJson,
|
OtelHttpProtocol::Json => Protocol::HttpJson,
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut exporter_builder = SpanExporter::builder()
|
let client = crate::otlp::build_http_client(
|
||||||
|
http_client_factory,
|
||||||
|
&endpoint,
|
||||||
|
tls.as_ref(),
|
||||||
|
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let exporter_builder = SpanExporter::builder()
|
||||||
.with_http()
|
.with_http()
|
||||||
.with_endpoint(endpoint)
|
.with_endpoint(endpoint)
|
||||||
.with_protocol(protocol)
|
.with_protocol(protocol)
|
||||||
.with_headers(headers);
|
.with_headers(headers)
|
||||||
|
.with_http_client(client);
|
||||||
if let Some(tls) = tls.as_ref() {
|
|
||||||
let client =
|
|
||||||
crate::otlp::build_http_client(tls, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT)?;
|
|
||||||
exporter_builder = exporter_builder.with_http_client(client);
|
|
||||||
}
|
|
||||||
|
|
||||||
exporter_builder.build()?
|
exporter_builder.build()?
|
||||||
}
|
}
|
||||||
@@ -596,6 +612,7 @@ mod shutdown_tests;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use codex_http_client::OutboundProxyPolicy;
|
||||||
use crate::metrics::API_CALL_COUNT_METRIC;
|
use crate::metrics::API_CALL_COUNT_METRIC;
|
||||||
use crate::metrics::API_CALL_DURATION_METRIC;
|
use crate::metrics::API_CALL_DURATION_METRIC;
|
||||||
use crate::metrics::MetricsExporter;
|
use crate::metrics::MetricsExporter;
|
||||||
@@ -717,6 +734,7 @@ mod tests {
|
|||||||
"codex-cli",
|
"codex-cli",
|
||||||
env!("CARGO_PKG_VERSION"),
|
env!("CARGO_PKG_VERSION"),
|
||||||
OtelExporter::Statsig,
|
OtelExporter::Statsig,
|
||||||
|
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
);
|
);
|
||||||
config.exporter = MetricsExporter::InMemory(exporter.clone());
|
config.exporter = MetricsExporter::InMemory(exporter.clone());
|
||||||
let metrics = MetricsClient::new(config)?;
|
let metrics = MetricsClient::new(config)?;
|
||||||
@@ -768,6 +786,7 @@ mod tests {
|
|||||||
exporter: OtelExporter::None,
|
exporter: OtelExporter::None,
|
||||||
trace_exporter: OtelExporter::None,
|
trace_exporter: OtelExporter::None,
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
|
use codex_http_client::OutboundProxyPolicy;
|
||||||
|
use codex_http_client::cache_system_proxy_route_for_test;
|
||||||
use codex_otel::MetricsClient;
|
use codex_otel::MetricsClient;
|
||||||
use codex_otel::MetricsConfig;
|
use codex_otel::MetricsConfig;
|
||||||
use codex_otel::OtelExporter;
|
use codex_otel::OtelExporter;
|
||||||
@@ -139,6 +142,45 @@ fn write_http_response(stream: &mut TcpStream, status: &str) -> std::io::Result<
|
|||||||
stream.flush()
|
stream.flush()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn otlp_http_metrics_exporter_uses_configured_system_proxy() -> Result<()> {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").expect("proxy should bind");
|
||||||
|
let address = listener.local_addr().expect("proxy should have address");
|
||||||
|
let endpoint = "http://otel-metrics-system-proxy.test/v1/metrics";
|
||||||
|
cache_system_proxy_route_for_test(endpoint, format!("http://{address}"));
|
||||||
|
|
||||||
|
let proxy = thread::spawn(move || {
|
||||||
|
let (mut stream, _) = listener.accept().expect("proxy should receive export");
|
||||||
|
let request = read_http_request(&mut stream).expect("proxy should read export");
|
||||||
|
write_http_response(&mut stream, "202 Accepted").expect("proxy should respond");
|
||||||
|
request
|
||||||
|
});
|
||||||
|
|
||||||
|
let metrics = MetricsClient::new(MetricsConfig::otlp(
|
||||||
|
"test",
|
||||||
|
"codex-cli",
|
||||||
|
env!("CARGO_PKG_VERSION"),
|
||||||
|
OtelExporter::OtlpHttp {
|
||||||
|
endpoint: endpoint.to_string(),
|
||||||
|
headers: HashMap::new(),
|
||||||
|
protocol: OtelHttpProtocol::Json,
|
||||||
|
tls: None,
|
||||||
|
},
|
||||||
|
HttpClientFactory::new(OutboundProxyPolicy::RespectSystemProxy),
|
||||||
|
))?;
|
||||||
|
metrics.counter("codex.proxy_test", /*inc*/ 1, &[])?;
|
||||||
|
metrics.shutdown()?;
|
||||||
|
|
||||||
|
let (path, _, body) = proxy.join().expect("proxy should complete");
|
||||||
|
assert_eq!(path, endpoint);
|
||||||
|
assert!(
|
||||||
|
String::from_utf8_lossy(&body).contains("codex.proxy_test"),
|
||||||
|
"proxied request should contain the exported metric"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
|
fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
|
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
|
||||||
@@ -183,6 +225,7 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
|
|||||||
protocol: OtelHttpProtocol::Json,
|
protocol: OtelHttpProtocol::Json,
|
||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
|
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?;
|
metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?;
|
||||||
@@ -356,6 +399,7 @@ fn otlp_http_exporter_sends_logs_to_collector()
|
|||||||
},
|
},
|
||||||
trace_exporter: OtelExporter::None,
|
trace_exporter: OtelExporter::None,
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
@@ -415,6 +459,7 @@ fn otel_provider_rejects_header_unsafe_configured_tracestate() {
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::from([(
|
tracestate: BTreeMap::from([(
|
||||||
@@ -480,6 +525,7 @@ fn otlp_http_exporter_sends_traces_to_collector()
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::from([(
|
span_attributes: BTreeMap::from([(
|
||||||
"test.configured_attribute".to_string(),
|
"test.configured_attribute".to_string(),
|
||||||
@@ -625,6 +671,7 @@ async fn otlp_http_exporter_sends_traces_to_collector_with_bounded_shutdown_in_t
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
@@ -718,6 +765,7 @@ fn otlp_http_exporter_times_out_when_collector_stalls_during_bounded_shutdown()
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
@@ -817,6 +865,7 @@ fn otlp_http_exporter_sends_traces_to_collector_in_current_thread_tokio_runtime(
|
|||||||
tls: None,
|
tls: None,
|
||||||
},
|
},
|
||||||
metrics_exporter: OtelExporter::None,
|
metrics_exporter: OtelExporter::None,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ chrono = { version = "0.4.42", default-features = false, features = [
|
|||||||
codex-utils-pty = { workspace = true }
|
codex-utils-pty = { workspace = true }
|
||||||
codex-utils-absolute-path = { workspace = true }
|
codex-utils-absolute-path = { workspace = true }
|
||||||
codex-utils-string = { workspace = true }
|
codex-utils-string = { workspace = true }
|
||||||
|
codex-http-client = { workspace = true }
|
||||||
codex-otel = { workspace = true }
|
codex-otel = { workspace = true }
|
||||||
dunce = "1.0"
|
dunce = "1.0"
|
||||||
glob = { workspace = true }
|
glob = { workspace = true }
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
use crate::install_wfp_filters_for_account;
|
use crate::install_wfp_filters_for_account;
|
||||||
use crate::setup_error::sanitize_setup_metric_tag_value;
|
use crate::setup_error::sanitize_setup_metric_tag_value;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use codex_http_client::HttpClientFactory;
|
||||||
|
use codex_http_client::OutboundProxyPolicy;
|
||||||
use codex_otel::OtelExporter;
|
use codex_otel::OtelExporter;
|
||||||
use codex_otel::OtelProvider;
|
use codex_otel::OtelProvider;
|
||||||
use codex_otel::OtelSettings;
|
use codex_otel::OtelSettings;
|
||||||
@@ -54,6 +56,7 @@ fn build_wfp_metrics_provider(
|
|||||||
exporter: OtelExporter::None,
|
exporter: OtelExporter::None,
|
||||||
trace_exporter: OtelExporter::None,
|
trace_exporter: OtelExporter::None,
|
||||||
metrics_exporter: OtelExporter::Statsig,
|
metrics_exporter: OtelExporter::Statsig,
|
||||||
|
http_client_factory: HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||||
runtime_metrics: false,
|
runtime_metrics: false,
|
||||||
span_attributes: BTreeMap::new(),
|
span_attributes: BTreeMap::new(),
|
||||||
tracestate: BTreeMap::new(),
|
tracestate: BTreeMap::new(),
|
||||||
|
|||||||
Reference in New Issue
Block a user