mirror of
https://github.com/openai/codex.git
synced 2026-11-05 20:10:33 +00:00
Prevent telemetry exporters from feeding back into themselves (#40658)
## Why Telemetry transport and storage can emit their own diagnostics, causing exports or SQLite writes to recursively generate more telemetry. ## What changed - Exclude `h2` spans from trace export so OTLP transport activity cannot trigger additional OTLP exports. - Exclude all `sqlx`, `sqlx_core`, and `sqlx_sqlite` diagnostics from the SQLite log sink, including when no external subscriber filter is installed. - Keep similarly named targets such as `sqlx_application` eligible for storage. ## Testing - Extend the SQLite log-filter test to cover the broader SQLx exclusion and the unrelated-target case. GitOrigin-RevId: 13a832e1f228e9564a31cc08f00add286342bac1
This commit is contained in:
committed by
copyberry
parent
9695e71519
commit
2f4b105514
@@ -328,7 +328,14 @@ impl OtelProvider {
|
||||
}
|
||||
|
||||
pub fn trace_export_filter(meta: &tracing::Metadata<'_>) -> bool {
|
||||
meta.is_span() || is_trace_safe_target(meta.target())
|
||||
let target = meta.target();
|
||||
if meta.is_span() {
|
||||
// h2 creates explicit-root spans that escape the SDK's telemetry suppression.
|
||||
// Exporting them would make OTLP transport generate more OTLP exports.
|
||||
target != "h2" && !target.starts_with("h2::")
|
||||
} else {
|
||||
is_trace_safe_target(target)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn metrics(&self) -> Option<&MetricsClient> {
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
//! formats each one into a `LogEntry`, and sends entries to a bounded background
|
||||
//! queue. The background task inserts into the dedicated `logs` SQLite database
|
||||
//! in batches to keep logging overhead low.
|
||||
//! SQLx diagnostics are always excluded so database writes cannot generate more
|
||||
//! database writes, even without an external subscriber filter.
|
||||
//!
|
||||
//! ## Usage
|
||||
//!
|
||||
@@ -49,15 +51,15 @@ use crate::StateRuntime;
|
||||
const LOG_QUEUE_CAPACITY: usize = 2048;
|
||||
const LOG_BATCH_SIZE: usize = 512;
|
||||
const LOG_FLUSH_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const SQLX_LOG_TARGETS: [&str; 3] = ["sqlx", "sqlx_core", "sqlx_sqlite"];
|
||||
|
||||
pub fn default_filter() -> Targets {
|
||||
Targets::new()
|
||||
.with_default(LevelFilter::TRACE)
|
||||
.with_target("hyper_util", LevelFilter::WARN)
|
||||
.with_target("log", LevelFilter::OFF)
|
||||
// SQLite warnings must not feed back into the same SQLite log writer.
|
||||
.with_target("sqlx::query", LevelFilter::OFF)
|
||||
.with_target("sqlx::pool::acquire", LevelFilter::OFF)
|
||||
// Avoid constructing backend diagnostics when no other layer needs them.
|
||||
.with_targets(SQLX_LOG_TARGETS.map(|target| (format!("{target}::"), LevelFilter::OFF)))
|
||||
.with_target("codex_rmcp_client", LevelFilter::INFO)
|
||||
.with_target("codex_otel.log_only", LevelFilter::OFF)
|
||||
.with_target("codex_otel.trace_safe", LevelFilter::OFF)
|
||||
@@ -214,16 +216,27 @@ where
|
||||
|
||||
fn on_event(&self, event: &Event<'_>, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let metadata = event.metadata();
|
||||
let target = metadata.target();
|
||||
// SQLx can emit from both the inserter task and its separate worker threads.
|
||||
// This guard must remain local to the sink and independent of optional filters.
|
||||
if target
|
||||
.split("::")
|
||||
.next()
|
||||
.is_some_and(|target| SQLX_LOG_TARGETS.contains(&target))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// `tracing-log` checks filters with the original log target before
|
||||
// dispatching an event whose tracing target is `log`, so the outer
|
||||
// target filter cannot reliably reject these bridged events.
|
||||
if metadata.target() == "log" {
|
||||
if target == "log" {
|
||||
return;
|
||||
}
|
||||
|
||||
// The SDK emits DEBUG timer meta-events every second per process; these
|
||||
// were over 30% of retained logs in measured high-fanout Codex environments.
|
||||
if metadata.target() == "opentelemetry_sdk"
|
||||
if target == "opentelemetry_sdk"
|
||||
&& matches!(
|
||||
*metadata.level(),
|
||||
tracing::Level::TRACE | tracing::Level::DEBUG
|
||||
|
||||
@@ -30,7 +30,8 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics()
|
||||
tracing::info!(target: "opentelemetry_sdk", "retained-info");
|
||||
tracing::warn!(target: "sqlx::query", "dropped-slow-query-warning");
|
||||
tracing::warn!(target: "sqlx::pool::acquire", "dropped-slow-acquire-warning");
|
||||
tracing::warn!(target: "sqlx::other", "retained-sqlx-warning");
|
||||
tracing::warn!(target: "sqlx::other", "dropped-sqlx-warning");
|
||||
tracing::info!(target: "sqlx_application", "retained-unrelated-target");
|
||||
tracing::debug!(target: "rmcp::transport", "dropped-rmcp-debug");
|
||||
tracing::info!(target: "rmcp::transport", "retained-rmcp-info");
|
||||
tracing::debug!(
|
||||
@@ -90,7 +91,11 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
("INFO", "opentelemetry_sdk", Some("retained-info")),
|
||||
("WARN", "sqlx::other", Some("retained-sqlx-warning")),
|
||||
(
|
||||
"INFO",
|
||||
"sqlx_application",
|
||||
Some("retained-unrelated-target")
|
||||
),
|
||||
("INFO", "rmcp::transport", Some("retained-rmcp-info")),
|
||||
(
|
||||
"INFO",
|
||||
|
||||
Reference in New Issue
Block a user