Prevent telemetry exporters from feeding back into themselves (#40658)

## Why

Telemetry transport and storage can emit their own diagnostics, causing exports
or SQLite writes to recursively generate more telemetry.

## What changed

- Exclude `h2` spans from trace export so OTLP transport activity cannot trigger
  additional OTLP exports.
- Exclude all `sqlx`, `sqlx_core`, and `sqlx_sqlite` diagnostics from the SQLite
  log sink, including when no external subscriber filter is installed.
- Keep similarly named targets such as `sqlx_application` eligible for storage.

## Testing

- Extend the SQLite log-filter test to cover the broader SQLx exclusion and the
  unrelated-target case.

GitOrigin-RevId: 13a832e1f228e9564a31cc08f00add286342bac1
This commit is contained in:
Adam Perry @ OpenAI
2026-08-25 17:59:32 +00:00
committed by copyberry
parent 9695e71519
commit 2f4b105514
3 changed files with 33 additions and 8 deletions
+8 -1
View File
@@ -328,7 +328,14 @@ impl OtelProvider {
}
pub fn trace_export_filter(meta: &tracing::Metadata<'_>) -> bool {
meta.is_span() || is_trace_safe_target(meta.target())
let target = meta.target();
if meta.is_span() {
// h2 creates explicit-root spans that escape the SDK's telemetry suppression.
// Exporting them would make OTLP transport generate more OTLP exports.
target != "h2" && !target.starts_with("h2::")
} else {
is_trace_safe_target(target)
}
}
pub fn metrics(&self) -> Option<&MetricsClient> {
+18 -5
View File
@@ -4,6 +4,8 @@
//! formats each one into a `LogEntry`, and sends entries to a bounded background
//! queue. The background task inserts into the dedicated `logs` SQLite database
//! in batches to keep logging overhead low.
//! SQLx diagnostics are always excluded so database writes cannot generate more
//! database writes, even without an external subscriber filter.
//!
//! ## Usage
//!
@@ -49,15 +51,15 @@ use crate::StateRuntime;
const LOG_QUEUE_CAPACITY: usize = 2048;
const LOG_BATCH_SIZE: usize = 512;
const LOG_FLUSH_INTERVAL: Duration = Duration::from_secs(10);
const SQLX_LOG_TARGETS: [&str; 3] = ["sqlx", "sqlx_core", "sqlx_sqlite"];
pub fn default_filter() -> Targets {
Targets::new()
.with_default(LevelFilter::TRACE)
.with_target("hyper_util", LevelFilter::WARN)
.with_target("log", LevelFilter::OFF)
// SQLite warnings must not feed back into the same SQLite log writer.
.with_target("sqlx::query", LevelFilter::OFF)
.with_target("sqlx::pool::acquire", LevelFilter::OFF)
// Avoid constructing backend diagnostics when no other layer needs them.
.with_targets(SQLX_LOG_TARGETS.map(|target| (format!("{target}::"), LevelFilter::OFF)))
.with_target("codex_rmcp_client", LevelFilter::INFO)
.with_target("codex_otel.log_only", LevelFilter::OFF)
.with_target("codex_otel.trace_safe", LevelFilter::OFF)
@@ -214,16 +216,27 @@ where
fn on_event(&self, event: &Event<'_>, ctx: tracing_subscriber::layer::Context<'_, S>) {
let metadata = event.metadata();
let target = metadata.target();
// SQLx can emit from both the inserter task and its separate worker threads.
// This guard must remain local to the sink and independent of optional filters.
if target
.split("::")
.next()
.is_some_and(|target| SQLX_LOG_TARGETS.contains(&target))
{
return;
}
// `tracing-log` checks filters with the original log target before
// dispatching an event whose tracing target is `log`, so the outer
// target filter cannot reliably reject these bridged events.
if metadata.target() == "log" {
if target == "log" {
return;
}
// The SDK emits DEBUG timer meta-events every second per process; these
// were over 30% of retained logs in measured high-fanout Codex environments.
if metadata.target() == "opentelemetry_sdk"
if target == "opentelemetry_sdk"
&& matches!(
*metadata.level(),
tracing::Level::TRACE | tracing::Level::DEBUG
+7 -2
View File
@@ -30,7 +30,8 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics()
tracing::info!(target: "opentelemetry_sdk", "retained-info");
tracing::warn!(target: "sqlx::query", "dropped-slow-query-warning");
tracing::warn!(target: "sqlx::pool::acquire", "dropped-slow-acquire-warning");
tracing::warn!(target: "sqlx::other", "retained-sqlx-warning");
tracing::warn!(target: "sqlx::other", "dropped-sqlx-warning");
tracing::info!(target: "sqlx_application", "retained-unrelated-target");
tracing::debug!(target: "rmcp::transport", "dropped-rmcp-debug");
tracing::info!(target: "rmcp::transport", "retained-rmcp-info");
tracing::debug!(
@@ -90,7 +91,11 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics()
.collect::<Vec<_>>(),
vec![
("INFO", "opentelemetry_sdk", Some("retained-info")),
("WARN", "sqlx::other", Some("retained-sqlx-warning")),
(
"INFO",
"sqlx_application",
Some("retained-unrelated-target")
),
("INFO", "rmcp::transport", Some("retained-rmcp-info")),
(
"INFO",