Files
codex/codex-rs/core/src/util.rs
Colin Young 0d2ff40a58 Add auth env observability (#14905)
CXC-410 Emit Env Var Status with `/feedback` report

Add more observability on top of #14611 

[Unset](https://openai.sentry.io/issues/7340419168/?project=4510195390611458&query=019cfa8d-c1ba-7002-96fa-e35fc340551d&referrer=issue-stream)

[Set](https://openai.sentry.io/issues/7340426331/?project=4510195390611458&query=019cfa91-aba1-7823-ab7e-762edfbc0ed4&referrer=issue-stream)
<img width="1063" height="610" alt="image"
src="https://github.com/user-attachments/assets/937ab026-1c2d-4757-81d5-5f31b853113e"
/>


###### Summary
- Adds auth-env telemetry that records whether key auth-related env
overrides were present on session start and request paths.
- Threads those auth-env fields through `/responses`, websocket, and
`/models` telemetry and feedback metadata.
- Buckets custom provider `env_key` configuration to a safe
`"configured"` value instead of emitting raw config text.
- Keeps the slice observability-only: no raw token values or raw URLs
are emitted.

###### Rationale (from spec findings)
- 401 and auth-path debugging needs a way to distinguish env-driven auth
paths from sessions with no auth env override.
- Startup and model-refresh failures need the same auth-env diagnostics
as normal request failures.
- Feedback and Sentry tags need the same auth-env signal as OTel events
so reports can be triaged consistently.
- Custom provider config is user-controlled text, so the telemetry
contract must stay presence-only / bucketed.

###### Scope
- Adds a small `AuthEnvTelemetry` bundle for env presence collection and
threads it through the main request/session telemetry paths.
- Does not add endpoint/base-url/provider-header/geo routing attribution
or broader telemetry API redesign.

###### Trade-offs
- `provider_env_key_name` is bucketed to `"configured"` instead of
preserving the literal configured env var name.
- `/models` is included because startup/model-refresh auth failures need
the same diagnostics, but broader parity work remains out of scope.
- This slice keeps the existing telemetry APIs and layers auth-env
fields onto them rather than redesigning the metadata model.

###### Client follow-up
- Add the separate endpoint/base-url attribution slice if routing-source
diagnosis is still needed.
- Add provider-header or residency attribution only if auth-env presence
proves insufficient in real reports.
- Revisit whether any additional auth-related env inputs need safe
bucketing after more 401 triage data.

###### Testing
- `cargo test -p codex-core emit_feedback_request_tags -- --nocapture`
- `cargo test -p codex-core
collect_auth_env_telemetry_buckets_provider_env_key_name -- --nocapture`
- `cargo test -p codex-core
models_request_telemetry_emits_auth_env_feedback_tags_on_failure --
--nocapture`
- `cargo test -p codex-otel
otel_export_routing_policy_routes_api_request_auth_observability --
--nocapture`
- `cargo test -p codex-otel
otel_export_routing_policy_routes_websocket_connect_auth_observability
-- --nocapture`
- `cargo test -p codex-otel
otel_export_routing_policy_routes_websocket_request_transport_observability
-- --nocapture`
- `cargo test -p codex-core --no-run --message-format short`
- `cargo test -p codex-otel --no-run --message-format short`

---------

Co-authored-by: Codex <noreply@openai.com>
2026-03-17 14:26:27 -07:00

272 lines
9.5 KiB
Rust

use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use codex_protocol::ThreadId;
use rand::Rng;
use tracing::debug;
use tracing::error;
use crate::auth_env_telemetry::AuthEnvTelemetry;
use crate::parse_command::shlex_join;
const INITIAL_DELAY_MS: u64 = 200;
const BACKOFF_FACTOR: f64 = 2.0;
/// Emit structured feedback metadata as key/value pairs.
///
/// This logs a tracing event with `target: "feedback_tags"`. If
/// `codex_feedback::CodexFeedback::metadata_layer()` is installed, these fields are captured and
/// later attached as tags when feedback is uploaded.
///
/// Values are wrapped with [`tracing::field::DebugValue`], so the expression only needs to
/// implement [`std::fmt::Debug`].
///
/// Example:
///
/// ```rust
/// codex_core::feedback_tags!(model = "gpt-5", cached = true);
/// codex_core::feedback_tags!(provider = provider_id, request_id = request_id);
/// ```
#[macro_export]
macro_rules! feedback_tags {
($( $key:ident = $value:expr ),+ $(,)?) => {
::tracing::info!(
target: "feedback_tags",
$( $key = ::tracing::field::debug(&$value) ),+
);
};
}
pub(crate) struct FeedbackRequestTags<'a> {
pub endpoint: &'a str,
pub auth_header_attached: bool,
pub auth_header_name: Option<&'a str>,
pub auth_mode: Option<&'a str>,
pub auth_retry_after_unauthorized: Option<bool>,
pub auth_recovery_mode: Option<&'a str>,
pub auth_recovery_phase: Option<&'a str>,
pub auth_connection_reused: Option<bool>,
pub auth_request_id: Option<&'a str>,
pub auth_cf_ray: Option<&'a str>,
pub auth_error: Option<&'a str>,
pub auth_error_code: Option<&'a str>,
pub auth_recovery_followup_success: Option<bool>,
pub auth_recovery_followup_status: Option<u16>,
}
struct FeedbackRequestSnapshot<'a> {
endpoint: &'a str,
auth_header_attached: bool,
auth_header_name: &'a str,
auth_mode: &'a str,
auth_retry_after_unauthorized: String,
auth_recovery_mode: &'a str,
auth_recovery_phase: &'a str,
auth_connection_reused: String,
auth_request_id: &'a str,
auth_cf_ray: &'a str,
auth_error: &'a str,
auth_error_code: &'a str,
auth_recovery_followup_success: String,
auth_recovery_followup_status: String,
}
struct Auth401FeedbackSnapshot<'a> {
request_id: &'a str,
cf_ray: &'a str,
error: &'a str,
error_code: &'a str,
}
impl<'a> Auth401FeedbackSnapshot<'a> {
fn from_optional_fields(
request_id: Option<&'a str>,
cf_ray: Option<&'a str>,
error: Option<&'a str>,
error_code: Option<&'a str>,
) -> Self {
Self {
request_id: request_id.unwrap_or(""),
cf_ray: cf_ray.unwrap_or(""),
error: error.unwrap_or(""),
error_code: error_code.unwrap_or(""),
}
}
}
impl<'a> FeedbackRequestSnapshot<'a> {
fn from_tags(tags: &'a FeedbackRequestTags<'a>) -> Self {
Self {
endpoint: tags.endpoint,
auth_header_attached: tags.auth_header_attached,
auth_header_name: tags.auth_header_name.unwrap_or(""),
auth_mode: tags.auth_mode.unwrap_or(""),
auth_retry_after_unauthorized: tags
.auth_retry_after_unauthorized
.map_or_else(String::new, |value| value.to_string()),
auth_recovery_mode: tags.auth_recovery_mode.unwrap_or(""),
auth_recovery_phase: tags.auth_recovery_phase.unwrap_or(""),
auth_connection_reused: tags
.auth_connection_reused
.map_or_else(String::new, |value| value.to_string()),
auth_request_id: tags.auth_request_id.unwrap_or(""),
auth_cf_ray: tags.auth_cf_ray.unwrap_or(""),
auth_error: tags.auth_error.unwrap_or(""),
auth_error_code: tags.auth_error_code.unwrap_or(""),
auth_recovery_followup_success: tags
.auth_recovery_followup_success
.map_or_else(String::new, |value| value.to_string()),
auth_recovery_followup_status: tags
.auth_recovery_followup_status
.map_or_else(String::new, |value| value.to_string()),
}
}
}
#[cfg(test)]
pub(crate) fn emit_feedback_request_tags(tags: &FeedbackRequestTags<'_>) {
let snapshot = FeedbackRequestSnapshot::from_tags(tags);
feedback_tags!(
endpoint = snapshot.endpoint,
auth_header_attached = snapshot.auth_header_attached,
auth_header_name = snapshot.auth_header_name,
auth_mode = snapshot.auth_mode,
auth_retry_after_unauthorized = snapshot.auth_retry_after_unauthorized,
auth_recovery_mode = snapshot.auth_recovery_mode,
auth_recovery_phase = snapshot.auth_recovery_phase,
auth_connection_reused = snapshot.auth_connection_reused,
auth_request_id = snapshot.auth_request_id,
auth_cf_ray = snapshot.auth_cf_ray,
auth_error = snapshot.auth_error,
auth_error_code = snapshot.auth_error_code,
auth_recovery_followup_success = snapshot.auth_recovery_followup_success,
auth_recovery_followup_status = snapshot.auth_recovery_followup_status
);
}
pub(crate) fn emit_feedback_request_tags_with_auth_env(
tags: &FeedbackRequestTags<'_>,
auth_env: &AuthEnvTelemetry,
) {
let snapshot = FeedbackRequestSnapshot::from_tags(tags);
feedback_tags!(
endpoint = snapshot.endpoint,
auth_header_attached = snapshot.auth_header_attached,
auth_header_name = snapshot.auth_header_name,
auth_mode = snapshot.auth_mode,
auth_retry_after_unauthorized = snapshot.auth_retry_after_unauthorized,
auth_recovery_mode = snapshot.auth_recovery_mode,
auth_recovery_phase = snapshot.auth_recovery_phase,
auth_connection_reused = snapshot.auth_connection_reused,
auth_request_id = snapshot.auth_request_id,
auth_cf_ray = snapshot.auth_cf_ray,
auth_error = snapshot.auth_error,
auth_error_code = snapshot.auth_error_code,
auth_recovery_followup_success = snapshot.auth_recovery_followup_success,
auth_recovery_followup_status = snapshot.auth_recovery_followup_status,
auth_env_openai_api_key_present = auth_env.openai_api_key_env_present,
auth_env_codex_api_key_present = auth_env.codex_api_key_env_present,
auth_env_codex_api_key_enabled = auth_env.codex_api_key_env_enabled,
auth_env_provider_key_name = auth_env.provider_env_key_name.as_deref().unwrap_or(""),
auth_env_provider_key_present = auth_env
.provider_env_key_present
.map_or_else(String::new, |value| value.to_string()),
auth_env_refresh_token_url_override_present = auth_env.refresh_token_url_override_present
);
}
pub(crate) fn emit_feedback_auth_recovery_tags(
auth_recovery_mode: &str,
auth_recovery_phase: &str,
auth_recovery_outcome: &str,
auth_request_id: Option<&str>,
auth_cf_ray: Option<&str>,
auth_error: Option<&str>,
auth_error_code: Option<&str>,
) {
let auth_401 = Auth401FeedbackSnapshot::from_optional_fields(
auth_request_id,
auth_cf_ray,
auth_error,
auth_error_code,
);
feedback_tags!(
auth_recovery_mode = auth_recovery_mode,
auth_recovery_phase = auth_recovery_phase,
auth_recovery_outcome = auth_recovery_outcome,
auth_401_request_id = auth_401.request_id,
auth_401_cf_ray = auth_401.cf_ray,
auth_401_error = auth_401.error,
auth_401_error_code = auth_401.error_code
);
}
pub fn backoff(attempt: u64) -> Duration {
let exp = BACKOFF_FACTOR.powi(attempt.saturating_sub(1) as i32);
let base = (INITIAL_DELAY_MS as f64 * exp) as u64;
let jitter = rand::rng().random_range(0.9..1.1);
Duration::from_millis((base as f64 * jitter) as u64)
}
pub(crate) fn error_or_panic(message: impl std::string::ToString) {
if cfg!(debug_assertions) {
panic!("{}", message.to_string());
} else {
error!("{}", message.to_string());
}
}
pub(crate) fn try_parse_error_message(text: &str) -> String {
debug!("Parsing server error response: {}", text);
let json = serde_json::from_str::<serde_json::Value>(text).unwrap_or_default();
if let Some(error) = json.get("error")
&& let Some(message) = error.get("message")
&& let Some(message_str) = message.as_str()
{
return message_str.to_string();
}
if text.is_empty() {
return "Unknown error".to_string();
}
text.to_string()
}
pub fn resolve_path(base: &Path, path: &PathBuf) -> PathBuf {
if path.is_absolute() {
path.clone()
} else {
base.join(path)
}
}
/// Trim a thread name and return `None` if it is empty after trimming.
pub fn normalize_thread_name(name: &str) -> Option<String> {
let trimmed = name.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_string())
}
}
pub fn resume_command(thread_name: Option<&str>, thread_id: Option<ThreadId>) -> Option<String> {
let resume_target = thread_name
.filter(|name| !name.is_empty())
.map(str::to_string)
.or_else(|| thread_id.map(|thread_id| thread_id.to_string()));
resume_target.map(|target| {
let needs_double_dash = target.starts_with('-');
let escaped = shlex_join(&[target]);
if needs_double_dash {
format!("codex resume -- {escaped}")
} else {
format!("codex resume {escaped}")
}
})
}
#[cfg(test)]
#[path = "util_tests.rs"]
mod tests;