[Codex][Codex CLI] Emit auth-failure Sentry telemetry without feedback

Add automatic auth-failure Sentry events for request-level and refresh-token failures, including the early /oauth/token path hit by short-lived CLI commands. Also allow a local DSN override so the non-/feedback path can be validated end to end with a real CLI 401 repro.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Colin Young
2026-03-31 12:19:47 -07:00
co-authored by Codex
parent e8de4ea953
commit 2b01b05bb9
12 changed files with 785 additions and 89 deletions
+55 -52
View File
@@ -112,6 +112,7 @@ use crate::response_debug_context::telemetry_transport_error_message;
use crate::util::FeedbackRequestTags;
use crate::util::emit_feedback_auth_recovery_tags;
use crate::util::emit_feedback_request_tags_with_auth_env;
use crate::util::emit_sentry_auth_failure_event_with_auth_env;
pub const OPENAI_BETA_HEADER: &str = "OpenAI-Beta";
pub const X_CODEX_TURN_STATE_HEADER: &str = "x-codex-turn-state";
@@ -1732,32 +1733,33 @@ impl RequestTelemetry for ApiTelemetry {
debug.auth_error.as_deref(),
debug.auth_error_code.as_deref(),
);
emit_feedback_request_tags_with_auth_env(
&FeedbackRequestTags {
endpoint: self.request_route_telemetry.endpoint,
auth_header_attached: self.auth_context.auth_header_attached,
auth_header_name: self.auth_context.auth_header_name,
auth_mode: self.auth_context.auth_mode,
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
auth_recovery_mode: self.auth_context.recovery_mode,
auth_recovery_phase: self.auth_context.recovery_phase,
auth_connection_reused: None,
auth_request_id: debug.request_id.as_deref(),
auth_cf_ray: debug.cf_ray.as_deref(),
auth_error: debug.auth_error.as_deref(),
auth_error_code: debug.auth_error_code.as_deref(),
auth_recovery_followup_success: self
.auth_context
.retry_after_unauthorized
.then_some(error.is_none()),
auth_recovery_followup_status: self
.auth_context
.retry_after_unauthorized
.then_some(status)
.flatten(),
},
&self.auth_env_telemetry,
);
let feedback_tags = FeedbackRequestTags {
endpoint: self.request_route_telemetry.endpoint,
auth_header_attached: self.auth_context.auth_header_attached,
auth_header_name: self.auth_context.auth_header_name,
auth_mode: self.auth_context.auth_mode,
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
auth_recovery_mode: self.auth_context.recovery_mode,
auth_recovery_phase: self.auth_context.recovery_phase,
auth_connection_reused: None,
auth_request_id: debug.request_id.as_deref(),
auth_cf_ray: debug.cf_ray.as_deref(),
auth_error: debug.auth_error.as_deref(),
auth_error_code: debug.auth_error_code.as_deref(),
auth_recovery_followup_success: self
.auth_context
.retry_after_unauthorized
.then_some(error.is_none()),
auth_recovery_followup_status: self
.auth_context
.retry_after_unauthorized
.then_some(status)
.flatten(),
};
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
}
}
}
@@ -1786,32 +1788,33 @@ impl WebsocketTelemetry for ApiTelemetry {
error_message.as_deref(),
connection_reused,
);
emit_feedback_request_tags_with_auth_env(
&FeedbackRequestTags {
endpoint: self.request_route_telemetry.endpoint,
auth_header_attached: self.auth_context.auth_header_attached,
auth_header_name: self.auth_context.auth_header_name,
auth_mode: self.auth_context.auth_mode,
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
auth_recovery_mode: self.auth_context.recovery_mode,
auth_recovery_phase: self.auth_context.recovery_phase,
auth_connection_reused: Some(connection_reused),
auth_request_id: debug.request_id.as_deref(),
auth_cf_ray: debug.cf_ray.as_deref(),
auth_error: debug.auth_error.as_deref(),
auth_error_code: debug.auth_error_code.as_deref(),
auth_recovery_followup_success: self
.auth_context
.retry_after_unauthorized
.then_some(error.is_none()),
auth_recovery_followup_status: self
.auth_context
.retry_after_unauthorized
.then_some(status)
.flatten(),
},
&self.auth_env_telemetry,
);
let feedback_tags = FeedbackRequestTags {
endpoint: self.request_route_telemetry.endpoint,
auth_header_attached: self.auth_context.auth_header_attached,
auth_header_name: self.auth_context.auth_header_name,
auth_mode: self.auth_context.auth_mode,
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
auth_recovery_mode: self.auth_context.recovery_mode,
auth_recovery_phase: self.auth_context.recovery_phase,
auth_connection_reused: Some(connection_reused),
auth_request_id: debug.request_id.as_deref(),
auth_cf_ray: debug.cf_ray.as_deref(),
auth_error: debug.auth_error.as_deref(),
auth_error_code: debug.auth_error_code.as_deref(),
auth_recovery_followup_success: self
.auth_context
.retry_after_unauthorized
.then_some(error.is_none()),
auth_recovery_followup_status: self
.auth_context
.retry_after_unauthorized
.then_some(status)
.flatten(),
};
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
}
}
fn on_ws_event(
+21 -19
View File
@@ -19,6 +19,7 @@ use crate::response_debug_context::extract_response_debug_context;
use crate::response_debug_context::telemetry_transport_error_message;
use crate::util::FeedbackRequestTags;
use crate::util::emit_feedback_request_tags_with_auth_env;
use crate::util::emit_sentry_auth_failure_event_with_auth_env;
use codex_api::ModelsClient;
use codex_api::RequestTelemetry;
use codex_api::ReqwestTransport;
@@ -114,25 +115,26 @@ impl RequestTelemetry for ModelsRequestTelemetry {
auth.error_code = response_debug.auth_error_code.as_deref(),
auth.mode = self.auth_mode.as_deref(),
);
emit_feedback_request_tags_with_auth_env(
&FeedbackRequestTags {
endpoint: MODELS_ENDPOINT,
auth_header_attached: self.auth_header_attached,
auth_header_name: self.auth_header_name,
auth_mode: self.auth_mode.as_deref(),
auth_retry_after_unauthorized: None,
auth_recovery_mode: None,
auth_recovery_phase: None,
auth_connection_reused: None,
auth_request_id: response_debug.request_id.as_deref(),
auth_cf_ray: response_debug.cf_ray.as_deref(),
auth_error: response_debug.auth_error.as_deref(),
auth_error_code: response_debug.auth_error_code.as_deref(),
auth_recovery_followup_success: None,
auth_recovery_followup_status: None,
},
&self.auth_env,
);
let feedback_tags = FeedbackRequestTags {
endpoint: MODELS_ENDPOINT,
auth_header_attached: self.auth_header_attached,
auth_header_name: self.auth_header_name,
auth_mode: self.auth_mode.as_deref(),
auth_retry_after_unauthorized: None,
auth_recovery_mode: None,
auth_recovery_phase: None,
auth_connection_reused: None,
auth_request_id: response_debug.request_id.as_deref(),
auth_cf_ray: response_debug.cf_ray.as_deref(),
auth_error: response_debug.auth_error.as_deref(),
auth_error_code: response_debug.auth_error_code.as_deref(),
auth_recovery_followup_success: None,
auth_recovery_followup_status: None,
};
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env);
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env);
}
}
}
@@ -215,6 +215,7 @@ impl Visit for TagCollectorVisitor {
#[derive(Clone)]
struct TagCollectorLayer {
target: &'static str,
tags: Arc<Mutex<BTreeMap<String, String>>>,
}
@@ -223,7 +224,7 @@ where
S: Subscriber + for<'a> LookupSpan<'a>,
{
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
if event.metadata().target() != "feedback_tags" {
if event.metadata().target() != self.target {
return;
}
let mut visitor = TagCollectorVisitor::default();
@@ -741,7 +742,10 @@ async fn refresh_available_models_skips_network_without_chatgpt_auth() {
fn models_request_telemetry_emits_auth_env_feedback_tags_on_failure() {
let tags = Arc::new(Mutex::new(BTreeMap::new()));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer { tags: tags.clone() })
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
})
.set_default();
let telemetry = ModelsRequestTelemetry {
@@ -835,6 +839,79 @@ fn models_request_telemetry_emits_auth_env_feedback_tags_on_failure() {
);
}
#[test]
fn models_request_telemetry_emits_sentry_auth_failure_event_on_401() {
let tags = Arc::new(Mutex::new(BTreeMap::new()));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: crate::util::SENTRY_AUTH_FAILURES_TARGET,
tags: tags.clone(),
})
.set_default();
let telemetry = ModelsRequestTelemetry {
auth_mode: Some(TelemetryAuthMode::Chatgpt.to_string()),
auth_header_attached: true,
auth_header_name: Some("authorization"),
auth_env: crate::auth_env_telemetry::AuthEnvTelemetry {
openai_api_key_env_present: false,
codex_api_key_env_present: false,
codex_api_key_env_enabled: false,
provider_env_key_name: Some("configured".to_string()),
provider_env_key_present: Some(false),
refresh_token_url_override_present: false,
},
};
let mut headers = HeaderMap::new();
headers.insert("x-request-id", "req-models-401".parse().unwrap());
headers.insert("cf-ray", "ray-models-401".parse().unwrap());
headers.insert(
"x-openai-authorization-error",
"missing_authorization_header".parse().unwrap(),
);
headers.insert(
"x-error-json",
base64::engine::general_purpose::STANDARD
.encode(r#"{"error":{"code":"token_expired"}}"#)
.parse()
.unwrap(),
);
telemetry.on_request(
1,
Some(StatusCode::UNAUTHORIZED),
Some(&TransportError::Http {
status: StatusCode::UNAUTHORIZED,
url: Some("https://example.test/models".to_string()),
headers: Some(headers),
body: Some("plain text error".to_string()),
}),
Duration::from_millis(17),
);
let tags = tags.lock().unwrap().clone();
assert_eq!(
tags.get("report_kind").map(String::as_str),
Some("auth_failure_auto")
);
assert_eq!(tags.get("endpoint").map(String::as_str), Some("/models"));
assert_eq!(
tags.get("auth_header_attached").map(String::as_str),
Some("true")
);
assert_eq!(
tags.get("auth_header_name").map(String::as_str),
Some("authorization")
);
assert_eq!(
tags.get("auth_request_id").map(String::as_str),
Some("req-models-401")
);
assert_eq!(
tags.get("auth_cf_ray").map(String::as_str),
Some("ray-models-401")
);
}
#[test]
fn build_available_models_picks_default_after_hiding_hidden_models() {
let codex_home = tempdir().expect("temp dir");
+39
View File
@@ -11,6 +11,7 @@ use crate::parse_command::shlex_join;
const INITIAL_DELAY_MS: u64 = 200;
const BACKOFF_FACTOR: f64 = 2.0;
pub(crate) const SENTRY_AUTH_FAILURES_TARGET: &str = "sentry_auth_failures";
/// Emit structured feedback metadata as key/value pairs.
///
@@ -201,6 +202,44 @@ pub(crate) fn emit_feedback_auth_recovery_tags(
);
}
pub(crate) fn emit_sentry_auth_failure_event_with_auth_env(
tags: &FeedbackRequestTags<'_>,
auth_env: &AuthEnvTelemetry,
) {
tracing::event!(
target: SENTRY_AUTH_FAILURES_TARGET,
tracing::Level::ERROR,
report_kind = "auth_failure_auto",
endpoint = tags.endpoint,
auth_header_attached = tags.auth_header_attached,
auth_header_name = tags.auth_header_name.unwrap_or(""),
auth_mode = tags.auth_mode.unwrap_or(""),
auth_retry_after_unauthorized = tags
.auth_retry_after_unauthorized
.unwrap_or(false),
auth_recovery_mode = tags.auth_recovery_mode.unwrap_or(""),
auth_recovery_phase = tags.auth_recovery_phase.unwrap_or(""),
auth_connection_reused = tags.auth_connection_reused.unwrap_or(false),
auth_request_id = tags.auth_request_id.unwrap_or(""),
auth_cf_ray = tags.auth_cf_ray.unwrap_or(""),
auth_error = tags.auth_error.unwrap_or(""),
auth_error_code = tags.auth_error_code.unwrap_or(""),
auth_recovery_followup_success = tags
.auth_recovery_followup_success
.unwrap_or(false),
auth_recovery_followup_status = tags
.auth_recovery_followup_status
.unwrap_or_default(),
auth_env_openai_api_key_present = auth_env.openai_api_key_env_present,
auth_env_codex_api_key_present = auth_env.codex_api_key_env_present,
auth_env_codex_api_key_enabled = auth_env.codex_api_key_env_enabled,
auth_env_provider_key_name = auth_env.provider_env_key_name.as_deref().unwrap_or(""),
auth_env_provider_key_present = auth_env.provider_env_key_present.unwrap_or(false),
auth_env_refresh_token_url_override_present = auth_env.refresh_token_url_override_present,
cli_version = env!("CARGO_PKG_VERSION"),
);
}
pub fn backoff(attempt: u64) -> Duration {
let exp = BACKOFF_FACTOR.powi(attempt.saturating_sub(1) as i32);
let base = (INITIAL_DELAY_MS as f64 * exp) as u64;
+70 -1
View File
@@ -44,6 +44,7 @@ impl Visit for TagCollectorVisitor {
#[derive(Clone)]
struct TagCollectorLayer {
target: &'static str,
tags: Arc<Mutex<BTreeMap<String, String>>>,
event_count: Arc<Mutex<usize>>,
}
@@ -53,7 +54,7 @@ where
S: Subscriber + for<'a> LookupSpan<'a>,
{
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
if event.metadata().target() != "feedback_tags" {
if event.metadata().target() != self.target {
return;
}
let mut visitor = TagCollectorVisitor::default();
@@ -69,6 +70,7 @@ fn emit_feedback_request_tags_records_sentry_feedback_fields() {
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
event_count: event_count.clone(),
})
@@ -172,6 +174,7 @@ fn emit_feedback_auth_recovery_tags_preserves_401_specific_fields() {
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
event_count: event_count.clone(),
})
@@ -213,6 +216,7 @@ fn emit_feedback_auth_recovery_tags_clears_stale_401_fields() {
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
event_count: event_count.clone(),
})
@@ -260,6 +264,7 @@ fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized()
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
event_count: event_count.clone(),
})
@@ -307,12 +312,76 @@ fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized()
assert_eq!(*event_count.lock().unwrap(), 1);
}
#[test]
fn emit_sentry_auth_failure_event_records_non_feedback_fields() {
let tags = Arc::new(Mutex::new(BTreeMap::new()));
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: SENTRY_AUTH_FAILURES_TARGET,
tags: tags.clone(),
event_count: event_count.clone(),
})
.set_default();
let auth_env = AuthEnvTelemetry {
openai_api_key_env_present: true,
codex_api_key_env_present: false,
codex_api_key_env_enabled: true,
provider_env_key_name: Some("configured".to_string()),
provider_env_key_present: Some(true),
refresh_token_url_override_present: true,
};
emit_sentry_auth_failure_event_with_auth_env(
&FeedbackRequestTags {
endpoint: "/responses",
auth_header_attached: true,
auth_header_name: Some("authorization"),
auth_mode: Some("chatgpt"),
auth_retry_after_unauthorized: Some(false),
auth_recovery_mode: Some("managed"),
auth_recovery_phase: Some("reload"),
auth_connection_reused: Some(false),
auth_request_id: Some("req-123"),
auth_cf_ray: Some("ray-123"),
auth_error: Some("missing_authorization_header"),
auth_error_code: Some("token_expired"),
auth_recovery_followup_success: None,
auth_recovery_followup_status: None,
},
&auth_env,
);
let tags = tags.lock().unwrap().clone();
assert_eq!(
tags.get("report_kind").map(String::as_str),
Some("auth_failure_auto")
);
assert_eq!(tags.get("endpoint").map(String::as_str), Some("/responses"));
assert_eq!(
tags.get("auth_header_attached").map(String::as_str),
Some("true")
);
assert_eq!(
tags.get("auth_header_name").map(String::as_str),
Some("authorization")
);
assert_eq!(
tags.get("auth_request_id").map(String::as_str),
Some("req-123")
);
assert_eq!(tags.get("auth_cf_ray").map(String::as_str), Some("ray-123"));
assert_eq!(*event_count.lock().unwrap(), 1);
}
#[test]
fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() {
let tags = Arc::new(Mutex::new(BTreeMap::new()));
let event_count = Arc::new(Mutex::new(0));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer {
target: "feedback_tags",
tags: tags.clone(),
event_count: event_count.clone(),
})