mirror of
https://github.com/openai/codex.git
synced 2026-11-12 21:20:34 +00:00
[Codex][Codex CLI] Emit auth-failure Sentry telemetry without feedback
Add automatic auth-failure Sentry events for request-level and refresh-token failures, including the early /oauth/token path hit by short-lived CLI commands. Also allow a local DSN override so the non-/feedback path can be validated end to end with a real CLI 401 repro. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
+55
-52
@@ -112,6 +112,7 @@ use crate::response_debug_context::telemetry_transport_error_message;
|
||||
use crate::util::FeedbackRequestTags;
|
||||
use crate::util::emit_feedback_auth_recovery_tags;
|
||||
use crate::util::emit_feedback_request_tags_with_auth_env;
|
||||
use crate::util::emit_sentry_auth_failure_event_with_auth_env;
|
||||
|
||||
pub const OPENAI_BETA_HEADER: &str = "OpenAI-Beta";
|
||||
pub const X_CODEX_TURN_STATE_HEADER: &str = "x-codex-turn-state";
|
||||
@@ -1732,32 +1733,33 @@ impl RequestTelemetry for ApiTelemetry {
|
||||
debug.auth_error.as_deref(),
|
||||
debug.auth_error_code.as_deref(),
|
||||
);
|
||||
emit_feedback_request_tags_with_auth_env(
|
||||
&FeedbackRequestTags {
|
||||
endpoint: self.request_route_telemetry.endpoint,
|
||||
auth_header_attached: self.auth_context.auth_header_attached,
|
||||
auth_header_name: self.auth_context.auth_header_name,
|
||||
auth_mode: self.auth_context.auth_mode,
|
||||
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
|
||||
auth_recovery_mode: self.auth_context.recovery_mode,
|
||||
auth_recovery_phase: self.auth_context.recovery_phase,
|
||||
auth_connection_reused: None,
|
||||
auth_request_id: debug.request_id.as_deref(),
|
||||
auth_cf_ray: debug.cf_ray.as_deref(),
|
||||
auth_error: debug.auth_error.as_deref(),
|
||||
auth_error_code: debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(error.is_none()),
|
||||
auth_recovery_followup_status: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(status)
|
||||
.flatten(),
|
||||
},
|
||||
&self.auth_env_telemetry,
|
||||
);
|
||||
let feedback_tags = FeedbackRequestTags {
|
||||
endpoint: self.request_route_telemetry.endpoint,
|
||||
auth_header_attached: self.auth_context.auth_header_attached,
|
||||
auth_header_name: self.auth_context.auth_header_name,
|
||||
auth_mode: self.auth_context.auth_mode,
|
||||
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
|
||||
auth_recovery_mode: self.auth_context.recovery_mode,
|
||||
auth_recovery_phase: self.auth_context.recovery_phase,
|
||||
auth_connection_reused: None,
|
||||
auth_request_id: debug.request_id.as_deref(),
|
||||
auth_cf_ray: debug.cf_ray.as_deref(),
|
||||
auth_error: debug.auth_error.as_deref(),
|
||||
auth_error_code: debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(error.is_none()),
|
||||
auth_recovery_followup_status: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(status)
|
||||
.flatten(),
|
||||
};
|
||||
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
|
||||
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
|
||||
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1786,32 +1788,33 @@ impl WebsocketTelemetry for ApiTelemetry {
|
||||
error_message.as_deref(),
|
||||
connection_reused,
|
||||
);
|
||||
emit_feedback_request_tags_with_auth_env(
|
||||
&FeedbackRequestTags {
|
||||
endpoint: self.request_route_telemetry.endpoint,
|
||||
auth_header_attached: self.auth_context.auth_header_attached,
|
||||
auth_header_name: self.auth_context.auth_header_name,
|
||||
auth_mode: self.auth_context.auth_mode,
|
||||
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
|
||||
auth_recovery_mode: self.auth_context.recovery_mode,
|
||||
auth_recovery_phase: self.auth_context.recovery_phase,
|
||||
auth_connection_reused: Some(connection_reused),
|
||||
auth_request_id: debug.request_id.as_deref(),
|
||||
auth_cf_ray: debug.cf_ray.as_deref(),
|
||||
auth_error: debug.auth_error.as_deref(),
|
||||
auth_error_code: debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(error.is_none()),
|
||||
auth_recovery_followup_status: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(status)
|
||||
.flatten(),
|
||||
},
|
||||
&self.auth_env_telemetry,
|
||||
);
|
||||
let feedback_tags = FeedbackRequestTags {
|
||||
endpoint: self.request_route_telemetry.endpoint,
|
||||
auth_header_attached: self.auth_context.auth_header_attached,
|
||||
auth_header_name: self.auth_context.auth_header_name,
|
||||
auth_mode: self.auth_context.auth_mode,
|
||||
auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized),
|
||||
auth_recovery_mode: self.auth_context.recovery_mode,
|
||||
auth_recovery_phase: self.auth_context.recovery_phase,
|
||||
auth_connection_reused: Some(connection_reused),
|
||||
auth_request_id: debug.request_id.as_deref(),
|
||||
auth_cf_ray: debug.cf_ray.as_deref(),
|
||||
auth_error: debug.auth_error.as_deref(),
|
||||
auth_error_code: debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(error.is_none()),
|
||||
auth_recovery_followup_status: self
|
||||
.auth_context
|
||||
.retry_after_unauthorized
|
||||
.then_some(status)
|
||||
.flatten(),
|
||||
};
|
||||
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
|
||||
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
|
||||
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env_telemetry);
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ws_event(
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::response_debug_context::extract_response_debug_context;
|
||||
use crate::response_debug_context::telemetry_transport_error_message;
|
||||
use crate::util::FeedbackRequestTags;
|
||||
use crate::util::emit_feedback_request_tags_with_auth_env;
|
||||
use crate::util::emit_sentry_auth_failure_event_with_auth_env;
|
||||
use codex_api::ModelsClient;
|
||||
use codex_api::RequestTelemetry;
|
||||
use codex_api::ReqwestTransport;
|
||||
@@ -114,25 +115,26 @@ impl RequestTelemetry for ModelsRequestTelemetry {
|
||||
auth.error_code = response_debug.auth_error_code.as_deref(),
|
||||
auth.mode = self.auth_mode.as_deref(),
|
||||
);
|
||||
emit_feedback_request_tags_with_auth_env(
|
||||
&FeedbackRequestTags {
|
||||
endpoint: MODELS_ENDPOINT,
|
||||
auth_header_attached: self.auth_header_attached,
|
||||
auth_header_name: self.auth_header_name,
|
||||
auth_mode: self.auth_mode.as_deref(),
|
||||
auth_retry_after_unauthorized: None,
|
||||
auth_recovery_mode: None,
|
||||
auth_recovery_phase: None,
|
||||
auth_connection_reused: None,
|
||||
auth_request_id: response_debug.request_id.as_deref(),
|
||||
auth_cf_ray: response_debug.cf_ray.as_deref(),
|
||||
auth_error: response_debug.auth_error.as_deref(),
|
||||
auth_error_code: response_debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: None,
|
||||
auth_recovery_followup_status: None,
|
||||
},
|
||||
&self.auth_env,
|
||||
);
|
||||
let feedback_tags = FeedbackRequestTags {
|
||||
endpoint: MODELS_ENDPOINT,
|
||||
auth_header_attached: self.auth_header_attached,
|
||||
auth_header_name: self.auth_header_name,
|
||||
auth_mode: self.auth_mode.as_deref(),
|
||||
auth_retry_after_unauthorized: None,
|
||||
auth_recovery_mode: None,
|
||||
auth_recovery_phase: None,
|
||||
auth_connection_reused: None,
|
||||
auth_request_id: response_debug.request_id.as_deref(),
|
||||
auth_cf_ray: response_debug.cf_ray.as_deref(),
|
||||
auth_error: response_debug.auth_error.as_deref(),
|
||||
auth_error_code: response_debug.auth_error_code.as_deref(),
|
||||
auth_recovery_followup_success: None,
|
||||
auth_recovery_followup_status: None,
|
||||
};
|
||||
emit_feedback_request_tags_with_auth_env(&feedback_tags, &self.auth_env);
|
||||
if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) {
|
||||
emit_sentry_auth_failure_event_with_auth_env(&feedback_tags, &self.auth_env);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -215,6 +215,7 @@ impl Visit for TagCollectorVisitor {
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TagCollectorLayer {
|
||||
target: &'static str,
|
||||
tags: Arc<Mutex<BTreeMap<String, String>>>,
|
||||
}
|
||||
|
||||
@@ -223,7 +224,7 @@ where
|
||||
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
|
||||
if event.metadata().target() != "feedback_tags" {
|
||||
if event.metadata().target() != self.target {
|
||||
return;
|
||||
}
|
||||
let mut visitor = TagCollectorVisitor::default();
|
||||
@@ -741,7 +742,10 @@ async fn refresh_available_models_skips_network_without_chatgpt_auth() {
|
||||
fn models_request_telemetry_emits_auth_env_feedback_tags_on_failure() {
|
||||
let tags = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer { tags: tags.clone() })
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
})
|
||||
.set_default();
|
||||
|
||||
let telemetry = ModelsRequestTelemetry {
|
||||
@@ -835,6 +839,79 @@ fn models_request_telemetry_emits_auth_env_feedback_tags_on_failure() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn models_request_telemetry_emits_sentry_auth_failure_event_on_401() {
|
||||
let tags = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: crate::util::SENTRY_AUTH_FAILURES_TARGET,
|
||||
tags: tags.clone(),
|
||||
})
|
||||
.set_default();
|
||||
|
||||
let telemetry = ModelsRequestTelemetry {
|
||||
auth_mode: Some(TelemetryAuthMode::Chatgpt.to_string()),
|
||||
auth_header_attached: true,
|
||||
auth_header_name: Some("authorization"),
|
||||
auth_env: crate::auth_env_telemetry::AuthEnvTelemetry {
|
||||
openai_api_key_env_present: false,
|
||||
codex_api_key_env_present: false,
|
||||
codex_api_key_env_enabled: false,
|
||||
provider_env_key_name: Some("configured".to_string()),
|
||||
provider_env_key_present: Some(false),
|
||||
refresh_token_url_override_present: false,
|
||||
},
|
||||
};
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-request-id", "req-models-401".parse().unwrap());
|
||||
headers.insert("cf-ray", "ray-models-401".parse().unwrap());
|
||||
headers.insert(
|
||||
"x-openai-authorization-error",
|
||||
"missing_authorization_header".parse().unwrap(),
|
||||
);
|
||||
headers.insert(
|
||||
"x-error-json",
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.encode(r#"{"error":{"code":"token_expired"}}"#)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
telemetry.on_request(
|
||||
1,
|
||||
Some(StatusCode::UNAUTHORIZED),
|
||||
Some(&TransportError::Http {
|
||||
status: StatusCode::UNAUTHORIZED,
|
||||
url: Some("https://example.test/models".to_string()),
|
||||
headers: Some(headers),
|
||||
body: Some("plain text error".to_string()),
|
||||
}),
|
||||
Duration::from_millis(17),
|
||||
);
|
||||
|
||||
let tags = tags.lock().unwrap().clone();
|
||||
assert_eq!(
|
||||
tags.get("report_kind").map(String::as_str),
|
||||
Some("auth_failure_auto")
|
||||
);
|
||||
assert_eq!(tags.get("endpoint").map(String::as_str), Some("/models"));
|
||||
assert_eq!(
|
||||
tags.get("auth_header_attached").map(String::as_str),
|
||||
Some("true")
|
||||
);
|
||||
assert_eq!(
|
||||
tags.get("auth_header_name").map(String::as_str),
|
||||
Some("authorization")
|
||||
);
|
||||
assert_eq!(
|
||||
tags.get("auth_request_id").map(String::as_str),
|
||||
Some("req-models-401")
|
||||
);
|
||||
assert_eq!(
|
||||
tags.get("auth_cf_ray").map(String::as_str),
|
||||
Some("ray-models-401")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_available_models_picks_default_after_hiding_hidden_models() {
|
||||
let codex_home = tempdir().expect("temp dir");
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::parse_command::shlex_join;
|
||||
|
||||
const INITIAL_DELAY_MS: u64 = 200;
|
||||
const BACKOFF_FACTOR: f64 = 2.0;
|
||||
pub(crate) const SENTRY_AUTH_FAILURES_TARGET: &str = "sentry_auth_failures";
|
||||
|
||||
/// Emit structured feedback metadata as key/value pairs.
|
||||
///
|
||||
@@ -201,6 +202,44 @@ pub(crate) fn emit_feedback_auth_recovery_tags(
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_sentry_auth_failure_event_with_auth_env(
|
||||
tags: &FeedbackRequestTags<'_>,
|
||||
auth_env: &AuthEnvTelemetry,
|
||||
) {
|
||||
tracing::event!(
|
||||
target: SENTRY_AUTH_FAILURES_TARGET,
|
||||
tracing::Level::ERROR,
|
||||
report_kind = "auth_failure_auto",
|
||||
endpoint = tags.endpoint,
|
||||
auth_header_attached = tags.auth_header_attached,
|
||||
auth_header_name = tags.auth_header_name.unwrap_or(""),
|
||||
auth_mode = tags.auth_mode.unwrap_or(""),
|
||||
auth_retry_after_unauthorized = tags
|
||||
.auth_retry_after_unauthorized
|
||||
.unwrap_or(false),
|
||||
auth_recovery_mode = tags.auth_recovery_mode.unwrap_or(""),
|
||||
auth_recovery_phase = tags.auth_recovery_phase.unwrap_or(""),
|
||||
auth_connection_reused = tags.auth_connection_reused.unwrap_or(false),
|
||||
auth_request_id = tags.auth_request_id.unwrap_or(""),
|
||||
auth_cf_ray = tags.auth_cf_ray.unwrap_or(""),
|
||||
auth_error = tags.auth_error.unwrap_or(""),
|
||||
auth_error_code = tags.auth_error_code.unwrap_or(""),
|
||||
auth_recovery_followup_success = tags
|
||||
.auth_recovery_followup_success
|
||||
.unwrap_or(false),
|
||||
auth_recovery_followup_status = tags
|
||||
.auth_recovery_followup_status
|
||||
.unwrap_or_default(),
|
||||
auth_env_openai_api_key_present = auth_env.openai_api_key_env_present,
|
||||
auth_env_codex_api_key_present = auth_env.codex_api_key_env_present,
|
||||
auth_env_codex_api_key_enabled = auth_env.codex_api_key_env_enabled,
|
||||
auth_env_provider_key_name = auth_env.provider_env_key_name.as_deref().unwrap_or(""),
|
||||
auth_env_provider_key_present = auth_env.provider_env_key_present.unwrap_or(false),
|
||||
auth_env_refresh_token_url_override_present = auth_env.refresh_token_url_override_present,
|
||||
cli_version = env!("CARGO_PKG_VERSION"),
|
||||
);
|
||||
}
|
||||
|
||||
pub fn backoff(attempt: u64) -> Duration {
|
||||
let exp = BACKOFF_FACTOR.powi(attempt.saturating_sub(1) as i32);
|
||||
let base = (INITIAL_DELAY_MS as f64 * exp) as u64;
|
||||
|
||||
@@ -44,6 +44,7 @@ impl Visit for TagCollectorVisitor {
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TagCollectorLayer {
|
||||
target: &'static str,
|
||||
tags: Arc<Mutex<BTreeMap<String, String>>>,
|
||||
event_count: Arc<Mutex<usize>>,
|
||||
}
|
||||
@@ -53,7 +54,7 @@ where
|
||||
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
|
||||
if event.metadata().target() != "feedback_tags" {
|
||||
if event.metadata().target() != self.target {
|
||||
return;
|
||||
}
|
||||
let mut visitor = TagCollectorVisitor::default();
|
||||
@@ -69,6 +70,7 @@ fn emit_feedback_request_tags_records_sentry_feedback_fields() {
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
@@ -172,6 +174,7 @@ fn emit_feedback_auth_recovery_tags_preserves_401_specific_fields() {
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
@@ -213,6 +216,7 @@ fn emit_feedback_auth_recovery_tags_clears_stale_401_fields() {
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
@@ -260,6 +264,7 @@ fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized()
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
@@ -307,12 +312,76 @@ fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized()
|
||||
assert_eq!(*event_count.lock().unwrap(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emit_sentry_auth_failure_event_records_non_feedback_fields() {
|
||||
let tags = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: SENTRY_AUTH_FAILURES_TARGET,
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
.set_default();
|
||||
|
||||
let auth_env = AuthEnvTelemetry {
|
||||
openai_api_key_env_present: true,
|
||||
codex_api_key_env_present: false,
|
||||
codex_api_key_env_enabled: true,
|
||||
provider_env_key_name: Some("configured".to_string()),
|
||||
provider_env_key_present: Some(true),
|
||||
refresh_token_url_override_present: true,
|
||||
};
|
||||
|
||||
emit_sentry_auth_failure_event_with_auth_env(
|
||||
&FeedbackRequestTags {
|
||||
endpoint: "/responses",
|
||||
auth_header_attached: true,
|
||||
auth_header_name: Some("authorization"),
|
||||
auth_mode: Some("chatgpt"),
|
||||
auth_retry_after_unauthorized: Some(false),
|
||||
auth_recovery_mode: Some("managed"),
|
||||
auth_recovery_phase: Some("reload"),
|
||||
auth_connection_reused: Some(false),
|
||||
auth_request_id: Some("req-123"),
|
||||
auth_cf_ray: Some("ray-123"),
|
||||
auth_error: Some("missing_authorization_header"),
|
||||
auth_error_code: Some("token_expired"),
|
||||
auth_recovery_followup_success: None,
|
||||
auth_recovery_followup_status: None,
|
||||
},
|
||||
&auth_env,
|
||||
);
|
||||
|
||||
let tags = tags.lock().unwrap().clone();
|
||||
assert_eq!(
|
||||
tags.get("report_kind").map(String::as_str),
|
||||
Some("auth_failure_auto")
|
||||
);
|
||||
assert_eq!(tags.get("endpoint").map(String::as_str), Some("/responses"));
|
||||
assert_eq!(
|
||||
tags.get("auth_header_attached").map(String::as_str),
|
||||
Some("true")
|
||||
);
|
||||
assert_eq!(
|
||||
tags.get("auth_header_name").map(String::as_str),
|
||||
Some("authorization")
|
||||
);
|
||||
assert_eq!(
|
||||
tags.get("auth_request_id").map(String::as_str),
|
||||
Some("req-123")
|
||||
);
|
||||
assert_eq!(tags.get("auth_cf_ray").map(String::as_str), Some("ray-123"));
|
||||
assert_eq!(*event_count.lock().unwrap(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() {
|
||||
let tags = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let event_count = Arc::new(Mutex::new(0));
|
||||
let _guard = tracing_subscriber::registry()
|
||||
.with(TagCollectorLayer {
|
||||
target: "feedback_tags",
|
||||
tags: tags.clone(),
|
||||
event_count: event_count.clone(),
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user