From 700388f7966a399abb8b6e6f15b621cf237a8c19 Mon Sep 17 00:00:00 2001 From: Colin Young Date: Mon, 16 Mar 2026 15:33:12 -0700 Subject: [PATCH] [Codex][Codex CLI] Add auth routing observability follow-up Checkpoint the PR2 remainder before rebasing onto main. Co-authored-by: Codex --- codex-rs/core/src/auth.rs | 4 + codex-rs/core/src/auth_env_telemetry.rs | 6 + codex-rs/core/src/client.rs | 673 +++++++++++------- codex-rs/core/src/codex.rs | 43 +- codex-rs/core/src/config/config_tests.rs | 2 +- codex-rs/core/src/default_client.rs | 44 +- .../core/src/endpoint_config_telemetry.rs | 336 +++++++++ codex-rs/core/src/lib.rs | 2 + codex-rs/core/src/model_provider_info.rs | 69 +- .../core/src/model_provider_info_tests.rs | 9 + codex-rs/core/src/models_manager/manager.rs | 287 +++++--- codex-rs/core/src/response_debug_context.rs | 151 +++- codex-rs/core/src/thread_manager.rs | 124 ++-- .../src/tools/handlers/multi_agents_tests.rs | 4 +- codex-rs/core/src/util.rs | 175 +++-- codex-rs/core/src/util_tests.rs | 276 +++---- codex-rs/core/tests/common/test_codex.rs | 71 +- codex-rs/core/tests/suite/client.rs | 6 +- codex-rs/core/tests/suite/compact.rs | 2 +- codex-rs/core/tests/suite/remote_models.rs | 14 +- codex-rs/otel/src/events/session_telemetry.rs | 355 +++++++-- .../tests/suite/otel_export_routing_policy.rs | 459 ++++++++++-- 22 files changed, 2146 insertions(+), 966 deletions(-) create mode 100644 codex-rs/core/src/endpoint_config_telemetry.rs diff --git a/codex-rs/core/src/auth.rs b/codex-rs/core/src/auth.rs index 90f0dcfdaf..beb005cbf7 100644 --- a/codex-rs/core/src/auth.rs +++ b/codex-rs/core/src/auth.rs @@ -1244,6 +1244,10 @@ impl AuthManager { .unwrap_or(false) } + pub(crate) fn codex_api_key_env_enabled(&self) -> bool { + self.enable_codex_api_key_env + } + pub fn is_external_auth_active(&self) -> bool { self.auth_cached() .as_ref() diff --git a/codex-rs/core/src/auth_env_telemetry.rs b/codex-rs/core/src/auth_env_telemetry.rs index 85cd23fe06..af0e83fbf4 100644 --- a/codex-rs/core/src/auth_env_telemetry.rs +++ b/codex-rs/core/src/auth_env_telemetry.rs @@ -26,6 +26,12 @@ impl AuthEnvTelemetry { refresh_token_url_override_present: self.refresh_token_url_override_present, } } + + pub(crate) fn has_api_key_override(&self) -> bool { + self.openai_api_key_env_present + || (self.codex_api_key_env_enabled && self.codex_api_key_env_present) + || self.provider_env_key_present == Some(true) + } } pub(crate) fn collect_auth_env_telemetry( diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 48e8d90ecf..d375e4c3f4 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -83,7 +83,6 @@ use tokio::sync::oneshot; use tokio::sync::oneshot::error::TryRecvError; use tokio_tungstenite::tungstenite::Error; use tokio_tungstenite::tungstenite::Message; -use tracing::instrument; use tracing::trace; use tracing::warn; @@ -96,6 +95,9 @@ use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; use crate::config::Config; use crate::default_client::build_reqwest_client; +use crate::default_client::current_residency_header_telemetry; +use crate::endpoint_config_telemetry::EndpointConfigTelemetry; +use crate::endpoint_config_telemetry::EndpointConfigTelemetrySource; use crate::error::CodexErr; use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; @@ -108,7 +110,7 @@ use crate::response_debug_context::telemetry_transport_error_message; use crate::tools::spec::create_tools_json_for_responses_api; use crate::util::FeedbackRequestTags; use crate::util::emit_feedback_auth_recovery_tags; -use crate::util::emit_feedback_request_tags_with_auth_env; +use crate::util::emit_feedback_request_tags; pub const OPENAI_BETA_HEADER: &str = "OpenAI-Beta"; pub const X_CODEX_TURN_STATE_HEADER: &str = "x-codex-turn-state"; @@ -119,9 +121,6 @@ const RESPONSES_WEBSOCKETS_V2_BETA_HEADER_VALUE: &str = "responses_websockets=20 const RESPONSES_ENDPOINT: &str = "/responses"; const RESPONSES_COMPACT_ENDPOINT: &str = "/responses/compact"; const MEMORIES_SUMMARIZE_ENDPOINT: &str = "/memories/trace_summarize"; -#[cfg(test)] -pub(crate) const WEBSOCKET_CONNECT_TIMEOUT: Duration = - Duration::from_millis(crate::model_provider_info::DEFAULT_WEBSOCKET_CONNECT_TIMEOUT_MS); pub fn ws_version_from_features(config: &Config) -> bool { config .features @@ -140,6 +139,7 @@ struct ModelClientState { auth_manager: Option>, conversation_id: ThreadId, provider: ModelProviderInfo, + endpoint_telemetry_source: EndpointConfigTelemetrySource, auth_env_telemetry: AuthEnvTelemetry, session_source: SessionSource, model_verbosity: Option, @@ -159,16 +159,25 @@ struct CurrentClientSetup { auth: Option, api_provider: codex_api::Provider, api_auth: CoreAuthProvider, + endpoint_telemetry: EndpointConfigTelemetry, + provider_header_names: Option, } #[derive(Clone, Copy)] struct RequestRouteTelemetry { endpoint: &'static str, + residency_header_attached: bool, + residency_header_value: Option<&'static str>, } impl RequestRouteTelemetry { fn for_endpoint(endpoint: &'static str) -> Self { - Self { endpoint } + let residency = current_residency_header_telemetry(); + Self { + endpoint, + residency_header_attached: residency.attached, + residency_header_value: residency.value, + } } } @@ -269,6 +278,64 @@ impl ModelClient { enable_request_compression: bool, include_timing_metrics: bool, beta_features_header: Option, + ) -> Self { + let endpoint_telemetry_source = + EndpointConfigTelemetrySource::for_provider_without_id(&provider); + Self::new_with_endpoint_telemetry_source( + auth_manager, + conversation_id, + provider, + endpoint_telemetry_source, + session_source, + model_verbosity, + responses_websockets_enabled_by_feature, + enable_request_compression, + include_timing_metrics, + beta_features_header, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_with_provider_id( + auth_manager: Option>, + conversation_id: ThreadId, + provider_id: &str, + provider: ModelProviderInfo, + session_source: SessionSource, + model_verbosity: Option, + responses_websockets_enabled_by_feature: bool, + enable_request_compression: bool, + include_timing_metrics: bool, + beta_features_header: Option, + ) -> Self { + let endpoint_telemetry_source = + EndpointConfigTelemetrySource::for_provider(provider_id, &provider); + Self::new_with_endpoint_telemetry_source( + auth_manager, + conversation_id, + provider, + endpoint_telemetry_source, + session_source, + model_verbosity, + responses_websockets_enabled_by_feature, + enable_request_compression, + include_timing_metrics, + beta_features_header, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn new_with_endpoint_telemetry_source( + auth_manager: Option>, + conversation_id: ThreadId, + provider: ModelProviderInfo, + endpoint_telemetry_source: EndpointConfigTelemetrySource, + session_source: SessionSource, + model_verbosity: Option, + responses_websockets_enabled_by_feature: bool, + enable_request_compression: bool, + include_timing_metrics: bool, + beta_features_header: Option, ) -> Self { let codex_api_key_env_enabled = auth_manager .as_ref() @@ -279,6 +346,7 @@ impl ModelClient { auth_manager, conversation_id, provider, + endpoint_telemetry_source, auth_env_telemetry, session_source, model_verbosity, @@ -321,27 +389,6 @@ impl ModelClient { .unwrap_or_else(std::sync::PoisonError::into_inner) = websocket_session; } - pub(crate) fn force_http_fallback( - &self, - session_telemetry: &SessionTelemetry, - model_info: &ModelInfo, - ) -> bool { - let websocket_enabled = self.responses_websocket_enabled(model_info); - let activated = - websocket_enabled && !self.state.disable_websockets.swap(true, Ordering::Relaxed); - if activated { - warn!("falling back to HTTP"); - session_telemetry.counter( - "codex.transport.fallback_to_http", - /*inc*/ 1, - &[("from_wire_api", "responses_websocket")], - ); - } - - self.store_cached_websocket_session(WebsocketSession::default()); - activated - } - /// Compacts the current conversation history using the Compact endpoint. /// /// This is a unary call (no streaming) that returns a new list of @@ -369,8 +416,10 @@ impl ModelClient { &client_setup.api_auth, PendingUnauthorizedRetry::default(), ), - RequestRouteTelemetry::for_endpoint(RESPONSES_COMPACT_ENDPOINT), + client_setup.endpoint_telemetry, self.state.auth_env_telemetry.clone(), + client_setup.provider_header_names.clone(), + RequestRouteTelemetry::for_endpoint(RESPONSES_COMPACT_ENDPOINT), ); let client = ApiCompactClient::new(transport, client_setup.api_provider, client_setup.api_auth) @@ -438,8 +487,10 @@ impl ModelClient { &client_setup.api_auth, PendingUnauthorizedRetry::default(), ), - RequestRouteTelemetry::for_endpoint(MEMORIES_SUMMARIZE_ENDPOINT), + client_setup.endpoint_telemetry, self.state.auth_env_telemetry.clone(), + client_setup.provider_header_names.clone(), + RequestRouteTelemetry::for_endpoint(MEMORIES_SUMMARIZE_ENDPOINT), ); let client = ApiMemoriesClient::new(transport, client_setup.api_provider, client_setup.api_auth) @@ -483,14 +534,18 @@ impl ModelClient { fn build_request_telemetry( session_telemetry: &SessionTelemetry, auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, + endpoint_telemetry: EndpointConfigTelemetry, auth_env_telemetry: AuthEnvTelemetry, + provider_header_names: Option, + request_route_telemetry: RequestRouteTelemetry, ) -> Arc { let telemetry = Arc::new(ApiTelemetry::new( session_telemetry.clone(), auth_context, - request_route_telemetry, + endpoint_telemetry, auth_env_telemetry, + provider_header_names, + request_route_telemetry, )); let request_telemetry: Arc = telemetry; request_telemetry @@ -546,10 +601,16 @@ impl ModelClient { .provider .to_api_provider(auth.as_ref().map(CodexAuth::auth_mode))?; let api_auth = auth_provider_from_auth(auth.clone(), &self.state.provider)?; + let endpoint_telemetry = self + .state + .endpoint_telemetry_source + .classify(api_provider.base_url.as_str()); Ok(CurrentClientSetup { auth, api_provider, api_auth, + endpoint_telemetry, + provider_header_names: self.state.provider.telemetry_header_names(), }) } @@ -566,31 +627,28 @@ impl ModelClient { turn_state: Option>>, turn_metadata_header: Option<&str>, auth_context: AuthRequestTelemetryContext, + endpoint_telemetry: EndpointConfigTelemetry, + provider_header_names: Option<&str>, request_route_telemetry: RequestRouteTelemetry, ) -> std::result::Result { let headers = self.build_websocket_headers(turn_state.as_ref(), turn_metadata_header); let websocket_telemetry = ModelClientSession::build_websocket_telemetry( session_telemetry, auth_context, - request_route_telemetry, + endpoint_telemetry, self.state.auth_env_telemetry.clone(), + provider_header_names.map(str::to_owned), + request_route_telemetry, ); - let websocket_connect_timeout = self.state.provider.websocket_connect_timeout(); let start = Instant::now(); - let result = match tokio::time::timeout( - websocket_connect_timeout, - ApiWebSocketResponsesClient::new(api_provider, api_auth).connect( + let result = ApiWebSocketResponsesClient::new(api_provider, api_auth) + .connect( headers, crate::default_client::default_headers(), turn_state, Some(websocket_telemetry), - ), - ) - .await - { - Ok(result) => result, - Err(_) => Err(ApiError::Transport(TransportError::Timeout)), - }; + ) + .await; let error_message = result.as_ref().err().map(telemetry_api_error_message); let response_debug = result .as_ref() @@ -598,7 +656,7 @@ impl ModelClient { .map(extract_response_debug_context_from_api_error) .unwrap_or_default(); let status = result.as_ref().err().and_then(api_error_http_status); - session_telemetry.record_websocket_connect( + session_telemetry.record_websocket_connect_with_endpoint_details( start.elapsed(), status, error_message.as_deref(), @@ -608,36 +666,96 @@ impl ModelClient { auth_context.recovery_mode, auth_context.recovery_phase, request_route_telemetry.endpoint, - /*connection_reused*/ false, + request_route_telemetry.residency_header_attached, + request_route_telemetry.residency_header_value, + provider_header_names, + endpoint_telemetry.base_url_origin, + endpoint_telemetry.host_class, + endpoint_telemetry.base_url_source, + endpoint_telemetry.base_url_is_default, + self.state.auth_env_telemetry.openai_api_key_env_present, + self.state.auth_env_telemetry.codex_api_key_env_present, + self.state.auth_env_telemetry.codex_api_key_env_enabled, + self.state + .auth_env_telemetry + .provider_env_key_name + .as_deref(), + self.state.auth_env_telemetry.provider_env_key_present, + self.state + .auth_env_telemetry + .refresh_token_url_override_present, + false, response_debug.request_id.as_deref(), response_debug.cf_ray.as_deref(), response_debug.auth_error.as_deref(), response_debug.auth_error_code.as_deref(), + response_debug.error_body_class, + response_debug.safe_error_message, ); - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: request_route_telemetry.endpoint, - auth_header_attached: auth_context.auth_header_attached, - auth_header_name: auth_context.auth_header_name, - auth_mode: auth_context.auth_mode, - auth_retry_after_unauthorized: Some(auth_context.retry_after_unauthorized), - auth_recovery_mode: auth_context.recovery_mode, - auth_recovery_phase: auth_context.recovery_phase, - auth_connection_reused: Some(false), - auth_request_id: response_debug.request_id.as_deref(), - auth_cf_ray: response_debug.cf_ray.as_deref(), - auth_error: response_debug.auth_error.as_deref(), - auth_error_code: response_debug.auth_error_code.as_deref(), - auth_recovery_followup_success: auth_context - .retry_after_unauthorized - .then_some(result.is_ok()), - auth_recovery_followup_status: auth_context - .retry_after_unauthorized - .then_some(status) - .flatten(), - }, - &self.state.auth_env_telemetry, - ); + emit_feedback_request_tags(&FeedbackRequestTags { + endpoint: request_route_telemetry.endpoint, + auth_header_attached: auth_context.auth_header_attached, + auth_header_name: auth_context.auth_header_name, + auth_mode: auth_context.auth_mode, + auth_env_openai_api_key_present: self + .state + .auth_env_telemetry + .openai_api_key_env_present, + auth_env_codex_api_key_present: self.state.auth_env_telemetry.codex_api_key_env_present, + auth_env_codex_api_key_enabled: self.state.auth_env_telemetry.codex_api_key_env_enabled, + auth_env_provider_key_name: self + .state + .auth_env_telemetry + .provider_env_key_name + .as_deref(), + auth_env_provider_key_present: self.state.auth_env_telemetry.provider_env_key_present, + auth_env_refresh_token_url_override_present: self + .state + .auth_env_telemetry + .refresh_token_url_override_present, + auth_retry_after_unauthorized: Some(auth_context.retry_after_unauthorized), + auth_recovery_mode: auth_context.recovery_mode, + auth_recovery_phase: auth_context.recovery_phase, + auth_connection_reused: Some(false), + provider_header_names, + base_url_origin: endpoint_telemetry.base_url_origin, + host_class: endpoint_telemetry.host_class, + base_url_source: endpoint_telemetry.base_url_source, + base_url_is_default: endpoint_telemetry.base_url_is_default, + residency_header_attached: Some(request_route_telemetry.residency_header_attached), + residency_header_value: request_route_telemetry.residency_header_value, + auth_request_id: response_debug.request_id.as_deref(), + auth_cf_ray: response_debug.cf_ray.as_deref(), + auth_error: response_debug.auth_error.as_deref(), + auth_error_code: response_debug.auth_error_code.as_deref(), + error_body_class: response_debug.error_body_class, + safe_error_message: response_debug.safe_error_message, + geo_denial_detected: Some(response_debug.geo_denial_detected), + auth_recovery_followup_success: auth_context + .retry_after_unauthorized + .then_some(result.is_ok()), + auth_recovery_followup_status: auth_context + .retry_after_unauthorized + .then_some(status) + .flatten(), + }); + if status == Some(StatusCode::UNAUTHORIZED.as_u16()) && response_debug.geo_denial_detected { + session_telemetry.record_geo_denial( + request_route_telemetry.endpoint, + auth_context.auth_header_attached, + auth_context.auth_header_name, + request_route_telemetry.residency_header_attached, + request_route_telemetry.residency_header_value, + provider_header_names, + status, + response_debug.request_id.as_deref(), + response_debug.cf_ray.as_deref(), + response_debug.auth_error.as_deref(), + response_debug.auth_error_code.as_deref(), + response_debug.error_body_class.unwrap_or_default(), + response_debug.safe_error_message, + ); + } result } @@ -684,12 +802,13 @@ impl Drop for ModelClientSession { } impl ModelClientSession { - fn reset_websocket_session(&mut self) { - self.websocket_session.connection = None; - self.websocket_session.last_request = None; - self.websocket_session.last_response_rx = None; - self.websocket_session - .set_connection_reused(/*connection_reused*/ false); + fn activate_http_fallback(&self, websocket_enabled: bool) -> bool { + websocket_enabled + && !self + .client + .state + .disable_websockets + .swap(true, Ordering::Relaxed) } fn build_responses_request( @@ -842,11 +961,9 @@ impl ModelClientSession { let Some(last_response) = self.get_last_response() else { return ResponsesWsRequest::ResponseCreate(payload); }; - let Some(incremental_items) = self.get_incremental_items( - request, - Some(&last_response), - /*allow_empty_delta*/ true, - ) else { + let Some(incremental_items) = + self.get_incremental_items(request, Some(&last_response), true) + else { return ResponsesWsRequest::ResponseCreate(payload); }; @@ -887,6 +1004,7 @@ impl ModelClientSession { &client_setup.api_auth, PendingUnauthorizedRetry::default(), ); + let endpoint_telemetry = client_setup.endpoint_telemetry; let connection = self .client .connect_websocket( @@ -894,42 +1012,31 @@ impl ModelClientSession { client_setup.api_provider, client_setup.api_auth, Some(Arc::clone(&self.turn_state)), - /*turn_metadata_header*/ None, + None, auth_context, + endpoint_telemetry, + client_setup.provider_header_names.as_deref(), RequestRouteTelemetry::for_endpoint(RESPONSES_ENDPOINT), ) .await?; self.websocket_session.connection = Some(connection); - self.websocket_session - .set_connection_reused(/*connection_reused*/ false); + self.websocket_session.set_connection_reused(false); Ok(()) } /// Returns a websocket connection for this turn. - #[instrument( - name = "model_client.websocket_connection", - level = "info", - skip_all, - fields( - provider = %self.client.state.provider.name, - wire_api = %self.client.state.provider.wire_api, - transport = "responses_websocket", - api.path = "responses", - turn.has_metadata_header = params.turn_metadata_header.is_some() - ) - )] + #[allow(clippy::too_many_arguments)] async fn websocket_connection( &mut self, - params: WebsocketConnectParams<'_>, + session_telemetry: &SessionTelemetry, + api_provider: codex_api::Provider, + api_auth: CoreAuthProvider, + turn_metadata_header: Option<&str>, + options: &ApiResponsesOptions, + auth_context: AuthRequestTelemetryContext, + endpoint_telemetry: EndpointConfigTelemetry, + provider_header_names: Option<&str>, + request_route_telemetry: RequestRouteTelemetry, ) -> std::result::Result<&ApiWebSocketConnection, ApiError> { - let WebsocketConnectParams { - session_telemetry, - api_provider, - api_auth, - turn_metadata_header, - options, - auth_context, - request_route_telemetry, - } = params; let needs_new = match self.websocket_session.connection.as_ref() { Some(conn) => conn.is_closed().await, None => true, @@ -942,7 +1049,7 @@ impl ModelClientSession { .turn_state .clone() .unwrap_or_else(|| Arc::clone(&self.turn_state)); - let new_conn = match self + let new_conn = self .client .connect_websocket( session_telemetry, @@ -951,24 +1058,15 @@ impl ModelClientSession { Some(turn_state), turn_metadata_header, auth_context, + endpoint_telemetry, + provider_header_names, request_route_telemetry, ) - .await - { - Ok(new_conn) => new_conn, - Err(err) => { - if matches!(err, ApiError::Transport(TransportError::Timeout)) { - self.reset_websocket_session(); - } - return Err(err); - } - }; + .await?; self.websocket_session.connection = Some(new_conn); - self.websocket_session - .set_connection_reused(/*connection_reused*/ false); + self.websocket_session.set_connection_reused(false); } else { - self.websocket_session - .set_connection_reused(/*connection_reused*/ true); + self.websocket_session.set_connection_reused(true); } self.websocket_session @@ -995,19 +1093,6 @@ impl ModelClientSession { /// Handles SSE fixtures, reasoning summaries, verbosity, and the /// `text` controls used for output schemas. #[allow(clippy::too_many_arguments)] - #[instrument( - name = "model_client.stream_responses_api", - level = "info", - skip_all, - fields( - model = %model_info.slug, - wire_api = %self.client.state.provider.wire_api, - transport = "responses_http", - http.method = "POST", - api.path = "responses", - turn.has_metadata_header = turn_metadata_header.is_some() - ) - )] async fn stream_responses_api( &self, prompt: &Prompt, @@ -1045,8 +1130,10 @@ impl ModelClientSession { let (request_telemetry, sse_telemetry) = Self::build_streaming_telemetry( session_telemetry, request_auth_context, - RequestRouteTelemetry::for_endpoint(RESPONSES_ENDPOINT), + client_setup.endpoint_telemetry, self.client.state.auth_env_telemetry.clone(), + client_setup.provider_header_names.clone(), + RequestRouteTelemetry::for_endpoint(RESPONSES_ENDPOINT), ); let compression = self.responses_request_compression(client_setup.auth.as_ref()); let options = self.build_responses_options(turn_metadata_header, compression); @@ -1092,19 +1179,6 @@ impl ModelClientSession { /// Streams a turn via the Responses API over WebSocket transport. #[allow(clippy::too_many_arguments)] - #[instrument( - name = "model_client.stream_responses_websocket", - level = "info", - skip_all, - fields( - model = %model_info.slug, - wire_api = %self.client.state.provider.wire_api, - transport = "responses_websocket", - api.path = "responses", - turn.has_metadata_header = turn_metadata_header.is_some(), - websocket.warmup = warmup - ) - )] async fn stream_responses_websocket( &mut self, prompt: &Prompt, @@ -1149,17 +1223,17 @@ impl ModelClientSession { } match self - .websocket_connection(WebsocketConnectParams { + .websocket_connection( session_telemetry, - api_provider: client_setup.api_provider, - api_auth: client_setup.api_auth, + client_setup.api_provider, + client_setup.api_auth, turn_metadata_header, - options: &options, - auth_context: request_auth_context, - request_route_telemetry: RequestRouteTelemetry::for_endpoint( - RESPONSES_ENDPOINT, - ), - }) + &options, + request_auth_context, + client_setup.endpoint_telemetry, + client_setup.provider_header_names.as_deref(), + RequestRouteTelemetry::for_endpoint(RESPONSES_ENDPOINT), + ) .await { Ok(_) => {} @@ -1186,12 +1260,15 @@ impl ModelClientSession { let ws_request = self.prepare_websocket_request(ws_payload, &request); self.websocket_session.last_request = Some(request); - let stream_result = self.websocket_session.connection.as_ref().ok_or_else(|| { - map_api_error(ApiError::Stream( - "websocket connection is unavailable".to_string(), - )) - })?; - let stream_result = stream_result + let stream_result = self + .websocket_session + .connection + .as_ref() + .ok_or_else(|| { + map_api_error(ApiError::Stream( + "websocket connection is unavailable".to_string(), + )) + })? .stream_request(ws_request, self.websocket_session.connection_reused()) .await .map_err(map_api_error)?; @@ -1206,14 +1283,18 @@ impl ModelClientSession { fn build_streaming_telemetry( session_telemetry: &SessionTelemetry, auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, + endpoint_telemetry: EndpointConfigTelemetry, auth_env_telemetry: AuthEnvTelemetry, + provider_header_names: Option, + request_route_telemetry: RequestRouteTelemetry, ) -> (Arc, Arc) { let telemetry = Arc::new(ApiTelemetry::new( session_telemetry.clone(), auth_context, - request_route_telemetry, + endpoint_telemetry, auth_env_telemetry, + provider_header_names, + request_route_telemetry, )); let request_telemetry: Arc = telemetry.clone(); let sse_telemetry: Arc = telemetry; @@ -1224,14 +1305,18 @@ impl ModelClientSession { fn build_websocket_telemetry( session_telemetry: &SessionTelemetry, auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, + endpoint_telemetry: EndpointConfigTelemetry, auth_env_telemetry: AuthEnvTelemetry, + provider_header_names: Option, + request_route_telemetry: RequestRouteTelemetry, ) -> Arc { let telemetry = Arc::new(ApiTelemetry::new( session_telemetry.clone(), auth_context, - request_route_telemetry, + endpoint_telemetry, auth_env_telemetry, + provider_header_names, + request_route_telemetry, )); let websocket_telemetry: Arc = telemetry; websocket_telemetry @@ -1264,7 +1349,7 @@ impl ModelClientSession { summary, service_tier, turn_metadata_header, - /*warmup*/ true, + true, ) .await { @@ -1317,7 +1402,7 @@ impl ModelClientSession { summary, service_tier, turn_metadata_header, - /*warmup*/ false, + false, ) .await? { @@ -1353,10 +1438,21 @@ impl ModelClientSession { session_telemetry: &SessionTelemetry, model_info: &ModelInfo, ) -> bool { - let activated = self - .client - .force_http_fallback(session_telemetry, model_info); - self.websocket_session = WebsocketSession::default(); + let websocket_enabled = self.client.responses_websocket_enabled(model_info); + let activated = self.activate_http_fallback(websocket_enabled); + if activated { + warn!("falling back to HTTP"); + session_telemetry.counter( + "codex.transport.fallback_to_http", + 1, + &[("from_wire_api", "responses_websocket")], + ); + + self.websocket_session.connection = None; + self.websocket_session.last_request = None; + self.websocket_session.last_response_rx = None; + self.websocket_session.set_connection_reused(false); + } activated } } @@ -1547,16 +1643,6 @@ impl AuthRequestTelemetryContext { } } -struct WebsocketConnectParams<'a> { - session_telemetry: &'a SessionTelemetry, - api_provider: codex_api::Provider, - api_auth: CoreAuthProvider, - turn_metadata_header: Option<&'a str>, - options: &'a ApiResponsesOptions, - auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, -} - async fn handle_unauthorized( transport: TransportError, auth_recovery: &mut Option, @@ -1578,7 +1664,7 @@ async fn handle_unauthorized( debug.cf_ray.as_deref(), debug.auth_error.as_deref(), debug.auth_error_code.as_deref(), - /*recovery_reason*/ None, + None, step_result.auth_state_changed(), ); emit_feedback_auth_recovery_tags( @@ -1601,8 +1687,8 @@ async fn handle_unauthorized( debug.cf_ray.as_deref(), debug.auth_error.as_deref(), debug.auth_error_code.as_deref(), - /*recovery_reason*/ None, - /*auth_state_changed*/ None, + None, + None, ); emit_feedback_auth_recovery_tags( mode, @@ -1624,8 +1710,8 @@ async fn handle_unauthorized( debug.cf_ray.as_deref(), debug.auth_error.as_deref(), debug.auth_error_code.as_deref(), - /*recovery_reason*/ None, - /*auth_state_changed*/ None, + None, + None, ); emit_feedback_auth_recovery_tags( mode, @@ -1658,7 +1744,7 @@ async fn handle_unauthorized( debug.auth_error.as_deref(), debug.auth_error_code.as_deref(), recovery_reason, - /*auth_state_changed*/ None, + None, ); emit_feedback_auth_recovery_tags( mode, @@ -1683,22 +1769,28 @@ fn api_error_http_status(error: &ApiError) -> Option { struct ApiTelemetry { session_telemetry: SessionTelemetry, auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, + endpoint_telemetry: EndpointConfigTelemetry, auth_env_telemetry: AuthEnvTelemetry, + provider_header_names: Option, + request_route_telemetry: RequestRouteTelemetry, } impl ApiTelemetry { fn new( session_telemetry: SessionTelemetry, auth_context: AuthRequestTelemetryContext, - request_route_telemetry: RequestRouteTelemetry, + endpoint_telemetry: EndpointConfigTelemetry, auth_env_telemetry: AuthEnvTelemetry, + provider_header_names: Option, + request_route_telemetry: RequestRouteTelemetry, ) -> Self { Self { session_telemetry, auth_context, - request_route_telemetry, + endpoint_telemetry, auth_env_telemetry, + provider_header_names, + request_route_telemetry, } } } @@ -1716,48 +1808,96 @@ impl RequestTelemetry for ApiTelemetry { let debug = error .map(extract_response_debug_context) .unwrap_or_default(); - self.session_telemetry.record_api_request( - attempt, - status, - error_message.as_deref(), - duration, - self.auth_context.auth_header_attached, - self.auth_context.auth_header_name, - self.auth_context.retry_after_unauthorized, - self.auth_context.recovery_mode, - self.auth_context.recovery_phase, - self.request_route_telemetry.endpoint, - debug.request_id.as_deref(), - debug.cf_ray.as_deref(), - debug.auth_error.as_deref(), - debug.auth_error_code.as_deref(), - ); - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: self.request_route_telemetry.endpoint, - auth_header_attached: self.auth_context.auth_header_attached, - auth_header_name: self.auth_context.auth_header_name, - auth_mode: self.auth_context.auth_mode, - auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized), - auth_recovery_mode: self.auth_context.recovery_mode, - auth_recovery_phase: self.auth_context.recovery_phase, - auth_connection_reused: None, - auth_request_id: debug.request_id.as_deref(), - auth_cf_ray: debug.cf_ray.as_deref(), - auth_error: debug.auth_error.as_deref(), - auth_error_code: debug.auth_error_code.as_deref(), - auth_recovery_followup_success: self - .auth_context - .retry_after_unauthorized - .then_some(error.is_none()), - auth_recovery_followup_status: self - .auth_context - .retry_after_unauthorized - .then_some(status) - .flatten(), - }, - &self.auth_env_telemetry, - ); + self.session_telemetry + .record_api_request_with_endpoint_details( + attempt, + status, + error_message.as_deref(), + duration, + self.auth_context.auth_header_attached, + self.auth_context.auth_header_name, + self.auth_context.retry_after_unauthorized, + self.auth_context.recovery_mode, + self.auth_context.recovery_phase, + self.request_route_telemetry.endpoint, + self.request_route_telemetry.residency_header_attached, + self.request_route_telemetry.residency_header_value, + self.provider_header_names.as_deref(), + self.endpoint_telemetry.base_url_origin, + self.endpoint_telemetry.host_class, + self.endpoint_telemetry.base_url_source, + self.endpoint_telemetry.base_url_is_default, + self.auth_env_telemetry.openai_api_key_env_present, + self.auth_env_telemetry.codex_api_key_env_present, + self.auth_env_telemetry.codex_api_key_env_enabled, + self.auth_env_telemetry.provider_env_key_name.as_deref(), + self.auth_env_telemetry.provider_env_key_present, + self.auth_env_telemetry.refresh_token_url_override_present, + debug.request_id.as_deref(), + debug.cf_ray.as_deref(), + debug.auth_error.as_deref(), + debug.auth_error_code.as_deref(), + debug.error_body_class, + debug.safe_error_message, + ); + emit_feedback_request_tags(&FeedbackRequestTags { + endpoint: self.request_route_telemetry.endpoint, + auth_header_attached: self.auth_context.auth_header_attached, + auth_header_name: self.auth_context.auth_header_name, + auth_mode: self.auth_context.auth_mode, + auth_env_openai_api_key_present: self.auth_env_telemetry.openai_api_key_env_present, + auth_env_codex_api_key_present: self.auth_env_telemetry.codex_api_key_env_present, + auth_env_codex_api_key_enabled: self.auth_env_telemetry.codex_api_key_env_enabled, + auth_env_provider_key_name: self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth_env_provider_key_present: self.auth_env_telemetry.provider_env_key_present, + auth_env_refresh_token_url_override_present: self + .auth_env_telemetry + .refresh_token_url_override_present, + auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized), + auth_recovery_mode: self.auth_context.recovery_mode, + auth_recovery_phase: self.auth_context.recovery_phase, + auth_connection_reused: None, + provider_header_names: self.provider_header_names.as_deref(), + base_url_origin: self.endpoint_telemetry.base_url_origin, + host_class: self.endpoint_telemetry.host_class, + base_url_source: self.endpoint_telemetry.base_url_source, + base_url_is_default: self.endpoint_telemetry.base_url_is_default, + residency_header_attached: Some(self.request_route_telemetry.residency_header_attached), + residency_header_value: self.request_route_telemetry.residency_header_value, + auth_request_id: debug.request_id.as_deref(), + auth_cf_ray: debug.cf_ray.as_deref(), + auth_error: debug.auth_error.as_deref(), + auth_error_code: debug.auth_error_code.as_deref(), + error_body_class: debug.error_body_class, + safe_error_message: debug.safe_error_message, + geo_denial_detected: Some(debug.geo_denial_detected), + auth_recovery_followup_success: self + .auth_context + .retry_after_unauthorized + .then_some(error.is_none()), + auth_recovery_followup_status: self + .auth_context + .retry_after_unauthorized + .then_some(status) + .flatten(), + }); + if status == Some(StatusCode::UNAUTHORIZED.as_u16()) && debug.geo_denial_detected { + self.session_telemetry.record_geo_denial( + self.request_route_telemetry.endpoint, + self.auth_context.auth_header_attached, + self.auth_context.auth_header_name, + self.request_route_telemetry.residency_header_attached, + self.request_route_telemetry.residency_header_value, + self.provider_header_names.as_deref(), + status, + debug.request_id.as_deref(), + debug.cf_ray.as_deref(), + debug.auth_error.as_deref(), + debug.auth_error_code.as_deref(), + debug.error_body_class.unwrap_or_default(), + debug.safe_error_message, + ); + } } } @@ -1786,32 +1926,47 @@ impl WebsocketTelemetry for ApiTelemetry { error_message.as_deref(), connection_reused, ); - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: self.request_route_telemetry.endpoint, - auth_header_attached: self.auth_context.auth_header_attached, - auth_header_name: self.auth_context.auth_header_name, - auth_mode: self.auth_context.auth_mode, - auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized), - auth_recovery_mode: self.auth_context.recovery_mode, - auth_recovery_phase: self.auth_context.recovery_phase, - auth_connection_reused: Some(connection_reused), - auth_request_id: debug.request_id.as_deref(), - auth_cf_ray: debug.cf_ray.as_deref(), - auth_error: debug.auth_error.as_deref(), - auth_error_code: debug.auth_error_code.as_deref(), - auth_recovery_followup_success: self - .auth_context - .retry_after_unauthorized - .then_some(error.is_none()), - auth_recovery_followup_status: self - .auth_context - .retry_after_unauthorized - .then_some(status) - .flatten(), - }, - &self.auth_env_telemetry, - ); + emit_feedback_request_tags(&FeedbackRequestTags { + endpoint: self.request_route_telemetry.endpoint, + auth_header_attached: self.auth_context.auth_header_attached, + auth_header_name: self.auth_context.auth_header_name, + auth_mode: self.auth_context.auth_mode, + auth_env_openai_api_key_present: self.auth_env_telemetry.openai_api_key_env_present, + auth_env_codex_api_key_present: self.auth_env_telemetry.codex_api_key_env_present, + auth_env_codex_api_key_enabled: self.auth_env_telemetry.codex_api_key_env_enabled, + auth_env_provider_key_name: self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth_env_provider_key_present: self.auth_env_telemetry.provider_env_key_present, + auth_env_refresh_token_url_override_present: self + .auth_env_telemetry + .refresh_token_url_override_present, + auth_retry_after_unauthorized: Some(self.auth_context.retry_after_unauthorized), + auth_recovery_mode: self.auth_context.recovery_mode, + auth_recovery_phase: self.auth_context.recovery_phase, + auth_connection_reused: Some(connection_reused), + provider_header_names: self.provider_header_names.as_deref(), + base_url_origin: self.endpoint_telemetry.base_url_origin, + host_class: self.endpoint_telemetry.host_class, + base_url_source: self.endpoint_telemetry.base_url_source, + base_url_is_default: self.endpoint_telemetry.base_url_is_default, + residency_header_attached: Some(self.request_route_telemetry.residency_header_attached), + residency_header_value: self.request_route_telemetry.residency_header_value, + auth_request_id: debug.request_id.as_deref(), + auth_cf_ray: debug.cf_ray.as_deref(), + auth_error: debug.auth_error.as_deref(), + auth_error_code: debug.auth_error_code.as_deref(), + error_body_class: debug.error_body_class, + safe_error_message: debug.safe_error_message, + geo_denial_detected: Some(debug.geo_denial_detected), + auth_recovery_followup_success: self + .auth_context + .retry_after_unauthorized + .then_some(error.is_none()), + auth_recovery_followup_status: self + .auth_context + .retry_after_unauthorized + .then_some(status) + .flatten(), + }); } fn on_ws_event( diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index d367479187..9023876a18 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -26,6 +26,7 @@ use crate::compact::should_use_remote_compact_task; use crate::compact_remote::run_inline_remote_auto_compact_task; use crate::config::ManagedFeatures; use crate::connectors; +use crate::endpoint_config_telemetry::resolve_endpoint_config_telemetry_source; use crate::exec_policy::ExecPolicyManager; use crate::features::FEATURES; use crate::features::Feature; @@ -153,6 +154,7 @@ use tracing::warn; use uuid::Uuid; use crate::ModelProviderInfo; +use crate::auth_env_telemetry::collect_auth_env_telemetry; use crate::client::ModelClient; use crate::client::ModelClientSession; use crate::client_common::Prompt; @@ -1611,8 +1613,41 @@ impl Session { )], ); - session_telemetry.conversation_starts( + let auth_env_telemetry = collect_auth_env_telemetry( + &config.model_provider, + auth_manager.codex_api_key_env_enabled(), + ); + let conversation_start_base_url = if auth.is_none() + && config.model_provider.requires_openai_auth + && config.model_provider.base_url.is_none() + && !auth_env_telemetry.has_api_key_override() + { + "https://chatgpt.com/backend-api/codex".to_string() + } else { + config + .model_provider + .to_api_provider(auth.map(CodexAuth::auth_mode)) + .map(|provider| provider.base_url) + .unwrap_or_default() + }; + let endpoint_telemetry = resolve_endpoint_config_telemetry_source( + &config, + session_configuration.session_source.clone(), + ) + .classify(&conversation_start_base_url); + + session_telemetry.conversation_starts_with_endpoint_details( config.model_provider.name.as_str(), + endpoint_telemetry.base_url_origin, + endpoint_telemetry.host_class, + endpoint_telemetry.base_url_source, + endpoint_telemetry.base_url_is_default, + auth_env_telemetry.openai_api_key_env_present, + auth_env_telemetry.codex_api_key_env_present, + auth_env_telemetry.codex_api_key_env_enabled, + auth_env_telemetry.provider_env_key_name.as_deref(), + auth_env_telemetry.provider_env_key_present, + auth_env_telemetry.refresh_token_url_override_present, session_configuration.collaboration_mode.reasoning_effort(), config .model_reasoning_summary @@ -1796,10 +1831,14 @@ impl Session { network_proxy, network_approval: Arc::clone(&network_approval), state_db: state_db_ctx.clone(), - model_client: ModelClient::new( + model_client: ModelClient::new_with_endpoint_telemetry_source( Some(Arc::clone(&auth_manager)), conversation_id, session_configuration.provider.clone(), + resolve_endpoint_config_telemetry_source( + &config, + session_configuration.session_source.clone(), + ), session_configuration.session_source.clone(), config.model_verbosity, ws_version_from_features(config.as_ref()), diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index c6372de258..e354441690 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -4136,7 +4136,7 @@ model_verbosity = "high" supports_websockets: false, }; let model_provider_map = { - let mut model_provider_map = built_in_model_providers(/* openai_base_url */ None); + let mut model_provider_map = built_in_model_providers(None); model_provider_map.insert("openai-custom".to_string(), openai_custom_provider.clone()); model_provider_map }; diff --git a/codex-rs/core/src/default_client.rs b/codex-rs/core/src/default_client.rs index 3ca653ba46..aed426d4c4 100644 --- a/codex-rs/core/src/default_client.rs +++ b/codex-rs/core/src/default_client.rs @@ -30,6 +30,12 @@ pub const DEFAULT_ORIGINATOR: &str = "codex_cli_rs"; pub const CODEX_INTERNAL_ORIGINATOR_OVERRIDE_ENV_VAR: &str = "CODEX_INTERNAL_ORIGINATOR_OVERRIDE"; pub const RESIDENCY_HEADER_NAME: &str = "x-openai-internal-codex-residency"; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ResidencyHeaderTelemetry { + pub attached: bool, + pub value: Option<&'static str>, +} + #[derive(Debug, Clone)] pub struct Originator { pub value: String, @@ -89,6 +95,20 @@ pub fn set_default_client_residency_requirement(enforce_residency: Option ResidencyHeaderTelemetry { + let Ok(guard) = REQUIREMENTS_RESIDENCY.read() else { + tracing::warn!("Failed to acquire requirements residency lock"); + return ResidencyHeaderTelemetry::default(); + }; + let Some(requirement) = guard.as_ref() else { + return ResidencyHeaderTelemetry::default(); + }; + ResidencyHeaderTelemetry { + attached: true, + value: Some(residency_header_value(*requirement)), + } +} + pub fn originator() -> Originator { if let Ok(guard) = ORIGINATOR.read() && let Some(originator) = guard.as_ref() @@ -184,12 +204,6 @@ pub fn create_client() -> CodexHttpClient { CodexHttpClient::new(inner) } -/// Builds the default reqwest client used for ordinary Codex HTTP traffic. -/// -/// This starts from the standard Codex user agent, default headers, and sandbox-specific proxy -/// policy, then layers in shared custom CA handling from `CODEX_CA_CERTIFICATE` / -/// `SSL_CERT_FILE`. The function remains infallible for compatibility with existing call sites, so -/// a custom-CA or builder failure is logged and falls back to `reqwest::Client::new()`. pub fn build_reqwest_client() -> reqwest::Client { try_build_reqwest_client().unwrap_or_else(|error| { tracing::warn!(error = %error, "failed to build default reqwest client"); @@ -197,10 +211,6 @@ pub fn build_reqwest_client() -> reqwest::Client { }) } -/// Tries to build the default reqwest client used for ordinary Codex HTTP traffic. -/// -/// Callers that need a structured CA-loading failure instead of the legacy logged fallback can use -/// this method directly. pub fn try_build_reqwest_client() -> Result { let ua = get_codex_user_agent(); @@ -222,14 +232,20 @@ pub fn default_headers() -> HeaderMap { && let Some(requirement) = guard.as_ref() && !headers.contains_key(RESIDENCY_HEADER_NAME) { - let value = match requirement { - ResidencyRequirement::Us => HeaderValue::from_static("us"), - }; - headers.insert(RESIDENCY_HEADER_NAME, value); + headers.insert( + RESIDENCY_HEADER_NAME, + HeaderValue::from_static(residency_header_value(*requirement)), + ); } headers } +fn residency_header_value(requirement: ResidencyRequirement) -> &'static str { + match requirement { + ResidencyRequirement::Us => "us", + } +} + fn is_sandboxed() -> bool { std::env::var(CODEX_SANDBOX_ENV_VAR).as_deref() == Ok("seatbelt") } diff --git a/codex-rs/core/src/endpoint_config_telemetry.rs b/codex-rs/core/src/endpoint_config_telemetry.rs new file mode 100644 index 0000000000..79cff8c6ad --- /dev/null +++ b/codex-rs/core/src/endpoint_config_telemetry.rs @@ -0,0 +1,336 @@ +use crate::config::Config; +use crate::model_provider_info::LMSTUDIO_OSS_PROVIDER_ID; +use crate::model_provider_info::ModelProviderInfo; +use crate::model_provider_info::OLLAMA_OSS_PROVIDER_ID; +use crate::model_provider_info::OPENAI_PROVIDER_ID; +use codex_app_server_protocol::ConfigLayerSource; +use codex_protocol::protocol::SessionSource; +use reqwest::Url; + +const BASE_URL_ORIGIN_CHATGPT: &str = "chatgpt.com"; +const BASE_URL_ORIGIN_OPENAI_API: &str = "api.openai.com"; +const BASE_URL_ORIGIN_OPENROUTER: &str = "openrouter.ai"; +const BASE_URL_ORIGIN_CUSTOM: &str = "custom"; + +const HOST_CLASS_OPENAI_CHATGPT: &str = "openai_chatgpt"; +const HOST_CLASS_OPENAI_API: &str = "openai_api"; +const HOST_CLASS_KNOWN_THIRD_PARTY: &str = "known_third_party"; +const HOST_CLASS_CUSTOM_UNKNOWN: &str = "custom_unknown"; + +const BASE_URL_SOURCE_DEFAULT: &str = "default"; +const BASE_URL_SOURCE_ENV: &str = "env"; +const BASE_URL_SOURCE_CONFIG_TOML: &str = "config_toml"; +const BASE_URL_SOURCE_IDE_SETTINGS: &str = "ide_settings"; +const BASE_URL_SOURCE_MANAGED_CONFIG: &str = "managed_config"; +const BASE_URL_SOURCE_SESSION_FLAGS: &str = "session_flags"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EndpointConfigTelemetrySource { + pub(crate) base_url_source: &'static str, + pub(crate) base_url_is_default: bool, +} + +impl EndpointConfigTelemetrySource { + pub(crate) const fn new(base_url_source: &'static str, base_url_is_default: bool) -> Self { + Self { + base_url_source, + base_url_is_default, + } + } + + pub(crate) fn classify(self, base_url: &str) -> EndpointConfigTelemetry { + let (base_url_origin, host_class) = classify_base_url(base_url); + EndpointConfigTelemetry { + base_url_origin, + host_class, + base_url_source: self.base_url_source, + base_url_is_default: self.base_url_is_default, + } + } + + pub(crate) fn for_provider( + provider_id: &str, + provider: &ModelProviderInfo, + ) -> EndpointConfigTelemetrySource { + endpoint_source_from_provider_defaults(provider_id, provider) + } + + pub(crate) fn for_provider_without_id(provider: &ModelProviderInfo) -> Self { + let base_url_is_default = provider.base_url.is_none(); + let base_url_source = if base_url_is_default { + BASE_URL_SOURCE_DEFAULT + } else { + BASE_URL_SOURCE_CONFIG_TOML + }; + EndpointConfigTelemetrySource::new(base_url_source, base_url_is_default) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EndpointConfigTelemetry { + pub(crate) base_url_origin: &'static str, + pub(crate) host_class: &'static str, + pub(crate) base_url_source: &'static str, + pub(crate) base_url_is_default: bool, +} + +impl Default for EndpointConfigTelemetry { + fn default() -> Self { + Self { + base_url_origin: BASE_URL_ORIGIN_CUSTOM, + host_class: HOST_CLASS_CUSTOM_UNKNOWN, + base_url_source: BASE_URL_SOURCE_DEFAULT, + base_url_is_default: false, + } + } +} + +pub(crate) fn resolve_endpoint_config_telemetry_source( + config: &Config, + session_source: SessionSource, +) -> EndpointConfigTelemetrySource { + resolve_endpoint_config_telemetry_source_for_provider( + config, + config.model_provider_id.as_str(), + &config.model_provider, + session_source, + ) +} + +pub(crate) fn resolve_endpoint_config_telemetry_source_for_provider( + config: &Config, + provider_id: &str, + provider: &ModelProviderInfo, + session_source: SessionSource, +) -> EndpointConfigTelemetrySource { + let origins = config.config_layer_stack.origins(); + if provider_id == OPENAI_PROVIDER_ID + && let Some(origin) = origins.get("openai_base_url") + { + return endpoint_source_from_layer(&origin.name, session_source); + } + + let key = format!("model_providers.{provider_id}.base_url"); + if let Some(origin) = origins.get(&key) { + return endpoint_source_from_layer(&origin.name, session_source); + } + + endpoint_source_from_provider_defaults(provider_id, provider) +} + +fn endpoint_source_from_layer( + layer: &ConfigLayerSource, + session_source: SessionSource, +) -> EndpointConfigTelemetrySource { + let base_url_source = match layer { + ConfigLayerSource::SessionFlags => match session_source { + SessionSource::VSCode | SessionSource::Mcp => BASE_URL_SOURCE_IDE_SETTINGS, + SessionSource::Cli + | SessionSource::Exec + | SessionSource::SubAgent(_) + | SessionSource::Unknown => BASE_URL_SOURCE_SESSION_FLAGS, + }, + ConfigLayerSource::User { .. } | ConfigLayerSource::Project { .. } => { + BASE_URL_SOURCE_CONFIG_TOML + } + ConfigLayerSource::System { .. } + | ConfigLayerSource::Mdm { .. } + | ConfigLayerSource::LegacyManagedConfigTomlFromFile { .. } + | ConfigLayerSource::LegacyManagedConfigTomlFromMdm => BASE_URL_SOURCE_MANAGED_CONFIG, + }; + + EndpointConfigTelemetrySource::new(base_url_source, false) +} + +fn endpoint_source_from_provider_defaults( + provider_id: &str, + provider: &ModelProviderInfo, +) -> EndpointConfigTelemetrySource { + let env_source = match provider_id { + "openai" => env_var_present("OPENAI_BASE_URL"), + OLLAMA_OSS_PROVIDER_ID | LMSTUDIO_OSS_PROVIDER_ID => { + env_var_present("CODEX_OSS_BASE_URL") || env_var_present("CODEX_OSS_PORT") + } + _ => false, + }; + if env_source { + return EndpointConfigTelemetrySource::new(BASE_URL_SOURCE_ENV, false); + } + + let base_url_is_default = match provider_id { + "openai" => provider.base_url.is_none(), + OLLAMA_OSS_PROVIDER_ID | LMSTUDIO_OSS_PROVIDER_ID => true, + _ => provider.base_url.is_none(), + }; + if base_url_is_default { + return EndpointConfigTelemetrySource::new(BASE_URL_SOURCE_DEFAULT, true); + } + + EndpointConfigTelemetrySource::new(BASE_URL_SOURCE_CONFIG_TOML, false) +} + +fn env_var_present(name: &str) -> bool { + std::env::var(name) + .ok() + .is_some_and(|value| !value.trim().is_empty()) +} + +fn classify_base_url(base_url: &str) -> (&'static str, &'static str) { + let Ok(url) = Url::parse(base_url) else { + return (BASE_URL_ORIGIN_CUSTOM, HOST_CLASS_CUSTOM_UNKNOWN); + }; + let Some(host) = url.host_str().map(str::to_ascii_lowercase) else { + return (BASE_URL_ORIGIN_CUSTOM, HOST_CLASS_CUSTOM_UNKNOWN); + }; + + if matches!(host.as_str(), "chatgpt.com" | "chat.openai.com") { + if is_chatgpt_codex_path(url.path()) { + return (BASE_URL_ORIGIN_CHATGPT, HOST_CLASS_OPENAI_CHATGPT); + } + return (BASE_URL_ORIGIN_CHATGPT, HOST_CLASS_CUSTOM_UNKNOWN); + } + + if host == BASE_URL_ORIGIN_OPENAI_API { + return (BASE_URL_ORIGIN_OPENAI_API, HOST_CLASS_OPENAI_API); + } + + if host == BASE_URL_ORIGIN_OPENROUTER || host.ends_with(".openrouter.ai") { + return (BASE_URL_ORIGIN_OPENROUTER, HOST_CLASS_KNOWN_THIRD_PARTY); + } + + (BASE_URL_ORIGIN_CUSTOM, HOST_CLASS_CUSTOM_UNKNOWN) +} + +fn is_chatgpt_codex_path(path: &str) -> bool { + path == "/backend-api/codex" || path.starts_with("/backend-api/codex/") +} + +#[cfg(test)] +mod tests { + use super::EndpointConfigTelemetry; + use super::EndpointConfigTelemetrySource; + use super::endpoint_source_from_layer; + use super::endpoint_source_from_provider_defaults; + use crate::model_provider_info::WireApi; + use crate::model_provider_info::create_oss_provider_with_base_url; + use codex_app_server_protocol::ConfigLayerSource; + use codex_protocol::protocol::SessionSource; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + fn provider(base_url: Option<&str>) -> crate::ModelProviderInfo { + crate::ModelProviderInfo { + name: "test-provider".to_string(), + base_url: base_url.map(str::to_string), + env_key: None, + env_key_instructions: None, + experimental_bearer_token: None, + wire_api: crate::WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_openai_auth: true, + supports_websockets: true, + } + } + + #[test] + fn endpoint_config_telemetry_classifies_known_hosts_without_logging_custom_values() { + let source = EndpointConfigTelemetrySource::new("config_toml", false); + + assert_eq!( + source.classify("https://chatgpt.com/backend-api/codex"), + EndpointConfigTelemetry { + base_url_origin: "chatgpt.com", + host_class: "openai_chatgpt", + base_url_source: "config_toml", + base_url_is_default: false, + } + ); + assert_eq!( + source.classify("https://api.openai.com/v1"), + EndpointConfigTelemetry { + base_url_origin: "api.openai.com", + host_class: "openai_api", + base_url_source: "config_toml", + base_url_is_default: false, + } + ); + assert_eq!( + source.classify("https://openrouter.ai/api/v1"), + EndpointConfigTelemetry { + base_url_origin: "openrouter.ai", + host_class: "known_third_party", + base_url_source: "config_toml", + base_url_is_default: false, + } + ); + assert_eq!( + source.classify("https://private.example.internal/v1?token=secret"), + EndpointConfigTelemetry { + base_url_origin: "custom", + host_class: "custom_unknown", + base_url_source: "config_toml", + base_url_is_default: false, + } + ); + assert_eq!( + source.classify("https://chatgpt.com/api/codex"), + EndpointConfigTelemetry { + base_url_origin: "chatgpt.com", + host_class: "custom_unknown", + base_url_source: "config_toml", + base_url_is_default: false, + } + ); + } + + #[test] + fn endpoint_config_telemetry_source_maps_layers_and_defaults() { + assert_eq!( + endpoint_source_from_layer(&ConfigLayerSource::SessionFlags, SessionSource::VSCode), + EndpointConfigTelemetrySource::new("ide_settings", false) + ); + assert_eq!( + endpoint_source_from_layer( + &ConfigLayerSource::Project { + dot_codex_folder: AbsolutePathBuf::try_from(std::path::PathBuf::from( + "/tmp/project/.codex", + )) + .expect("absolute path"), + }, + SessionSource::Cli, + ), + EndpointConfigTelemetrySource::new("config_toml", false) + ); + assert_eq!( + endpoint_source_from_provider_defaults("openai", &provider(None)), + EndpointConfigTelemetrySource::new("default", true) + ); + assert_eq!( + endpoint_source_from_provider_defaults( + "custom", + &provider(Some("https://example.com/v1")) + ), + EndpointConfigTelemetrySource::new("config_toml", false) + ); + } + + #[test] + fn endpoint_config_telemetry_source_requires_explicit_provider_id_for_builtin_oss_defaults() { + let provider = + create_oss_provider_with_base_url("http://localhost:1234/v1", WireApi::Responses); + + assert_eq!( + EndpointConfigTelemetrySource::for_provider("lmstudio", &provider), + EndpointConfigTelemetrySource::new("default", true) + ); + assert_eq!( + EndpointConfigTelemetrySource::for_provider_without_id(&provider), + EndpointConfigTelemetrySource::new("config_toml", false) + ); + } +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 51d9fcf8d6..2dc601ac44 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -32,6 +32,7 @@ pub mod connectors; mod context_manager; mod contextual_user_message; pub mod custom_prompts; +mod endpoint_config_telemetry; pub mod env; mod environment_context; pub mod error; @@ -109,6 +110,7 @@ pub type CodexConversation = CodexThread; pub use analytics_client::AnalyticsEventsClient; pub use auth::AuthManager; pub use auth::CodexAuth; +mod auth_env_telemetry; pub mod default_client; pub mod project_doc; mod rollout; diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 737a47780d..1f9fcece19 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -15,6 +15,7 @@ use http::header::HeaderValue; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use std::collections::BTreeSet; use std::collections::HashMap; use std::fmt; use std::time::Duration; @@ -22,7 +23,6 @@ use std::time::Duration; const DEFAULT_STREAM_IDLE_TIMEOUT_MS: u64 = 300_000; const DEFAULT_STREAM_MAX_RETRIES: u64 = 5; const DEFAULT_REQUEST_MAX_RETRIES: u64 = 4; -pub(crate) const DEFAULT_WEBSOCKET_CONNECT_TIMEOUT_MS: u64 = 15_000; /// Hard cap for user-configured `stream_max_retries`. const MAX_STREAM_MAX_RETRIES: u64 = 100; /// Hard cap for user-configured `request_max_retries`. @@ -43,15 +43,6 @@ pub enum WireApi { Responses, } -impl fmt::Display for WireApi { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let value = match self { - Self::Responses => "responses", - }; - f.write_str(value) - } -} - impl<'de> Deserialize<'de> for WireApi { fn deserialize(deserializer: D) -> Result where @@ -66,6 +57,15 @@ impl<'de> Deserialize<'de> for WireApi { } } +impl fmt::Display for WireApi { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = match self { + Self::Responses => "responses", + }; + f.write_str(value) + } +} + /// Serializable representation of a provider definition. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema)] #[schemars(deny_unknown_fields)] @@ -113,10 +113,6 @@ pub struct ModelProviderInfo { /// the connection as lost. pub stream_idle_timeout_ms: Option, - /// Maximum time (in milliseconds) to wait for a websocket connection attempt before treating - /// it as failed. - pub websocket_connect_timeout_ms: Option, - /// Does this provider require an OpenAI API Key or ChatGPT login token? If true, /// user is presented with login screen on first run, and login preference and token/key /// are stored in auth.json. If false (which is the default), login screen is skipped, @@ -130,6 +126,38 @@ pub struct ModelProviderInfo { } impl ModelProviderInfo { + pub(crate) fn telemetry_header_names(&self) -> Option { + let mut names = BTreeSet::new(); + + if let Some(headers) = &self.http_headers { + for (name, value) in headers { + if let (Ok(name), Ok(_value)) = + (HeaderName::try_from(name), HeaderValue::try_from(value)) + { + names.insert(name.as_str().to_string()); + } + } + } + + if let Some(env_headers) = &self.env_http_headers { + for (header, env_var) in env_headers { + if let Ok(value) = std::env::var(env_var) + && !value.trim().is_empty() + && let (Ok(name), Ok(_value)) = + (HeaderName::try_from(header), HeaderValue::try_from(value)) + { + names.insert(name.as_str().to_string()); + } + } + } + + if names.is_empty() { + None + } else { + Some(names.into_iter().collect::>().join(",")) + } + } + fn build_header_map(&self) -> crate::error::Result { let capacity = self.http_headers.as_ref().map_or(0, HashMap::len) + self.env_http_headers.as_ref().map_or(0, HashMap::len); @@ -231,14 +259,6 @@ impl ModelProviderInfo { .map(Duration::from_millis) .unwrap_or(Duration::from_millis(DEFAULT_STREAM_IDLE_TIMEOUT_MS)) } - - /// Effective timeout for websocket connect attempts. - pub fn websocket_connect_timeout(&self) -> Duration { - self.websocket_connect_timeout_ms - .map(Duration::from_millis) - .unwrap_or(Duration::from_millis(DEFAULT_WEBSOCKET_CONNECT_TIMEOUT_MS)) - } - pub fn create_openai_provider(base_url: Option) -> ModelProviderInfo { ModelProviderInfo { name: OPENAI_PROVIDER_NAME.into(), @@ -268,7 +288,6 @@ impl ModelProviderInfo { request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - websocket_connect_timeout_ms: None, requires_openai_auth: true, supports_websockets: true, } @@ -290,14 +309,13 @@ pub fn built_in_model_providers( openai_base_url: Option, ) -> HashMap { use ModelProviderInfo as P; - let openai_provider = P::create_openai_provider(openai_base_url); // We do not want to be in the business of adjucating which third-party // providers are bundled with Codex CLI, so we only include the OpenAI and // open source ("oss") providers by default. Users are encouraged to add to // `model_providers` in config.toml to add their own providers. [ - (OPENAI_PROVIDER_ID, openai_provider), + ("openai", P::create_openai_provider(openai_base_url)), ( OLLAMA_OSS_PROVIDER_ID, create_oss_provider(DEFAULT_OLLAMA_PORT, WireApi::Responses), @@ -345,7 +363,6 @@ pub fn create_oss_provider_with_base_url(base_url: &str, wire_api: WireApi) -> M request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - websocket_connect_timeout_ms: None, requires_openai_auth: false, supports_websockets: false, } diff --git a/codex-rs/core/src/model_provider_info_tests.rs b/codex-rs/core/src/model_provider_info_tests.rs index a5309117ae..8034672493 100644 --- a/codex-rs/core/src/model_provider_info_tests.rs +++ b/codex-rs/core/src/model_provider_info_tests.rs @@ -1,6 +1,15 @@ use super::*; use pretty_assertions::assert_eq; +#[test] +fn openai_auth_provider_defaults_to_api_before_login() { + let provider = ModelProviderInfo::create_openai_provider(None); + + let api_provider = provider.to_api_provider(None).unwrap(); + + assert_eq!(api_provider.base_url, "https://api.openai.com/v1"); +} + #[test] fn test_deserialize_ollama_model_provider_toml() { let azure_provider_toml = r#" diff --git a/codex-rs/core/src/models_manager/manager.rs b/codex-rs/core/src/models_manager/manager.rs index 29a1a85767..7c736bf268 100644 --- a/codex-rs/core/src/models_manager/manager.rs +++ b/codex-rs/core/src/models_manager/manager.rs @@ -3,11 +3,12 @@ use crate::api_bridge::auth_provider_from_auth; use crate::api_bridge::map_api_error; use crate::auth::AuthManager; use crate::auth::AuthMode; -use crate::auth::CodexAuth; use crate::auth_env_telemetry::AuthEnvTelemetry; use crate::auth_env_telemetry::collect_auth_env_telemetry; use crate::config::Config; use crate::default_client::build_reqwest_client; +use crate::default_client::current_residency_header_telemetry; +use crate::endpoint_config_telemetry::EndpointConfigTelemetrySource; use crate::error::CodexErr; use crate::error::Result as CoreResult; use crate::model_provider_info::ModelProviderInfo; @@ -17,7 +18,7 @@ use crate::models_manager::model_info; use crate::response_debug_context::extract_response_debug_context; use crate::response_debug_context::telemetry_transport_error_message; use crate::util::FeedbackRequestTags; -use crate::util::emit_feedback_request_tags_with_auth_env; +use crate::util::emit_feedback_request_tags; use codex_api::ModelsClient; use codex_api::RequestTelemetry; use codex_api::ReqwestTransport; @@ -28,7 +29,6 @@ use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; use codex_protocol::openai_models::ModelsResponse; use http::HeaderMap; -use std::fmt; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -37,18 +37,26 @@ use tokio::sync::TryLockError; use tokio::time::timeout; use tracing::error; use tracing::info; -use tracing::instrument; const MODEL_CACHE_FILE: &str = "models_cache.json"; const DEFAULT_MODEL_CACHE_TTL: Duration = Duration::from_secs(300); const MODELS_REFRESH_TIMEOUT: Duration = Duration::from_secs(5); const MODELS_ENDPOINT: &str = "/models"; +const OPENAI_PROVIDER_ID: &str = "openai"; + #[derive(Clone)] struct ModelsRequestTelemetry { auth_mode: Option, auth_header_attached: bool, auth_header_name: Option<&'static str>, - auth_env: AuthEnvTelemetry, + auth_env_telemetry: AuthEnvTelemetry, + residency_header_attached: bool, + residency_header_value: Option<&'static str>, + provider_header_names: Option, + base_url_origin: &'static str, + host_class: &'static str, + base_url_source: &'static str, + base_url_is_default: bool, } impl RequestTelemetry for ModelsRequestTelemetry { @@ -59,34 +67,44 @@ impl RequestTelemetry for ModelsRequestTelemetry { error: Option<&TransportError>, duration: Duration, ) { - let success = status.is_some_and(|code| code.is_success()) && error.is_none(); let error_message = error.map(telemetry_transport_error_message); let response_debug = error .map(extract_response_debug_context) .unwrap_or_default(); let status = status.map(|status| status.as_u16()); + let success = status.is_some_and(|code| (200..=299).contains(&code)) && error.is_none(); + let success_str = if success { "true" } else { "false" }; tracing::event!( target: "codex_otel.log_only", tracing::Level::INFO, event.name = "codex.api_request", duration_ms = %duration.as_millis(), http.response.status_code = status, - success = success, + success = success_str, error.message = error_message.as_deref(), attempt = attempt, endpoint = MODELS_ENDPOINT, auth.header_attached = self.auth_header_attached, auth.header_name = self.auth_header_name, - auth.env_openai_api_key_present = self.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.auth_env.refresh_token_url_override_present, + auth.env_openai_api_key_present = self.auth_env_telemetry.openai_api_key_env_present, + auth.env_codex_api_key_present = self.auth_env_telemetry.codex_api_key_env_present, + auth.env_codex_api_key_enabled = self.auth_env_telemetry.codex_api_key_env_enabled, + auth.env_provider_key_name = self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth.env_provider_key_present = self.auth_env_telemetry.provider_env_key_present, + auth.env_refresh_token_url_override_present = self.auth_env_telemetry.refresh_token_url_override_present, + residency_header_attached = self.residency_header_attached, + residency_header_value = self.residency_header_value, + provider_header_names = self.provider_header_names.as_deref(), + base_url_origin = self.base_url_origin, + host_class = self.host_class, + base_url_source = self.base_url_source, + base_url_is_default = self.base_url_is_default, auth.request_id = response_debug.request_id.as_deref(), auth.cf_ray = response_debug.cf_ray.as_deref(), auth.error = response_debug.auth_error.as_deref(), auth.error_code = response_debug.auth_error_code.as_deref(), + error_body_class = response_debug.error_body_class, + safe_error_message = response_debug.safe_error_message, auth.mode = self.auth_mode.as_deref(), ); tracing::event!( @@ -95,43 +113,132 @@ impl RequestTelemetry for ModelsRequestTelemetry { event.name = "codex.api_request", duration_ms = %duration.as_millis(), http.response.status_code = status, - success = success, + success = success_str, error.message = error_message.as_deref(), attempt = attempt, endpoint = MODELS_ENDPOINT, auth.header_attached = self.auth_header_attached, auth.header_name = self.auth_header_name, - auth.env_openai_api_key_present = self.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.auth_env.refresh_token_url_override_present, + auth.env_openai_api_key_present = self.auth_env_telemetry.openai_api_key_env_present, + auth.env_codex_api_key_present = self.auth_env_telemetry.codex_api_key_env_present, + auth.env_codex_api_key_enabled = self.auth_env_telemetry.codex_api_key_env_enabled, + auth.env_provider_key_name = self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth.env_provider_key_present = self.auth_env_telemetry.provider_env_key_present, + auth.env_refresh_token_url_override_present = self.auth_env_telemetry.refresh_token_url_override_present, + residency_header_attached = self.residency_header_attached, + residency_header_value = self.residency_header_value, + provider_header_names = self.provider_header_names.as_deref(), + base_url_origin = self.base_url_origin, + host_class = self.host_class, + base_url_source = self.base_url_source, + base_url_is_default = self.base_url_is_default, auth.request_id = response_debug.request_id.as_deref(), auth.cf_ray = response_debug.cf_ray.as_deref(), auth.error = response_debug.auth_error.as_deref(), auth.error_code = response_debug.auth_error_code.as_deref(), + error_body_class = response_debug.error_body_class, + safe_error_message = response_debug.safe_error_message, auth.mode = self.auth_mode.as_deref(), ); - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: MODELS_ENDPOINT, - auth_header_attached: self.auth_header_attached, - auth_header_name: self.auth_header_name, - auth_mode: self.auth_mode.as_deref(), - auth_retry_after_unauthorized: None, - auth_recovery_mode: None, - auth_recovery_phase: None, - auth_connection_reused: None, - auth_request_id: response_debug.request_id.as_deref(), - auth_cf_ray: response_debug.cf_ray.as_deref(), - auth_error: response_debug.auth_error.as_deref(), - auth_error_code: response_debug.auth_error_code.as_deref(), - auth_recovery_followup_success: None, - auth_recovery_followup_status: None, - }, - &self.auth_env, - ); + emit_feedback_request_tags(&FeedbackRequestTags { + endpoint: MODELS_ENDPOINT, + auth_header_attached: self.auth_header_attached, + auth_header_name: self.auth_header_name, + auth_mode: self.auth_mode.as_deref(), + auth_env_openai_api_key_present: self.auth_env_telemetry.openai_api_key_env_present, + auth_env_codex_api_key_present: self.auth_env_telemetry.codex_api_key_env_present, + auth_env_codex_api_key_enabled: self.auth_env_telemetry.codex_api_key_env_enabled, + auth_env_provider_key_name: self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth_env_provider_key_present: self.auth_env_telemetry.provider_env_key_present, + auth_env_refresh_token_url_override_present: self + .auth_env_telemetry + .refresh_token_url_override_present, + auth_retry_after_unauthorized: None, + auth_recovery_mode: None, + auth_recovery_phase: None, + auth_connection_reused: None, + provider_header_names: self.provider_header_names.as_deref(), + base_url_origin: self.base_url_origin, + host_class: self.host_class, + base_url_source: self.base_url_source, + base_url_is_default: self.base_url_is_default, + residency_header_attached: Some(self.residency_header_attached), + residency_header_value: self.residency_header_value, + auth_request_id: response_debug.request_id.as_deref(), + auth_cf_ray: response_debug.cf_ray.as_deref(), + auth_error: response_debug.auth_error.as_deref(), + auth_error_code: response_debug.auth_error_code.as_deref(), + error_body_class: response_debug.error_body_class, + safe_error_message: response_debug.safe_error_message, + geo_denial_detected: Some(response_debug.geo_denial_detected), + auth_recovery_followup_success: None, + auth_recovery_followup_status: None, + }); + + if status == Some(http::StatusCode::UNAUTHORIZED.as_u16()) + && response_debug.geo_denial_detected + { + tracing::event!( + target: "codex_otel.log_only", + tracing::Level::INFO, + event.name = "codex.geo_denial", + geo_denial_detected = true, + request_id = response_debug.request_id.as_deref(), + cf_ray = response_debug.cf_ray.as_deref(), + endpoint = MODELS_ENDPOINT, + auth.header_attached = self.auth_header_attached, + auth.header_name = self.auth_header_name, + auth.mode = self.auth_mode.as_deref(), + auth.env_openai_api_key_present = self.auth_env_telemetry.openai_api_key_env_present, + auth.env_codex_api_key_present = self.auth_env_telemetry.codex_api_key_env_present, + auth.env_codex_api_key_enabled = self.auth_env_telemetry.codex_api_key_env_enabled, + auth.env_provider_key_name = self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth.env_provider_key_present = self.auth_env_telemetry.provider_env_key_present, + auth.env_refresh_token_url_override_present = self.auth_env_telemetry.refresh_token_url_override_present, + residency_header_attached = self.residency_header_attached, + residency_header_value = self.residency_header_value, + provider_header_names = self.provider_header_names.as_deref(), + base_url_origin = self.base_url_origin, + host_class = self.host_class, + base_url_source = self.base_url_source, + base_url_is_default = self.base_url_is_default, + http_status = status, + auth.error = response_debug.auth_error.as_deref(), + auth.error_code = response_debug.auth_error_code.as_deref(), + error_body_class = response_debug.error_body_class.unwrap_or_default(), + safe_error_message = response_debug.safe_error_message, + ); + tracing::event!( + target: "codex_otel.trace_safe", + tracing::Level::INFO, + event.name = "codex.geo_denial", + geo_denial_detected = true, + request_id = response_debug.request_id.as_deref(), + cf_ray = response_debug.cf_ray.as_deref(), + endpoint = MODELS_ENDPOINT, + auth.header_attached = self.auth_header_attached, + auth.header_name = self.auth_header_name, + auth.mode = self.auth_mode.as_deref(), + auth.env_openai_api_key_present = self.auth_env_telemetry.openai_api_key_env_present, + auth.env_codex_api_key_present = self.auth_env_telemetry.codex_api_key_env_present, + auth.env_codex_api_key_enabled = self.auth_env_telemetry.codex_api_key_env_enabled, + auth.env_provider_key_name = self.auth_env_telemetry.provider_env_key_name.as_deref(), + auth.env_provider_key_present = self.auth_env_telemetry.provider_env_key_present, + auth.env_refresh_token_url_override_present = self.auth_env_telemetry.refresh_token_url_override_present, + residency_header_attached = self.residency_header_attached, + residency_header_value = self.residency_header_value, + provider_header_names = self.provider_header_names.as_deref(), + base_url_origin = self.base_url_origin, + host_class = self.host_class, + base_url_source = self.base_url_source, + base_url_is_default = self.base_url_is_default, + http_status = status, + auth.error = response_debug.auth_error.as_deref(), + auth.error_code = response_debug.auth_error_code.as_deref(), + error_body_class = response_debug.error_body_class.unwrap_or_default(), + safe_error_message = response_debug.safe_error_message, + ); + } } } @@ -146,22 +253,6 @@ pub enum RefreshStrategy { OnlineIfUncached, } -impl RefreshStrategy { - const fn as_str(self) -> &'static str { - match self { - Self::Online => "online", - Self::Offline => "offline", - Self::OnlineIfUncached => "online_if_uncached", - } - } -} - -impl fmt::Display for RefreshStrategy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - /// How the manager's base catalog is sourced for the lifetime of the process. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CatalogMode { @@ -181,6 +272,8 @@ pub struct ModelsManager { etag: RwLock>, cache_manager: ModelsCacheManager, provider: ModelProviderInfo, + endpoint_telemetry_source: EndpointConfigTelemetrySource, + auth_env_telemetry: AuthEnvTelemetry, } impl ModelsManager { @@ -195,16 +288,16 @@ impl ModelsManager { model_catalog: Option, collaboration_modes_config: CollaborationModesConfig, ) -> Self { + let provider = ModelProviderInfo::create_openai_provider(None); Self::new_with_provider( codex_home, auth_manager, model_catalog, collaboration_modes_config, - ModelProviderInfo::create_openai_provider(/*base_url*/ None), + provider, ) } - /// Construct a manager with an explicit provider used for remote model refreshes. pub fn new_with_provider( codex_home: PathBuf, auth_manager: Arc, @@ -212,6 +305,26 @@ impl ModelsManager { collaboration_modes_config: CollaborationModesConfig, provider: ModelProviderInfo, ) -> Self { + Self::new_with_provider_and_endpoint_telemetry_source( + codex_home, + auth_manager, + model_catalog, + collaboration_modes_config, + provider.clone(), + EndpointConfigTelemetrySource::for_provider(OPENAI_PROVIDER_ID, &provider), + ) + } + + pub(crate) fn new_with_provider_and_endpoint_telemetry_source( + codex_home: PathBuf, + auth_manager: Arc, + model_catalog: Option, + collaboration_modes_config: CollaborationModesConfig, + provider: ModelProviderInfo, + endpoint_telemetry_source: EndpointConfigTelemetrySource, + ) -> Self { + let auth_env_telemetry = + collect_auth_env_telemetry(&provider, auth_manager.codex_api_key_env_enabled()); let cache_path = codex_home.join(MODEL_CACHE_FILE); let cache_manager = ModelsCacheManager::new(cache_path, DEFAULT_MODEL_CACHE_TTL); let catalog_mode = if model_catalog.is_some() { @@ -233,17 +346,14 @@ impl ModelsManager { etag: RwLock::new(None), cache_manager, provider, + endpoint_telemetry_source, + auth_env_telemetry, } } /// List all available models, refreshing according to the specified strategy. /// /// Returns model presets sorted by priority and filtered by auth mode and visibility. - #[instrument( - level = "info", - skip(self), - fields(refresh_strategy = %refresh_strategy) - )] pub async fn list_models(&self, refresh_strategy: RefreshStrategy) -> Vec { if let Err(err) = self.refresh_available_models(refresh_strategy).await { error!("failed to refresh available models: {err}"); @@ -279,14 +389,6 @@ impl ModelsManager { /// /// If `model` is provided, returns it directly. Otherwise selects the default based on /// auth mode and available models. - #[instrument( - level = "info", - skip(self, model), - fields( - model.provided = model.is_some(), - refresh_strategy = %refresh_strategy - ) - )] pub async fn get_default_model( &self, model: &Option, @@ -310,7 +412,6 @@ impl ModelsManager { // todo(aibrahim): look if we can tighten it to pub(crate) /// Look up model metadata, applying remote overrides and config adjustments. - #[instrument(level = "info", skip(self, config), fields(model = model))] pub async fn get_model_info(&self, model: &str, config: &Config) -> ModelInfo { let remote_models = self.get_remote_models().await; Self::construct_model_info_from_candidates(model, &remote_models, config) @@ -432,19 +533,26 @@ impl ModelsManager { let _timer = codex_otel::start_global_timer("codex.remote_models.fetch_update.duration_ms", &[]); let auth = self.auth_manager.auth().await; - let auth_mode = auth.as_ref().map(CodexAuth::auth_mode); + let auth_mode = self.auth_manager.auth_mode(); let api_provider = self.provider.to_api_provider(auth_mode)?; let api_auth = auth_provider_from_auth(auth.clone(), &self.provider)?; - let auth_env = collect_auth_env_telemetry( - &self.provider, - self.auth_manager.codex_api_key_env_enabled(), - ); let transport = ReqwestTransport::new(build_reqwest_client()); + let endpoint_telemetry = self + .endpoint_telemetry_source + .classify(api_provider.base_url.as_str()); + let residency = current_residency_header_telemetry(); let request_telemetry: Arc = Arc::new(ModelsRequestTelemetry { auth_mode: auth_mode.map(|mode| TelemetryAuthMode::from(mode).to_string()), auth_header_attached: api_auth.auth_header_attached(), auth_header_name: api_auth.auth_header_name(), - auth_env, + auth_env_telemetry: self.auth_env_telemetry.clone(), + residency_header_attached: residency.attached, + residency_header_value: residency.value, + provider_header_names: self.provider.telemetry_header_names(), + base_url_origin: endpoint_telemetry.base_url_origin, + host_class: endpoint_telemetry.host_class, + base_url_source: endpoint_telemetry.base_url_source, + base_url_is_default: endpoint_telemetry.base_url_is_default, }); let client = ModelsClient::new(transport, api_provider, api_auth) .with_telemetry(Some(request_telemetry)); @@ -543,13 +651,28 @@ impl ModelsManager { auth_manager: Arc, provider: ModelProviderInfo, ) -> Self { - Self::new_with_provider( - codex_home, + let cache_path = codex_home.join(MODEL_CACHE_FILE); + let cache_manager = ModelsCacheManager::new(cache_path, DEFAULT_MODEL_CACHE_TTL); + let auth_env_telemetry = + collect_auth_env_telemetry(&provider, auth_manager.codex_api_key_env_enabled()); + Self { + remote_models: RwLock::new( + Self::load_remote_models_from_file() + .unwrap_or_else(|err| panic!("failed to load bundled models.json: {err}")), + ), + catalog_mode: CatalogMode::Default, + collaboration_modes_config: CollaborationModesConfig::default(), auth_manager, - /*model_catalog*/ None, - CollaborationModesConfig::default(), + etag: RwLock::new(None), + cache_manager, + endpoint_telemetry_source: if provider.is_openai() { + EndpointConfigTelemetrySource::for_provider(OPENAI_PROVIDER_ID, &provider) + } else { + EndpointConfigTelemetrySource::for_provider_without_id(&provider) + }, + auth_env_telemetry, provider, - ) + } } /// Get model identifier without consulting remote state or cache. diff --git a/codex-rs/core/src/response_debug_context.rs b/codex-rs/core/src/response_debug_context.rs index bc7eab172b..1fd57ef66d 100644 --- a/codex-rs/core/src/response_debug_context.rs +++ b/codex-rs/core/src/response_debug_context.rs @@ -7,6 +7,11 @@ const OAI_REQUEST_ID_HEADER: &str = "x-oai-request-id"; const CF_RAY_HEADER: &str = "cf-ray"; const AUTH_ERROR_HEADER: &str = "x-openai-authorization-error"; const X_ERROR_JSON_HEADER: &str = "x-error-json"; +const WORKSPACE_NOT_AUTHORIZED_IN_REGION_MESSAGE: &str = + "Workspace is not authorized in this region."; +pub(crate) const WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS: &str = + "workspace_not_authorized_in_region"; +const MAX_ERROR_BODY_BYTES: usize = 1000; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub(crate) struct ResponseDebugContext { @@ -14,15 +19,15 @@ pub(crate) struct ResponseDebugContext { pub(crate) cf_ray: Option, pub(crate) auth_error: Option, pub(crate) auth_error_code: Option, + pub(crate) safe_error_message: Option<&'static str>, + pub(crate) error_body_class: Option<&'static str>, + pub(crate) geo_denial_detected: bool, } pub(crate) fn extract_response_debug_context(transport: &TransportError) -> ResponseDebugContext { let mut context = ResponseDebugContext::default(); - let TransportError::Http { - headers, body: _, .. - } = transport - else { + let TransportError::Http { headers, body, .. } = transport else { return context; }; @@ -49,6 +54,14 @@ pub(crate) fn extract_response_debug_context(transport: &TransportError) -> Resp .and_then(serde_json::Value::as_str) .map(str::to_string) }); + let error_body = extract_error_body(body.as_deref()); + context.safe_error_message = error_body + .as_deref() + .and_then(allowlisted_error_body_message); + context.error_body_class = error_body.as_deref().and_then(classify_error_body_message); + context.geo_denial_detected = context.error_body_class + == Some(WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS) + || context.auth_error_code.as_deref() == Some(WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS); context } @@ -87,9 +100,75 @@ pub(crate) fn telemetry_api_error_message(error: &ApiError) -> String { } } +fn extract_error_body(body: Option<&str>) -> Option { + let body = body?; + if let Some(message) = extract_error_message(body) { + return Some(message); + } + + let trimmed = body.trim(); + if trimmed.is_empty() { + return None; + } + + Some(truncate_with_ellipsis(trimmed, MAX_ERROR_BODY_BYTES)) +} + +fn extract_error_message(body: &str) -> Option { + let json = serde_json::from_str::(body).ok()?; + let message = json + .get("error") + .and_then(|error| error.get("message")) + .and_then(serde_json::Value::as_str)?; + let message = message.trim(); + if message.is_empty() { + None + } else { + Some(message.to_string()) + } +} + +fn classify_error_body_message(message: &str) -> Option<&'static str> { + if message == WORKSPACE_NOT_AUTHORIZED_IN_REGION_MESSAGE { + Some(WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS) + } else { + None + } +} + +fn allowlisted_error_body_message(message: &str) -> Option<&'static str> { + if message == WORKSPACE_NOT_AUTHORIZED_IN_REGION_MESSAGE { + Some(WORKSPACE_NOT_AUTHORIZED_IN_REGION_MESSAGE) + } else { + None + } +} + +fn truncate_with_ellipsis(input: &str, max_bytes: usize) -> String { + if input.len() <= max_bytes { + return input.to_string(); + } + + let ellipsis = "..."; + let keep = max_bytes.saturating_sub(ellipsis.len()); + let mut truncated = String::new(); + let mut used = 0usize; + for ch in input.chars() { + let len = ch.len_utf8(); + if used + len > keep { + break; + } + truncated.push(ch); + used += len; + } + truncated.push_str(ellipsis); + truncated +} + #[cfg(test)] mod tests { use super::ResponseDebugContext; + use super::WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS; use super::extract_response_debug_context; use super::telemetry_api_error_message; use super::telemetry_transport_error_message; @@ -101,33 +180,69 @@ mod tests { use pretty_assertions::assert_eq; #[test] - fn extract_response_debug_context_decodes_identity_headers() { + fn extract_response_debug_context_decodes_geo_denial_details() { let mut headers = HeaderMap::new(); - headers.insert("x-oai-request-id", HeaderValue::from_static("req-auth")); - headers.insert("cf-ray", HeaderValue::from_static("ray-auth")); - headers.insert( - "x-openai-authorization-error", - HeaderValue::from_static("missing_authorization_header"), - ); + headers.insert("x-oai-request-id", HeaderValue::from_static("req-geo")); + headers.insert("cf-ray", HeaderValue::from_static("ray-geo")); headers.insert( "x-error-json", - HeaderValue::from_static("eyJlcnJvciI6eyJjb2RlIjoidG9rZW5fZXhwaXJlZCJ9fQ=="), + HeaderValue::from_static( + "eyJlcnJvciI6eyJjb2RlIjoid29ya3NwYWNlX25vdF9hdXRob3JpemVkX2luX3JlZ2lvbiJ9fQ==", + ), ); let context = extract_response_debug_context(&TransportError::Http { status: StatusCode::UNAUTHORIZED, - url: Some("https://chatgpt.com/backend-api/codex/models".to_string()), + url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()), headers: Some(headers), - body: Some(r#"{"error":{"message":"plain text error"},"status":401}"#.to_string()), + body: Some( + r#"{"error":{"message":"Workspace is not authorized in this region."},"status":401}"# + .to_string(), + ), }); assert_eq!( context, ResponseDebugContext { - request_id: Some("req-auth".to_string()), - cf_ray: Some("ray-auth".to_string()), - auth_error: Some("missing_authorization_header".to_string()), - auth_error_code: Some("token_expired".to_string()), + request_id: Some("req-geo".to_string()), + cf_ray: Some("ray-geo".to_string()), + auth_error: None, + auth_error_code: Some("workspace_not_authorized_in_region".to_string()), + safe_error_message: Some("Workspace is not authorized in this region."), + error_body_class: Some(WORKSPACE_NOT_AUTHORIZED_IN_REGION_CLASS), + geo_denial_detected: true, + } + ); + } + + #[test] + fn extract_response_debug_context_detects_geo_denial_from_error_code_without_body_message() { + let mut headers = HeaderMap::new(); + headers.insert("x-oai-request-id", HeaderValue::from_static("req-geo-code")); + headers.insert( + "x-error-json", + HeaderValue::from_static( + "eyJlcnJvciI6eyJjb2RlIjoid29ya3NwYWNlX25vdF9hdXRob3JpemVkX2luX3JlZ2lvbiJ9fQ==", + ), + ); + + let context = extract_response_debug_context(&TransportError::Http { + status: StatusCode::UNAUTHORIZED, + url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()), + headers: Some(headers), + body: Some(String::new()), + }); + + assert_eq!( + context, + ResponseDebugContext { + request_id: Some("req-geo-code".to_string()), + cf_ray: None, + auth_error: None, + auth_error_code: Some("workspace_not_authorized_in_region".to_string()), + safe_error_message: None, + error_body_class: None, + geo_denial_detected: true, } ); } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 58b3e30c0d..f725a04084 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1,7 +1,6 @@ use crate::AuthManager; use crate::CodexAuth; use crate::ModelProviderInfo; -use crate::OPENAI_PROVIDER_ID; use crate::agent::AgentControl; use crate::codex::Codex; use crate::codex::CodexSpawnArgs; @@ -9,6 +8,7 @@ use crate::codex::CodexSpawnOk; use crate::codex::INITIAL_SUBMIT_ID; use crate::codex_thread::CodexThread; use crate::config::Config; +use crate::endpoint_config_telemetry::resolve_endpoint_config_telemetry_source_for_provider; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::file_watcher::FileWatcher; @@ -47,6 +47,8 @@ use tokio::sync::RwLock; use tokio::sync::broadcast; use tracing::warn; +use crate::model_provider_info::OPENAI_PROVIDER_ID; + const THREAD_CREATED_CHANNEL_CAPACITY: usize = 1024; /// Test-only override for enabling thread-manager behaviors used by integration /// tests. @@ -173,7 +175,14 @@ impl ThreadManager { .model_providers .get(OPENAI_PROVIDER_ID) .cloned() - .unwrap_or_else(|| ModelProviderInfo::create_openai_provider(/*base_url*/ None)); + .unwrap_or_else(|| ModelProviderInfo::create_openai_provider(None)); + let openai_endpoint_telemetry_source = + resolve_endpoint_config_telemetry_source_for_provider( + config, + OPENAI_PROVIDER_ID, + &openai_models_provider, + session_source.clone(), + ); let (thread_created_tx, _) = broadcast::channel(THREAD_CREATED_CHANNEL_CAPACITY); let plugins_manager = Arc::new(PluginsManager::new(codex_home.clone())); let mcp_manager = Arc::new(McpManager::new(Arc::clone(&plugins_manager))); @@ -187,13 +196,16 @@ impl ThreadManager { state: Arc::new(ThreadManagerState { threads: Arc::new(RwLock::new(HashMap::new())), thread_created_tx, - models_manager: Arc::new(ModelsManager::new_with_provider( - codex_home, - auth_manager.clone(), - config.model_catalog.clone(), - collaboration_modes_config, - openai_models_provider, - )), + models_manager: Arc::new( + ModelsManager::new_with_provider_and_endpoint_telemetry_source( + codex_home, + auth_manager.clone(), + config.model_catalog.clone(), + collaboration_modes_config, + openai_models_provider, + openai_endpoint_telemetry_source, + ), + ), skills_manager, plugins_manager, mcp_manager, @@ -213,7 +225,7 @@ impl ThreadManager { auth: CodexAuth, provider: ModelProviderInfo, ) -> Self { - set_thread_manager_test_mode_for_tests(/*enabled*/ true); + set_thread_manager_test_mode_for_tests(true); let codex_home = std::env::temp_dir().join(format!( "codex-thread-manager-test-{}", uuid::Uuid::new_v4() @@ -233,7 +245,7 @@ impl ThreadManager { provider: ModelProviderInfo, codex_home: PathBuf, ) -> Self { - set_thread_manager_test_mode_for_tests(/*enabled*/ true); + set_thread_manager_test_mode_for_tests(true); let auth_manager = AuthManager::from_auth_for_testing(auth); let (thread_created_tx, _) = broadcast::channel(THREAD_CREATED_CHANNEL_CAPACITY); let plugins_manager = Arc::new(PluginsManager::new(codex_home.clone())); @@ -241,7 +253,7 @@ impl ThreadManager { let skills_manager = Arc::new(SkillsManager::new( codex_home.clone(), Arc::clone(&plugins_manager), - /*bundled_skills_enabled*/ true, + true, )); let file_watcher = build_file_watcher(codex_home.clone(), Arc::clone(&skills_manager)); Self { @@ -340,12 +352,7 @@ impl ThreadManager { pub async fn start_thread(&self, config: Config) -> CodexResult { // Box delegated thread-spawn futures so these convenience wrappers do // not inline the full spawn path into every caller's async state. - Box::pin(self.start_thread_with_tools( - config, - Vec::new(), - /*persist_extended_history*/ false, - )) - .await + Box::pin(self.start_thread_with_tools(config, Vec::new(), false)).await } pub async fn start_thread_with_tools( @@ -358,8 +365,8 @@ impl ThreadManager { config, dynamic_tools, persist_extended_history, - /*metrics_service_name*/ None, - /*parent_trace*/ None, + None, + None, )) .await } @@ -381,7 +388,6 @@ impl ThreadManager { persist_extended_history, metrics_service_name, parent_trace, - /*user_shell_override*/ None, )) .await } @@ -398,7 +404,7 @@ impl ThreadManager { config, initial_history, auth_manager, - /*persist_extended_history*/ false, + false, parent_trace, )) .await @@ -419,50 +425,8 @@ impl ThreadManager { self.agent_control(), Vec::new(), persist_extended_history, - /*metrics_service_name*/ None, + None, parent_trace, - /*user_shell_override*/ None, - )) - .await - } - - pub(crate) async fn start_thread_with_user_shell_override_for_tests( - &self, - config: Config, - user_shell_override: crate::shell::Shell, - ) -> CodexResult { - Box::pin(self.state.spawn_thread( - config, - InitialHistory::New, - Arc::clone(&self.state.auth_manager), - self.agent_control(), - Vec::new(), - /*persist_extended_history*/ false, - /*metrics_service_name*/ None, - /*parent_trace*/ None, - /*user_shell_override*/ Some(user_shell_override), - )) - .await - } - - pub(crate) async fn resume_thread_from_rollout_with_user_shell_override_for_tests( - &self, - config: Config, - rollout_path: PathBuf, - auth_manager: Arc, - user_shell_override: crate::shell::Shell, - ) -> CodexResult { - let initial_history = RolloutRecorder::get_rollout_history(&rollout_path).await?; - Box::pin(self.state.spawn_thread( - config, - initial_history, - auth_manager, - self.agent_control(), - Vec::new(), - /*persist_extended_history*/ false, - /*metrics_service_name*/ None, - /*parent_trace*/ None, - /*user_shell_override*/ Some(user_shell_override), )) .await } @@ -546,9 +510,8 @@ impl ThreadManager { self.agent_control(), Vec::new(), persist_extended_history, - /*metrics_service_name*/ None, + None, parent_trace, - /*user_shell_override*/ None, )) .await } @@ -607,9 +570,9 @@ impl ThreadManagerState { config, agent_control, self.session_source.clone(), - /*persist_extended_history*/ false, - /*metrics_service_name*/ None, - /*inherited_shell_snapshot*/ None, + false, + None, + None, )) .await } @@ -633,8 +596,7 @@ impl ThreadManagerState { persist_extended_history, metrics_service_name, inherited_shell_snapshot, - /*parent_trace*/ None, - /*user_shell_override*/ None, + None, )) .await } @@ -655,11 +617,10 @@ impl ThreadManagerState { agent_control, session_source, Vec::new(), - /*persist_extended_history*/ false, - /*metrics_service_name*/ None, + false, + None, inherited_shell_snapshot, - /*parent_trace*/ None, - /*user_shell_override*/ None, + None, )) .await } @@ -681,10 +642,9 @@ impl ThreadManagerState { session_source, Vec::new(), persist_extended_history, - /*metrics_service_name*/ None, + None, inherited_shell_snapshot, - /*parent_trace*/ None, - /*user_shell_override*/ None, + None, )) .await } @@ -701,7 +661,6 @@ impl ThreadManagerState { persist_extended_history: bool, metrics_service_name: Option, parent_trace: Option, - user_shell_override: Option, ) -> CodexResult { Box::pin(self.spawn_thread_with_source( config, @@ -712,9 +671,8 @@ impl ThreadManagerState { dynamic_tools, persist_extended_history, metrics_service_name, - /*inherited_shell_snapshot*/ None, + None, parent_trace, - user_shell_override, )) .await } @@ -732,7 +690,6 @@ impl ThreadManagerState { metrics_service_name: Option, inherited_shell_snapshot: Option>, parent_trace: Option, - user_shell_override: Option, ) -> CodexResult { let watch_registration = self .file_watcher @@ -754,7 +711,6 @@ impl ThreadManagerState { persist_extended_history, metrics_service_name, inherited_shell_snapshot, - user_shell_override, parent_trace, }) .await?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index 99afe8ac25..e4596febf1 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -59,7 +59,7 @@ fn function_payload(args: serde_json::Value) -> ToolPayload { fn thread_manager() -> ThreadManager { ThreadManager::with_models_provider_for_tests( CodexAuth::from_api_key("dummy"), - built_in_model_providers(/* openai_base_url */ None)["openai"].clone(), + built_in_model_providers(None)["openai"].clone(), ) } @@ -164,7 +164,7 @@ async fn spawn_agent_uses_explorer_role_and_preserves_approval_policy() { let manager = thread_manager(); session.services.agent_control = manager.agent_control(); let mut config = (*turn.config).clone(); - let provider = built_in_model_providers(/* openai_base_url */ None)["ollama"].clone(); + let provider = built_in_model_providers(None)["ollama"].clone(); config.model_provider_id = "ollama".to_string(); config.model_provider = provider.clone(); config diff --git a/codex-rs/core/src/util.rs b/codex-rs/core/src/util.rs index 1dbd6a84fc..c7dac9ee78 100644 --- a/codex-rs/core/src/util.rs +++ b/codex-rs/core/src/util.rs @@ -7,7 +7,6 @@ use rand::Rng; use tracing::debug; use tracing::error; -use crate::auth_env_telemetry::AuthEnvTelemetry; use crate::parse_command::shlex_join; const INITIAL_DELAY_MS: u64 = 200; @@ -43,35 +42,34 @@ pub(crate) struct FeedbackRequestTags<'a> { pub auth_header_attached: bool, pub auth_header_name: Option<&'a str>, pub auth_mode: Option<&'a str>, + pub auth_env_openai_api_key_present: bool, + pub auth_env_codex_api_key_present: bool, + pub auth_env_codex_api_key_enabled: bool, + pub auth_env_provider_key_name: Option<&'a str>, + pub auth_env_provider_key_present: Option, + pub auth_env_refresh_token_url_override_present: bool, pub auth_retry_after_unauthorized: Option, pub auth_recovery_mode: Option<&'a str>, pub auth_recovery_phase: Option<&'a str>, pub auth_connection_reused: Option, + pub provider_header_names: Option<&'a str>, + pub base_url_origin: &'a str, + pub host_class: &'a str, + pub base_url_source: &'a str, + pub base_url_is_default: bool, + pub residency_header_attached: Option, + pub residency_header_value: Option<&'a str>, pub auth_request_id: Option<&'a str>, pub auth_cf_ray: Option<&'a str>, pub auth_error: Option<&'a str>, pub auth_error_code: Option<&'a str>, + pub error_body_class: Option<&'a str>, + pub safe_error_message: Option<&'a str>, + pub geo_denial_detected: Option, pub auth_recovery_followup_success: Option, pub auth_recovery_followup_status: Option, } -struct FeedbackRequestSnapshot<'a> { - endpoint: &'a str, - auth_header_attached: bool, - auth_header_name: &'a str, - auth_mode: &'a str, - auth_retry_after_unauthorized: String, - auth_recovery_mode: &'a str, - auth_recovery_phase: &'a str, - auth_connection_reused: String, - auth_request_id: &'a str, - auth_cf_ray: &'a str, - auth_error: &'a str, - auth_error_code: &'a str, - auth_recovery_followup_success: String, - auth_recovery_followup_status: String, -} - struct Auth401FeedbackSnapshot<'a> { request_id: &'a str, cf_ray: &'a str, @@ -95,84 +93,73 @@ impl<'a> Auth401FeedbackSnapshot<'a> { } } -impl<'a> FeedbackRequestSnapshot<'a> { - fn from_tags(tags: &'a FeedbackRequestTags<'a>) -> Self { - Self { - endpoint: tags.endpoint, - auth_header_attached: tags.auth_header_attached, - auth_header_name: tags.auth_header_name.unwrap_or(""), - auth_mode: tags.auth_mode.unwrap_or(""), - auth_retry_after_unauthorized: tags - .auth_retry_after_unauthorized - .map_or_else(String::new, |value| value.to_string()), - auth_recovery_mode: tags.auth_recovery_mode.unwrap_or(""), - auth_recovery_phase: tags.auth_recovery_phase.unwrap_or(""), - auth_connection_reused: tags - .auth_connection_reused - .map_or_else(String::new, |value| value.to_string()), - auth_request_id: tags.auth_request_id.unwrap_or(""), - auth_cf_ray: tags.auth_cf_ray.unwrap_or(""), - auth_error: tags.auth_error.unwrap_or(""), - auth_error_code: tags.auth_error_code.unwrap_or(""), - auth_recovery_followup_success: tags - .auth_recovery_followup_success - .map_or_else(String::new, |value| value.to_string()), - auth_recovery_followup_status: tags - .auth_recovery_followup_status - .map_or_else(String::new, |value| value.to_string()), - } - } -} - -#[cfg(test)] pub(crate) fn emit_feedback_request_tags(tags: &FeedbackRequestTags<'_>) { - let snapshot = FeedbackRequestSnapshot::from_tags(tags); + let auth_header_name = tags.auth_header_name.unwrap_or(""); + let auth_mode = tags.auth_mode.unwrap_or(""); + let auth_env_provider_key_name = tags.auth_env_provider_key_name.unwrap_or(""); + let auth_env_provider_key_present = tags + .auth_env_provider_key_present + .map_or_else(String::new, |value| value.to_string()); + let auth_retry_after_unauthorized = tags + .auth_retry_after_unauthorized + .map_or_else(String::new, |value| value.to_string()); + let auth_recovery_mode = tags.auth_recovery_mode.unwrap_or(""); + let auth_recovery_phase = tags.auth_recovery_phase.unwrap_or(""); + let auth_connection_reused = tags + .auth_connection_reused + .map_or_else(String::new, |value| value.to_string()); + let provider_header_names = tags.provider_header_names.unwrap_or(""); + let residency_header_attached = tags + .residency_header_attached + .map_or_else(String::new, |value| value.to_string()); + let residency_header_value = tags.residency_header_value.unwrap_or(""); + let auth_request_id = tags.auth_request_id.unwrap_or(""); + let auth_cf_ray = tags.auth_cf_ray.unwrap_or(""); + let auth_error = tags.auth_error.unwrap_or(""); + let auth_error_code = tags.auth_error_code.unwrap_or(""); + let error_body_class = tags.error_body_class.unwrap_or(""); + let safe_error_message = tags.safe_error_message.unwrap_or(""); + let geo_denial_detected = tags + .geo_denial_detected + .map_or_else(String::new, |value| value.to_string()); + let auth_recovery_followup_success = tags + .auth_recovery_followup_success + .map_or_else(String::new, |value| value.to_string()); + let auth_recovery_followup_status = tags + .auth_recovery_followup_status + .map_or_else(String::new, |value| value.to_string()); feedback_tags!( - endpoint = snapshot.endpoint, - auth_header_attached = snapshot.auth_header_attached, - auth_header_name = snapshot.auth_header_name, - auth_mode = snapshot.auth_mode, - auth_retry_after_unauthorized = snapshot.auth_retry_after_unauthorized, - auth_recovery_mode = snapshot.auth_recovery_mode, - auth_recovery_phase = snapshot.auth_recovery_phase, - auth_connection_reused = snapshot.auth_connection_reused, - auth_request_id = snapshot.auth_request_id, - auth_cf_ray = snapshot.auth_cf_ray, - auth_error = snapshot.auth_error, - auth_error_code = snapshot.auth_error_code, - auth_recovery_followup_success = snapshot.auth_recovery_followup_success, - auth_recovery_followup_status = snapshot.auth_recovery_followup_status - ); -} - -pub(crate) fn emit_feedback_request_tags_with_auth_env( - tags: &FeedbackRequestTags<'_>, - auth_env: &AuthEnvTelemetry, -) { - let snapshot = FeedbackRequestSnapshot::from_tags(tags); - feedback_tags!( - endpoint = snapshot.endpoint, - auth_header_attached = snapshot.auth_header_attached, - auth_header_name = snapshot.auth_header_name, - auth_mode = snapshot.auth_mode, - auth_retry_after_unauthorized = snapshot.auth_retry_after_unauthorized, - auth_recovery_mode = snapshot.auth_recovery_mode, - auth_recovery_phase = snapshot.auth_recovery_phase, - auth_connection_reused = snapshot.auth_connection_reused, - auth_request_id = snapshot.auth_request_id, - auth_cf_ray = snapshot.auth_cf_ray, - auth_error = snapshot.auth_error, - auth_error_code = snapshot.auth_error_code, - auth_recovery_followup_success = snapshot.auth_recovery_followup_success, - auth_recovery_followup_status = snapshot.auth_recovery_followup_status, - auth_env_openai_api_key_present = auth_env.openai_api_key_env_present, - auth_env_codex_api_key_present = auth_env.codex_api_key_env_present, - auth_env_codex_api_key_enabled = auth_env.codex_api_key_env_enabled, - auth_env_provider_key_name = auth_env.provider_env_key_name.as_deref().unwrap_or(""), - auth_env_provider_key_present = auth_env - .provider_env_key_present - .map_or_else(String::new, |value| value.to_string()), - auth_env_refresh_token_url_override_present = auth_env.refresh_token_url_override_present + endpoint = tags.endpoint, + auth_header_attached = tags.auth_header_attached, + auth_header_name = auth_header_name, + auth_mode = auth_mode, + auth_env_openai_api_key_present = tags.auth_env_openai_api_key_present, + auth_env_codex_api_key_present = tags.auth_env_codex_api_key_present, + auth_env_codex_api_key_enabled = tags.auth_env_codex_api_key_enabled, + auth_env_provider_key_name = auth_env_provider_key_name, + auth_env_provider_key_present = auth_env_provider_key_present, + auth_env_refresh_token_url_override_present = + tags.auth_env_refresh_token_url_override_present, + auth_retry_after_unauthorized = auth_retry_after_unauthorized, + auth_recovery_mode = auth_recovery_mode, + auth_recovery_phase = auth_recovery_phase, + auth_connection_reused = auth_connection_reused, + provider_header_names = provider_header_names, + base_url_origin = tags.base_url_origin, + host_class = tags.host_class, + base_url_source = tags.base_url_source, + base_url_is_default = tags.base_url_is_default, + residency_header_attached = residency_header_attached, + residency_header_value = residency_header_value, + auth_request_id = auth_request_id, + auth_cf_ray = auth_cf_ray, + auth_error = auth_error, + auth_error_code = auth_error_code, + error_body_class = error_body_class, + safe_error_message = safe_error_message, + geo_denial_detected = geo_denial_detected, + auth_recovery_followup_success = auth_recovery_followup_success, + auth_recovery_followup_status = auth_recovery_followup_status ); } diff --git a/codex-rs/core/src/util_tests.rs b/codex-rs/core/src/util_tests.rs index 0e9979309f..6dfdb2f499 100644 --- a/codex-rs/core/src/util_tests.rs +++ b/codex-rs/core/src/util_tests.rs @@ -1,5 +1,4 @@ use super::*; -use crate::auth_env_telemetry::AuthEnvTelemetry; use std::collections::BTreeMap; use std::sync::Arc; use std::sync::Mutex; @@ -69,7 +68,6 @@ impl Visit for TagCollectorVisitor { #[derive(Clone)] struct TagCollectorLayer { tags: Arc>>, - event_count: Arc>, } impl Layer for TagCollectorLayer @@ -83,49 +81,48 @@ where let mut visitor = TagCollectorVisitor::default(); event.record(&mut visitor); self.tags.lock().unwrap().extend(visitor.tags); - *self.event_count.lock().unwrap() += 1; } } #[test] fn emit_feedback_request_tags_records_sentry_feedback_fields() { let tags = Arc::new(Mutex::new(BTreeMap::new())); - let event_count = Arc::new(Mutex::new(0)); let _guard = tracing_subscriber::registry() - .with(TagCollectorLayer { - tags: tags.clone(), - event_count: event_count.clone(), - }) + .with(TagCollectorLayer { tags: tags.clone() }) .set_default(); - let auth_env = AuthEnvTelemetry { - openai_api_key_env_present: true, - codex_api_key_env_present: false, - codex_api_key_env_enabled: true, - provider_env_key_name: Some("configured".to_string()), - provider_env_key_present: Some(true), - refresh_token_url_override_present: true, - }; - - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: "/responses", - auth_header_attached: true, - auth_header_name: Some("authorization"), - auth_mode: Some("chatgpt"), - auth_retry_after_unauthorized: Some(false), - auth_recovery_mode: Some("managed"), - auth_recovery_phase: Some("refresh_token"), - auth_connection_reused: Some(true), - auth_request_id: Some("req-123"), - auth_cf_ray: Some("ray-123"), - auth_error: Some("missing_authorization_header"), - auth_error_code: Some("token_expired"), - auth_recovery_followup_success: Some(true), - auth_recovery_followup_status: Some(200), - }, - &auth_env, - ); + emit_feedback_request_tags(&FeedbackRequestTags { + endpoint: "/responses", + auth_header_attached: true, + auth_header_name: Some("authorization"), + auth_mode: Some("chatgpt"), + auth_env_openai_api_key_present: true, + auth_env_codex_api_key_present: false, + auth_env_codex_api_key_enabled: true, + auth_env_provider_key_name: Some("OPENAI_API_KEY"), + auth_env_provider_key_present: Some(true), + auth_env_refresh_token_url_override_present: true, + auth_retry_after_unauthorized: Some(false), + auth_recovery_mode: Some("managed"), + auth_recovery_phase: Some("refresh_token"), + auth_connection_reused: Some(true), + provider_header_names: Some("openai-project"), + base_url_origin: "chatgpt.com", + host_class: "openai_chatgpt", + base_url_source: "default", + base_url_is_default: true, + residency_header_attached: Some(true), + residency_header_value: Some("us"), + auth_request_id: Some("req-123"), + auth_cf_ray: Some("ray-123"), + auth_error: Some("missing_authorization_header"), + auth_error_code: Some("token_expired"), + error_body_class: Some("workspace_not_authorized_in_region"), + safe_error_message: Some("Workspace is not authorized in this region."), + geo_denial_detected: Some(true), + auth_recovery_followup_success: Some(true), + auth_recovery_followup_status: Some(200), + }); let tags = tags.lock().unwrap().clone(); assert_eq!( @@ -141,42 +138,26 @@ fn emit_feedback_request_tags_records_sentry_feedback_fields() { Some("\"authorization\"") ); assert_eq!( - tags.get("auth_env_openai_api_key_present") - .map(String::as_str), - Some("true") - ); - assert_eq!( - tags.get("auth_env_codex_api_key_present") - .map(String::as_str), - Some("false") - ); - assert_eq!( - tags.get("auth_env_codex_api_key_enabled") - .map(String::as_str), - Some("true") + tags.get("auth_request_id").map(String::as_str), + Some("\"req-123\"") ); assert_eq!( tags.get("auth_env_provider_key_name").map(String::as_str), - Some("\"configured\"") + Some("\"OPENAI_API_KEY\"") ); assert_eq!( tags.get("auth_env_provider_key_present") .map(String::as_str), Some("\"true\"") ); - assert_eq!( - tags.get("auth_env_refresh_token_url_override_present") - .map(String::as_str), - Some("true") - ); - assert_eq!( - tags.get("auth_request_id").map(String::as_str), - Some("\"req-123\"") - ); assert_eq!( tags.get("auth_error_code").map(String::as_str), Some("\"token_expired\"") ); + assert_eq!( + tags.get("geo_denial_detected").map(String::as_str), + Some("\"true\"") + ); assert_eq!( tags.get("auth_recovery_followup_success") .map(String::as_str), @@ -187,18 +168,13 @@ fn emit_feedback_request_tags_records_sentry_feedback_fields() { .map(String::as_str), Some("\"200\"") ); - assert_eq!(*event_count.lock().unwrap(), 1); } #[test] fn emit_feedback_auth_recovery_tags_preserves_401_specific_fields() { let tags = Arc::new(Mutex::new(BTreeMap::new())); - let event_count = Arc::new(Mutex::new(0)); let _guard = tracing_subscriber::registry() - .with(TagCollectorLayer { - tags: tags.clone(), - event_count: event_count.clone(), - }) + .with(TagCollectorLayer { tags: tags.clone() }) .set_default(); emit_feedback_auth_recovery_tags( @@ -228,18 +204,13 @@ fn emit_feedback_auth_recovery_tags_preserves_401_specific_fields() { tags.get("auth_401_error_code").map(String::as_str), Some("\"token_expired\"") ); - assert_eq!(*event_count.lock().unwrap(), 1); } #[test] fn emit_feedback_auth_recovery_tags_clears_stale_401_fields() { let tags = Arc::new(Mutex::new(BTreeMap::new())); - let event_count = Arc::new(Mutex::new(0)); let _guard = tracing_subscriber::registry() - .with(TagCollectorLayer { - tags: tags.clone(), - event_count: event_count.clone(), - }) + .with(TagCollectorLayer { tags: tags.clone() }) .set_default(); emit_feedback_auth_recovery_tags( @@ -275,18 +246,13 @@ fn emit_feedback_auth_recovery_tags_clears_stale_401_fields() { tags.get("auth_401_error_code").map(String::as_str), Some("\"\"") ); - assert_eq!(*event_count.lock().unwrap(), 2); } #[test] -fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized() { +fn emit_feedback_request_tags_clears_stale_latest_auth_fields() { let tags = Arc::new(Mutex::new(BTreeMap::new())); - let event_count = Arc::new(Mutex::new(0)); let _guard = tracing_subscriber::registry() - .with(TagCollectorLayer { - tags: tags.clone(), - event_count: event_count.clone(), - }) + .with(TagCollectorLayer { tags: tags.clone() }) .set_default(); emit_feedback_request_tags(&FeedbackRequestTags { @@ -294,95 +260,62 @@ fn emit_feedback_request_tags_preserves_latest_auth_fields_after_unauthorized() auth_header_attached: true, auth_header_name: Some("authorization"), auth_mode: Some("chatgpt"), + auth_env_openai_api_key_present: true, + auth_env_codex_api_key_present: true, + auth_env_codex_api_key_enabled: true, + auth_env_provider_key_name: Some("OPENAI_API_KEY"), + auth_env_provider_key_present: Some(true), + auth_env_refresh_token_url_override_present: true, auth_retry_after_unauthorized: Some(true), auth_recovery_mode: Some("managed"), auth_recovery_phase: Some("refresh_token"), - auth_connection_reused: None, + auth_connection_reused: Some(true), + provider_header_names: Some("openai-project"), + base_url_origin: "chatgpt.com", + host_class: "openai_chatgpt", + base_url_source: "default", + base_url_is_default: true, + residency_header_attached: Some(true), + residency_header_value: Some("us"), auth_request_id: Some("req-123"), auth_cf_ray: Some("ray-123"), auth_error: Some("missing_authorization_header"), auth_error_code: Some("token_expired"), - auth_recovery_followup_success: Some(false), - auth_recovery_followup_status: Some(401), + error_body_class: Some("workspace_not_authorized_in_region"), + safe_error_message: Some("Workspace is not authorized in this region."), + geo_denial_detected: Some(true), + auth_recovery_followup_success: Some(true), + auth_recovery_followup_status: Some(200), }); - - let tags = tags.lock().unwrap().clone(); - assert_eq!( - tags.get("auth_request_id").map(String::as_str), - Some("\"req-123\"") - ); - assert_eq!( - tags.get("auth_cf_ray").map(String::as_str), - Some("\"ray-123\"") - ); - assert_eq!( - tags.get("auth_error").map(String::as_str), - Some("\"missing_authorization_header\"") - ); - assert_eq!( - tags.get("auth_error_code").map(String::as_str), - Some("\"token_expired\"") - ); - assert_eq!( - tags.get("auth_recovery_followup_success") - .map(String::as_str), - Some("\"false\"") - ); - assert_eq!(*event_count.lock().unwrap(), 1); -} - -#[test] -fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() { - let tags = Arc::new(Mutex::new(BTreeMap::new())); - let event_count = Arc::new(Mutex::new(0)); - let _guard = tracing_subscriber::registry() - .with(TagCollectorLayer { - tags: tags.clone(), - event_count: event_count.clone(), - }) - .set_default(); - - let auth_env = AuthEnvTelemetry { - openai_api_key_env_present: true, - codex_api_key_env_present: true, - codex_api_key_env_enabled: true, - provider_env_key_name: Some("configured".to_string()), - provider_env_key_present: Some(true), - refresh_token_url_override_present: true, - }; - - emit_feedback_request_tags_with_auth_env( - &FeedbackRequestTags { - endpoint: "/responses", - auth_header_attached: true, - auth_header_name: Some("authorization"), - auth_mode: Some("chatgpt"), - auth_retry_after_unauthorized: Some(false), - auth_recovery_mode: Some("managed"), - auth_recovery_phase: Some("refresh_token"), - auth_connection_reused: Some(true), - auth_request_id: Some("req-123"), - auth_cf_ray: Some("ray-123"), - auth_error: Some("missing_authorization_header"), - auth_error_code: Some("token_expired"), - auth_recovery_followup_success: Some(true), - auth_recovery_followup_status: Some(200), - }, - &auth_env, - ); emit_feedback_request_tags(&FeedbackRequestTags { endpoint: "/responses", auth_header_attached: true, auth_header_name: None, auth_mode: None, + auth_env_openai_api_key_present: false, + auth_env_codex_api_key_present: false, + auth_env_codex_api_key_enabled: false, + auth_env_provider_key_name: None, + auth_env_provider_key_present: None, + auth_env_refresh_token_url_override_present: false, auth_retry_after_unauthorized: None, auth_recovery_mode: None, auth_recovery_phase: None, auth_connection_reused: None, + provider_header_names: None, + base_url_origin: "chatgpt.com", + host_class: "openai_chatgpt", + base_url_source: "default", + base_url_is_default: true, + residency_header_attached: None, + residency_header_value: None, auth_request_id: None, auth_cf_ray: None, auth_error: None, auth_error_code: None, + error_body_class: None, + safe_error_message: None, + geo_denial_detected: None, auth_recovery_followup_success: None, auth_recovery_followup_status: None, }); @@ -393,6 +326,27 @@ fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() { Some("\"\"") ); assert_eq!(tags.get("auth_mode").map(String::as_str), Some("\"\"")); + assert_eq!( + tags.get("provider_header_names").map(String::as_str), + Some("\"\"") + ); + assert_eq!( + tags.get("auth_env_provider_key_name").map(String::as_str), + Some("\"\"") + ); + assert_eq!( + tags.get("auth_env_provider_key_present") + .map(String::as_str), + Some("\"\"") + ); + assert_eq!( + tags.get("residency_header_attached").map(String::as_str), + Some("\"\"") + ); + assert_eq!( + tags.get("residency_header_value").map(String::as_str), + Some("\"\"") + ); assert_eq!( tags.get("auth_request_id").map(String::as_str), Some("\"\"") @@ -404,33 +358,16 @@ fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() { Some("\"\"") ); assert_eq!( - tags.get("auth_env_openai_api_key_present") - .map(String::as_str), - Some("true") + tags.get("error_body_class").map(String::as_str), + Some("\"\"") ); assert_eq!( - tags.get("auth_env_codex_api_key_present") - .map(String::as_str), - Some("true") + tags.get("safe_error_message").map(String::as_str), + Some("\"\"") ); assert_eq!( - tags.get("auth_env_codex_api_key_enabled") - .map(String::as_str), - Some("true") - ); - assert_eq!( - tags.get("auth_env_provider_key_name").map(String::as_str), - Some("\"configured\"") - ); - assert_eq!( - tags.get("auth_env_provider_key_present") - .map(String::as_str), - Some("\"true\"") - ); - assert_eq!( - tags.get("auth_env_refresh_token_url_override_present") - .map(String::as_str), - Some("true") + tags.get("geo_denial_detected").map(String::as_str), + Some("\"\"") ); assert_eq!( tags.get("auth_recovery_followup_success") @@ -442,7 +379,6 @@ fn emit_feedback_request_tags_preserves_auth_env_fields_for_legacy_emitters() { .map(String::as_str), Some("\"\"") ); - assert_eq!(*event_count.lock().unwrap(), 2); } #[test] diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index f66855a6a3..a78cd66bc9 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -13,8 +13,6 @@ use codex_core::built_in_model_providers; use codex_core::config::Config; use codex_core::features::Feature; use codex_core::models_manager::collaboration_mode_presets::CollaborationModesConfig; -use codex_core::shell::Shell; -use codex_core::shell::get_shell_by_model_provided_path; use codex_protocol::config_types::ServiceTier; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::protocol::AskForApproval; @@ -66,7 +64,6 @@ pub struct TestCodexBuilder { auth: CodexAuth, pre_build_hooks: Vec>, home: Option>, - user_shell_override: Option, } impl TestCodexBuilder { @@ -103,25 +100,12 @@ impl TestCodexBuilder { self } - pub fn with_user_shell(mut self, user_shell: Shell) -> Self { - self.user_shell_override = Some(user_shell); - self - } - - pub fn with_windows_cmd_shell(self) -> Self { - if cfg!(windows) { - self.with_user_shell(get_shell_by_model_provided_path(&PathBuf::from("cmd.exe"))) - } else { - self - } - } - pub async fn build(&mut self, server: &wiremock::MockServer) -> anyhow::Result { let home = match self.home.clone() { Some(home) => home, None => Arc::new(TempDir::new()?), }; - Box::pin(self.build_with_home(server, home, /*resume_from*/ None)).await + Box::pin(self.build_with_home(server, home, None)).await } pub async fn build_with_streaming_server( @@ -133,12 +117,7 @@ impl TestCodexBuilder { Some(home) => home, None => Arc::new(TempDir::new()?), }; - Box::pin(self.build_with_home_and_base_url( - format!("{base_url}/v1"), - home, - /*resume_from*/ None, - )) - .await + Box::pin(self.build_with_home_and_base_url(format!("{base_url}/v1"), home, None)).await } pub async fn build_with_websocket_server( @@ -159,7 +138,7 @@ impl TestCodexBuilder { .enable(Feature::ResponsesWebsockets) .expect("test config should allow feature update"); })); - Box::pin(self.build_with_home_and_base_url(base_url, home, /*resume_from*/ None)).await + Box::pin(self.build_with_home_and_base_url(base_url, home, None)).await } pub async fn resume( @@ -215,43 +194,19 @@ impl TestCodexBuilder { ) }; let thread_manager = Arc::new(thread_manager); - let user_shell_override = self.user_shell_override.clone(); - let new_conversation = match (resume_from, user_shell_override) { - (Some(path), Some(user_shell_override)) => { - let auth_manager = codex_core::test_support::auth_manager_from_auth(auth); - Box::pin( - codex_core::test_support::resume_thread_from_rollout_with_user_shell_override( - thread_manager.as_ref(), - config.clone(), - path, - auth_manager, - user_shell_override, - ), - ) - .await? - } - (Some(path), None) => { + let new_conversation = match resume_from { + Some(path) => { let auth_manager = codex_core::test_support::auth_manager_from_auth(auth); Box::pin(thread_manager.resume_thread_from_rollout( config.clone(), path, auth_manager, - /*parent_trace*/ None, + None, )) .await? } - (None, Some(user_shell_override)) => { - Box::pin( - codex_core::test_support::start_thread_with_user_shell_override( - thread_manager.as_ref(), - config.clone(), - user_shell_override, - ), - ) - .await? - } - (None, None) => Box::pin(thread_manager.start_thread(config.clone())).await?, + None => Box::pin(thread_manager.start_thread(config.clone())).await?, }; Ok(TestCodex { @@ -271,7 +226,7 @@ impl TestCodexBuilder { ) -> anyhow::Result<(Config, Arc)> { let model_provider = ModelProviderInfo { base_url: Some(base_url), - ..built_in_model_providers(/*openai_base_url*/ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let cwd = Arc::new(TempDir::new()?); let mut config = load_default_config_for_test(home).await; @@ -407,13 +362,8 @@ impl TestCodex { approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, ) -> Result<()> { - self.submit_turn_with_context( - prompt, - approval_policy, - sandbox_policy, - /*service_tier*/ None, - ) - .await + self.submit_turn_with_context(prompt, approval_policy, sandbox_policy, None) + .await } async fn submit_turn_with_context( @@ -602,7 +552,6 @@ pub fn test_codex() -> TestCodexBuilder { auth: CodexAuth::from_api_key("dummy"), pre_build_hooks: vec![], home: None, - user_shell_override: None, } } diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 7d785694dc..f6ab8e6a3a 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -715,7 +715,7 @@ async fn chatgpt_auth_sends_correct_request() { ) .await; - let mut model_provider = built_in_model_providers(/* openai_base_url */ None)["openai"].clone(); + let mut model_provider = built_in_model_providers(None)["openai"].clone(); model_provider.base_url = Some(format!("{}/api/codex", server.uri())); let mut builder = test_codex() .with_auth(create_dummy_codex_auth()) @@ -791,7 +791,7 @@ async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { let model_provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; // Init session @@ -1966,7 +1966,7 @@ async fn token_count_includes_rate_limits_snapshot() { .mount(&server) .await; - let mut provider = built_in_model_providers(/* openai_base_url */ None)["openai"].clone(); + let mut provider = built_in_model_providers(None)["openai"].clone(); provider.base_url = Some(format!("{}/v1", server.uri())); let mut builder = test_codex() diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 6cf1c275a7..b2f06d50d5 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -93,7 +93,7 @@ fn json_fragment(text: &str) -> String { } fn non_openai_model_provider(server: &MockServer) -> ModelProviderInfo { - let mut provider = built_in_model_providers(/* openai_base_url */ None)["openai"].clone(); + let mut provider = built_in_model_providers(None)["openai"].clone(); provider.name = "OpenAI (test)".into(); provider.base_url = Some(format!("{}/v1", server.uri())); provider diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 589c6b9d5d..f43b94c2f2 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -95,7 +95,7 @@ async fn remote_models_get_model_info_uses_longest_matching_prefix() -> Result<( let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -654,7 +654,7 @@ async fn remote_models_do_not_append_removed_builtin_presets() -> Result<()> { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -709,7 +709,7 @@ async fn remote_models_merge_adds_new_high_priority_first() -> Result<()> { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -756,7 +756,7 @@ async fn remote_models_merge_replaces_overlapping_model() -> Result<()> { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -800,7 +800,7 @@ async fn remote_models_merge_preserves_bundled_models_on_empty_response() -> Res let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -841,7 +841,7 @@ async fn remote_models_request_times_out_after_5s() -> Result<()> { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), @@ -907,7 +907,7 @@ async fn remote_models_hide_picker_only_models() -> Result<()> { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + ..built_in_model_providers(None)["openai"].clone() }; let manager = codex_core::test_support::models_manager_with_provider( codex_home.path().to_path_buf(), diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index e2c86a6e63..6075fd2158 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -62,21 +62,10 @@ const RESPONSES_API_ENGINE_SERVICE_TTFT_FIELD: &str = "engine_service_ttft_total const RESPONSES_API_ENGINE_IAPI_TBT_FIELD: &str = "engine_iapi_tbt_across_engine_calls_ms"; const RESPONSES_API_ENGINE_SERVICE_TBT_FIELD: &str = "engine_service_tbt_across_engine_calls_ms"; -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AuthEnvTelemetryMetadata { - pub openai_api_key_env_present: bool, - pub codex_api_key_env_present: bool, - pub codex_api_key_env_enabled: bool, - pub provider_env_key_name: Option, - pub provider_env_key_present: Option, - pub refresh_token_url_override_present: bool, -} - #[derive(Debug, Clone)] pub struct SessionTelemetryMetadata { pub(crate) conversation_id: ThreadId, pub(crate) auth_mode: Option, - pub(crate) auth_env: AuthEnvTelemetryMetadata, pub(crate) account_id: Option, pub(crate) account_email: Option, pub(crate) originator: String, @@ -97,11 +86,6 @@ pub struct SessionTelemetry { } impl SessionTelemetry { - pub fn with_auth_env(mut self, auth_env: AuthEnvTelemetryMetadata) -> Self { - self.metadata.auth_env = auth_env; - self - } - pub fn with_model(mut self, model: &str, slug: &str) -> Self { self.metadata.model = model.to_owned(); self.metadata.slug = slug.to_owned(); @@ -271,7 +255,6 @@ impl SessionTelemetry { metadata: SessionTelemetryMetadata { conversation_id, auth_mode: auth_mode.map(|m| m.to_string()), - auth_env: AuthEnvTelemetryMetadata::default(), account_id, account_email, originator: sanitize_metric_tag_value(originator.as_str()), @@ -320,18 +303,68 @@ impl SessionTelemetry { sandbox_policy: SandboxPolicy, mcp_servers: Vec<&str>, active_profile: Option, + ) { + self.conversation_starts_with_endpoint_details( + provider_name, + "unknown", + "unknown", + "unknown", + false, + false, + false, + false, + None, + None, + false, + reasoning_effort, + reasoning_summary, + context_window, + auto_compact_token_limit, + approval_policy, + sandbox_policy, + mcp_servers, + active_profile, + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn conversation_starts_with_endpoint_details( + &self, + provider_name: &str, + base_url_origin: &str, + host_class: &str, + base_url_source: &str, + base_url_is_default: bool, + auth_env_openai_api_key_present: bool, + auth_env_codex_api_key_present: bool, + auth_env_codex_api_key_enabled: bool, + auth_env_provider_key_name: Option<&str>, + auth_env_provider_key_present: Option, + auth_env_refresh_token_url_override_present: bool, + reasoning_effort: Option, + reasoning_summary: ReasoningSummary, + context_window: Option, + auto_compact_token_limit: Option, + approval_policy: AskForApproval, + sandbox_policy: SandboxPolicy, + mcp_servers: Vec<&str>, + active_profile: Option, ) { log_and_trace_event!( self, common: { event.name = "codex.conversation_starts", provider_name = %provider_name, - auth.env_openai_api_key_present = self.metadata.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.metadata.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.metadata.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.metadata.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.metadata.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.metadata.auth_env.refresh_token_url_override_present, + base_url_origin = base_url_origin, + host_class = host_class, + base_url_source = base_url_source, + base_url_is_default = base_url_is_default, + auth.env_openai_api_key_present = auth_env_openai_api_key_present, + auth.env_codex_api_key_present = auth_env_codex_api_key_present, + auth.env_codex_api_key_enabled = auth_env_codex_api_key_enabled, + auth.env_provider_key_name = auth_env_provider_key_name, + auth.env_provider_key_present = auth_env_provider_key_present, + auth.env_refresh_token_url_override_present = auth_env_refresh_token_url_override_present, reasoning_effort = reasoning_effort.map(|e| e.to_string()), reasoning_summary = %reasoning_summary, context_window = context_window, @@ -363,21 +396,36 @@ impl SessionTelemetry { Ok(response) => (Some(response.status().as_u16()), None), Err(error) => (error.status().map(|s| s.as_u16()), Some(error.to_string())), }; - self.record_api_request( + self.record_api_request_with_endpoint_details( attempt, status, error.as_deref(), duration, - /*auth_header_attached*/ false, - /*auth_header_name*/ None, - /*retry_after_unauthorized*/ false, - /*recovery_mode*/ None, - /*recovery_phase*/ None, + false, + None, + false, + None, + None, "unknown", - /*request_id*/ None, - /*cf_ray*/ None, - /*auth_error*/ None, - /*auth_error_code*/ None, + false, + None, + None, + "custom", + "custom_unknown", + "default", + false, + false, + false, + false, + None, + None, + false, + None, + None, + None, + None, + None, + None, ); response @@ -400,6 +448,72 @@ impl SessionTelemetry { cf_ray: Option<&str>, auth_error: Option<&str>, auth_error_code: Option<&str>, + ) { + self.record_api_request_with_endpoint_details( + attempt, + status, + error, + duration, + auth_header_attached, + auth_header_name, + retry_after_unauthorized, + recovery_mode, + recovery_phase, + endpoint, + false, + None, + None, + "unknown", + "unknown", + "unknown", + false, + false, + false, + false, + None, + None, + false, + request_id, + cf_ray, + auth_error, + auth_error_code, + None, + None, + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn record_api_request_with_endpoint_details( + &self, + attempt: u64, + status: Option, + error: Option<&str>, + duration: Duration, + auth_header_attached: bool, + auth_header_name: Option<&str>, + retry_after_unauthorized: bool, + recovery_mode: Option<&str>, + recovery_phase: Option<&str>, + endpoint: &str, + residency_header_attached: bool, + residency_header_value: Option<&str>, + provider_header_names: Option<&str>, + base_url_origin: &str, + host_class: &str, + base_url_source: &str, + base_url_is_default: bool, + auth_env_openai_api_key_present: bool, + auth_env_codex_api_key_present: bool, + auth_env_codex_api_key_enabled: bool, + auth_env_provider_key_name: Option<&str>, + auth_env_provider_key_present: Option, + auth_env_refresh_token_url_override_present: bool, + request_id: Option<&str>, + cf_ray: Option<&str>, + auth_error: Option<&str>, + auth_error_code: Option<&str>, + error_body_class: Option<&str>, + safe_error_message: Option<&str>, ) { let success = status.is_some_and(|code| (200..=299).contains(&code)) && error.is_none(); let success_str = if success { "true" } else { "false" }; @@ -408,7 +522,7 @@ impl SessionTelemetry { .unwrap_or_else(|| "none".to_string()); self.counter( API_CALL_COUNT_METRIC, - /*inc*/ 1, + 1, &[("status", status_str.as_str()), ("success", success_str)], ); self.record_duration( @@ -430,16 +544,25 @@ impl SessionTelemetry { auth.recovery_mode = recovery_mode, auth.recovery_phase = recovery_phase, endpoint = endpoint, - auth.env_openai_api_key_present = self.metadata.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.metadata.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.metadata.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.metadata.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.metadata.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.metadata.auth_env.refresh_token_url_override_present, + residency_header_attached = residency_header_attached, + residency_header_value = residency_header_value, + provider_header_names = provider_header_names, + base_url_origin = base_url_origin, + host_class = host_class, + base_url_source = base_url_source, + base_url_is_default = base_url_is_default, + auth.env_openai_api_key_present = auth_env_openai_api_key_present, + auth.env_codex_api_key_present = auth_env_codex_api_key_present, + auth.env_codex_api_key_enabled = auth_env_codex_api_key_enabled, + auth.env_provider_key_name = auth_env_provider_key_name, + auth.env_provider_key_present = auth_env_provider_key_present, + auth.env_refresh_token_url_override_present = auth_env_refresh_token_url_override_present, auth.request_id = request_id, auth.cf_ray = cf_ray, auth.error = auth_error, auth.error_code = auth_error_code, + error_body_class = error_body_class, + safe_error_message = safe_error_message, }, log: {}, trace: {}, @@ -463,6 +586,72 @@ impl SessionTelemetry { cf_ray: Option<&str>, auth_error: Option<&str>, auth_error_code: Option<&str>, + ) { + self.record_websocket_connect_with_endpoint_details( + duration, + status, + error, + auth_header_attached, + auth_header_name, + retry_after_unauthorized, + recovery_mode, + recovery_phase, + endpoint, + false, + None, + None, + "unknown", + "unknown", + "unknown", + false, + false, + false, + false, + None, + None, + false, + connection_reused, + request_id, + cf_ray, + auth_error, + auth_error_code, + None, + None, + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn record_websocket_connect_with_endpoint_details( + &self, + duration: Duration, + status: Option, + error: Option<&str>, + auth_header_attached: bool, + auth_header_name: Option<&str>, + retry_after_unauthorized: bool, + recovery_mode: Option<&str>, + recovery_phase: Option<&str>, + endpoint: &str, + residency_header_attached: bool, + residency_header_value: Option<&str>, + provider_header_names: Option<&str>, + base_url_origin: &str, + host_class: &str, + base_url_source: &str, + base_url_is_default: bool, + auth_env_openai_api_key_present: bool, + auth_env_codex_api_key_present: bool, + auth_env_codex_api_key_enabled: bool, + auth_env_provider_key_name: Option<&str>, + auth_env_provider_key_present: Option, + auth_env_refresh_token_url_override_present: bool, + connection_reused: bool, + request_id: Option<&str>, + cf_ray: Option<&str>, + auth_error: Option<&str>, + auth_error_code: Option<&str>, + error_body_class: Option<&str>, + safe_error_message: Option<&str>, ) { let success = error.is_none() && status @@ -483,17 +672,67 @@ impl SessionTelemetry { auth.recovery_mode = recovery_mode, auth.recovery_phase = recovery_phase, endpoint = endpoint, - auth.env_openai_api_key_present = self.metadata.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.metadata.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.metadata.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.metadata.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.metadata.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.metadata.auth_env.refresh_token_url_override_present, + residency_header_attached = residency_header_attached, + residency_header_value = residency_header_value, + provider_header_names = provider_header_names, + base_url_origin = base_url_origin, + host_class = host_class, + base_url_source = base_url_source, + base_url_is_default = base_url_is_default, + auth.env_openai_api_key_present = auth_env_openai_api_key_present, + auth.env_codex_api_key_present = auth_env_codex_api_key_present, + auth.env_codex_api_key_enabled = auth_env_codex_api_key_enabled, + auth.env_provider_key_name = auth_env_provider_key_name, + auth.env_provider_key_present = auth_env_provider_key_present, + auth.env_refresh_token_url_override_present = auth_env_refresh_token_url_override_present, auth.connection_reused = connection_reused, auth.request_id = request_id, auth.cf_ray = cf_ray, auth.error = auth_error, auth.error_code = auth_error_code, + error_body_class = error_body_class, + safe_error_message = safe_error_message, + }, + log: {}, + trace: {}, + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn record_geo_denial( + &self, + endpoint: &str, + auth_header_attached: bool, + auth_header_name: Option<&str>, + residency_header_attached: bool, + residency_header_value: Option<&str>, + provider_header_names: Option<&str>, + http_status: Option, + request_id: Option<&str>, + cf_ray: Option<&str>, + auth_error: Option<&str>, + auth_error_code: Option<&str>, + error_body_class: &str, + safe_error_message: Option<&str>, + ) { + log_and_trace_event!( + self, + common: { + event.name = "codex.geo_denial", + geo_denial_detected = true, + request_id = request_id, + cf_ray = cf_ray, + endpoint = endpoint, + auth.header_attached = auth_header_attached, + auth.header_name = auth_header_name, + residency_header_attached = residency_header_attached, + residency_header_value = residency_header_value, + provider_header_names = provider_header_names, + http_status = http_status, + auth.error = auth_error, + auth.error_code = auth_error_code, + error_body_class = error_body_class, + safe_error_message = safe_error_message, }, log: {}, trace: {}, @@ -509,7 +748,7 @@ impl SessionTelemetry { let success_str = if error.is_none() { "true" } else { "false" }; self.counter( WEBSOCKET_REQUEST_COUNT_METRIC, - /*inc*/ 1, + 1, &[("success", success_str)], ); self.record_duration( @@ -524,12 +763,6 @@ impl SessionTelemetry { duration_ms = %duration.as_millis(), success = success_str, error.message = error, - auth.env_openai_api_key_present = self.metadata.auth_env.openai_api_key_env_present, - auth.env_codex_api_key_present = self.metadata.auth_env.codex_api_key_env_present, - auth.env_codex_api_key_enabled = self.metadata.auth_env.codex_api_key_env_enabled, - auth.env_provider_key_name = self.metadata.auth_env.provider_env_key_name.as_deref(), - auth.env_provider_key_present = self.metadata.auth_env.provider_env_key_present, - auth.env_refresh_token_url_override_present = self.metadata.auth_env.refresh_token_url_override_present, auth.connection_reused = connection_reused, }, log: {}, @@ -649,7 +882,7 @@ impl SessionTelemetry { let kind_str = kind.as_deref().unwrap_or(WEBSOCKET_UNKNOWN_KIND); let success_str = if success { "true" } else { "false" }; let tags = [("kind", kind_str), ("success", success_str)]; - self.counter(WEBSOCKET_EVENT_COUNT_METRIC, /*inc*/ 1, &tags); + self.counter(WEBSOCKET_EVENT_COUNT_METRIC, 1, &tags); self.record_duration(WEBSOCKET_EVENT_DURATION_METRIC, duration, &tags); log_and_trace_event!( self, @@ -703,15 +936,11 @@ impl SessionTelemetry { } } Ok(Some(Err(error))) => { - self.sse_event_failed(/*kind*/ None, duration, error); + self.sse_event_failed(None, duration, error); } Ok(None) => {} Err(_) => { - self.sse_event_failed( - /*kind*/ None, - duration, - &"idle timeout waiting for SSE", - ); + self.sse_event_failed(None, duration, &"idle timeout waiting for SSE"); } } } @@ -719,7 +948,7 @@ impl SessionTelemetry { fn sse_event(&self, kind: &str, duration: Duration) { self.counter( SSE_EVENT_COUNT_METRIC, - /*inc*/ 1, + 1, &[("kind", kind), ("success", "true")], ); self.record_duration( @@ -742,7 +971,7 @@ impl SessionTelemetry { let kind_str = kind.map_or(SSE_UNKNOWN_KIND, String::as_str); self.counter( SSE_EVENT_COUNT_METRIC, - /*inc*/ 1, + 1, &[("kind", kind_str), ("success", "false")], ); self.record_duration( @@ -956,7 +1185,7 @@ impl SessionTelemetry { tags.push(("tool", tool_name)); tags.push(("success", success_str)); tags.extend_from_slice(extra_tags); - self.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &tags); + self.counter(TOOL_CALL_COUNT_METRIC, 1, &tags); self.record_duration(TOOL_CALL_DURATION_METRIC, duration, &tags); let mcp_server = mcp_server.unwrap_or(""); let mcp_server_origin = mcp_server_origin.unwrap_or(""); diff --git a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs index b7fce86145..b924ef31cf 100644 --- a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs +++ b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs @@ -1,4 +1,3 @@ -use codex_otel::AuthEnvTelemetryMetadata; use codex_otel::OtelProvider; use codex_otel::SessionTelemetry; use codex_otel::TelemetryAuthMode; @@ -19,9 +18,6 @@ use tracing_subscriber::filter::filter_fn; use tracing_subscriber::layer::SubscriberExt; use codex_protocol::ThreadId; -use codex_protocol::config_types::ReasoningSummary; -use codex_protocol::protocol::AskForApproval; -use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; @@ -80,17 +76,6 @@ fn find_span_event_by_name_attr<'a>( .unwrap_or_else(|| panic!("missing span event: {event_name}")) } -fn auth_env_metadata() -> AuthEnvTelemetryMetadata { - AuthEnvTelemetryMetadata { - openai_api_key_env_present: true, - codex_api_key_env_present: false, - codex_api_key_env_enabled: true, - provider_env_key_name: Some("configured".to_string()), - provider_env_key_present: Some(true), - refresh_token_url_override_present: true, - } -} - #[test] fn otel_export_routing_policy_routes_user_prompt_log_and_trace_events() { let log_exporter = InMemoryLogExporter::default(); @@ -497,22 +482,10 @@ fn otel_export_routing_policy_routes_api_request_auth_observability() { true, "tty".to_string(), SessionSource::Cli, - ) - .with_auth_env(auth_env_metadata()); + ); let root_span = tracing::info_span!("root"); let _root_guard = root_span.enter(); - manager.conversation_starts( - "openai", - None, - ReasoningSummary::Auto, - None, - None, - AskForApproval::Never, - SandboxPolicy::DangerFullAccess, - Vec::new(), - None, - ); - manager.record_api_request( + manager.record_api_request_with_endpoint_details( 1, Some(401), Some("http 401"), @@ -523,10 +496,25 @@ fn otel_export_routing_policy_routes_api_request_auth_observability() { Some("managed"), Some("refresh_token"), "/responses", + true, + Some("us"), + Some("openai-project,version"), + "chatgpt.com", + "openai_chatgpt", + "ide_settings", + false, + true, + false, + true, + Some("OPENAI_API_KEY"), + Some(true), + true, Some("req-401"), Some("ray-401"), Some("missing_authorization_header"), Some("token_expired"), + Some("workspace_not_authorized_in_region"), + Some("Workspace is not authorized in this region."), ); }); @@ -534,20 +522,6 @@ fn otel_export_routing_policy_routes_api_request_auth_observability() { tracer_provider.force_flush().expect("flush traces"); let logs = log_exporter.get_emitted_logs().expect("log export"); - let conversation_log = find_log_by_event_name(&logs, "codex.conversation_starts"); - let conversation_log_attrs = log_attributes(&conversation_log.record); - assert_eq!( - conversation_log_attrs - .get("auth.env_openai_api_key_present") - .map(String::as_str), - Some("true") - ); - assert_eq!( - conversation_log_attrs - .get("auth.env_provider_key_name") - .map(String::as_str), - Some("configured") - ); let request_log = find_log_by_event_name(&logs, "codex.api_request"); let request_log_attrs = log_attributes(&request_log.record); assert_eq!( @@ -584,33 +558,73 @@ fn otel_export_routing_policy_routes_api_request_auth_observability() { request_log_attrs.get("endpoint").map(String::as_str), Some("/responses") ); + assert_eq!( + request_log_attrs + .get("residency_header_attached") + .map(String::as_str), + Some("true") + ); + assert_eq!( + request_log_attrs + .get("residency_header_value") + .map(String::as_str), + Some("us") + ); + assert_eq!( + request_log_attrs.get("base_url_origin").map(String::as_str), + Some("chatgpt.com") + ); + assert_eq!( + request_log_attrs.get("host_class").map(String::as_str), + Some("openai_chatgpt") + ); + assert_eq!( + request_log_attrs.get("base_url_source").map(String::as_str), + Some("ide_settings") + ); + assert_eq!( + request_log_attrs + .get("auth.env_openai_api_key_present") + .map(String::as_str), + Some("true") + ); + assert_eq!( + request_log_attrs + .get("auth.env_provider_key_name") + .map(String::as_str), + Some("OPENAI_API_KEY") + ); + assert_eq!( + request_log_attrs + .get("provider_header_names") + .map(String::as_str), + Some("openai-project,version") + ); + assert_eq!( + request_log_attrs + .get("base_url_is_default") + .map(String::as_str), + Some("false") + ); assert_eq!( request_log_attrs.get("auth.error").map(String::as_str), Some("missing_authorization_header") ); assert_eq!( request_log_attrs - .get("auth.env_codex_api_key_enabled") + .get("error_body_class") .map(String::as_str), - Some("true") + Some("workspace_not_authorized_in_region") ); assert_eq!( request_log_attrs - .get("auth.env_refresh_token_url_override_present") + .get("safe_error_message") .map(String::as_str), - Some("true") + Some("Workspace is not authorized in this region.") ); + assert!(!request_log_attrs.contains_key("error_body")); let spans = span_exporter.get_finished_spans().expect("span export"); - let conversation_trace_event = - find_span_event_by_name_attr(&spans[0].events.events, "codex.conversation_starts"); - let conversation_trace_attrs = span_event_attributes(conversation_trace_event); - assert_eq!( - conversation_trace_attrs - .get("auth.env_provider_key_present") - .map(String::as_str), - Some("true") - ); let request_trace_event = find_span_event_by_name_attr(&spans[0].events.events, "codex.api_request"); let request_trace_attrs = span_event_attributes(request_trace_event); @@ -632,15 +646,21 @@ fn otel_export_routing_policy_routes_api_request_auth_observability() { .map(String::as_str), Some("true") ); + assert_eq!( + request_trace_attrs + .get("base_url_origin") + .map(String::as_str), + Some("chatgpt.com") + ); assert_eq!( request_trace_attrs.get("endpoint").map(String::as_str), Some("/responses") ); assert_eq!( request_trace_attrs - .get("auth.env_openai_api_key_present") + .get("safe_error_message") .map(String::as_str), - Some("true") + Some("Workspace is not authorized in this region.") ); } @@ -682,11 +702,10 @@ fn otel_export_routing_policy_routes_websocket_connect_auth_observability() { true, "tty".to_string(), SessionSource::Cli, - ) - .with_auth_env(auth_env_metadata()); + ); let root_span = tracing::info_span!("root"); let _root_guard = root_span.enter(); - manager.record_websocket_connect( + manager.record_websocket_connect_with_endpoint_details( std::time::Duration::from_millis(17), Some(401), Some("http 401"), @@ -696,11 +715,26 @@ fn otel_export_routing_policy_routes_websocket_connect_auth_observability() { Some("managed"), Some("reload"), "/responses", + true, + Some("us"), + Some("x-api-key"), + "openrouter.ai", + "known_third_party", + "config_toml", + false, + false, + true, + false, + Some("OPENROUTER_API_KEY"), + Some(true), + false, false, Some("req-ws-401"), Some("ray-ws-401"), Some("missing_authorization_header"), Some("token_expired"), + Some("workspace_not_authorized_in_region"), + Some("Workspace is not authorized in this region."), ); }); @@ -716,6 +750,12 @@ fn otel_export_routing_policy_routes_websocket_connect_auth_observability() { .map(String::as_str), Some("true") ); + assert_eq!( + connect_log_attrs + .get("auth.env_provider_key_name") + .map(String::as_str), + Some("OPENROUTER_API_KEY") + ); assert_eq!( connect_log_attrs .get("auth.header_name") @@ -726,22 +766,42 @@ fn otel_export_routing_policy_routes_websocket_connect_auth_observability() { connect_log_attrs.get("auth.error").map(String::as_str), Some("missing_authorization_header") ); + assert_eq!( + connect_log_attrs + .get("provider_header_names") + .map(String::as_str), + Some("x-api-key") + ); + assert_eq!( + connect_log_attrs.get("base_url_origin").map(String::as_str), + Some("openrouter.ai") + ); + assert_eq!( + connect_log_attrs.get("host_class").map(String::as_str), + Some("known_third_party") + ); assert_eq!( connect_log_attrs.get("endpoint").map(String::as_str), Some("/responses") ); + assert_eq!( + connect_log_attrs + .get("residency_header_value") + .map(String::as_str), + Some("us") + ); + assert_eq!( + connect_log_attrs + .get("safe_error_message") + .map(String::as_str), + Some("Workspace is not authorized in this region.") + ); assert_eq!( connect_log_attrs .get("auth.connection_reused") .map(String::as_str), Some("false") ); - assert_eq!( - connect_log_attrs - .get("auth.env_provider_key_name") - .map(String::as_str), - Some("configured") - ); let spans = span_exporter.get_finished_spans().expect("span export"); let connect_trace_event = @@ -755,9 +815,21 @@ fn otel_export_routing_policy_routes_websocket_connect_auth_observability() { ); assert_eq!( connect_trace_attrs - .get("auth.env_refresh_token_url_override_present") + .get("base_url_source") .map(String::as_str), - Some("true") + Some("config_toml") + ); + assert_eq!( + connect_trace_attrs + .get("error_body_class") + .map(String::as_str), + Some("workspace_not_authorized_in_region") + ); + assert_eq!( + connect_trace_attrs + .get("safe_error_message") + .map(String::as_str), + Some("Workspace is not authorized in this region.") ); } @@ -799,8 +871,7 @@ fn otel_export_routing_policy_routes_websocket_request_transport_observability() true, "tty".to_string(), SessionSource::Cli, - ) - .with_auth_env(auth_env_metadata()); + ); let root_span = tracing::info_span!("root"); let _root_guard = root_span.enter(); manager.record_websocket_request( @@ -826,12 +897,6 @@ fn otel_export_routing_policy_routes_websocket_request_transport_observability() request_log_attrs.get("error.message").map(String::as_str), Some("stream error") ); - assert_eq!( - request_log_attrs - .get("auth.env_openai_api_key_present") - .map(String::as_str), - Some("true") - ); let spans = span_exporter.get_finished_spans().expect("span export"); let request_trace_event = @@ -843,10 +908,246 @@ fn otel_export_routing_policy_routes_websocket_request_transport_observability() .map(String::as_str), Some("true") ); +} + +#[test] +fn otel_export_routing_policy_routes_geo_denial_log_and_trace_events() { + let log_exporter = InMemoryLogExporter::default(); + let logger_provider = SdkLoggerProvider::builder() + .with_simple_exporter(log_exporter.clone()) + .build(); + let span_exporter = InMemorySpanExporter::default(); + let tracer_provider = SdkTracerProvider::builder() + .with_simple_exporter(span_exporter.clone()) + .build(); + let tracer = tracer_provider.tracer("sink-split-test"); + + let subscriber = tracing_subscriber::registry() + .with( + opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new( + &logger_provider, + ) + .with_filter(filter_fn(OtelProvider::log_export_filter)), + ) + .with( + tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter_fn(OtelProvider::trace_export_filter)), + ); + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let manager = SessionTelemetry::new( + ThreadId::new(), + "gpt-5.1", + "gpt-5.1", + Some("account-id".to_string()), + Some("engineer@example.com".to_string()), + Some(TelemetryAuthMode::Chatgpt), + "codex_exec".to_string(), + true, + "tty".to_string(), + SessionSource::Cli, + ); + let root_span = tracing::info_span!("root"); + let _root_guard = root_span.enter(); + manager.record_geo_denial( + "/responses", + true, + Some("authorization"), + true, + Some("us"), + Some("x-api-key"), + Some(401), + Some("req-geo"), + Some("ray-geo"), + Some("missing_authorization_header"), + Some("workspace_not_authorized_in_region"), + "workspace_not_authorized_in_region", + Some("Workspace is not authorized in this region."), + ); + }); + + logger_provider.force_flush().expect("flush logs"); + tracer_provider.force_flush().expect("flush traces"); + + let logs = log_exporter.get_emitted_logs().expect("log export"); + let geo_log = find_log_by_event_name(&logs, "codex.geo_denial"); + let geo_log_attrs = log_attributes(&geo_log.record); assert_eq!( - request_trace_attrs - .get("auth.env_provider_key_present") + geo_log_attrs.get("geo_denial_detected").map(String::as_str), + Some("true") + ); + assert_eq!( + geo_log_attrs.get("request_id").map(String::as_str), + Some("req-geo") + ); + assert_eq!( + geo_log_attrs + .get("auth.header_attached") .map(String::as_str), Some("true") ); + assert_eq!( + geo_log_attrs.get("auth.header_name").map(String::as_str), + Some("authorization") + ); + assert_eq!( + geo_log_attrs.get("endpoint").map(String::as_str), + Some("/responses") + ); + assert_eq!( + geo_log_attrs + .get("provider_header_names") + .map(String::as_str), + Some("x-api-key") + ); + assert_eq!( + geo_log_attrs.get("auth.error_code").map(String::as_str), + Some("workspace_not_authorized_in_region") + ); + assert_eq!( + geo_log_attrs + .get("residency_header_value") + .map(String::as_str), + Some("us") + ); + assert_eq!( + geo_log_attrs.get("error_body_class").map(String::as_str), + Some("workspace_not_authorized_in_region") + ); + assert_eq!( + geo_log_attrs.get("safe_error_message").map(String::as_str), + Some("Workspace is not authorized in this region.") + ); + assert!(!geo_log_attrs.contains_key("error_body")); + + let spans = span_exporter.get_finished_spans().expect("span export"); + let geo_trace_event = find_span_event_by_name_attr(&spans[0].events.events, "codex.geo_denial"); + let geo_trace_attrs = span_event_attributes(geo_trace_event); + assert_eq!( + geo_trace_attrs + .get("auth.header_attached") + .map(String::as_str), + Some("true") + ); + assert_eq!( + geo_trace_attrs.get("cf_ray").map(String::as_str), + Some("ray-geo") + ); + assert_eq!( + geo_trace_attrs.get("auth.error").map(String::as_str), + Some("missing_authorization_header") + ); + assert_eq!( + geo_trace_attrs.get("http_status").map(String::as_str), + Some("401") + ); + assert_eq!( + geo_trace_attrs + .get("safe_error_message") + .map(String::as_str), + Some("Workspace is not authorized in this region.") + ); +} + +#[test] +fn otel_export_routing_policy_routes_conversation_start_endpoint_config() { + let log_exporter = InMemoryLogExporter::default(); + let logger_provider = SdkLoggerProvider::builder() + .with_simple_exporter(log_exporter.clone()) + .build(); + let span_exporter = InMemorySpanExporter::default(); + let tracer_provider = SdkTracerProvider::builder() + .with_simple_exporter(span_exporter.clone()) + .build(); + let tracer = tracer_provider.tracer("sink-split-test"); + + let subscriber = tracing_subscriber::registry() + .with( + opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new( + &logger_provider, + ) + .with_filter(filter_fn(OtelProvider::log_export_filter)), + ) + .with( + tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter_fn(OtelProvider::trace_export_filter)), + ); + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let manager = SessionTelemetry::new( + ThreadId::new(), + "gpt-5.1", + "gpt-5.1", + Some("account-id".to_string()), + Some("engineer@example.com".to_string()), + Some(TelemetryAuthMode::Chatgpt), + "codex_exec".to_string(), + true, + "tty".to_string(), + SessionSource::Cli, + ); + let root_span = tracing::info_span!("root"); + let _root_guard = root_span.enter(); + manager.conversation_starts_with_endpoint_details( + "OpenAI", + "custom", + "custom_unknown", + "env", + false, + true, + false, + true, + Some("OPENAI_API_KEY"), + Some(true), + true, + None, + codex_protocol::config_types::ReasoningSummary::Auto, + None, + None, + codex_protocol::protocol::AskForApproval::OnRequest, + codex_protocol::protocol::SandboxPolicy::new_read_only_policy(), + Vec::new(), + None, + ); + }); + + logger_provider.force_flush().expect("flush logs"); + tracer_provider.force_flush().expect("flush traces"); + + let logs = log_exporter.get_emitted_logs().expect("log export"); + let start_log = find_log_by_event_name(&logs, "codex.conversation_starts"); + let start_log_attrs = log_attributes(&start_log.record); + assert_eq!( + start_log_attrs.get("base_url_origin").map(String::as_str), + Some("custom") + ); + assert_eq!( + start_log_attrs.get("host_class").map(String::as_str), + Some("custom_unknown") + ); + assert_eq!( + start_log_attrs.get("base_url_source").map(String::as_str), + Some("env") + ); + assert_eq!( + start_log_attrs + .get("auth.env_provider_key_name") + .map(String::as_str), + Some("OPENAI_API_KEY") + ); + + let spans = span_exporter.get_finished_spans().expect("span export"); + let start_trace_event = + find_span_event_by_name_attr(&spans[0].events.events, "codex.conversation_starts"); + let start_trace_attrs = span_event_attributes(start_trace_event); + assert_eq!( + start_trace_attrs + .get("base_url_is_default") + .map(String::as_str), + Some("false") + ); }