diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 04d98c6255..9a0d551632 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2477,6 +2477,7 @@ dependencies = [ "codex-login", "codex-model-provider", "codex-protocol", + "codex-response-debug-context", "codex-runtime-agent-instance", "http 1.4.0", "pretty_assertions", diff --git a/codex-rs/app-server/src/request_processors/account_processor.rs b/codex-rs/app-server/src/request_processors/account_processor.rs index f85b6bcd92..38bdda4260 100644 --- a/codex-rs/app-server/src/request_processors/account_processor.rs +++ b/codex-rs/app-server/src/request_processors/account_processor.rs @@ -1710,6 +1710,7 @@ mod tests { status, content_type: "application/json".to_string(), body: "{}".to_string(), + identity_error_code: None, }; assert_eq!(workspace_messages_feature_disabled(&err), expected); } diff --git a/codex-rs/backend-client/Cargo.toml b/codex-rs/backend-client/Cargo.toml index 0de0d35336..d1d476e30a 100644 --- a/codex-rs/backend-client/Cargo.toml +++ b/codex-rs/backend-client/Cargo.toml @@ -26,6 +26,7 @@ codex-login = { workspace = true } codex-model-provider = { workspace = true } codex-runtime-agent-instance = { workspace = true } codex-protocol = { workspace = true } +codex-response-debug-context = { workspace = true } tokio = { workspace = true, features = ["time"] } [dev-dependencies] diff --git a/codex-rs/backend-client/src/client.rs b/codex-rs/backend-client/src/client.rs index 05a2493a9f..6dafe2e1af 100644 --- a/codex-rs/backend-client/src/client.rs +++ b/codex-rs/backend-client/src/client.rs @@ -66,6 +66,7 @@ pub enum RequestError { status: StatusCode, content_type: String, body: String, + identity_error_code: Option, }, Other(anyhow::Error), } @@ -79,7 +80,13 @@ impl RequestError { } pub fn is_unauthorized(&self) -> bool { - self.status() == Some(StatusCode::UNAUTHORIZED) + matches!(self, Self::UnexpectedStatus { status, body, identity_error_code, .. } + if *status == StatusCode::UNAUTHORIZED + && !codex_protocol::error::is_runtime_auth_denial( + *status, + body, + identity_error_code.as_deref(), + )) } } @@ -93,6 +100,7 @@ impl fmt::Display for RequestError { status, content_type, body, + .. } => write!( f, "{method} {url} failed: {status}; content-type={content_type}; body={body}" @@ -436,6 +444,11 @@ impl Client { .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); + let identity_error_code = + codex_response_debug_context::extract_response_debug_context_from_headers( + res.headers(), + ) + .auth_error_code; let body = res.text().await?; if !status.is_success() { return Err(RequestError::UnexpectedStatus { @@ -444,6 +457,7 @@ impl Client { status, content_type, body, + identity_error_code, }); } Ok((body, content_type)) diff --git a/codex-rs/backend-client/src/client_request_tests.rs b/codex-rs/backend-client/src/client_request_tests.rs index 210ce43319..9583b62f7e 100644 --- a/codex-rs/backend-client/src/client_request_tests.rs +++ b/codex-rs/backend-client/src/client_request_tests.rs @@ -6,6 +6,7 @@ use std::process::Command; use std::sync::Arc; use std::time::Duration; +use base64::Engine; use codex_http_client::HttpClientFactory; use codex_http_client::OutboundProxyPolicy; use codex_http_client::cache_system_proxy_route_for_test; @@ -439,3 +440,35 @@ async fn bootstrap_get_recovers_from_stalled_body_before_cloud_startup_timeout() .expect("cloud bundle via proxy"); proxy.verify().await; } + +#[tokio::test] +async fn identity_header_errors_preserve_credential_recovery() { + for (code, recoverable) in [ + ("invalid_agent_instance_assertion", false), + ("token_expired", true), + ] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/codex/accounts/check")) + .respond_with( + ResponseTemplate::new(401) + .insert_header( + "x-error-json", + base64::engine::general_purpose::STANDARD + .encode(serde_json::json!({"error": {"code": code}}).to_string()), + ) + .set_body_json(serde_json::json!({"detail": "Unauthorized"})), + ) + .expect(1) + .mount(&server) + .await; + let client = Client::new( + server.uri(), + HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), + ); + let error = client.get_accounts_check().await.unwrap_err(); + assert_eq!(error.status(), Some(StatusCode::UNAUTHORIZED)); + assert_eq!(error.is_unauthorized(), recoverable, "{code}"); + server.verify().await; + } +} diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index b2809a4c0b..7fd24f67a1 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -839,3 +839,57 @@ fn map_api_error_extracts_identity_auth_details_from_headers() { ); assert_eq!(err.identity_error_code.as_deref(), Some("token_expired")); } + +#[test] +fn runtime_auth_denials_stop_retries_without_losing_http_diagnostics() { + for (status, code, retryable) in [ + (403, "runtime_assertion_invalid", false), + (403, "runtime_instance_paused", false), + (403, "runtime_instance_revoked", false), + (401, "invalid_agent_instance_assertion", false), + (503, "runtime_verification_unavailable", true), + (503, "agent_instance_authentication_unavailable", true), + (401, "token_expired", true), + (403, "ordinary_policy_denial", true), + ] { + for in_header in [false, true] { + let status = http::StatusCode::from_u16(status).unwrap(); + let details = serde_json::json!({"error": {"code": code}}).to_string(); + let mut headers = http::HeaderMap::from_iter([( + "x-request-id".parse().unwrap(), + "request-runtime".parse().unwrap(), + )]); + let body = if in_header { + headers.insert( + X_ERROR_JSON_HEADER, + base64::engine::general_purpose::STANDARD + .encode(details) + .parse() + .unwrap(), + ); + r#"{"detail":"Unauthorized"}"#.to_owned() + } else { + details + }; + let error = map_api_error(ApiError::Transport(TransportError::Http { + status, + retry_after: None, + url: Some("https://api.openai.com/v1/responses".into()), + headers: Some(headers), + body: Some(body.clone()), + })); + assert_eq!( + error.retry_delay(/*retry_count*/ 1).is_some(), + retryable, + "{code}, in_header={in_header}" + ); + let CodexErrorDetails::UnexpectedStatus(details) = error.details() else { + panic!("missing HTTP diagnostics: {error:?}"); + }; + assert_eq!( + (&details.body, details.status, details.request_id.as_deref()), + (&body, status, Some("request-runtime")) + ); + } + } +} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 7a06e6e90e..86310dd789 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -896,8 +896,7 @@ impl ModelClient { expected_auth, &setup.api_provider.headers, ) - .await - .map_err(|error| codex_api::map_api_error(error.into()))?; + .await?; setup.api_auth = provider .compose_auth(codex_model_provider::ResolvedProviderAuth { credential: Some(selected.auth().clone()), diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index d5f355e0be..6545ed97d3 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -1552,8 +1552,7 @@ async fn prepare_realtime_start( &CodexAuth::from_api_key(&realtime_api_key), &api_provider.headers, ) - .await - .map_err(|error| codex_api::map_api_error(error.into()))?; + .await?; codex_model_provider::validate_runtime_selectors( &api_provider.headers, selected.runtime().actor().info(), diff --git a/codex-rs/core/tests/suite/client_websockets_runtime_auth.rs b/codex-rs/core/tests/suite/client_websockets_runtime_auth.rs index 9e0d588ce8..2c3620480d 100644 --- a/codex-rs/core/tests/suite/client_websockets_runtime_auth.rs +++ b/codex-rs/core/tests/suite/client_websockets_runtime_auth.rs @@ -339,6 +339,8 @@ async fn websocket_reconnects_for_replacement_and_account_switch_within_a_turn( vec![response.clone(), response.clone()], vec![response.clone(), response.clone()], vec![response.clone(), response.clone()], + // The account switches after this handshake, before its request is sent. + vec![response.clone()], vec![response], ]) .await; @@ -395,7 +397,7 @@ async fn websocket_reconnects_for_replacement_and_account_switch_within_a_turn( input.push(message_item("fourth")); let prompt = prompt_with_input(input); let gate = Arc::new(Barrier::new(2)); - // Allow the pre-connect check, then pause after Core selects the existing socket. + // Allow the pre-connect check, then pause after the old account's socket opens. budget.0.lock().unwrap().extend([None, Some(gate.clone())]); let generation = stream_until_complete(&mut session, &harness, &prompt); tokio::pin!(generation); @@ -409,10 +411,10 @@ async fn websocket_reconnects_for_replacement_and_account_switch_within_a_turn( assert!(!previous_owner.is_current()); tokio::join!(generation, gate.wait()); let handshakes = server.handshakes(); - assert_eq!(handshakes.len(), 4); + assert_eq!(handshakes.len(), 5); let runtime_ids: Vec<_> = handshakes .iter() - .zip(["key-a", "key-a", "key-b", "key-other-user"]) + .zip(["key-a", "key-a", "key-b", "key-b", "key-other-user"]) .map(|(h, credential)| { assert_eq!( h.header("cookie").as_deref(), @@ -434,17 +436,18 @@ async fn websocket_reconnects_for_replacement_and_account_switch_within_a_turn( .collect(); assert_eq!(runtime_ids[0], runtime_ids[1]); assert_ne!(runtime_ids[1], runtime_ids[2]); - assert_ne!(runtime_ids[2], runtime_ids[3]); + assert_eq!(runtime_ids[2], runtime_ids[3]); + assert_ne!(runtime_ids[3], runtime_ids[4]); assert_eq!( server .connections() .iter() .map(Vec::len) .collect::>(), - vec![1, 1, 1, 1] + vec![1, 1, 1, 0, 1] ); - for (index, requests) in server.connections().iter().enumerate() { - let body = requests[0].body_json(); + for (index, request) in server.connections().iter().flatten().enumerate() { + let body = request.body_json(); assert_eq!(body["previous_response_id"], Value::Null); // Reconnecting with the same actor preserves turn routing. let expected_turn_state = if index == 1 { @@ -641,7 +644,7 @@ async fn paused_runtime_cannot_continue_on_an_admitted_websocket(mode: RuntimeAu listener.local_addr().unwrap() ))); provider.request_max_retries = Some(0); - let (harness, _) = runtime_harness(®istration, provider, mode).await; + let (harness, _) = runtime_harness(®istration, provider, mode, Vec::new()).await; let requests = Arc::new(AtomicUsize::new(0)); let proofs = Arc::new(std::sync::Mutex::new(Vec::new())); let server_requests = Arc::clone(&requests); diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 59eb54f175..163107b8ec 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -214,6 +214,7 @@ mod rollout_budget; mod rollout_compression; mod rollout_list_find; mod runtime_auth; +mod runtime_auth_denials; mod safety_buffering; mod safety_check_downgrade; mod scenarios; diff --git a/codex-rs/core/tests/suite/runtime_auth_denials.rs b/codex-rs/core/tests/suite/runtime_auth_denials.rs new file mode 100644 index 0000000000..d6c8dfb635 --- /dev/null +++ b/codex-rs/core/tests/suite/runtime_auth_denials.rs @@ -0,0 +1,215 @@ +//! Check terminal runtime failures and credential recovery through complete turns. + +use anyhow::Result; +use base64::Engine; +use codex_core::TurnInputRequest; +use codex_features::Feature; +use codex_features::RuntimeAuthMode; +use codex_login::CodexAuth; +use codex_login::ExternalAuth; +use codex_login::ExternalAuthFuture; +use codex_login::ExternalAuthRefreshContext; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; +use core_test_support::responses; +use core_test_support::runtime_auth::PROBE_SERVER; +use core_test_support::runtime_auth::run_harness_through_proxy; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use std::collections::BTreeSet; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::body_string_contains; +use wiremock::matchers::method; +use wiremock::matchers::path; + +const TEST_NAME: &str = + "suite::runtime_auth_denials::turns_stop_on_runtime_denial_but_recover_ordinary_401"; +const CASES: [&str; 3] = ["revoked", "header-denial", "expired"]; +const MODES: [(&str, RuntimeAuthMode); 2] = [ + ("dual", RuntimeAuthMode::DualHeader), + ("assertion", RuntimeAuthMode::AssertionOnly), +]; + +#[derive(Default)] +struct Credentials { + refreshes: AtomicUsize, +} + +impl ExternalAuth for Credentials { + fn resolve(&self) -> ExternalAuthFuture<'_, CodexAuth> { + Box::pin(async { Ok(CodexAuth::from_api_key("key-a")) }) + } + + fn refresh(&self, _: ExternalAuthRefreshContext) -> ExternalAuthFuture<'_, CodexAuth> { + self.refreshes.fetch_add(1, Ordering::SeqCst); + self.resolve() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn turns_stop_on_runtime_denial_but_recover_ordinary_401() -> Result<()> { + skip_if_no_network!(Ok(())); + if std::env::var_os(PROBE_SERVER).is_some() { + return exercise_turns().await; + } + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/agent/register")) + .respond_with(|request: &wiremock::Request| { + ResponseTemplate::new(200).set_body_json(json!({ + "runtime_agent_instance_id": request.headers["idempotency-key"].to_str().unwrap(), + "agent_host_id": "host-a", + "actor": {"kind": "platform", "user_id": "user-a", + "organization_id": "org-a", "project_id": null}, + })) + }) + .expect(6) + .mount(&server) + .await; + for (mode, _) in MODES { + for case in CASES { + let label = format!("{mode}-{case}"); + let denial = match case { + "revoked" => ResponseTemplate::new(403).set_body_json(json!({ + "error": {"code": "runtime_instance_revoked", "message": "Runtime revoked"} + })), + "header-denial" => ResponseTemplate::new(401) + .insert_header( + "x-error-json", + base64::engine::general_purpose::STANDARD + .encode(br#"{"error":{"code":"invalid_agent_instance_assertion"}}"#), + ) + .set_body_json(json!({"error": {"message": "Runtime rejected"}})), + _ => ResponseTemplate::new(401).set_body_json(json!({ + "error": {"code": "token_expired", "message": "Credential expired"} + })), + }; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(body_string_contains(label.clone())) + .respond_with(denial) + .with_priority(1) + .up_to_n_times(1) + .expect(1) + .mount(&server) + .await; + // A mistaken retry would complete the turn, making either denial assertion fail. + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(body_string_contains(label)) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(responses::sse_completed("completed")), + ) + .with_priority(2) + .expect(u64::from(case == "expired")) + .mount(&server) + .await; + } + } + + run_harness_through_proxy(&server, TEST_NAME, /*realtime_target*/ None).await?; + let mut proof_ids = BTreeSet::new(); + let requests = server.received_requests().await.unwrap(); + let attempts: Vec<_> = requests + .iter() + .filter(|request| request.url.path() == "/v1/responses") + .collect(); + assert_eq!(attempts.len(), 8); + for request in attempts { + let authorization = request.headers["authorization"].to_str()?; + let proof = if let Some(proof) = authorization.strip_prefix("AgentAssertion ") { + assert!(!request.headers.contains_key("x-openai-agent-assertion")); + proof + } else { + assert_eq!(authorization, "Bearer key-a"); + request.headers["x-openai-agent-assertion"].to_str()? + }; + let payload: Value = serde_json::from_slice( + &base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(proof.split('.').nth(2).unwrap())?, + )?; + assert!(proof_ids.insert(payload["id"].as_str().unwrap().to_owned())); + } + server.verify().await; + Ok(()) +} + +async fn exercise_turns() -> Result<()> { + let server = MockServer::start().await; + for (mode_label, mode) in MODES { + for case in CASES { + let test = test_codex() + .with_auth(CodexAuth::from_api_key("key-a")) + .with_config(move |config| { + config.model_provider.base_url = Some("https://api.openai.com/v1".into()); + config.model_provider.env_http_headers = None; + config.model_provider.supports_websockets = false; + config.model_provider.request_max_retries = Some(2); + config.model_provider.stream_max_retries = Some(2); + config + .features + .enable(Feature::RuntimeAgentInstances) + .expect("enable runtime agent instances"); + config.runtime_auth_mode = mode; + config.model_catalog = Some( + codex_models_manager::bundled_models_response() + .expect("load bundled models"), + ); + }) + .build_with_auto_env(&server) + .await?; + let credentials = Arc::new(Credentials::default()); + test.thread_manager + .auth_manager() + .set_external_auth(credentials.clone()) + .await?; + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: format!("{mode_label}-{case}"), + text_elements: Vec::new(), + }])) + .await?; + let mut errors = Vec::new(); + let mut stream_retries = 0; + wait_for_event(&test.codex, |event| { + match event { + EventMsg::Error(error) => errors.push(error.message.clone()), + EventMsg::StreamError(_) => stream_retries += 1, + _ => {} + } + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + assert_eq!( + errors.len(), + usize::from(case != "expired"), + "{mode_label}-{case}" + ); + assert_eq!(stream_retries, 0, "{mode_label}-{case}"); + assert_eq!( + credentials.refreshes.load(Ordering::SeqCst), + usize::from(case == "expired"), + "{mode_label}-{case}" + ); + test.codex.submit(Op::Shutdown).await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + } + } + Ok(()) +} diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rs index daf14c8ddd..e0c73083b7 100644 --- a/codex-rs/model-provider/src/provider.rs +++ b/codex-rs/model-provider/src/provider.rs @@ -529,6 +529,21 @@ impl ConfiguredModelProvider { } impl ModelProvider for ConfiguredModelProvider { + fn is_recoverable_auth_error(&self, error: &TransportError) -> bool { + let TransportError::Http { status, body, .. } = error else { + return false; + }; + *status == http::StatusCode::UNAUTHORIZED + && (self.runtime_auth.is_none() + || !codex_protocol::error::is_runtime_auth_denial( + *status, + body.as_deref().unwrap_or_default(), + codex_response_debug_context::extract_response_debug_context(error) + .auth_error_code + .as_deref(), + )) + } + fn info(&self) -> &ModelProviderInfo { &self.info } diff --git a/codex-rs/model-provider/src/request_setup.rs b/codex-rs/model-provider/src/request_setup.rs index 106e4c87cb..2fc3ce4a8d 100644 --- a/codex-rs/model-provider/src/request_setup.rs +++ b/codex-rs/model-provider/src/request_setup.rs @@ -46,9 +46,8 @@ pub(crate) async fn resolve_request_auth( let runtime = crate::select_runtime_provider(&info, manager.as_ref(), &runtime)?; let headers = info.build_header_map()?; - let selected = crate::runtime_auth::select_runtime_auth(&runtime, &headers) - .await - .map_err(|error| codex_api::map_api_error(error.into()))?; + let selected = + crate::runtime_auth::select_runtime_auth(&runtime, &headers).await?; let resolved = ResolvedProviderAuth::for_runtime(selected, mode); resolved.validate_headers(&headers)?; tracing::debug!(runtime_auth_mode = ?mode, "selected runtime authentication"); diff --git a/codex-rs/model-provider/src/runtime_auth.rs b/codex-rs/model-provider/src/runtime_auth.rs index 7caae6d7fa..801655ee55 100644 --- a/codex-rs/model-provider/src/runtime_auth.rs +++ b/codex-rs/model-provider/src/runtime_auth.rs @@ -25,11 +25,13 @@ use std::sync::Arc; pub(crate) async fn select_runtime_auth( manager: &Arc, headers: &HeaderMap, -) -> Result { +) -> CodexResult { + let selection = + resource_selection(headers).map_err(|error| codex_api::map_api_error(error.into()))?; manager - .auth_with_selection(&resource_selection(headers)?) + .auth_with_selection(&selection) .await - .map_err(AuthError::from) + .map_err(map_runtime_auth_error) } /// Select the request's resource scope while preserving captured credential ownership. @@ -37,11 +39,20 @@ pub async fn select_runtime_auth_for( binding: &Arc, auth: &CodexAuth, headers: &HeaderMap, -) -> Result { +) -> CodexResult { + let selection = + resource_selection(headers).map_err(|error| codex_api::map_api_error(error.into()))?; binding - .auth_for(auth, &resource_selection(headers)?) + .auth_for(auth, &selection) .await - .map_err(AuthError::from) + .map_err(map_runtime_auth_error) +} + +fn map_runtime_auth_error(error: RuntimeAuthError) -> CodexErr { + match error { + RuntimeAuthError::RefreshTokenFailed(error) => CodexErr::RefreshTokenFailed(error), + error => codex_api::map_api_error(AuthError::from(error).into()), + } } /// Select fresh request auth without allowing a reusable client's source to change. diff --git a/codex-rs/model-provider/src/runtime_auth_tests.rs b/codex-rs/model-provider/src/runtime_auth_tests.rs index 65bca05e98..0a322a19c1 100644 --- a/codex-rs/model-provider/src/runtime_auth_tests.rs +++ b/codex-rs/model-provider/src/runtime_auth_tests.rs @@ -1,9 +1,14 @@ use super::*; use crate::ProviderAuthContext; +use base64::Engine; use codex_features::RuntimeAuthMode; +use codex_login::AuthCredentialsStoreMode; +use codex_login::AuthKeyringBackendKind; use codex_login::CodexAuth; use codex_login::auth::BedrockApiKeyAuth; +use codex_login::auth::RefreshTokenFailedReason; use codex_protocol::config_types::ModelProviderAuthInfo; +use codex_protocol::error::CodexErrorDetails; use codex_runtime_agent_instance::AgentHostAuthorization; use codex_runtime_agent_instance::RuntimeAuthManager; use codex_runtime_agent_instance::RuntimeRegistrationMetadata; @@ -15,6 +20,7 @@ use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; use wiremock::matchers::method; +use wiremock::matchers::path; async fn context( server: &MockServer, @@ -241,3 +247,140 @@ async fn bedrock_auth_stays_independent_when_openai_runtime_auth_is_enabled() { "openai-key" ); } + +#[tokio::test] +async fn runtime_denial_recovery_is_scoped_to_openai() { + let server = MockServer::start().await; + let credentials = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("actor-key")); + let context = context(&server, credentials, RuntimeAuthMode::AssertionOnly).await; + for (url, has_runtime) in [ + ("https://api.openai.com/v1", true), + ("https://provider.example/v1", false), + ] { + let provider = context.model_provider(ModelProviderInfo::create_openai_provider(Some( + url.to_string(), + ))); + for code in ["invalid_agent_instance_assertion", "token_expired"] { + let error = codex_api::TransportError::Http { + status: http::StatusCode::UNAUTHORIZED, + retry_after: None, + url: Some(format!("{url}/responses")), + headers: None, + body: Some(json!({"error": {"code": code}}).to_string()), + }; + assert_eq!( + provider.is_recoverable_auth_error(&error), + !has_runtime || code == "token_expired" + ); + } + } +} + +#[tokio::test] +async fn permanent_runtime_refresh_failure_preserves_sign_in_error() { + for mode in [RuntimeAuthMode::DualHeader, RuntimeAuthMode::AssertionOnly] { + // Exercise proactive refresh and registration's existing 401 recovery. + for last_refresh in ["2000-01-01T00:00:00Z", "2099-01-01T00:00:00Z"] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": {"code": "refresh_token_invalidated"}, + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/register")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + let home = tempfile::tempdir().expect("create auth home"); + let claims = json!({ + "https://api.openai.com/auth": { + "chatgpt_user_id": "user-a", + "chatgpt_account_id": "workspace-a", + }, + }); + let token = format!( + "e30.{}.signature", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(claims.to_string()) + ); + let stored = serde_json::from_value(json!({ + "auth_mode": "chatgpt", + "tokens": { + "id_token": token, "access_token": "initial-token", + "refresh_token": "revoked-refresh-token", "account_id": "workspace-a", + "oauth_provenance": { + "issuer": server.uri(), "client_id": "runtime-refresh-test", + }, + }, + "last_refresh": last_refresh, + })) + .expect("construct stored OAuth credentials"); + codex_login::save_auth( + home.path(), + &stored, + AuthCredentialsStoreMode::File, + AuthKeyringBackendKind::Direct, + ) + .expect("save OAuth credentials"); + let credentials = Arc::new( + AuthManager::new( + home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*forced_chatgpt_workspace_id*/ None, + /*chatgpt_base_url*/ None, + AuthKeyringBackendKind::Direct, + codex_login::test_support::transport_default_auth_route_config(), + ) + .await, + ); + let original = credentials.auth_cached().expect("load OAuth credentials"); + let runtime = Arc::new(RuntimeAuthManager::new( + Arc::clone(&credentials), + AgentHostAuthorization::InstallationId { + installation_id: "00000000-0000-0000-0000-000000000000" + .parse() + .expect("parse installation ID"), + }, + RuntimeRegistrationMetadata { + harness: None, + harness_version: None, + role: RuntimeRole::Orchestrator, + }, + format!("{}/register", server.uri()) + .parse() + .expect("parse registration URL"), + )); + let provider = ProviderAuthContext { + credentials: Arc::clone(&credentials), + runtime_auth: Some((Arc::clone(&runtime), mode)), + } + .model_provider(ModelProviderInfo::create_openai_provider( + /*base_url*/ None, + )); + + // A later request must preserve the cached rejection, not retry it as a stream error. + for _ in 0..2 { + let error = provider + .request_auth(/*scope*/ None) + .await + .err() + .expect("revoked refresh must reject request preparation"); + let failure = credentials + .refresh_failure_for_auth(&original) + .expect("permanent refresh rejection is cached"); + assert_eq!(failure.reason, RefreshTokenFailedReason::Revoked); + let CodexErrorDetails::RefreshTokenFailed(actual) = error.details() else { + panic!("expected sign-in error for {mode:?}/{last_refresh}, got {error:?}"); + }; + assert_eq!(actual, &failure); + assert_eq!(error.retry_delay(/*retry_count*/ 1), None); + } + + server.verify().await; + } + } +} diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rs index 3dff3b4a75..bc47eedb2d 100644 --- a/codex-rs/protocol/src/error.rs +++ b/codex-rs/protocol/src/error.rs @@ -429,6 +429,15 @@ impl CodexErr { /// otherwise use exponential backoff with jitter. Callers enforce their own retry budgets. pub fn retry_delay(&self, retry_count: u64) -> Option { match self.details() { + CodexErrorDetails::UnexpectedStatus(error) + if is_runtime_auth_denial( + error.status, + &error.body, + error.identity_error_code.as_deref(), + ) => + { + None + } CodexErrorDetails::TurnAborted | CodexErrorDetails::SessionBudgetExceeded | CodexErrorDetails::Interrupted @@ -629,6 +638,39 @@ pub struct UnexpectedResponseError { pub identity_error_code: Option, } +/// Runtime denial cannot be repaired by retrying the request or refreshing its original credential. +/// Keep unavailable verification state and ordinary credential errors on their existing paths. +pub fn is_runtime_auth_denial( + status: StatusCode, + body: &str, + identity_error_code: Option<&str>, +) -> bool { + if !matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + return false; + } + let is_terminal = |code| { + matches!( + code, + Some( + "runtime_assertion_invalid" + | "runtime_instance_paused" + | "runtime_instance_revoked" + | "invalid_agent_instance_assertion" + ) + ) + }; + if is_terminal(identity_error_code) { + return true; + } + let Ok(body) = serde_json::from_str::(body) else { + return false; + }; + is_terminal( + body.pointer("/error/code") + .and_then(serde_json::Value::as_str), + ) +} + const UNEXPECTED_RESPONSE_BODY_MAX_BYTES: usize = 1000; impl UnexpectedResponseError { diff --git a/codex-rs/response-debug-context/src/lib.rs b/codex-rs/response-debug-context/src/lib.rs index 3446b1c2a3..aca010b013 100644 --- a/codex-rs/response-debug-context/src/lib.rs +++ b/codex-rs/response-debug-context/src/lib.rs @@ -17,16 +17,22 @@ pub struct ResponseDebugContext { } pub fn extract_response_debug_context(transport: &TransportError) -> ResponseDebugContext { + match transport { + TransportError::Http { + headers: Some(headers), + .. + } => extract_response_debug_context_from_headers(headers), + _ => ResponseDebugContext::default(), + } +} + +pub fn extract_response_debug_context_from_headers( + headers: &http::HeaderMap, +) -> ResponseDebugContext { let mut context = ResponseDebugContext::default(); - - let TransportError::Http { headers, .. } = transport else { - return context; - }; - let extract_header = |name: &str| { headers - .as_ref() - .and_then(|headers| headers.get(name)) + .get(name) .and_then(|value| value.to_str().ok()) .map(str::to_string) }; diff --git a/codex-rs/runtime-agent-instance/src/auth.rs b/codex-rs/runtime-agent-instance/src/auth.rs index a59952e526..4f0475a567 100644 --- a/codex-rs/runtime-agent-instance/src/auth.rs +++ b/codex-rs/runtime-agent-instance/src/auth.rs @@ -183,6 +183,7 @@ impl From for AuthError { | RuntimeAuthError::UnsupportedResourceSelection | RuntimeAuthError::Changed | RuntimeAuthError::RefreshFailed + | RuntimeAuthError::RefreshTokenFailed(_) | RuntimeAuthError::KeyGeneration | RuntimeAuthError::Registration( RegistrationError::InvalidEndpoint diff --git a/codex-rs/runtime-agent-instance/src/binding.rs b/codex-rs/runtime-agent-instance/src/binding.rs index 5e4e93e66e..d810d5f7c7 100644 --- a/codex-rs/runtime-agent-instance/src/binding.rs +++ b/codex-rs/runtime-agent-instance/src/binding.rs @@ -17,6 +17,7 @@ use codex_login::AuthManager; use codex_login::AuthOwner; use codex_login::CodexAuth; use codex_login::RefreshTokenError; +use codex_login::auth::RefreshTokenFailedError; use http::StatusCode; use std::sync::Arc; use tokio::sync::Mutex; @@ -169,9 +170,8 @@ impl RuntimeAuthBinding { self.check_source(expected_source)?; recovery.next().await.map_err(|error| match error { RefreshTokenError::Policy(denied) => RuntimeAuthError::Policy(denied), - RefreshTokenError::Permanent(_) | RefreshTokenError::Transient(_) => { - RuntimeAuthError::RefreshFailed - } + RefreshTokenError::Permanent(error) => RuntimeAuthError::RefreshTokenFailed(error), + RefreshTokenError::Transient(_) => RuntimeAuthError::RefreshFailed, })?; let (auth, http_client_factory) = self .credentials @@ -340,8 +340,8 @@ impl RuntimeAuthBinding { .credentials .auth_change_state_for(auth) .ok_or(RuntimeAuthError::Changed)?; - if self.credentials.refresh_failure_for_auth(auth).is_some() { - return Err(RuntimeAuthError::RefreshFailed); + if let Some(error) = self.credentials.refresh_failure_for_auth(auth) { + return Err(RuntimeAuthError::RefreshTokenFailed(error)); } Ok(state) } @@ -390,6 +390,8 @@ pub enum RuntimeAuthError { Changed, #[error("runtime registration could not refresh actor credentials")] RefreshFailed, + #[error(transparent)] + RefreshTokenFailed(#[from] RefreshTokenFailedError), #[error("failed to generate runtime signing key")] KeyGeneration, #[error(transparent)]