diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index c4bc064a47..f733bb8920 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -604,7 +604,7 @@ impl Session { let mcp_manager_for_mcp = Arc::clone(&mcp_manager); let mcp_thread_init_for_startup = &mcp_thread_init; let auth_and_mcp_fut = async move { - let auth = auth_manager_clone.auth().await; + let auth = auth_manager_clone.auth_result().await?; let mcp_config = mcp_manager_for_mcp .runtime_config_for_thread(&config_for_mcp, mcp_thread_init_for_startup) .await; @@ -617,7 +617,12 @@ impl Session { auth.as_ref(), ) .await; - (auth, mcp_servers, auth_statuses, tool_plugin_provenance) + Ok::<_, std::io::Error>(( + auth, + mcp_servers, + auth_statuses, + tool_plugin_provenance, + )) } .instrument(info_span!( "session_init.auth_mcp", @@ -628,8 +633,9 @@ impl Session { let ( thread_persistence_result, state_db_ctx, - (auth, mcp_servers, auth_statuses, tool_plugin_provenance), + auth_and_mcp_result, ) = tokio::join!(thread_persistence_fut, state_db_fut, auth_and_mcp_fut); + let (auth, mcp_servers, auth_statuses, tool_plugin_provenance) = auth_and_mcp_result?; let mut live_thread_init = LiveThreadInitGuard::new(thread_persistence_result.map_err(|e| { diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 6690201bff..e2865364ff 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -555,6 +555,26 @@ struct TestExternalChatgptAuth { refresh_count: AtomicUsize, } +struct FailingExternalChatgptAuth; + +#[async_trait] +impl ExternalAuth for FailingExternalChatgptAuth { + fn auth_mode(&self) -> AuthMode { + AuthMode::Chatgpt + } + + async fn resolve(&self) -> std::io::Result> { + Err(std::io::Error::other("external token exchange failed")) + } + + async fn refresh( + &self, + _context: ExternalAuthRefreshContext, + ) -> std::io::Result { + Err(std::io::Error::other("external token exchange failed")) + } +} + #[async_trait] impl ExternalAuth for TestExternalChatgptAuth { fn auth_mode(&self) -> AuthMode { @@ -629,6 +649,26 @@ async fn external_chatgpt_auth_resolves_and_refreshes_without_writing_auth_file( assert!(!get_auth_file(codex_home.path()).exists()); } +#[tokio::test] +async fn auth_result_preserves_external_auth_resolution_errors() { + let manager = AuthManager::shared( + PathBuf::from("non-existent"), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*chatgpt_base_url*/ None, + ) + .await; + manager.set_external_auth(Arc::new(FailingExternalChatgptAuth)); + + let error = manager + .auth_result() + .await + .expect_err("external auth failure should be preserved"); + + assert_eq!(error.to_string(), "external token exchange failed"); + assert_eq!(manager.auth().await, None); +} + #[tokio::test] async fn external_bearer_only_auth_manager_uses_cached_provider_token() { let script = ProviderAuthScript::new(&["provider-token", "next-token"]).unwrap(); diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 2e1b8eabbf..50f09a3d0a 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -1689,18 +1689,31 @@ impl AuthManager { /// For managed ChatGPT auth that needs a proactive refresh, first performs /// a guarded reload and then refreshes only if the on-disk auth is unchanged. pub async fn auth(&self) -> Option { + match self.auth_result().await { + Ok(auth) => auth, + Err(error) => { + tracing::error!("Failed to resolve external auth: {error}"); + None + } + } + } + + /// Resolves current auth while preserving errors from explicitly configured external auth. + pub async fn auth_result(&self) -> std::io::Result> { if self.has_external_auth() { return self.resolve_external_auth().await; } - let auth = self.auth_cached()?; + let Some(auth) = self.auth_cached() else { + return Ok(None); + }; if Self::should_refresh_proactively(&auth) && let Err(err) = self.refresh_token().await { tracing::error!("Failed to refresh token: {}", err); - return Some(auth); + return Ok(Some(auth)); } - self.auth_cached() + Ok(self.auth_cached()) } /// Force a reload of the auth information from auth.json. Returns @@ -1945,28 +1958,22 @@ impl AuthManager { self.external_auth_mode() == Some(AuthMode::ApiKey) } - async fn resolve_external_auth(&self) -> Option { - let external_auth = self.external_auth()?; + async fn resolve_external_auth(&self) -> std::io::Result> { + let Some(external_auth) = self.external_auth() else { + return Ok(None); + }; match external_auth.resolve().await { Ok(Some(tokens)) if external_auth.auth_mode() == AuthMode::ApiKey => { - Some(CodexAuth::from_api_key(&tokens.access_token)) + Ok(Some(CodexAuth::from_api_key(&tokens.access_token))) } - Ok(Some(tokens)) => match self.external_chatgpt_auth(&tokens).await { - Ok(auth) => { - self.set_cached_auth(Some(auth.clone())); - Some(auth) - } - Err(error) => { - tracing::error!("Failed to resolve external ChatGPT auth: {error}"); - None - } - }, - Ok(None) => self.auth_cached(), - Err(err) => { - tracing::error!("Failed to resolve external auth: {err}"); - None + Ok(Some(tokens)) => { + let auth = self.external_chatgpt_auth(&tokens).await?; + self.set_cached_auth(Some(auth.clone())); + Ok(Some(auth)) } + Ok(None) => Ok(self.auth_cached()), + Err(error) => Err(error), } } diff --git a/codex-rs/workload-identity/Cargo.toml b/codex-rs/workload-identity/Cargo.toml index c9ee639aa5..2f4082286c 100644 --- a/codex-rs/workload-identity/Cargo.toml +++ b/codex-rs/workload-identity/Cargo.toml @@ -16,7 +16,7 @@ reqwest = { workspace = true, features = ["json"] } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } +tokio = { workspace = true, features = ["fs", "sync"] } url = { workspace = true } [dev-dependencies] diff --git a/codex-rs/workload-identity/src/client.rs b/codex-rs/workload-identity/src/client.rs index c7da7d36b9..c65cbb5d31 100644 --- a/codex-rs/workload-identity/src/client.rs +++ b/codex-rs/workload-identity/src/client.rs @@ -11,12 +11,14 @@ use reqwest::StatusCode; use serde::Deserialize; use serde::Serialize; use thiserror::Error; +use tokio::sync::Semaphore; const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; const JWT_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:jwt"; const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_token"; const MAX_REFRESH_LEAD: Duration = Duration::from_secs(5 * 60); const TOKEN_EXCHANGE_TIMEOUT: Duration = Duration::from_secs(10); +const FAILED_EXCHANGE_RETRY_DELAY: Duration = Duration::from_secs(5); #[derive(Clone, PartialEq, Eq)] pub struct WorkloadIdentityAccessToken { @@ -62,6 +64,10 @@ pub enum WorkloadIdentityError { InvalidLifetime, #[error("workload identity token exchange returned an unexpected token type")] UnexpectedTokenType, + #[error("workload identity token exchange is unavailable")] + ExchangeUnavailable, + #[error("{0}")] + RecentFailure(String), } enum SubjectTokenSource { @@ -103,6 +109,17 @@ struct CachedAccessToken { refresh_at: Instant, } +struct CachedExchangeFailure { + message: String, + retry_at: Instant, +} + +#[derive(Default)] +struct CacheState { + token: Option, + failure: Option, +} + pub struct WorkloadIdentityClient { identity_provider_id: String, identity_provider_mapping_id: String, @@ -110,7 +127,8 @@ pub struct WorkloadIdentityClient { client_id: String, source: SubjectTokenSource, http: reqwest::Client, - cache: Mutex>, + cache: Mutex, + exchange_lock: Semaphore, } impl WorkloadIdentityClient { @@ -126,27 +144,69 @@ impl WorkloadIdentityClient { client_id: client_id.into(), source: SubjectTokenSource::from_config(config.credential_source), http, - cache: Mutex::new(None), + cache: Mutex::new(CacheState::default()), + exchange_lock: Semaphore::new(/*permits*/ 1), } } pub async fn resolve(&self) -> Result { - { - let cache = self - .cache - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(cached) = cache.as_ref() - && Instant::now() < cached.refresh_at - { - return Ok(cached.token.clone()); - } + if let Some(cached) = self.cached_result() { + return cached; } - self.exchange_and_cache().await + let _permit = self + .exchange_lock + .acquire() + .await + .map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?; + if let Some(cached) = self.cached_result() { + return cached; + } + self.exchange_and_record().await } pub async fn refresh(&self) -> Result { - self.exchange_and_cache().await + let _permit = self + .exchange_lock + .acquire() + .await + .map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?; + self.exchange_and_record().await + } + + fn cached_result(&self) -> Option> { + let cache = self + .cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + if let Some(failure) = cache.failure.as_ref() + && now < failure.retry_at + { + return Some(Err(WorkloadIdentityError::RecentFailure( + failure.message.clone(), + ))); + } + cache + .token + .as_ref() + .filter(|cached| now < cached.refresh_at) + .map(|cached| Ok(cached.token.clone())) + } + + async fn exchange_and_record( + &self, + ) -> Result { + let result = self.exchange_and_cache().await; + if let Err(error) = &result { + self.cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .failure = Some(CachedExchangeFailure { + message: error.to_string(), + retry_at: Instant::now() + FAILED_EXCHANGE_RETRY_DELAY, + }); + } + result } async fn exchange_and_cache( @@ -209,10 +269,13 @@ impl WorkloadIdentityClient { *self .cache .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(CachedAccessToken { - token: token.clone(), - refresh_at, - }); + .unwrap_or_else(std::sync::PoisonError::into_inner) = CacheState { + failure: None, + token: Some(CachedAccessToken { + token: token.clone(), + refresh_at, + }), + }; Ok(token) } } diff --git a/codex-rs/workload-identity/src/client_tests.rs b/codex-rs/workload-identity/src/client_tests.rs index 79210ffdf9..41dee413bd 100644 --- a/codex-rs/workload-identity/src/client_tests.rs +++ b/codex-rs/workload-identity/src/client_tests.rs @@ -74,6 +74,54 @@ async fn exchanges_azure_subject_token_and_caches_access_token() -> anyhow::Resu Ok(()) } +#[tokio::test] +async fn concurrent_resolves_share_one_successful_exchange() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(exchange_response("shared.access.token")) + .expect(1) + .mount(&server) + .await; + let client = test_client(&server).await?; + + let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve()); + + assert_eq!(first?, second?); + assert_eq!(client.resolve().await?, third?); + Ok(()) +} + +#[tokio::test] +async fn concurrent_resolves_share_one_rejected_exchange() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(401).set_body_json(json!({"error": "invalid_grant"}))) + .expect(1) + .mount(&server) + .await; + let client = test_client(&server).await?; + + let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve()); + let expected = + "workload identity token exchange was rejected with HTTP 401 Unauthorized: invalid_grant"; + + assert_eq!( + first.expect_err("first exchange should fail").to_string(), + expected + ); + assert_eq!( + second.expect_err("second exchange should fail").to_string(), + expected + ); + assert_eq!( + third.expect_err("third exchange should fail").to_string(), + expected + ); + Ok(()) +} + #[tokio::test] async fn forced_refresh_performs_a_new_exchange() -> anyhow::Result<()> { let server = MockServer::start().await;