[codex] Harden workload identity startup [ci changed_files]

This commit is contained in:
Cooper Gamble
2026-06-17 17:00:14 +00:00
parent 042b7dcfd3
commit 7050f9e1c6
6 changed files with 206 additions and 42 deletions
+9 -3
View File
@@ -604,7 +604,7 @@ impl Session {
let mcp_manager_for_mcp = Arc::clone(&mcp_manager);
let mcp_thread_init_for_startup = &mcp_thread_init;
let auth_and_mcp_fut = async move {
let auth = auth_manager_clone.auth().await;
let auth = auth_manager_clone.auth_result().await?;
let mcp_config = mcp_manager_for_mcp
.runtime_config_for_thread(&config_for_mcp, mcp_thread_init_for_startup)
.await;
@@ -617,7 +617,12 @@ impl Session {
auth.as_ref(),
)
.await;
(auth, mcp_servers, auth_statuses, tool_plugin_provenance)
Ok::<_, std::io::Error>((
auth,
mcp_servers,
auth_statuses,
tool_plugin_provenance,
))
}
.instrument(info_span!(
"session_init.auth_mcp",
@@ -628,8 +633,9 @@ impl Session {
let (
thread_persistence_result,
state_db_ctx,
(auth, mcp_servers, auth_statuses, tool_plugin_provenance),
auth_and_mcp_result,
) = tokio::join!(thread_persistence_fut, state_db_fut, auth_and_mcp_fut);
let (auth, mcp_servers, auth_statuses, tool_plugin_provenance) = auth_and_mcp_result?;
let mut live_thread_init =
LiveThreadInitGuard::new(thread_persistence_result.map_err(|e| {
+40
View File
@@ -555,6 +555,26 @@ struct TestExternalChatgptAuth {
refresh_count: AtomicUsize,
}
struct FailingExternalChatgptAuth;
#[async_trait]
impl ExternalAuth for FailingExternalChatgptAuth {
fn auth_mode(&self) -> AuthMode {
AuthMode::Chatgpt
}
async fn resolve(&self) -> std::io::Result<Option<ExternalAuthTokens>> {
Err(std::io::Error::other("external token exchange failed"))
}
async fn refresh(
&self,
_context: ExternalAuthRefreshContext,
) -> std::io::Result<ExternalAuthTokens> {
Err(std::io::Error::other("external token exchange failed"))
}
}
#[async_trait]
impl ExternalAuth for TestExternalChatgptAuth {
fn auth_mode(&self) -> AuthMode {
@@ -629,6 +649,26 @@ async fn external_chatgpt_auth_resolves_and_refreshes_without_writing_auth_file(
assert!(!get_auth_file(codex_home.path()).exists());
}
#[tokio::test]
async fn auth_result_preserves_external_auth_resolution_errors() {
let manager = AuthManager::shared(
PathBuf::from("non-existent"),
/*enable_codex_api_key_env*/ false,
AuthCredentialsStoreMode::File,
/*chatgpt_base_url*/ None,
)
.await;
manager.set_external_auth(Arc::new(FailingExternalChatgptAuth));
let error = manager
.auth_result()
.await
.expect_err("external auth failure should be preserved");
assert_eq!(error.to_string(), "external token exchange failed");
assert_eq!(manager.auth().await, None);
}
#[tokio::test]
async fn external_bearer_only_auth_manager_uses_cached_provider_token() {
let script = ProviderAuthScript::new(&["provider-token", "next-token"]).unwrap();
+27 -20
View File
@@ -1689,18 +1689,31 @@ impl AuthManager {
/// For managed ChatGPT auth that needs a proactive refresh, first performs
/// a guarded reload and then refreshes only if the on-disk auth is unchanged.
pub async fn auth(&self) -> Option<CodexAuth> {
match self.auth_result().await {
Ok(auth) => auth,
Err(error) => {
tracing::error!("Failed to resolve external auth: {error}");
None
}
}
}
/// Resolves current auth while preserving errors from explicitly configured external auth.
pub async fn auth_result(&self) -> std::io::Result<Option<CodexAuth>> {
if self.has_external_auth() {
return self.resolve_external_auth().await;
}
let auth = self.auth_cached()?;
let Some(auth) = self.auth_cached() else {
return Ok(None);
};
if Self::should_refresh_proactively(&auth)
&& let Err(err) = self.refresh_token().await
{
tracing::error!("Failed to refresh token: {}", err);
return Some(auth);
return Ok(Some(auth));
}
self.auth_cached()
Ok(self.auth_cached())
}
/// Force a reload of the auth information from auth.json. Returns
@@ -1945,28 +1958,22 @@ impl AuthManager {
self.external_auth_mode() == Some(AuthMode::ApiKey)
}
async fn resolve_external_auth(&self) -> Option<CodexAuth> {
let external_auth = self.external_auth()?;
async fn resolve_external_auth(&self) -> std::io::Result<Option<CodexAuth>> {
let Some(external_auth) = self.external_auth() else {
return Ok(None);
};
match external_auth.resolve().await {
Ok(Some(tokens)) if external_auth.auth_mode() == AuthMode::ApiKey => {
Some(CodexAuth::from_api_key(&tokens.access_token))
Ok(Some(CodexAuth::from_api_key(&tokens.access_token)))
}
Ok(Some(tokens)) => match self.external_chatgpt_auth(&tokens).await {
Ok(auth) => {
self.set_cached_auth(Some(auth.clone()));
Some(auth)
}
Err(error) => {
tracing::error!("Failed to resolve external ChatGPT auth: {error}");
None
}
},
Ok(None) => self.auth_cached(),
Err(err) => {
tracing::error!("Failed to resolve external auth: {err}");
None
Ok(Some(tokens)) => {
let auth = self.external_chatgpt_auth(&tokens).await?;
self.set_cached_auth(Some(auth.clone()));
Ok(Some(auth))
}
Ok(None) => Ok(self.auth_cached()),
Err(error) => Err(error),
}
}
+1 -1
View File
@@ -16,7 +16,7 @@ reqwest = { workspace = true, features = ["json"] }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["fs"] }
tokio = { workspace = true, features = ["fs", "sync"] }
url = { workspace = true }
[dev-dependencies]
+81 -18
View File
@@ -11,12 +11,14 @@ use reqwest::StatusCode;
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
use tokio::sync::Semaphore;
const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange";
const JWT_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:jwt";
const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_token";
const MAX_REFRESH_LEAD: Duration = Duration::from_secs(5 * 60);
const TOKEN_EXCHANGE_TIMEOUT: Duration = Duration::from_secs(10);
const FAILED_EXCHANGE_RETRY_DELAY: Duration = Duration::from_secs(5);
#[derive(Clone, PartialEq, Eq)]
pub struct WorkloadIdentityAccessToken {
@@ -62,6 +64,10 @@ pub enum WorkloadIdentityError {
InvalidLifetime,
#[error("workload identity token exchange returned an unexpected token type")]
UnexpectedTokenType,
#[error("workload identity token exchange is unavailable")]
ExchangeUnavailable,
#[error("{0}")]
RecentFailure(String),
}
enum SubjectTokenSource {
@@ -103,6 +109,17 @@ struct CachedAccessToken {
refresh_at: Instant,
}
struct CachedExchangeFailure {
message: String,
retry_at: Instant,
}
#[derive(Default)]
struct CacheState {
token: Option<CachedAccessToken>,
failure: Option<CachedExchangeFailure>,
}
pub struct WorkloadIdentityClient {
identity_provider_id: String,
identity_provider_mapping_id: String,
@@ -110,7 +127,8 @@ pub struct WorkloadIdentityClient {
client_id: String,
source: SubjectTokenSource,
http: reqwest::Client,
cache: Mutex<Option<CachedAccessToken>>,
cache: Mutex<CacheState>,
exchange_lock: Semaphore,
}
impl WorkloadIdentityClient {
@@ -126,27 +144,69 @@ impl WorkloadIdentityClient {
client_id: client_id.into(),
source: SubjectTokenSource::from_config(config.credential_source),
http,
cache: Mutex::new(None),
cache: Mutex::new(CacheState::default()),
exchange_lock: Semaphore::new(/*permits*/ 1),
}
}
pub async fn resolve(&self) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
{
let cache = self
.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(cached) = cache.as_ref()
&& Instant::now() < cached.refresh_at
{
return Ok(cached.token.clone());
}
if let Some(cached) = self.cached_result() {
return cached;
}
self.exchange_and_cache().await
let _permit = self
.exchange_lock
.acquire()
.await
.map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?;
if let Some(cached) = self.cached_result() {
return cached;
}
self.exchange_and_record().await
}
pub async fn refresh(&self) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
self.exchange_and_cache().await
let _permit = self
.exchange_lock
.acquire()
.await
.map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?;
self.exchange_and_record().await
}
fn cached_result(&self) -> Option<Result<WorkloadIdentityAccessToken, WorkloadIdentityError>> {
let cache = self
.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = Instant::now();
if let Some(failure) = cache.failure.as_ref()
&& now < failure.retry_at
{
return Some(Err(WorkloadIdentityError::RecentFailure(
failure.message.clone(),
)));
}
cache
.token
.as_ref()
.filter(|cached| now < cached.refresh_at)
.map(|cached| Ok(cached.token.clone()))
}
async fn exchange_and_record(
&self,
) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
let result = self.exchange_and_cache().await;
if let Err(error) = &result {
self.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.failure = Some(CachedExchangeFailure {
message: error.to_string(),
retry_at: Instant::now() + FAILED_EXCHANGE_RETRY_DELAY,
});
}
result
}
async fn exchange_and_cache(
@@ -209,10 +269,13 @@ impl WorkloadIdentityClient {
*self
.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(CachedAccessToken {
token: token.clone(),
refresh_at,
});
.unwrap_or_else(std::sync::PoisonError::into_inner) = CacheState {
failure: None,
token: Some(CachedAccessToken {
token: token.clone(),
refresh_at,
}),
};
Ok(token)
}
}
@@ -74,6 +74,54 @@ async fn exchanges_azure_subject_token_and_caches_access_token() -> anyhow::Resu
Ok(())
}
#[tokio::test]
async fn concurrent_resolves_share_one_successful_exchange() -> anyhow::Result<()> {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/oauth/token"))
.respond_with(exchange_response("shared.access.token"))
.expect(1)
.mount(&server)
.await;
let client = test_client(&server).await?;
let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve());
assert_eq!(first?, second?);
assert_eq!(client.resolve().await?, third?);
Ok(())
}
#[tokio::test]
async fn concurrent_resolves_share_one_rejected_exchange() -> anyhow::Result<()> {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/oauth/token"))
.respond_with(ResponseTemplate::new(401).set_body_json(json!({"error": "invalid_grant"})))
.expect(1)
.mount(&server)
.await;
let client = test_client(&server).await?;
let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve());
let expected =
"workload identity token exchange was rejected with HTTP 401 Unauthorized: invalid_grant";
assert_eq!(
first.expect_err("first exchange should fail").to_string(),
expected
);
assert_eq!(
second.expect_err("second exchange should fail").to_string(),
expected
);
assert_eq!(
third.expect_err("third exchange should fail").to_string(),
expected
);
Ok(())
}
#[tokio::test]
async fn forced_refresh_performs_a_new_exchange() -> anyhow::Result<()> {
let server = MockServer::start().await;