mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
[codex] Harden workload identity startup [ci changed_files]
This commit is contained in:
@@ -604,7 +604,7 @@ impl Session {
|
||||
let mcp_manager_for_mcp = Arc::clone(&mcp_manager);
|
||||
let mcp_thread_init_for_startup = &mcp_thread_init;
|
||||
let auth_and_mcp_fut = async move {
|
||||
let auth = auth_manager_clone.auth().await;
|
||||
let auth = auth_manager_clone.auth_result().await?;
|
||||
let mcp_config = mcp_manager_for_mcp
|
||||
.runtime_config_for_thread(&config_for_mcp, mcp_thread_init_for_startup)
|
||||
.await;
|
||||
@@ -617,7 +617,12 @@ impl Session {
|
||||
auth.as_ref(),
|
||||
)
|
||||
.await;
|
||||
(auth, mcp_servers, auth_statuses, tool_plugin_provenance)
|
||||
Ok::<_, std::io::Error>((
|
||||
auth,
|
||||
mcp_servers,
|
||||
auth_statuses,
|
||||
tool_plugin_provenance,
|
||||
))
|
||||
}
|
||||
.instrument(info_span!(
|
||||
"session_init.auth_mcp",
|
||||
@@ -628,8 +633,9 @@ impl Session {
|
||||
let (
|
||||
thread_persistence_result,
|
||||
state_db_ctx,
|
||||
(auth, mcp_servers, auth_statuses, tool_plugin_provenance),
|
||||
auth_and_mcp_result,
|
||||
) = tokio::join!(thread_persistence_fut, state_db_fut, auth_and_mcp_fut);
|
||||
let (auth, mcp_servers, auth_statuses, tool_plugin_provenance) = auth_and_mcp_result?;
|
||||
|
||||
let mut live_thread_init =
|
||||
LiveThreadInitGuard::new(thread_persistence_result.map_err(|e| {
|
||||
|
||||
@@ -555,6 +555,26 @@ struct TestExternalChatgptAuth {
|
||||
refresh_count: AtomicUsize,
|
||||
}
|
||||
|
||||
struct FailingExternalChatgptAuth;
|
||||
|
||||
#[async_trait]
|
||||
impl ExternalAuth for FailingExternalChatgptAuth {
|
||||
fn auth_mode(&self) -> AuthMode {
|
||||
AuthMode::Chatgpt
|
||||
}
|
||||
|
||||
async fn resolve(&self) -> std::io::Result<Option<ExternalAuthTokens>> {
|
||||
Err(std::io::Error::other("external token exchange failed"))
|
||||
}
|
||||
|
||||
async fn refresh(
|
||||
&self,
|
||||
_context: ExternalAuthRefreshContext,
|
||||
) -> std::io::Result<ExternalAuthTokens> {
|
||||
Err(std::io::Error::other("external token exchange failed"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExternalAuth for TestExternalChatgptAuth {
|
||||
fn auth_mode(&self) -> AuthMode {
|
||||
@@ -629,6 +649,26 @@ async fn external_chatgpt_auth_resolves_and_refreshes_without_writing_auth_file(
|
||||
assert!(!get_auth_file(codex_home.path()).exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auth_result_preserves_external_auth_resolution_errors() {
|
||||
let manager = AuthManager::shared(
|
||||
PathBuf::from("non-existent"),
|
||||
/*enable_codex_api_key_env*/ false,
|
||||
AuthCredentialsStoreMode::File,
|
||||
/*chatgpt_base_url*/ None,
|
||||
)
|
||||
.await;
|
||||
manager.set_external_auth(Arc::new(FailingExternalChatgptAuth));
|
||||
|
||||
let error = manager
|
||||
.auth_result()
|
||||
.await
|
||||
.expect_err("external auth failure should be preserved");
|
||||
|
||||
assert_eq!(error.to_string(), "external token exchange failed");
|
||||
assert_eq!(manager.auth().await, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_bearer_only_auth_manager_uses_cached_provider_token() {
|
||||
let script = ProviderAuthScript::new(&["provider-token", "next-token"]).unwrap();
|
||||
|
||||
@@ -1689,18 +1689,31 @@ impl AuthManager {
|
||||
/// For managed ChatGPT auth that needs a proactive refresh, first performs
|
||||
/// a guarded reload and then refreshes only if the on-disk auth is unchanged.
|
||||
pub async fn auth(&self) -> Option<CodexAuth> {
|
||||
match self.auth_result().await {
|
||||
Ok(auth) => auth,
|
||||
Err(error) => {
|
||||
tracing::error!("Failed to resolve external auth: {error}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves current auth while preserving errors from explicitly configured external auth.
|
||||
pub async fn auth_result(&self) -> std::io::Result<Option<CodexAuth>> {
|
||||
if self.has_external_auth() {
|
||||
return self.resolve_external_auth().await;
|
||||
}
|
||||
|
||||
let auth = self.auth_cached()?;
|
||||
let Some(auth) = self.auth_cached() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if Self::should_refresh_proactively(&auth)
|
||||
&& let Err(err) = self.refresh_token().await
|
||||
{
|
||||
tracing::error!("Failed to refresh token: {}", err);
|
||||
return Some(auth);
|
||||
return Ok(Some(auth));
|
||||
}
|
||||
self.auth_cached()
|
||||
Ok(self.auth_cached())
|
||||
}
|
||||
|
||||
/// Force a reload of the auth information from auth.json. Returns
|
||||
@@ -1945,28 +1958,22 @@ impl AuthManager {
|
||||
self.external_auth_mode() == Some(AuthMode::ApiKey)
|
||||
}
|
||||
|
||||
async fn resolve_external_auth(&self) -> Option<CodexAuth> {
|
||||
let external_auth = self.external_auth()?;
|
||||
async fn resolve_external_auth(&self) -> std::io::Result<Option<CodexAuth>> {
|
||||
let Some(external_auth) = self.external_auth() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
match external_auth.resolve().await {
|
||||
Ok(Some(tokens)) if external_auth.auth_mode() == AuthMode::ApiKey => {
|
||||
Some(CodexAuth::from_api_key(&tokens.access_token))
|
||||
Ok(Some(CodexAuth::from_api_key(&tokens.access_token)))
|
||||
}
|
||||
Ok(Some(tokens)) => match self.external_chatgpt_auth(&tokens).await {
|
||||
Ok(auth) => {
|
||||
self.set_cached_auth(Some(auth.clone()));
|
||||
Some(auth)
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::error!("Failed to resolve external ChatGPT auth: {error}");
|
||||
None
|
||||
}
|
||||
},
|
||||
Ok(None) => self.auth_cached(),
|
||||
Err(err) => {
|
||||
tracing::error!("Failed to resolve external auth: {err}");
|
||||
None
|
||||
Ok(Some(tokens)) => {
|
||||
let auth = self.external_chatgpt_auth(&tokens).await?;
|
||||
self.set_cached_auth(Some(auth.clone()));
|
||||
Ok(Some(auth))
|
||||
}
|
||||
Ok(None) => Ok(self.auth_cached()),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ reqwest = { workspace = true, features = ["json"] }
|
||||
schemars = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs"] }
|
||||
tokio = { workspace = true, features = ["fs", "sync"] }
|
||||
url = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -11,12 +11,14 @@ use reqwest::StatusCode;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange";
|
||||
const JWT_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:jwt";
|
||||
const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_token";
|
||||
const MAX_REFRESH_LEAD: Duration = Duration::from_secs(5 * 60);
|
||||
const TOKEN_EXCHANGE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const FAILED_EXCHANGE_RETRY_DELAY: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct WorkloadIdentityAccessToken {
|
||||
@@ -62,6 +64,10 @@ pub enum WorkloadIdentityError {
|
||||
InvalidLifetime,
|
||||
#[error("workload identity token exchange returned an unexpected token type")]
|
||||
UnexpectedTokenType,
|
||||
#[error("workload identity token exchange is unavailable")]
|
||||
ExchangeUnavailable,
|
||||
#[error("{0}")]
|
||||
RecentFailure(String),
|
||||
}
|
||||
|
||||
enum SubjectTokenSource {
|
||||
@@ -103,6 +109,17 @@ struct CachedAccessToken {
|
||||
refresh_at: Instant,
|
||||
}
|
||||
|
||||
struct CachedExchangeFailure {
|
||||
message: String,
|
||||
retry_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CacheState {
|
||||
token: Option<CachedAccessToken>,
|
||||
failure: Option<CachedExchangeFailure>,
|
||||
}
|
||||
|
||||
pub struct WorkloadIdentityClient {
|
||||
identity_provider_id: String,
|
||||
identity_provider_mapping_id: String,
|
||||
@@ -110,7 +127,8 @@ pub struct WorkloadIdentityClient {
|
||||
client_id: String,
|
||||
source: SubjectTokenSource,
|
||||
http: reqwest::Client,
|
||||
cache: Mutex<Option<CachedAccessToken>>,
|
||||
cache: Mutex<CacheState>,
|
||||
exchange_lock: Semaphore,
|
||||
}
|
||||
|
||||
impl WorkloadIdentityClient {
|
||||
@@ -126,27 +144,69 @@ impl WorkloadIdentityClient {
|
||||
client_id: client_id.into(),
|
||||
source: SubjectTokenSource::from_config(config.credential_source),
|
||||
http,
|
||||
cache: Mutex::new(None),
|
||||
cache: Mutex::new(CacheState::default()),
|
||||
exchange_lock: Semaphore::new(/*permits*/ 1),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn resolve(&self) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
|
||||
{
|
||||
let cache = self
|
||||
.cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(cached) = cache.as_ref()
|
||||
&& Instant::now() < cached.refresh_at
|
||||
{
|
||||
return Ok(cached.token.clone());
|
||||
}
|
||||
if let Some(cached) = self.cached_result() {
|
||||
return cached;
|
||||
}
|
||||
self.exchange_and_cache().await
|
||||
let _permit = self
|
||||
.exchange_lock
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?;
|
||||
if let Some(cached) = self.cached_result() {
|
||||
return cached;
|
||||
}
|
||||
self.exchange_and_record().await
|
||||
}
|
||||
|
||||
pub async fn refresh(&self) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
|
||||
self.exchange_and_cache().await
|
||||
let _permit = self
|
||||
.exchange_lock
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| WorkloadIdentityError::ExchangeUnavailable)?;
|
||||
self.exchange_and_record().await
|
||||
}
|
||||
|
||||
fn cached_result(&self) -> Option<Result<WorkloadIdentityAccessToken, WorkloadIdentityError>> {
|
||||
let cache = self
|
||||
.cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let now = Instant::now();
|
||||
if let Some(failure) = cache.failure.as_ref()
|
||||
&& now < failure.retry_at
|
||||
{
|
||||
return Some(Err(WorkloadIdentityError::RecentFailure(
|
||||
failure.message.clone(),
|
||||
)));
|
||||
}
|
||||
cache
|
||||
.token
|
||||
.as_ref()
|
||||
.filter(|cached| now < cached.refresh_at)
|
||||
.map(|cached| Ok(cached.token.clone()))
|
||||
}
|
||||
|
||||
async fn exchange_and_record(
|
||||
&self,
|
||||
) -> Result<WorkloadIdentityAccessToken, WorkloadIdentityError> {
|
||||
let result = self.exchange_and_cache().await;
|
||||
if let Err(error) = &result {
|
||||
self.cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.failure = Some(CachedExchangeFailure {
|
||||
message: error.to_string(),
|
||||
retry_at: Instant::now() + FAILED_EXCHANGE_RETRY_DELAY,
|
||||
});
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn exchange_and_cache(
|
||||
@@ -209,10 +269,13 @@ impl WorkloadIdentityClient {
|
||||
*self
|
||||
.cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(CachedAccessToken {
|
||||
token: token.clone(),
|
||||
refresh_at,
|
||||
});
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = CacheState {
|
||||
failure: None,
|
||||
token: Some(CachedAccessToken {
|
||||
token: token.clone(),
|
||||
refresh_at,
|
||||
}),
|
||||
};
|
||||
Ok(token)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +74,54 @@ async fn exchanges_azure_subject_token_and_caches_access_token() -> anyhow::Resu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_resolves_share_one_successful_exchange() -> anyhow::Result<()> {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/oauth/token"))
|
||||
.respond_with(exchange_response("shared.access.token"))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = test_client(&server).await?;
|
||||
|
||||
let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve());
|
||||
|
||||
assert_eq!(first?, second?);
|
||||
assert_eq!(client.resolve().await?, third?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_resolves_share_one_rejected_exchange() -> anyhow::Result<()> {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/oauth/token"))
|
||||
.respond_with(ResponseTemplate::new(401).set_body_json(json!({"error": "invalid_grant"})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = test_client(&server).await?;
|
||||
|
||||
let (first, second, third) = tokio::join!(client.resolve(), client.resolve(), client.resolve());
|
||||
let expected =
|
||||
"workload identity token exchange was rejected with HTTP 401 Unauthorized: invalid_grant";
|
||||
|
||||
assert_eq!(
|
||||
first.expect_err("first exchange should fail").to_string(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
second.expect_err("second exchange should fail").to_string(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
third.expect_err("third exchange should fail").to_string(),
|
||||
expected
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_refresh_performs_a_new_exchange() -> anyhow::Result<()> {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
Reference in New Issue
Block a user