diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index dc0cab3d95..aedc6f44cf 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -662,6 +662,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index abe1824a97..9fc6373c67 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -8413,6 +8413,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index ec34fdcb54..887dd47583 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -4269,6 +4269,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json index 69187dd539..bd27cd304b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json @@ -10,6 +10,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 9f61cb9a2b..a4f4702daf 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -60,6 +60,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index d91152ca01..324dfbcabf 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -141,6 +141,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 61cdd1bed2..2b5713ec4d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index f358d1d8bf..dec9a2b430 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 2c3e187bb1..7d9935ab2c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 1ddad74a58..76efd1e76e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -141,6 +141,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index 0d1a6d4cff..51bc9f5eb0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index e46750676a..66a0b378b9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -141,6 +141,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index dc40dfe804..743f729bc5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index 99ff6281f9..ee92a8a2a5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -60,6 +60,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 2158ee699a..779ee79907 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -63,6 +63,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index a219cf31e4..1596283e07 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -60,6 +60,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 259f3cf717..cc196125b0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -60,6 +60,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index b8bb6240fa..00233950ae 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -60,6 +60,7 @@ "sessionBudgetExceeded", "usageLimitExceeded", "rateLimitExceeded", + "flexUnavailable", "serverOverloaded", "cyberPolicy", "misalignmentPolicyViolation", diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 4f97707655..07f3a62e3d 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index bc896e3ea9..cdc852f453 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts b/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts index e93d7fd59a..90aa55ff97 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts @@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind"; * When an upstream HTTP status is available (for example, from the Responses API or a provider), * it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant. */ -export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other"; +export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "flexUnavailable" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/shared.rs b/codex-rs/app-server-protocol/src/protocol/v2/shared.rs index a1ba74b0e4..f231df1f28 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/shared.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/shared.rs @@ -79,6 +79,7 @@ pub enum CodexErrorInfo { SessionBudgetExceeded, UsageLimitExceeded, RateLimitExceeded, + FlexUnavailable, ServerOverloaded, CyberPolicy, MisalignmentPolicyViolation, @@ -127,6 +128,7 @@ impl From for CodexErrorInfo { CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded, CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded, CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded, + CoreCodexErrorInfo::FlexUnavailable => CodexErrorInfo::FlexUnavailable, CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded, CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy, CoreCodexErrorInfo::BioPolicy => CodexErrorInfo::Other, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index a1923f58d0..7fcc7ce5e0 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -4732,6 +4732,10 @@ fn core_error_info_converts_to_camel_case() { (CoreCodexErrorInfo::CyberPolicy, json!("cyberPolicy")), (CoreCodexErrorInfo::BioPolicy, json!("other")), (CoreCodexErrorInfo::InvalidPrompt, json!("other")), + ( + CoreCodexErrorInfo::FlexUnavailable, + json!("flexUnavailable"), + ), ( CoreCodexErrorInfo::RateLimitExceeded, json!("rateLimitExceeded"), diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs index 8af977effb..d7014312c5 100644 --- a/codex-rs/cli/src/doctor.rs +++ b/codex-rs/cli/src/doctor.rs @@ -2440,6 +2440,7 @@ fn websocket_error_detail(err: &ApiError) -> String { | ApiError::CyberPolicy { .. } | ApiError::BioPolicy { .. } | ApiError::MisalignmentPolicyViolation { .. } + | ApiError::FlexUnavailable | ApiError::ServerOverloaded { .. } => format!("handshake error: {err}"), } } diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 2f6fe9a827..0570bbf0a2 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -1,5 +1,6 @@ use crate::TransportError; use crate::error::ApiError; +use crate::error::parse_flex_unavailable; use crate::rate_limits::parse_promo_message; use crate::rate_limits::parse_rate_limit_for_limit; use crate::rate_limits::parse_rate_limit_reached_type; @@ -30,6 +31,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr { | ApiError::ContextWindowExceeded | ApiError::QuotaExceeded | ApiError::UsageNotIncluded + | ApiError::FlexUnavailable | ApiError::RateLimit(_) | ApiError::InvalidRequest { .. } | ApiError::InvalidPrompt { .. } @@ -55,6 +57,7 @@ fn map_api_error_details(err: ApiError) -> CodexErr { } ApiError::Stream(msg) => CodexErr::Stream(msg), ApiError::ServerOverloaded { .. } => CodexErr::ServerOverloaded, + ApiError::FlexUnavailable => CodexErr::new(CodexErrorDetails::FlexUnavailable), ApiError::Api { status, message } => { let user_message = api_error_user_message(status, &message); CodexErr::UnexpectedStatus(UnexpectedResponseError { @@ -174,6 +177,11 @@ fn map_api_error_details(err: ApiError) -> CodexErr { } else if status == http::StatusCode::INTERNAL_SERVER_ERROR { CodexErr::InternalServerError } else if status == http::StatusCode::TOO_MANY_REQUESTS { + if let Ok(body) = serde_json::from_str::(&body_text) + && let Some(error) = body.get("error").and_then(parse_flex_unavailable) + { + return map_api_error(error); + } if let Ok(err) = serde_json::from_str::(&body_text) { if err.error.error_type.as_deref() == Some("usage_limit_reached") { let limit_id = extract_header(headers.as_ref(), ACTIVE_LIMIT_HEADER); diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index b7ed61ccf5..e8088cebc6 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -64,7 +64,13 @@ fn map_api_error_distinguishes_capacity_from_slow_down() { false, ), ("slow_down", CodexErrorInfo::RateLimitExceeded, true), - ("unknown_error", CodexErrorInfo::Other, true), + ( + "unknown_error", + CodexErrorInfo::HttpConnectionFailed { + http_status_code: Some(503), + }, + true, + ), ] { let err = map_api_error(ApiError::Transport(TransportError::Http { retry_after: None, diff --git a/codex-rs/codex-api/src/error.rs b/codex-rs/codex-api/src/error.rs index b4163b795a..e2c934ce8e 100644 --- a/codex-rs/codex-api/src/error.rs +++ b/codex-rs/codex-api/src/error.rs @@ -3,6 +3,7 @@ use codex_client::TransportError; use codex_http_client::RetryAfter; use codex_protocol::protocol::MisalignmentErrorDetails; use http::StatusCode; +use serde_json::Value; use thiserror::Error; #[derive(Debug, Error)] @@ -44,6 +45,8 @@ pub enum ApiError { message: String, misalignment: Option, }, + #[error("Flex capacity unavailable.")] + FlexUnavailable, #[error("server overloaded")] ServerOverloaded { retry_after: Option }, } @@ -53,3 +56,8 @@ impl From for ApiError { Self::RateLimit(err.to_string()) } } + +pub(crate) fn parse_flex_unavailable(error: &Value) -> Option { + (error.get("code").and_then(Value::as_str) == Some("flex_unavailable")) + .then_some(ApiError::FlexUnavailable) +} diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs index 1fb8835318..24a81bcc14 100644 --- a/codex-rs/codex-api/src/sse/responses.rs +++ b/codex-rs/codex-api/src/sse/responses.rs @@ -3,6 +3,7 @@ use crate::common::ResponseStream; use crate::common::SafetyBuffering; use crate::common::SafetyBufferingTreatment; use crate::error::ApiError; +use crate::error::parse_flex_unavailable; use crate::rate_limits::parse_all_rate_limits; use crate::safety_buffering::treatment_from_headers; use crate::telemetry::SseTelemetry; @@ -173,6 +174,7 @@ pub struct ResponsesStreamEvent { pub(crate) headers: Option, metadata: Option, response: Option, + error: Option, item: Option, item_id: Option, call_id: Option, @@ -355,6 +357,11 @@ pub fn process_responses_event( event: ResponsesStreamEvent, ) -> std::result::Result, ResponsesEventError> { match event.kind.as_str() { + "error" => { + if let Some(error) = event.error.as_ref().and_then(parse_flex_unavailable) { + return Err(ResponsesEventError::Api(error)); + } + } "response.output_item.done" => { if let Some(item_val) = event.item { if let Ok(item) = serde_json::from_value::(item_val) { @@ -418,6 +425,9 @@ pub fn process_responses_event( "response.failed" => { if let Some(resp_val) = event.response { let mut response_error = ApiError::Stream("response.failed event received".into()); + if let Some(error) = resp_val.get("error").and_then(parse_flex_unavailable) { + return Err(ResponsesEventError::Api(error)); + } if let Some(error) = resp_val.get("error") && let Ok(error) = serde_json::from_value::(error.clone()) { @@ -697,7 +707,12 @@ async fn process_sse_with_treatment( } Ok(None) => {} Err(error) => { - response_error = Some(error.into_api_error()); + let error = error.into_api_error(); + if matches!(error, ApiError::FlexUnavailable) { + let _ = tx_event.send(Err(error)).await; + return; + } + response_error = Some(error); } }; } diff --git a/codex-rs/core/tests/suite/retry_after.rs b/codex-rs/core/tests/suite/retry_after.rs index bc4f1f0360..4bb5da9a8d 100644 --- a/codex-rs/core/tests/suite/retry_after.rs +++ b/codex-rs/core/tests/suite/retry_after.rs @@ -496,6 +496,35 @@ async fn responses_http_overload_without_retry_after_exhausts_request_retries() Ok(()) } +#[tokio::test(flavor = "current_thread")] +async fn responses_http_504_preserves_status_after_stream_retry() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = + responses::mount_response_sequence(&server, vec![ResponseTemplate::new(504); 2]).await; + let test = test_codex() + .with_config(|config| { + config.service_tier = Some("flex".to_string()); + config.model_provider.request_max_retries = Some(0); + config.model_provider.stream_max_retries = Some(1); + }) + .build_with_auto_env(&server) + .await?; + + submit_user_input(&test, "surface the timeout after one retry").await?; + assert_terminal_failure( + &test, + CodexErrorInfo::HttpConnectionFailed { + http_status_code: Some(504), + }, + /*expected_retries*/ 1, + ) + .await?; + assert_eq!(response_mock.requests().len(), 2); + Ok(()) +} + /// Remote compaction v2 uses the upstream header before another HTTP request. #[tokio::test(flavor = "current_thread")] async fn compact_v2_uses_retry_after() -> Result<()> { @@ -1411,6 +1440,135 @@ async fn sse_overload_without_retry_after_is_terminal() -> Result<()> { Ok(()) } +async fn assert_terminal_failure( + test: &TestCodex, + expected: CodexErrorInfo, + expected_retries: usize, +) -> Result<()> { + let mut errors = Vec::new(); + let mut completions = Vec::new(); + let mut retries = 0; + loop { + match wait_for_event(&test.codex, |_| true).await { + EventMsg::Error(error) => errors.push(error.codex_error_info), + EventMsg::StreamError(_) => retries += 1, + EventMsg::TurnComplete(event) => { + completions.push(event.error.and_then(|error| error.codex_error_info)); + test.codex.submit(Op::Shutdown).await?; + } + EventMsg::ShutdownComplete => break, + _ => {} + } + } + assert_eq!(errors, vec![Some(expected.clone())]); + assert_eq!(completions, vec![Some(expected)]); + assert_eq!(retries, expected_retries); + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn http_and_sse_flex_unavailable_are_terminal() -> Result<()> { + skip_if_no_network!(Ok(())); + + let error = json!({ + "type": "resource_unavailable", "code": "flex_unavailable", + "message": "Flex capacity unavailable." + }); + for response in [ + ResponseTemplate::new(429) + .insert_header("Retry-After", "300") + .set_body_json(json!({"error": error})), + responses::sse_response(responses::sse(vec![ + json!({"type": "error", "error": error}), + ])), + responses::sse_response(responses::sse_failed( + "failed", + "flex_unavailable", + "No capacity", + )), + ] { + let server = responses::start_mock_server().await; + let response_mock = responses::mount_response_once(&server, response).await; + let test = test_codex() + .with_config(|config| { + config.service_tier = Some("flex".to_string()); + config.model_provider.request_max_retries = Some(2); + config.model_provider.stream_max_retries = Some(2); + }) + .build_with_auto_env(&server) + .await?; + + submit_user_input(&test, "surface the Flex capacity failure").await?; + assert_terminal_failure( + &test, + CodexErrorInfo::FlexUnavailable, + /*expected_retries*/ 0, + ) + .await?; + assert_eq!( + response_mock.single_request().body_json()["service_tier"], + json!("flex") + ); + assert_eq!( + server + .received_requests() + .await + .unwrap() + .iter() + .filter(|request| request.url.path() == "/v1/responses") + .count(), + 1 + ); + } + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn websocket_streamed_flex_unavailable_is_terminal() -> Result<()> { + skip_if_no_network!(Ok(())); + + let error = json!({ + "type": "resource_unavailable", "code": "flex_unavailable", + "message": "Flex capacity unavailable." + }); + for event in [ + json!({"type": "error", "status": 429, "headers": {"retry-after": "300"}, "error": error}), + json!({"type": "response.failed", "response": {"error": error}}), + ] { + let server = responses::start_websocket_server(vec![vec![ + vec![ + responses::ev_response_created("prewarm"), + responses::ev_completed("prewarm"), + ], + vec![event], + ]]) + .await; + let test = test_codex() + .with_config(|config| { + config.service_tier = Some("flex".to_string()); + config.model_catalog = + Some(bundled_models_response().expect("bundled models.json should parse")); + config.model_provider.request_max_retries = Some(2); + config.model_provider.stream_max_retries = Some(2); + }) + .build_with_websocket_server(&server) + .await?; + + submit_user_input(&test, "surface the Flex capacity failure").await?; + assert_terminal_failure( + &test, + CodexErrorInfo::FlexUnavailable, + /*expected_retries*/ 0, + ) + .await?; + let requests = server.single_connection(); + assert_eq!(requests.len(), 2, "prewarm and failed turn"); + assert_eq!(requests[1].body_json()["service_tier"], json!("flex")); + server.shutdown().await; + } + Ok(()) +} + /// Network reconnects keep their own attempt count without consuming stream retry budget. #[tokio::test(flavor = "current_thread")] async fn connection_failures_increment_retry_telemetry_without_consuming_retry_budget() -> Result<()> diff --git a/codex-rs/ext/guardian-reviewer/src/retry.rs b/codex-rs/ext/guardian-reviewer/src/retry.rs index f3b20d8bf5..82cb29fc32 100644 --- a/codex-rs/ext/guardian-reviewer/src/retry.rs +++ b/codex-rs/ext/guardian-reviewer/src/retry.rs @@ -101,6 +101,7 @@ fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool { .. }) => match error { CodexErrorInfo::RateLimitExceeded + | CodexErrorInfo::FlexUnavailable | CodexErrorInfo::ServerOverloaded | CodexErrorInfo::InternalServerError => true, CodexErrorInfo::HttpConnectionFailed { http_status_code } diff --git a/codex-rs/ext/guardian-reviewer/src/retry_tests.rs b/codex-rs/ext/guardian-reviewer/src/retry_tests.rs index cc751a0b31..4f998b8782 100644 --- a/codex-rs/ext/guardian-reviewer/src/retry_tests.rs +++ b/codex-rs/ext/guardian-reviewer/src/retry_tests.rs @@ -46,6 +46,7 @@ fn guardian_review_retry_only_retries_recoverable_errors() { }; let transient_error_info = [ CodexErrorInfo::ServerOverloaded, + CodexErrorInfo::FlexUnavailable, CodexErrorInfo::HttpConnectionFailed { http_status_code: Some(502), }, diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs index 574270492f..33c59d5478 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs @@ -54,6 +54,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str { | ApiError::BioPolicy { .. } | ApiError::MisalignmentPolicyViolation { .. } => "policy_error", ApiError::ServerOverloaded { .. } => "server_overloaded", + ApiError::FlexUnavailable => "flex_unavailable", }, } } diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs b/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs index 3a05f0c271..e0fc66eb8c 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs @@ -54,7 +54,8 @@ impl SamplingExecution { ApiError::Retryable { .. } | ApiError::RateLimitExceeded { .. } | ApiError::Stream(_) - | ApiError::ServerOverloaded { .. }, + | ApiError::ServerOverloaded { .. } + | ApiError::FlexUnavailable, ) | LunaSamplerError::Api(ApiError::Transport( TransportError::RetryLimit diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/sampler_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/sampler_tests.rs index ac2a4f2539..c4ea6b2d74 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/sampler_tests.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/sampler_tests.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use codex_api::ApiError; use codex_context_fragments::RenderedFragment; use codex_extension_api::ContextualUserFragment; use codex_extension_api::ExtensionMetrics; @@ -1094,7 +1095,11 @@ async fn sampler_reconnects_after_transient_service_failures() -> Result<()> { })]]] }; let first = responses::start_websocket_server(unavailable()).await; - let second = responses::start_websocket_server(unavailable()).await; + let second = responses::start_websocket_server(vec![vec![vec![json!({ + "type": "response.failed", + "response": {"error": {"code": "flex_unavailable", "message": "capacity unavailable"}} + })]]]) + .await; let http = responses::start_mock_server().await; let recovered = responses::mount_sse_once( &http, @@ -1145,13 +1150,17 @@ async fn sampler_limits_transient_recovery_attempts() -> Result<()> { })]]] }; let first = responses::start_websocket_server(unavailable()).await; - let second = responses::start_websocket_server(unavailable()).await; + let second = responses::start_websocket_server(vec![vec![vec![json!({ + "type": "response.failed", + "response": {"error": {"code": "flex_unavailable", "message": "capacity unavailable"}} + })]]]) + .await; let http = responses::start_mock_server().await; let third = responses::mount_sse_once( &http, responses::sse(vec![json!({ "type": "response.failed", "response": { - "error": {"code": "internal_server_error", "message": "HTTP sampling failed"} + "error": {"code": "flex_unavailable", "message": "HTTP sampling failed"} } })]), ) @@ -1171,7 +1180,10 @@ async fn sampler_limits_transient_recovery_attempts() -> Result<()> { .await .expect_err("sampling should stop after the bounded retries"); - assert!(error.to_string().contains("HTTP sampling failed")); + assert!(matches!( + error, + LunaSamplerError::Api(ApiError::FlexUnavailable) + )); assert_eq!(first.single_connection().len(), 1); assert_eq!(second.single_connection().len(), 1); assert_eq!(third.requests().len(), 1); diff --git a/codex-rs/protocol/src/codex_error_info.rs b/codex-rs/protocol/src/codex_error_info.rs index fd883e0558..c018fae107 100644 --- a/codex-rs/protocol/src/codex_error_info.rs +++ b/codex-rs/protocol/src/codex_error_info.rs @@ -21,6 +21,7 @@ enum CodexErrorInfoWire { SessionBudgetExceeded, UsageLimitExceeded, RateLimitExceeded, + FlexUnavailable, ServerOverloaded, CyberPolicy, BioPolicy, diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rs index 814cb63fc5..9838a50a37 100644 --- a/codex-rs/protocol/src/error.rs +++ b/codex-rs/protocol/src/error.rs @@ -139,6 +139,8 @@ pub enum CodexErrorDetails { UsageLimitReached(UsageLimitReachedError), #[error("Selected model is at capacity. Please try a different model.")] ServerOverloaded, + #[error("Flex capacity unavailable.")] + FlexUnavailable, #[error("{message}")] CyberPolicy { message: String }, #[error("{message}")] @@ -404,6 +406,7 @@ impl CodexErr { | CodexErrorDetails::SessionConfiguredNotFirstEvent | CodexErrorDetails::UsageLimitReached(_) | CodexErrorDetails::ServerOverloaded + | CodexErrorDetails::FlexUnavailable | CodexErrorDetails::CyberPolicy { .. } | CodexErrorDetails::BioPolicy { .. } | CodexErrorDetails::MisalignmentPolicyViolation { .. } => None, @@ -461,6 +464,7 @@ impl CodexErr { | CodexErrorDetails::QuotaExceeded | CodexErrorDetails::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded, CodexErrorDetails::ServerOverloaded => CodexErrorInfo::ServerOverloaded, + CodexErrorDetails::FlexUnavailable => CodexErrorInfo::FlexUnavailable, CodexErrorDetails::CyberPolicy { .. } => CodexErrorInfo::CyberPolicy, CodexErrorDetails::BioPolicy { .. } => CodexErrorInfo::BioPolicy, CodexErrorDetails::InvalidPrompt { .. } => CodexErrorInfo::InvalidPrompt, @@ -470,9 +474,11 @@ impl CodexErr { CodexErrorDetails::RetryLimit(_) => CodexErrorInfo::ResponseTooManyFailedAttempts { http_status_code: self.http_status_code_value(), }, - CodexErrorDetails::ConnectionFailed(_) => CodexErrorInfo::HttpConnectionFailed { - http_status_code: self.http_status_code_value(), - }, + CodexErrorDetails::ConnectionFailed(_) | CodexErrorDetails::UnexpectedStatus(_) => { + CodexErrorInfo::HttpConnectionFailed { + http_status_code: self.http_status_code_value(), + } + } CodexErrorDetails::ResponseStreamFailed(_) => { CodexErrorInfo::ResponseStreamConnectionFailed { http_status_code: self.http_status_code_value(), @@ -510,6 +516,7 @@ impl CodexErr { pub fn http_status_code_value(&self) -> Option { let http_status_code = match &self.details { + CodexErrorDetails::FlexUnavailable => Some(StatusCode::TOO_MANY_REQUESTS), CodexErrorDetails::RetryLimit(err) => Some(err.status), CodexErrorDetails::UnexpectedStatus(err) => Some(err.status), CodexErrorDetails::ConnectionFailed(err) => err.source.status(), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 3c2f05ac2b..da9ba1ad09 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1856,6 +1856,7 @@ pub enum CodexErrorInfo { SessionBudgetExceeded, UsageLimitExceeded, RateLimitExceeded, + FlexUnavailable, ServerOverloaded, CyberPolicy, BioPolicy, @@ -1899,6 +1900,7 @@ impl CodexErrorInfo { | Self::SessionBudgetExceeded | Self::UsageLimitExceeded | Self::RateLimitExceeded + | Self::FlexUnavailable | Self::ServerOverloaded | Self::CyberPolicy | Self::BioPolicy diff --git a/codex-rs/response-debug-context/src/lib.rs b/codex-rs/response-debug-context/src/lib.rs index 98e2a6892c..9b512cf348 100644 --- a/codex-rs/response-debug-context/src/lib.rs +++ b/codex-rs/response-debug-context/src/lib.rs @@ -88,6 +88,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String { ApiError::BioPolicy { .. } => "bio policy".to_string(), ApiError::MisalignmentPolicyViolation { .. } => "misalignment policy violation".to_string(), ApiError::ServerOverloaded { .. } => "server overloaded".to_string(), + ApiError::FlexUnavailable => "flex capacity unavailable".to_string(), } } diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 19c7471b98..4dd9a70968 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -543,6 +543,7 @@ class CodexErrorInfoValue(Enum): session_budget_exceeded = "sessionBudgetExceeded" usage_limit_exceeded = "usageLimitExceeded" rate_limit_exceeded = "rateLimitExceeded" + flex_unavailable = "flexUnavailable" server_overloaded = "serverOverloaded" cyber_policy = "cyberPolicy" misalignment_policy_violation = "misalignmentPolicyViolation"