mirror of
https://github.com/openai/codex.git
synced 2026-10-18 17:18:52 +00:00
Surface Flex capacity failures as a distinct terminal error (#47967)
## What changed - Recognize `flex_unavailable` in HTTP 429 responses and streamed `error` and `response.failed` events. End normal turns without retries and report `Flex capacity unavailable.` - Expose `flexUnavailable` through the core and app-server protocols and generated schemas and SDK types. - Keep Flex capacity failures eligible for Guardian review and sampler retries, with a distinct telemetry classification. - Map unexpected HTTP responses to `HttpConnectionFailed`, preserving their status codes. ## Testing Add regression coverage for terminal Flex failures over HTTP, SSE, and WebSocket, HTTP 504 status preservation after a stream retry, and protocol serialization. Update Guardian retry and sampler recovery tests to cover Flex failures. GitOrigin-RevId: f2e03a6d2d36a6d9748c35952e40397f8670d121
This commit is contained in:
@@ -662,6 +662,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
+1
@@ -8413,6 +8413,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
+1
@@ -4269,6 +4269,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -141,6 +141,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
+1
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -141,6 +141,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -141,6 +141,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"sessionBudgetExceeded",
|
||||
"usageLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
"flexUnavailable",
|
||||
"serverOverloaded",
|
||||
"cyberPolicy",
|
||||
"misalignmentPolicyViolation",
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind";
|
||||
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
|
||||
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
|
||||
*/
|
||||
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
||||
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "flexUnavailable" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
||||
|
||||
@@ -79,6 +79,7 @@ pub enum CodexErrorInfo {
|
||||
SessionBudgetExceeded,
|
||||
UsageLimitExceeded,
|
||||
RateLimitExceeded,
|
||||
FlexUnavailable,
|
||||
ServerOverloaded,
|
||||
CyberPolicy,
|
||||
MisalignmentPolicyViolation,
|
||||
@@ -127,6 +128,7 @@ impl From<CoreCodexErrorInfo> for CodexErrorInfo {
|
||||
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
||||
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
|
||||
CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded,
|
||||
CoreCodexErrorInfo::FlexUnavailable => CodexErrorInfo::FlexUnavailable,
|
||||
CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
|
||||
CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy,
|
||||
CoreCodexErrorInfo::BioPolicy => CodexErrorInfo::Other,
|
||||
|
||||
@@ -4732,6 +4732,10 @@ fn core_error_info_converts_to_camel_case() {
|
||||
(CoreCodexErrorInfo::CyberPolicy, json!("cyberPolicy")),
|
||||
(CoreCodexErrorInfo::BioPolicy, json!("other")),
|
||||
(CoreCodexErrorInfo::InvalidPrompt, json!("other")),
|
||||
(
|
||||
CoreCodexErrorInfo::FlexUnavailable,
|
||||
json!("flexUnavailable"),
|
||||
),
|
||||
(
|
||||
CoreCodexErrorInfo::RateLimitExceeded,
|
||||
json!("rateLimitExceeded"),
|
||||
|
||||
@@ -2440,6 +2440,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
|
||||
| ApiError::CyberPolicy { .. }
|
||||
| ApiError::BioPolicy { .. }
|
||||
| ApiError::MisalignmentPolicyViolation { .. }
|
||||
| ApiError::FlexUnavailable
|
||||
| ApiError::ServerOverloaded { .. } => format!("handshake error: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::TransportError;
|
||||
use crate::error::ApiError;
|
||||
use crate::error::parse_flex_unavailable;
|
||||
use crate::rate_limits::parse_promo_message;
|
||||
use crate::rate_limits::parse_rate_limit_for_limit;
|
||||
use crate::rate_limits::parse_rate_limit_reached_type;
|
||||
@@ -30,6 +31,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
|
||||
| ApiError::ContextWindowExceeded
|
||||
| ApiError::QuotaExceeded
|
||||
| ApiError::UsageNotIncluded
|
||||
| ApiError::FlexUnavailable
|
||||
| ApiError::RateLimit(_)
|
||||
| ApiError::InvalidRequest { .. }
|
||||
| ApiError::InvalidPrompt { .. }
|
||||
@@ -55,6 +57,7 @@ fn map_api_error_details(err: ApiError) -> CodexErr {
|
||||
}
|
||||
ApiError::Stream(msg) => CodexErr::Stream(msg),
|
||||
ApiError::ServerOverloaded { .. } => CodexErr::ServerOverloaded,
|
||||
ApiError::FlexUnavailable => CodexErr::new(CodexErrorDetails::FlexUnavailable),
|
||||
ApiError::Api { status, message } => {
|
||||
let user_message = api_error_user_message(status, &message);
|
||||
CodexErr::UnexpectedStatus(UnexpectedResponseError {
|
||||
@@ -174,6 +177,11 @@ fn map_api_error_details(err: ApiError) -> CodexErr {
|
||||
} else if status == http::StatusCode::INTERNAL_SERVER_ERROR {
|
||||
CodexErr::InternalServerError
|
||||
} else if status == http::StatusCode::TOO_MANY_REQUESTS {
|
||||
if let Ok(body) = serde_json::from_str::<Value>(&body_text)
|
||||
&& let Some(error) = body.get("error").and_then(parse_flex_unavailable)
|
||||
{
|
||||
return map_api_error(error);
|
||||
}
|
||||
if let Ok(err) = serde_json::from_str::<UsageErrorResponse>(&body_text) {
|
||||
if err.error.error_type.as_deref() == Some("usage_limit_reached") {
|
||||
let limit_id = extract_header(headers.as_ref(), ACTIVE_LIMIT_HEADER);
|
||||
|
||||
@@ -64,7 +64,13 @@ fn map_api_error_distinguishes_capacity_from_slow_down() {
|
||||
false,
|
||||
),
|
||||
("slow_down", CodexErrorInfo::RateLimitExceeded, true),
|
||||
("unknown_error", CodexErrorInfo::Other, true),
|
||||
(
|
||||
"unknown_error",
|
||||
CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: Some(503),
|
||||
},
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let err = map_api_error(ApiError::Transport(TransportError::Http {
|
||||
retry_after: None,
|
||||
|
||||
@@ -3,6 +3,7 @@ use codex_client::TransportError;
|
||||
use codex_http_client::RetryAfter;
|
||||
use codex_protocol::protocol::MisalignmentErrorDetails;
|
||||
use http::StatusCode;
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
@@ -44,6 +45,8 @@ pub enum ApiError {
|
||||
message: String,
|
||||
misalignment: Option<MisalignmentErrorDetails>,
|
||||
},
|
||||
#[error("Flex capacity unavailable.")]
|
||||
FlexUnavailable,
|
||||
#[error("server overloaded")]
|
||||
ServerOverloaded { retry_after: Option<RetryAfter> },
|
||||
}
|
||||
@@ -53,3 +56,8 @@ impl From<RateLimitError> for ApiError {
|
||||
Self::RateLimit(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_flex_unavailable(error: &Value) -> Option<ApiError> {
|
||||
(error.get("code").and_then(Value::as_str) == Some("flex_unavailable"))
|
||||
.then_some(ApiError::FlexUnavailable)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use crate::common::ResponseStream;
|
||||
use crate::common::SafetyBuffering;
|
||||
use crate::common::SafetyBufferingTreatment;
|
||||
use crate::error::ApiError;
|
||||
use crate::error::parse_flex_unavailable;
|
||||
use crate::rate_limits::parse_all_rate_limits;
|
||||
use crate::safety_buffering::treatment_from_headers;
|
||||
use crate::telemetry::SseTelemetry;
|
||||
@@ -173,6 +174,7 @@ pub struct ResponsesStreamEvent {
|
||||
pub(crate) headers: Option<Value>,
|
||||
metadata: Option<Value>,
|
||||
response: Option<Value>,
|
||||
error: Option<Value>,
|
||||
item: Option<Value>,
|
||||
item_id: Option<String>,
|
||||
call_id: Option<String>,
|
||||
@@ -355,6 +357,11 @@ pub fn process_responses_event(
|
||||
event: ResponsesStreamEvent,
|
||||
) -> std::result::Result<Option<ResponseEvent>, ResponsesEventError> {
|
||||
match event.kind.as_str() {
|
||||
"error" => {
|
||||
if let Some(error) = event.error.as_ref().and_then(parse_flex_unavailable) {
|
||||
return Err(ResponsesEventError::Api(error));
|
||||
}
|
||||
}
|
||||
"response.output_item.done" => {
|
||||
if let Some(item_val) = event.item {
|
||||
if let Ok(item) = serde_json::from_value::<ResponseItem>(item_val) {
|
||||
@@ -418,6 +425,9 @@ pub fn process_responses_event(
|
||||
"response.failed" => {
|
||||
if let Some(resp_val) = event.response {
|
||||
let mut response_error = ApiError::Stream("response.failed event received".into());
|
||||
if let Some(error) = resp_val.get("error").and_then(parse_flex_unavailable) {
|
||||
return Err(ResponsesEventError::Api(error));
|
||||
}
|
||||
if let Some(error) = resp_val.get("error")
|
||||
&& let Ok(error) = serde_json::from_value::<Error>(error.clone())
|
||||
{
|
||||
@@ -697,7 +707,12 @@ async fn process_sse_with_treatment(
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
response_error = Some(error.into_api_error());
|
||||
let error = error.into_api_error();
|
||||
if matches!(error, ApiError::FlexUnavailable) {
|
||||
let _ = tx_event.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
response_error = Some(error);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -496,6 +496,35 @@ async fn responses_http_overload_without_retry_after_exhausts_request_retries()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn responses_http_504_preserves_status_after_stream_retry() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock =
|
||||
responses::mount_response_sequence(&server, vec![ResponseTemplate::new(504); 2]).await;
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.service_tier = Some("flex".to_string());
|
||||
config.model_provider.request_max_retries = Some(0);
|
||||
config.model_provider.stream_max_retries = Some(1);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
|
||||
submit_user_input(&test, "surface the timeout after one retry").await?;
|
||||
assert_terminal_failure(
|
||||
&test,
|
||||
CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: Some(504),
|
||||
},
|
||||
/*expected_retries*/ 1,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response_mock.requests().len(), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remote compaction v2 uses the upstream header before another HTTP request.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn compact_v2_uses_retry_after() -> Result<()> {
|
||||
@@ -1411,6 +1440,135 @@ async fn sse_overload_without_retry_after_is_terminal() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_terminal_failure(
|
||||
test: &TestCodex,
|
||||
expected: CodexErrorInfo,
|
||||
expected_retries: usize,
|
||||
) -> Result<()> {
|
||||
let mut errors = Vec::new();
|
||||
let mut completions = Vec::new();
|
||||
let mut retries = 0;
|
||||
loop {
|
||||
match wait_for_event(&test.codex, |_| true).await {
|
||||
EventMsg::Error(error) => errors.push(error.codex_error_info),
|
||||
EventMsg::StreamError(_) => retries += 1,
|
||||
EventMsg::TurnComplete(event) => {
|
||||
completions.push(event.error.and_then(|error| error.codex_error_info));
|
||||
test.codex.submit(Op::Shutdown).await?;
|
||||
}
|
||||
EventMsg::ShutdownComplete => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!(errors, vec![Some(expected.clone())]);
|
||||
assert_eq!(completions, vec![Some(expected)]);
|
||||
assert_eq!(retries, expected_retries);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn http_and_sse_flex_unavailable_are_terminal() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let error = json!({
|
||||
"type": "resource_unavailable", "code": "flex_unavailable",
|
||||
"message": "Flex capacity unavailable."
|
||||
});
|
||||
for response in [
|
||||
ResponseTemplate::new(429)
|
||||
.insert_header("Retry-After", "300")
|
||||
.set_body_json(json!({"error": error})),
|
||||
responses::sse_response(responses::sse(vec![
|
||||
json!({"type": "error", "error": error}),
|
||||
])),
|
||||
responses::sse_response(responses::sse_failed(
|
||||
"failed",
|
||||
"flex_unavailable",
|
||||
"No capacity",
|
||||
)),
|
||||
] {
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock = responses::mount_response_once(&server, response).await;
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.service_tier = Some("flex".to_string());
|
||||
config.model_provider.request_max_retries = Some(2);
|
||||
config.model_provider.stream_max_retries = Some(2);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
|
||||
submit_user_input(&test, "surface the Flex capacity failure").await?;
|
||||
assert_terminal_failure(
|
||||
&test,
|
||||
CodexErrorInfo::FlexUnavailable,
|
||||
/*expected_retries*/ 0,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
response_mock.single_request().body_json()["service_tier"],
|
||||
json!("flex")
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/v1/responses")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn websocket_streamed_flex_unavailable_is_terminal() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let error = json!({
|
||||
"type": "resource_unavailable", "code": "flex_unavailable",
|
||||
"message": "Flex capacity unavailable."
|
||||
});
|
||||
for event in [
|
||||
json!({"type": "error", "status": 429, "headers": {"retry-after": "300"}, "error": error}),
|
||||
json!({"type": "response.failed", "response": {"error": error}}),
|
||||
] {
|
||||
let server = responses::start_websocket_server(vec![vec![
|
||||
vec![
|
||||
responses::ev_response_created("prewarm"),
|
||||
responses::ev_completed("prewarm"),
|
||||
],
|
||||
vec![event],
|
||||
]])
|
||||
.await;
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.service_tier = Some("flex".to_string());
|
||||
config.model_catalog =
|
||||
Some(bundled_models_response().expect("bundled models.json should parse"));
|
||||
config.model_provider.request_max_retries = Some(2);
|
||||
config.model_provider.stream_max_retries = Some(2);
|
||||
})
|
||||
.build_with_websocket_server(&server)
|
||||
.await?;
|
||||
|
||||
submit_user_input(&test, "surface the Flex capacity failure").await?;
|
||||
assert_terminal_failure(
|
||||
&test,
|
||||
CodexErrorInfo::FlexUnavailable,
|
||||
/*expected_retries*/ 0,
|
||||
)
|
||||
.await?;
|
||||
let requests = server.single_connection();
|
||||
assert_eq!(requests.len(), 2, "prewarm and failed turn");
|
||||
assert_eq!(requests[1].body_json()["service_tier"], json!("flex"));
|
||||
server.shutdown().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Network reconnects keep their own attempt count without consuming stream retry budget.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn connection_failures_increment_retry_telemetry_without_consuming_retry_budget() -> Result<()>
|
||||
|
||||
@@ -101,6 +101,7 @@ fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool {
|
||||
..
|
||||
}) => match error {
|
||||
CodexErrorInfo::RateLimitExceeded
|
||||
| CodexErrorInfo::FlexUnavailable
|
||||
| CodexErrorInfo::ServerOverloaded
|
||||
| CodexErrorInfo::InternalServerError => true,
|
||||
CodexErrorInfo::HttpConnectionFailed { http_status_code }
|
||||
|
||||
@@ -46,6 +46,7 @@ fn guardian_review_retry_only_retries_recoverable_errors() {
|
||||
};
|
||||
let transient_error_info = [
|
||||
CodexErrorInfo::ServerOverloaded,
|
||||
CodexErrorInfo::FlexUnavailable,
|
||||
CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: Some(502),
|
||||
},
|
||||
|
||||
@@ -54,6 +54,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str {
|
||||
| ApiError::BioPolicy { .. }
|
||||
| ApiError::MisalignmentPolicyViolation { .. } => "policy_error",
|
||||
ApiError::ServerOverloaded { .. } => "server_overloaded",
|
||||
ApiError::FlexUnavailable => "flex_unavailable",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,8 @@ impl SamplingExecution {
|
||||
ApiError::Retryable { .. }
|
||||
| ApiError::RateLimitExceeded { .. }
|
||||
| ApiError::Stream(_)
|
||||
| ApiError::ServerOverloaded { .. },
|
||||
| ApiError::ServerOverloaded { .. }
|
||||
| ApiError::FlexUnavailable,
|
||||
)
|
||||
| LunaSamplerError::Api(ApiError::Transport(
|
||||
TransportError::RetryLimit
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use codex_api::ApiError;
|
||||
use codex_context_fragments::RenderedFragment;
|
||||
use codex_extension_api::ContextualUserFragment;
|
||||
use codex_extension_api::ExtensionMetrics;
|
||||
@@ -1094,7 +1095,11 @@ async fn sampler_reconnects_after_transient_service_failures() -> Result<()> {
|
||||
})]]]
|
||||
};
|
||||
let first = responses::start_websocket_server(unavailable()).await;
|
||||
let second = responses::start_websocket_server(unavailable()).await;
|
||||
let second = responses::start_websocket_server(vec![vec![vec![json!({
|
||||
"type": "response.failed",
|
||||
"response": {"error": {"code": "flex_unavailable", "message": "capacity unavailable"}}
|
||||
})]]])
|
||||
.await;
|
||||
let http = responses::start_mock_server().await;
|
||||
let recovered = responses::mount_sse_once(
|
||||
&http,
|
||||
@@ -1145,13 +1150,17 @@ async fn sampler_limits_transient_recovery_attempts() -> Result<()> {
|
||||
})]]]
|
||||
};
|
||||
let first = responses::start_websocket_server(unavailable()).await;
|
||||
let second = responses::start_websocket_server(unavailable()).await;
|
||||
let second = responses::start_websocket_server(vec![vec![vec![json!({
|
||||
"type": "response.failed",
|
||||
"response": {"error": {"code": "flex_unavailable", "message": "capacity unavailable"}}
|
||||
})]]])
|
||||
.await;
|
||||
let http = responses::start_mock_server().await;
|
||||
let third = responses::mount_sse_once(
|
||||
&http,
|
||||
responses::sse(vec![json!({
|
||||
"type": "response.failed", "response": {
|
||||
"error": {"code": "internal_server_error", "message": "HTTP sampling failed"}
|
||||
"error": {"code": "flex_unavailable", "message": "HTTP sampling failed"}
|
||||
}
|
||||
})]),
|
||||
)
|
||||
@@ -1171,7 +1180,10 @@ async fn sampler_limits_transient_recovery_attempts() -> Result<()> {
|
||||
.await
|
||||
.expect_err("sampling should stop after the bounded retries");
|
||||
|
||||
assert!(error.to_string().contains("HTTP sampling failed"));
|
||||
assert!(matches!(
|
||||
error,
|
||||
LunaSamplerError::Api(ApiError::FlexUnavailable)
|
||||
));
|
||||
assert_eq!(first.single_connection().len(), 1);
|
||||
assert_eq!(second.single_connection().len(), 1);
|
||||
assert_eq!(third.requests().len(), 1);
|
||||
|
||||
@@ -21,6 +21,7 @@ enum CodexErrorInfoWire {
|
||||
SessionBudgetExceeded,
|
||||
UsageLimitExceeded,
|
||||
RateLimitExceeded,
|
||||
FlexUnavailable,
|
||||
ServerOverloaded,
|
||||
CyberPolicy,
|
||||
BioPolicy,
|
||||
|
||||
@@ -139,6 +139,8 @@ pub enum CodexErrorDetails {
|
||||
UsageLimitReached(UsageLimitReachedError),
|
||||
#[error("Selected model is at capacity. Please try a different model.")]
|
||||
ServerOverloaded,
|
||||
#[error("Flex capacity unavailable.")]
|
||||
FlexUnavailable,
|
||||
#[error("{message}")]
|
||||
CyberPolicy { message: String },
|
||||
#[error("{message}")]
|
||||
@@ -404,6 +406,7 @@ impl CodexErr {
|
||||
| CodexErrorDetails::SessionConfiguredNotFirstEvent
|
||||
| CodexErrorDetails::UsageLimitReached(_)
|
||||
| CodexErrorDetails::ServerOverloaded
|
||||
| CodexErrorDetails::FlexUnavailable
|
||||
| CodexErrorDetails::CyberPolicy { .. }
|
||||
| CodexErrorDetails::BioPolicy { .. }
|
||||
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => None,
|
||||
@@ -461,6 +464,7 @@ impl CodexErr {
|
||||
| CodexErrorDetails::QuotaExceeded
|
||||
| CodexErrorDetails::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded,
|
||||
CodexErrorDetails::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
|
||||
CodexErrorDetails::FlexUnavailable => CodexErrorInfo::FlexUnavailable,
|
||||
CodexErrorDetails::CyberPolicy { .. } => CodexErrorInfo::CyberPolicy,
|
||||
CodexErrorDetails::BioPolicy { .. } => CodexErrorInfo::BioPolicy,
|
||||
CodexErrorDetails::InvalidPrompt { .. } => CodexErrorInfo::InvalidPrompt,
|
||||
@@ -470,9 +474,11 @@ impl CodexErr {
|
||||
CodexErrorDetails::RetryLimit(_) => CodexErrorInfo::ResponseTooManyFailedAttempts {
|
||||
http_status_code: self.http_status_code_value(),
|
||||
},
|
||||
CodexErrorDetails::ConnectionFailed(_) => CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: self.http_status_code_value(),
|
||||
},
|
||||
CodexErrorDetails::ConnectionFailed(_) | CodexErrorDetails::UnexpectedStatus(_) => {
|
||||
CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: self.http_status_code_value(),
|
||||
}
|
||||
}
|
||||
CodexErrorDetails::ResponseStreamFailed(_) => {
|
||||
CodexErrorInfo::ResponseStreamConnectionFailed {
|
||||
http_status_code: self.http_status_code_value(),
|
||||
@@ -510,6 +516,7 @@ impl CodexErr {
|
||||
|
||||
pub fn http_status_code_value(&self) -> Option<u16> {
|
||||
let http_status_code = match &self.details {
|
||||
CodexErrorDetails::FlexUnavailable => Some(StatusCode::TOO_MANY_REQUESTS),
|
||||
CodexErrorDetails::RetryLimit(err) => Some(err.status),
|
||||
CodexErrorDetails::UnexpectedStatus(err) => Some(err.status),
|
||||
CodexErrorDetails::ConnectionFailed(err) => err.source.status(),
|
||||
|
||||
@@ -1856,6 +1856,7 @@ pub enum CodexErrorInfo {
|
||||
SessionBudgetExceeded,
|
||||
UsageLimitExceeded,
|
||||
RateLimitExceeded,
|
||||
FlexUnavailable,
|
||||
ServerOverloaded,
|
||||
CyberPolicy,
|
||||
BioPolicy,
|
||||
@@ -1899,6 +1900,7 @@ impl CodexErrorInfo {
|
||||
| Self::SessionBudgetExceeded
|
||||
| Self::UsageLimitExceeded
|
||||
| Self::RateLimitExceeded
|
||||
| Self::FlexUnavailable
|
||||
| Self::ServerOverloaded
|
||||
| Self::CyberPolicy
|
||||
| Self::BioPolicy
|
||||
|
||||
@@ -88,6 +88,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
|
||||
ApiError::BioPolicy { .. } => "bio policy".to_string(),
|
||||
ApiError::MisalignmentPolicyViolation { .. } => "misalignment policy violation".to_string(),
|
||||
ApiError::ServerOverloaded { .. } => "server overloaded".to_string(),
|
||||
ApiError::FlexUnavailable => "flex capacity unavailable".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -543,6 +543,7 @@ class CodexErrorInfoValue(Enum):
|
||||
session_budget_exceeded = "sessionBudgetExceeded"
|
||||
usage_limit_exceeded = "usageLimitExceeded"
|
||||
rate_limit_exceeded = "rateLimitExceeded"
|
||||
flex_unavailable = "flexUnavailable"
|
||||
server_overloaded = "serverOverloaded"
|
||||
cyber_policy = "cyberPolicy"
|
||||
misalignment_policy_violation = "misalignmentPolicyViolation"
|
||||
|
||||
Reference in New Issue
Block a user