Classify streaming rate-limit errors (#40931)

## What changed

- Classify `response.failed` events with the `rate_limit_exceeded` code as a distinct retryable error while preserving any parsed retry delay.
- Expose the error as `rateLimitExceeded` through the core protocol and app-server schemas after stream retries are exhausted.
- Preserve the upstream message for TUI display while keeping it out of telemetry summaries.

## Testing

- Cover SSE classification, retry metadata, protocol conversion and serialization, exhausted stream retries, telemetry redaction, and TUI rendering.

GitOrigin-RevId: 02dab4d3477dcd7653a58c49c4bd38687a616579
This commit is contained in:
Steve Coffey
2026-08-26 17:45:42 +00:00
committed by copyberry
parent 7625bd5665
commit e0c727de04
38 changed files with 198 additions and 26 deletions
@@ -617,6 +617,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -8013,6 +8013,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -3974,6 +3974,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -9,6 +9,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -38,6 +38,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -119,6 +119,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -119,6 +119,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -119,6 +119,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -38,6 +38,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -41,6 +41,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -38,6 +38,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -38,6 +38,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -38,6 +38,7 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"rateLimitExceeded",
"serverOverloaded",
"cyberPolicy",
"misalignmentPolicyViolation",
@@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind";
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
*/
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
@@ -78,6 +78,7 @@ pub enum CodexErrorInfo {
ContextWindowExceeded,
SessionBudgetExceeded,
UsageLimitExceeded,
RateLimitExceeded,
ServerOverloaded,
CyberPolicy,
MisalignmentPolicyViolation,
@@ -125,6 +126,7 @@ impl From<CoreCodexErrorInfo> for CodexErrorInfo {
CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded,
CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy,
CoreCodexErrorInfo::MisalignmentPolicyViolation => {
@@ -38,6 +38,7 @@ use codex_protocol::permissions::FileSystemSandboxEntry as CoreFileSystemSandbox
use codex_protocol::permissions::FileSystemSpecialPath as CoreFileSystemSpecialPath;
use codex_protocol::protocol::AgentStatus as CoreAgentStatus;
use codex_protocol::protocol::AskForApproval as CoreAskForApproval;
use codex_protocol::protocol::CodexErrorInfo as CoreCodexErrorInfo;
use codex_protocol::protocol::ConversationTextRole;
use codex_protocol::protocol::ExecCommandSource as CoreExecCommandSource;
use codex_protocol::protocol::GranularApprovalConfig as CoreGranularApprovalConfig;
@@ -4488,11 +4489,19 @@ fn codex_error_info_serializes_http_status_code_in_camel_case() {
}
#[test]
fn codex_error_info_serializes_cyber_policy_in_camel_case() {
assert_eq!(
serde_json::to_value(CodexErrorInfo::CyberPolicy).unwrap(),
json!("cyberPolicy")
);
fn core_error_info_converts_to_camel_case() {
for (core, expected) in [
(CoreCodexErrorInfo::CyberPolicy, json!("cyberPolicy")),
(
CoreCodexErrorInfo::RateLimitExceeded,
json!("rateLimitExceeded"),
),
] {
assert_eq!(
serde_json::to_value(CodexErrorInfo::from(core)).unwrap(),
expected
);
}
}
#[test]
+1
View File
@@ -1767,6 +1767,7 @@ Ownership rejections for parent-owned Multi-Agent V2 subagents return JSON-RPC e
- `ContextWindowExceeded`
- `SessionBudgetExceeded`
- `UsageLimitExceeded`
- `rateLimitExceeded`: an upstream rate limit received inside a streaming response; the turn fails with this category only after its existing stream retry budget is exhausted
- `misalignmentPolicyViolation`: a non-retryable request blocked by the misalignment policy
- `HttpConnectionFailed { httpStatusCode? }`: upstream HTTP failures including 4xx/5xx
- `ResponseStreamConnectionFailed { httpStatusCode? }`: failure to connect to the response SSE stream
+1
View File
@@ -2505,6 +2505,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
| ApiError::QuotaExceeded
| ApiError::UsageNotIncluded
| ApiError::Retryable { .. }
| ApiError::RateLimitExceeded { .. }
| ApiError::RateLimit(_)
| ApiError::InvalidRequest { .. }
| ApiError::CyberPolicy { .. }
+7
View File
@@ -29,6 +29,13 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
None => error,
}
}
ApiError::RateLimitExceeded { message, delay } => {
let error = CodexErr::new(CodexErrorDetails::RateLimitExceeded(message));
match delay {
Some(delay) => error.with_retry_delay(delay),
None => error,
}
}
ApiError::Stream(msg) => CodexErr::Stream(msg),
ApiError::ServerOverloaded => CodexErr::ServerOverloaded,
ApiError::Api { status, message } => {
+37 -10
View File
@@ -1,5 +1,6 @@
use super::*;
use base64::Engine;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::RateLimitReachedType;
use pretty_assertions::assert_eq;
@@ -12,16 +13,42 @@ fn map_api_error_maps_server_overloaded() {
#[test]
fn map_api_error_preserves_retry_delay() {
let retry_delay = std::time::Duration::from_secs(17);
let err = map_api_error(ApiError::Retryable {
message: "retry later".to_string(),
delay: Some(retry_delay),
});
assert!(matches!(
err.details(),
CodexErrorDetails::Stream(message) if message == "retry later"
));
assert_eq!(err.retry_delay(), Some(retry_delay));
for (error, expected_code, expected_message) in [
(
ApiError::Retryable {
message: "retry later".to_string(),
delay: Some(retry_delay),
},
CodexErrorInfo::Other,
"stream disconnected before completion: retry later",
),
(
ApiError::RateLimitExceeded {
message: "retry later".to_string(),
delay: Some(retry_delay),
},
CodexErrorInfo::RateLimitExceeded,
"rate limit exceeded: retry later",
),
] {
let err = map_api_error(error);
assert_eq!(
(
err.to_codex_protocol_error(),
err.retry_delay(),
err.is_retryable(),
err.http_status_code_value(),
err.to_string(),
),
(
expected_code,
Some(retry_delay),
true,
None,
expected_message.to_string(),
)
);
}
}
#[test]
+5
View File
@@ -23,6 +23,11 @@ pub enum ApiError {
message: String,
delay: Option<Duration>,
},
#[error("rate limit exceeded: {message}")]
RateLimitExceeded {
message: String,
delay: Option<Duration>,
},
#[error("rate limit: {0}")]
RateLimit(String),
#[error("invalid request: {message}")]
+50 -4
View File
@@ -440,7 +440,12 @@ pub fn process_responses_event(
} else {
let delay = try_parse_retry_after(&error);
let message = error.message.unwrap_or_default();
response_error = ApiError::Retryable { message, delay };
response_error = match error.code.as_deref() {
Some("rate_limit_exceeded") => {
ApiError::RateLimitExceeded { message, delay }
}
_ => ApiError::Retryable { message, delay },
};
}
}
return Err(ResponsesEventError::Api(response_error));
@@ -1058,7 +1063,7 @@ mod tests {
}
#[tokio::test]
async fn error_when_error_event() {
async fn rate_limit_error_preserves_retry_delay() {
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#;
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
@@ -1068,14 +1073,55 @@ mod tests {
assert_eq!(events.len(), 1);
match &events[0] {
Err(ApiError::Retryable { message, delay }) => {
Err(ApiError::RateLimitExceeded { message, delay }) => {
assert_eq!(
message,
"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."
);
assert_eq!(*delay, Some(Duration::from_secs_f64(11.054)));
}
other => panic!("unexpected second event: {other:?}"),
other => panic!("unexpected rate-limit event: {other:?}"),
}
}
#[tokio::test]
async fn failed_response_classification_uses_error_code() {
for (code, message) in [
("rate_limit_exceeded", "Temporary limit."),
(
"unknown_error",
"Rate limit reached. Please try again in 1s.",
),
] {
let event = json!({
"type": "response.failed",
"response": { "error": { "code": code, "message": message } },
});
let sse = format!("event: response.failed\ndata: {event}\n\n");
let events = collect_events(&[sse.as_bytes()]).await;
match (code, events.as_slice()) {
(
"rate_limit_exceeded",
[
Err(ApiError::RateLimitExceeded {
message: actual,
delay,
}),
],
)
| (
"unknown_error",
[
Err(ApiError::Retryable {
message: actual,
delay,
}),
],
) => {
assert_eq!((actual.as_str(), *delay), (message, None));
}
_ => panic!("unexpected events for {code}: {events:?}"),
}
}
}
+10 -4
View File
@@ -587,14 +587,17 @@ async fn compact_v2_stream_failure_without_retry_after_exhausts_stream_retries()
match wait_for_event(&test.codex, |_| true).await {
EventMsg::Error(error) => {
error_events += 1;
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
assert_eq!(
error.codex_error_info,
Some(CodexErrorInfo::RateLimitExceeded)
);
assert!(error.message.contains("Rate limit exceeded."));
}
EventMsg::StreamError(_) => stream_error_events += 1,
EventMsg::TurnComplete(event) => {
assert_eq!(
event.error.and_then(|error| error.codex_error_info),
Some(CodexErrorInfo::Other)
Some(CodexErrorInfo::RateLimitExceeded)
);
break;
}
@@ -1001,14 +1004,17 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries() -> Result<()>
match wait_for_event(&test.codex, |_| true).await {
EventMsg::Error(error) => {
error_events += 1;
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
assert_eq!(
error.codex_error_info,
Some(CodexErrorInfo::RateLimitExceeded)
);
assert!(error.message.contains("Rate limit exceeded."));
}
EventMsg::StreamError(_) => stream_error_events += 1,
EventMsg::TurnComplete(event) => {
assert_eq!(
event.error.and_then(|error| error.codex_error_info),
Some(CodexErrorInfo::Other)
Some(CodexErrorInfo::RateLimitExceeded)
);
break;
}
@@ -359,7 +359,10 @@ impl LunaSampler {
let retryable = match error {
LunaSamplerError::ConnectionTimeout
| LunaSamplerError::Api(
ApiError::Retryable { .. } | ApiError::Stream(_) | ApiError::ServerOverloaded,
ApiError::Retryable { .. }
| ApiError::RateLimitExceeded { .. }
| ApiError::Stream(_)
| ApiError::ServerOverloaded,
)
| LunaSamplerError::Api(ApiError::Transport(
TransportError::RetryLimit
+5
View File
@@ -91,6 +91,9 @@ pub enum CodexErrorDetails {
/// The Session loop treats this as a transient error and will automatically retry the turn.
#[error("stream disconnected before completion: {0}")]
Stream(String),
/// A retryable upstream rate limit received inside the response stream.
#[error("rate limit exceeded: {0}")]
RateLimitExceeded(String),
#[error(
"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."
)]
@@ -388,6 +391,7 @@ impl CodexErr {
| CodexErrorDetails::CyberPolicy { .. }
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => false,
CodexErrorDetails::Stream(..)
| CodexErrorDetails::RateLimitExceeded(_)
| CodexErrorDetails::Timeout
| CodexErrorDetails::RequestTimeout
| CodexErrorDetails::UnexpectedStatus(_)
@@ -424,6 +428,7 @@ impl CodexErr {
match &self.details {
CodexErrorDetails::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
CodexErrorDetails::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
CodexErrorDetails::RateLimitExceeded(_) => CodexErrorInfo::RateLimitExceeded,
CodexErrorDetails::UsageLimitReached(_)
| CodexErrorDetails::QuotaExceeded
| CodexErrorDetails::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded,
+4
View File
@@ -36,6 +36,10 @@ fn codex_err_debug_preserves_legacy_shape() {
fn retryability_preserves_error_details_distinctions() {
let errors = [
(CodexErr::ServerOverloaded, false),
(
CodexErr::new(CodexErrorDetails::RateLimitExceeded("retry later".into())),
true,
),
(
CodexErr::RetryLimit(RetryLimitReachedError {
status: StatusCode::TOO_MANY_REQUESTS,
+2
View File
@@ -1820,6 +1820,7 @@ pub enum CodexErrorInfo {
ContextWindowExceeded,
SessionBudgetExceeded,
UsageLimitExceeded,
RateLimitExceeded,
ServerOverloaded,
CyberPolicy,
MisalignmentPolicyViolation,
@@ -1859,6 +1860,7 @@ impl CodexErrorInfo {
Self::ContextWindowExceeded
| Self::SessionBudgetExceeded
| Self::UsageLimitExceeded
| Self::RateLimitExceeded
| Self::ServerOverloaded
| Self::CyberPolicy
| Self::MisalignmentPolicyViolation
+9 -1
View File
@@ -80,6 +80,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
ApiError::QuotaExceeded => "quota exceeded".to_string(),
ApiError::UsageNotIncluded => "usage not included".to_string(),
ApiError::Retryable { .. } => "retryable error".to_string(),
ApiError::RateLimitExceeded { .. } => "rate limit exceeded".to_string(),
ApiError::RateLimit(_) => "rate limit".to_string(),
ApiError::InvalidRequest { .. } => "invalid request".to_string(),
ApiError::CyberPolicy { .. } => "cyber policy".to_string(),
@@ -134,7 +135,7 @@ mod tests {
}
#[test]
fn telemetry_error_messages_omit_http_bodies() {
fn telemetry_error_messages_omit_upstream_bodies() {
let transport = TransportError::Http {
status: StatusCode::UNAUTHORIZED,
url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()),
@@ -147,6 +148,13 @@ mod tests {
telemetry_api_error_message(&ApiError::Transport(transport)),
"http 401"
);
assert_eq!(
telemetry_api_error_message(&ApiError::RateLimitExceeded {
message: "private upstream diagnostic".to_string(),
delay: Some(std::time::Duration::from_secs(1)),
}),
"rate limit exceeded"
);
}
#[test]
@@ -0,0 +1,5 @@
---
source: tui/src/chatwidget/tests/app_server.rs
expression: lines_to_single_string(&lines)
---
■ rate limit exceeded: Please try again in 10s.
@@ -1,4 +1,6 @@
use super::*;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use pretty_assertions::assert_eq;
const SAFETY_BUFFERING_HEADER_TEXT: &str =
@@ -1361,6 +1363,26 @@ async fn live_app_server_stream_recovery_restores_previous_status_header() {
assert!(chat.status_state.retry_status_header.is_none());
}
#[tokio::test]
async fn live_app_server_rate_limit_error_renders_upstream_message() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
let error = CodexErr::from(CodexErrorDetails::RateLimitExceeded(
"Please try again in 10s.".to_string(),
));
handle_error(
&mut chat,
error.to_string(),
Some(error.to_codex_protocol_error().into()),
);
let lines = drain_insert_history(&mut rx)
.into_iter()
.flatten()
.collect::<Vec<_>>();
assert_chatwidget_snapshot!("rate_limit_exceeded_error", lines_to_single_string(&lines));
}
#[tokio::test]
async fn live_app_server_server_overloaded_error_renders_warning() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;