mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Surface Responses stream errors as soon as they arrive (#62356)
## Why If a provider keeps an SSE connection open after reporting a failure, Codex can hit the idle timeout and lose the original error and its retry advice. ## What changed End the SSE stream immediately on `response.failed` or a recognized `error` event. Classify statusless quota, plan usage, invalid prompt, and misalignment errors consistently across both event forms. Preserve `Retry-After` for overload and capacity errors, preferring error-level headers over event-level headers when both provide valid advice. ## Testing Add SSE regression tests that hold the connection open and verify immediate error delivery, terminal error handling, and retry delays. Extend WebSocket coverage for statusless errors received before the close handshake. GitOrigin-RevId: 69afd1d3b18ca3139e9b7ad98ae1481a432179f0
This commit is contained in:
@@ -103,6 +103,23 @@ pub(crate) fn parse_terminal_provider_error(
|
||||
.and_then(Value::as_str)
|
||||
.filter(|message| !message.trim().is_empty());
|
||||
match (code, status) {
|
||||
("insufficient_quota" | "credit_balance_exhausted", None) => Some(ApiError::QuotaExceeded),
|
||||
("usage_not_included", None) => Some(ApiError::UsageNotIncluded),
|
||||
("invalid_prompt", None) => Some(ApiError::InvalidPrompt {
|
||||
message: error
|
||||
.get("message")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("Invalid request.")
|
||||
.to_owned(),
|
||||
}),
|
||||
("misalignment_policy_violation", None) => Some(ApiError::MisalignmentPolicyViolation {
|
||||
message: message
|
||||
.unwrap_or("This request was blocked due to a misalignment policy violation.")
|
||||
.to_owned(),
|
||||
misalignment: error.get("misalignment").cloned().and_then(|details| {
|
||||
serde_json::from_value::<MisalignmentErrorDetails>(details).ok()
|
||||
}),
|
||||
}),
|
||||
("api_key_spend_limit_exceeded", None | Some(StatusCode::TOO_MANY_REQUESTS)) => {
|
||||
Some(ApiError::SpendLimitExceeded(SpendLimitScope::ApiKey))
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::responses_error::parse_error_event;
|
||||
use super::responses_error::parse_failed_response;
|
||||
use crate::common::ReasoningRemovalReason;
|
||||
use crate::common::ResponseEvent;
|
||||
@@ -5,7 +6,6 @@ use crate::common::ResponseStream;
|
||||
use crate::common::SafetyBuffering;
|
||||
use crate::common::SafetyBufferingTreatment;
|
||||
use crate::error::ApiError;
|
||||
use crate::error::parse_terminal_provider_error;
|
||||
use crate::guardian_assessment::parse_guardian_assessment;
|
||||
use crate::rate_limits::parse_all_rate_limits;
|
||||
use crate::safety_buffering::treatment_from_headers;
|
||||
@@ -402,9 +402,7 @@ pub fn process_responses_event(
|
||||
) -> std::result::Result<Option<ResponseEvent>, ResponsesEventError> {
|
||||
match event.kind.as_str() {
|
||||
"error" => {
|
||||
if let Some(error) =
|
||||
parse_terminal_provider_error(event.error.as_ref(), /*status*/ None)
|
||||
{
|
||||
if let Some(error) = parse_error_event(event.error.as_ref(), event.headers.as_ref()) {
|
||||
return Err(ResponsesEventError::Api(error));
|
||||
}
|
||||
}
|
||||
@@ -614,7 +612,6 @@ async fn process_sse_with_treatment(
|
||||
safety_buffering_treatment: SafetyBufferingTreatment,
|
||||
) {
|
||||
let mut stream = stream.eventsource();
|
||||
let mut response_error: Option<ApiError> = None;
|
||||
let mut last_server_model: Option<String> = None;
|
||||
|
||||
loop {
|
||||
@@ -641,9 +638,7 @@ async fn process_sse_with_treatment(
|
||||
return;
|
||||
}
|
||||
Ok(None) => {
|
||||
let error = response_error.unwrap_or(ApiError::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
));
|
||||
let error = ApiError::Stream("stream closed before response.completed".into());
|
||||
let _ = tx_event.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
@@ -703,25 +698,8 @@ async fn process_sse_with_treatment(
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
let error = error.into_api_error();
|
||||
if matches!(
|
||||
error,
|
||||
ApiError::FlexUnavailable
|
||||
| ApiError::ContextWindowExceeded
|
||||
| ApiError::InvalidRequest { .. }
|
||||
| ApiError::SessionBudgetExceeded
|
||||
| ApiError::SpendLimitExceeded(_)
|
||||
| ApiError::InferenceHookDenied { .. }
|
||||
| ApiError::AccessProgramNotEnabled { .. }
|
||||
| ApiError::ContentPolicyViolation { .. }
|
||||
| ApiError::BioPolicy { .. }
|
||||
| ApiError::CyberPolicy { .. }
|
||||
| ApiError::ResponseIncomplete(_)
|
||||
) {
|
||||
let _ = tx_event.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
response_error = Some(error);
|
||||
let _ = tx_event.send(Err(error.into_api_error())).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ use crate::error::ApiError;
|
||||
use crate::error::parse_terminal_provider_error;
|
||||
use crate::responses_headers::json_headers_to_http_headers;
|
||||
use codex_http_client::RetryAfter;
|
||||
use codex_protocol::protocol::MisalignmentErrorDetails;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use std::time::Duration;
|
||||
@@ -23,12 +22,39 @@ struct Error {
|
||||
misalignment: Option<Value>,
|
||||
}
|
||||
|
||||
fn retry_after_header(headers: Option<&Value>) -> Option<RetryAfter> {
|
||||
headers
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|headers| RetryAfter::from_headers(&json_headers_to_http_headers(headers)))
|
||||
}
|
||||
|
||||
/// Classifies recognized statusless error events without treating unknown notifications as failures.
|
||||
pub(super) fn parse_error_event(
|
||||
error: Option<&Value>,
|
||||
headers: Option<&Value>,
|
||||
) -> Option<ApiError> {
|
||||
if let Some(error) = parse_terminal_provider_error(error, /*status*/ None) {
|
||||
return Some(error);
|
||||
}
|
||||
let error = error?;
|
||||
let code = error.get("code")?.as_str()?;
|
||||
match code {
|
||||
"policy_capacity_unavailable" | "server_is_overloaded" => {
|
||||
let retry_after =
|
||||
retry_after_header(error.get("headers")).or_else(|| retry_after_header(headers));
|
||||
Some(if code == "policy_capacity_unavailable" {
|
||||
ApiError::PolicyCapacityUnavailable { retry_after }
|
||||
} else {
|
||||
ApiError::ServerOverloaded { retry_after }
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
|
||||
let error = response.as_ref().and_then(|response| response.get("error"));
|
||||
let retry_after_header = error
|
||||
.and_then(|error| error.get("headers"))
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|headers| RetryAfter::from_headers(&json_headers_to_http_headers(headers)));
|
||||
let retry_after_header = retry_after_header(error.and_then(|error| error.get("headers")));
|
||||
if let Some(error) = parse_terminal_provider_error(error, /*status*/ None) {
|
||||
return error;
|
||||
}
|
||||
@@ -45,27 +71,6 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
|
||||
};
|
||||
|
||||
match error.code.as_deref() {
|
||||
Some("insufficient_quota" | "credit_balance_exhausted") => ApiError::QuotaExceeded,
|
||||
Some("usage_not_included") => ApiError::UsageNotIncluded,
|
||||
Some("misalignment_policy_violation") => {
|
||||
let message = error
|
||||
.message
|
||||
.filter(|message| !message.trim().is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
"This request was blocked due to a misalignment policy violation.".to_string()
|
||||
});
|
||||
ApiError::MisalignmentPolicyViolation {
|
||||
message,
|
||||
misalignment: error.misalignment.and_then(|details| {
|
||||
serde_json::from_value::<MisalignmentErrorDetails>(details).ok()
|
||||
}),
|
||||
}
|
||||
}
|
||||
Some("invalid_prompt") => ApiError::InvalidPrompt {
|
||||
message: error
|
||||
.message
|
||||
.unwrap_or_else(|| "Invalid request.".to_string()),
|
||||
},
|
||||
Some("policy_capacity_unavailable") => ApiError::PolicyCapacityUnavailable {
|
||||
retry_after: retry_after_header,
|
||||
},
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::common::ResponseEvent;
|
||||
use bytes::Bytes;
|
||||
use codex_client::StreamResponse;
|
||||
use codex_http_client::RetryAfter;
|
||||
use codex_protocol::error::SpendLimitScope;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use futures::StreamExt;
|
||||
use futures::stream;
|
||||
@@ -298,3 +299,155 @@ async fn retry_header_does_not_override_terminal_errors() {
|
||||
assert_eq!(error.retry_delay(/*retry_count*/ 1), None, "{code}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Exercise the SSE delivery path while the provider leaves its connection open.
|
||||
async fn keep_open_error(event: serde_json::Value) -> crate::ApiError {
|
||||
let bytes =
|
||||
stream::iter([Ok(Bytes::from(format!("data: {event}\n\n")))]).chain(stream::pending());
|
||||
let mut response = spawn_response_stream(
|
||||
StreamResponse {
|
||||
status: StatusCode::OK,
|
||||
headers: HeaderMap::new(),
|
||||
bytes: Box::pin(bytes),
|
||||
},
|
||||
Duration::from_secs(60),
|
||||
/*telemetry*/ None,
|
||||
/*turn_state*/ None,
|
||||
);
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = loop {
|
||||
match response.next().await {
|
||||
Some(Ok(ResponseEvent::RateLimits(_))) => {}
|
||||
Some(Err(error)) => break error,
|
||||
other => panic!("expected an error, got {other:?}"),
|
||||
}
|
||||
};
|
||||
assert_eq!(started.elapsed(), Duration::ZERO, "{event}");
|
||||
assert!(response.next().await.is_none());
|
||||
error
|
||||
}
|
||||
|
||||
fn error_forms(detail: serde_json::Value) -> [serde_json::Value; 2] {
|
||||
[
|
||||
json!({"type": "error", "error": detail}),
|
||||
json!({"type": "response.failed", "response": {"error": detail}}),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn terminal_codes_exit_keep_open_sse_in_both_event_forms() {
|
||||
use crate::ApiError;
|
||||
for code in [
|
||||
"audio_too_large",
|
||||
"audio_too_short",
|
||||
"invalid_audio",
|
||||
"invalid_audio_data",
|
||||
"invalid_audio_format",
|
||||
"missing_audio_sample_rate",
|
||||
"model_not_found",
|
||||
"insufficient_quota",
|
||||
"credit_balance_exhausted",
|
||||
"api_key_spend_limit_exceeded",
|
||||
"organization_spend_limit_exceeded",
|
||||
"project_spend_limit_exceeded",
|
||||
"usage_not_included",
|
||||
"invalid_prompt",
|
||||
"misalignment_policy_violation",
|
||||
] {
|
||||
for event in error_forms(
|
||||
json!({"code": code, "message": "Rejected.", "headers": {"Retry-After": "7"}}),
|
||||
) {
|
||||
let error = keep_open_error(event).await;
|
||||
match (code, &error) {
|
||||
(
|
||||
"audio_too_large",
|
||||
ApiError::InvalidAudioInput {
|
||||
per_recording: false,
|
||||
..
|
||||
},
|
||||
)
|
||||
| (
|
||||
"audio_too_short"
|
||||
| "invalid_audio"
|
||||
| "invalid_audio_data"
|
||||
| "invalid_audio_format"
|
||||
| "missing_audio_sample_rate",
|
||||
ApiError::InvalidAudioInput {
|
||||
per_recording: true,
|
||||
..
|
||||
},
|
||||
)
|
||||
| ("model_not_found", ApiError::ModelNotFound { status: None, .. })
|
||||
| ("insufficient_quota" | "credit_balance_exhausted", ApiError::QuotaExceeded)
|
||||
| (
|
||||
"api_key_spend_limit_exceeded",
|
||||
ApiError::SpendLimitExceeded(SpendLimitScope::ApiKey),
|
||||
)
|
||||
| (
|
||||
"project_spend_limit_exceeded",
|
||||
ApiError::SpendLimitExceeded(SpendLimitScope::Project),
|
||||
)
|
||||
| (
|
||||
"organization_spend_limit_exceeded",
|
||||
ApiError::SpendLimitExceeded(SpendLimitScope::Organization),
|
||||
)
|
||||
| ("usage_not_included", ApiError::UsageNotIncluded)
|
||||
| ("invalid_prompt", ApiError::InvalidPrompt { .. })
|
||||
| ("misalignment_policy_violation", ApiError::MisalignmentPolicyViolation { .. }) =>
|
||||
{}
|
||||
_ => panic!("wrong error for {code}: {error:?}"),
|
||||
}
|
||||
let error = map_api_error(error);
|
||||
assert_eq!(error.retry_delay(/*retry_count*/ 1), None, "{code}");
|
||||
assert!(!error.requires_http_fallback(), "{code}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn overload_advice_survives_keep_open_sse_in_both_event_forms() {
|
||||
for (code, expected) in [
|
||||
("server_is_overloaded", CodexErrorInfo::ServerOverloaded),
|
||||
(
|
||||
"policy_capacity_unavailable",
|
||||
CodexErrorInfo::PolicyCapacityUnavailable,
|
||||
),
|
||||
] {
|
||||
for (header, delay) in [
|
||||
(Some("7"), Some(Duration::from_secs(7))),
|
||||
(Some("bad"), None),
|
||||
(None, None),
|
||||
] {
|
||||
let mut detail = json!({"code": code});
|
||||
if let Some(value) = header {
|
||||
detail["headers"] = json!({"Retry-After": value});
|
||||
}
|
||||
for event in error_forms(detail) {
|
||||
let error = map_api_error(keep_open_error(event).await);
|
||||
assert_eq!(error.to_codex_protocol_error(), expected);
|
||||
assert_eq!(error.retry_delay(/*retry_count*/ 1), delay, "{code}");
|
||||
}
|
||||
}
|
||||
let event = json!({"type": "error", "headers": {"Retry-After": "9"}, "error": {"code": code, "headers": {"Retry-After": "3"}}});
|
||||
assert_eq!(
|
||||
map_api_error(keep_open_error(event).await).retry_delay(/*retry_count*/ 1),
|
||||
Some(Duration::from_secs(3))
|
||||
);
|
||||
let event =
|
||||
json!({"type": "error", "headers": {"Retry-After": "9"}, "error": {"code": code}});
|
||||
assert_eq!(
|
||||
map_api_error(keep_open_error(event).await).retry_delay(/*retry_count*/ 1),
|
||||
Some(Duration::from_secs(9))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retryable_failed_response_preserves_advice_before_sse_eof() {
|
||||
let event = json!({"type": "response.failed", "response": {"error": {"code": "rate_limit_exceeded", "message": "Try again in 35 seconds.", "headers": {"Retry-After": "2"}}}});
|
||||
let error = map_api_error(keep_open_error(event).await);
|
||||
assert_eq!(
|
||||
error.retry_delay(/*retry_count*/ 1),
|
||||
Some(Duration::from_secs(2))
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user