mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Fall back to HTTP immediately for oversized WebSocket requests (#60840)
## Why A WebSocket close with code `1009` before any text arrives indicates that the request exceeds the peer's message size limit. Retrying the request over another WebSocket does not address that limit. ## What changed - Classify these early size rejections as requiring HTTP fallback and switch transports without exhausting WebSocket retries, including during prewarming. - Preserve ordinary stream retry behavior for size closes after any text frame and for other close codes. ## Testing Add integration coverage for early size rejections during generation and prewarming, size closes after metadata, rate limits, or response creation, and ordinary closes. GitOrigin-RevId: 4550d289bb414fec1faae2a195bcc635d394117b
This commit is contained in:
@@ -2512,6 +2512,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
|
||||
}
|
||||
// These variants format fixed text; retry metadata is typed and never printed.
|
||||
ApiError::ContentFilter
|
||||
| ApiError::WebsocketMessageTooLarge
|
||||
| ApiError::ContextWindowExceeded
|
||||
| ApiError::QuotaExceeded
|
||||
| ApiError::UsageNotIncluded
|
||||
|
||||
@@ -30,6 +30,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
|
||||
ApiError::Transport(_)
|
||||
| ApiError::Api { .. }
|
||||
| ApiError::Stream(_)
|
||||
| ApiError::WebsocketMessageTooLarge
|
||||
| ApiError::ContentFilter
|
||||
| ApiError::ResponseIncomplete(_)
|
||||
| ApiError::ContextWindowExceeded
|
||||
@@ -67,6 +68,9 @@ fn map_api_error_details(err: ApiError) -> CodexErr {
|
||||
CodexErr::new(CodexErrorDetails::RateLimitExceeded(message))
|
||||
}
|
||||
ApiError::Stream(msg) => CodexErr::Stream(msg),
|
||||
ApiError::WebsocketMessageTooLarge => {
|
||||
CodexErr::new(CodexErrorDetails::WebsocketMessageTooLarge)
|
||||
}
|
||||
ApiError::ContentFilter => CodexErr::new(CodexErrorDetails::ContentFilter),
|
||||
ApiError::ResponseIncomplete(incomplete) => {
|
||||
CodexErr::new(CodexErrorDetails::ResponseIncomplete(incomplete))
|
||||
|
||||
@@ -47,6 +47,7 @@ use tokio_tungstenite::tungstenite::handshake::client::Response;
|
||||
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::Fragment;
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::Frame;
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::Data;
|
||||
use tracing::Instrument;
|
||||
use tracing::Span;
|
||||
@@ -848,6 +849,7 @@ async fn run_websocket_response_stream(
|
||||
let mut interrupt_request = interrupt.clone().fuse();
|
||||
let mut interruption = None;
|
||||
let mut response_id = None;
|
||||
let mut received_text = false;
|
||||
let mut awaiting_interrupt_ack = false;
|
||||
loop {
|
||||
let poll_start = Instant::now();
|
||||
@@ -896,6 +898,7 @@ async fn run_websocket_response_stream(
|
||||
|
||||
match message {
|
||||
Message::Text(text) => {
|
||||
received_text = true;
|
||||
let event = serde_json::from_str::<ResponsesStreamEvent>(&text);
|
||||
let kind = event.as_ref().ok().map(ResponsesStreamEvent::kind);
|
||||
if let Some(t) = telemetry.as_ref() {
|
||||
@@ -1033,6 +1036,9 @@ async fn run_websocket_response_stream(
|
||||
Message::Binary(_) => {
|
||||
return Err(ApiError::Stream("unexpected binary websocket event".into()));
|
||||
}
|
||||
Message::Close(Some(frame)) if frame.code == CloseCode::Size && !received_text => {
|
||||
return Err(ApiError::WebsocketMessageTooLarge);
|
||||
}
|
||||
Message::Close(_) => {
|
||||
return Err(ApiError::Stream(
|
||||
"websocket closed by server before response.completed".into(),
|
||||
|
||||
@@ -15,6 +15,8 @@ pub enum ApiError {
|
||||
Api { status: StatusCode, message: String },
|
||||
#[error("stream error: {0}")]
|
||||
Stream(String),
|
||||
#[error("websocket request exceeds the peer's message size limit (close code 1009)")]
|
||||
WebsocketMessageTooLarge,
|
||||
#[error("stream error: Incomplete response returned, reason: content_filter")]
|
||||
ContentFilter,
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -2571,6 +2571,10 @@ impl ModelClientSession {
|
||||
while let Some(event) = stream.next().await {
|
||||
match event {
|
||||
Ok(ResponseEvent::Completed { .. }) => break,
|
||||
Err(err) if err.requires_http_fallback() => {
|
||||
self.try_switch_fallback_transport(session_telemetry, model_info);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
@@ -118,12 +118,13 @@ pub(crate) async fn handle_response_stream_error(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if retry_state.retries >= max_retries
|
||||
let requires_http_fallback = err.requires_http_fallback();
|
||||
let switched = (requires_http_fallback || retry_state.retries >= max_retries)
|
||||
&& client_session.try_switch_fallback_transport(
|
||||
&turn_context.session_telemetry,
|
||||
turn_context.model_info(),
|
||||
)
|
||||
{
|
||||
);
|
||||
if switched {
|
||||
// Changing transport must not bypass the server's retry deadline.
|
||||
if let Some(retry_after) = retry_after {
|
||||
tokio::time::sleep_until(retry_after.deadline()).await;
|
||||
@@ -135,6 +136,9 @@ pub(crate) async fn handle_response_stream_error(
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// Another request can already have switched this session to HTTP.
|
||||
if switched || requires_http_fallback {
|
||||
retry_state.retries = 0;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -3158,7 +3158,10 @@ async fn try_run_sampling_request(
|
||||
.await;
|
||||
should_emit_token_count |= incomplete.token_usage.is_some();
|
||||
}
|
||||
if *draining_for_mailbox && matches!(err.details(), CodexErrorDetails::Stream(_)) {
|
||||
if *draining_for_mailbox
|
||||
&& (matches!(err.details(), CodexErrorDetails::Stream(_))
|
||||
|| err.requires_http_fallback())
|
||||
{
|
||||
// Mail already requires a follow-up, even if the producer exited before
|
||||
// receiving the interrupt. Do not retry without that pending input.
|
||||
break Ok(SamplingRequestResult {
|
||||
|
||||
@@ -30,6 +30,9 @@ use wiremock::http::Method;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path_regex;
|
||||
|
||||
#[path = "websocket_fallback_close.rs"]
|
||||
mod close_code;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn websocket_fallback_switches_to_http_on_upgrade_required_connect() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
//! Test which WebSocket closes switch the session to HTTP.
|
||||
|
||||
use anyhow::Result;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::responses::ev_completed;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use futures::SinkExt;
|
||||
use futures::StreamExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Rejection {
|
||||
Oversized,
|
||||
OversizedWarmup,
|
||||
AfterModelMetadata,
|
||||
AfterRateLimits,
|
||||
AfterResponseCreated,
|
||||
GoingAway,
|
||||
}
|
||||
|
||||
#[test_case::test_case(Rejection::Oversized, 1, 1; "size_rejection_before_any_text")]
|
||||
#[test_case::test_case(Rejection::OversizedWarmup, 0, 1; "size_rejection_during_prewarm")]
|
||||
#[test_case::test_case(Rejection::AfterModelMetadata, 2, 0; "size_close_after_model_metadata")]
|
||||
#[test_case::test_case(Rejection::AfterRateLimits, 2, 0; "size_close_after_rate_limits")]
|
||||
#[test_case::test_case(Rejection::AfterResponseCreated, 2, 0; "size_close_after_response_created")]
|
||||
#[test_case::test_case(Rejection::GoingAway, 2, 0; "ordinary_close")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn retry_transport(
|
||||
rejection: Rejection,
|
||||
expected_ws_generations: usize,
|
||||
expected_posts: usize,
|
||||
) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let http = responses::start_mock_server().await;
|
||||
responses::mount_sse_once(
|
||||
&http,
|
||||
sse(vec![ev_response_created("http"), ev_completed("http")]),
|
||||
)
|
||||
.await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let base_url = format!("http://{}/v1", listener.local_addr()?);
|
||||
let http_address = *http.address();
|
||||
let generations = Arc::new(AtomicUsize::new(0));
|
||||
let captured = Arc::clone(&generations);
|
||||
let mut server = JoinSet::new();
|
||||
server.spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
while let Ok((stream, _)) = listener.accept().await {
|
||||
connections.spawn(serve_connection(
|
||||
stream,
|
||||
http_address,
|
||||
Arc::clone(&captured),
|
||||
rejection,
|
||||
));
|
||||
}
|
||||
});
|
||||
|
||||
let mut builder = test_codex().with_config(move |config| {
|
||||
config.model_provider.base_url = Some(base_url);
|
||||
config.model_provider.supports_websockets = true;
|
||||
config.model_provider.stream_max_retries = Some(1);
|
||||
config.model_provider.request_max_retries = Some(0);
|
||||
});
|
||||
let test = builder.build_with_auto_env(&http).await?;
|
||||
test.submit_turn("first").await?;
|
||||
assert_eq!(generations.load(Ordering::Relaxed), expected_ws_generations);
|
||||
assert_eq!(
|
||||
responses::received_responses_requests(&http).await.len(),
|
||||
expected_posts
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn serve_connection(
|
||||
mut stream: TcpStream,
|
||||
http_address: SocketAddr,
|
||||
generations: Arc<AtomicUsize>,
|
||||
rejection: Rejection,
|
||||
) -> Result<()> {
|
||||
let mut method = [0];
|
||||
stream.peek(&mut method).await?;
|
||||
if method == *b"P" {
|
||||
let mut upstream = TcpStream::connect(http_address).await?;
|
||||
tokio::io::copy_bidirectional(&mut stream, &mut upstream).await?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut socket = tokio_tungstenite::accept_async(stream).await?;
|
||||
while let Some(message) = socket.next().await.transpose()? {
|
||||
let Message::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let request: Value = serde_json::from_str(&text)?;
|
||||
let warmup = request["generate"] == false;
|
||||
let reject = if warmup {
|
||||
rejection == Rejection::OversizedWarmup
|
||||
} else {
|
||||
generations.fetch_add(1, Ordering::Relaxed) == 0
|
||||
};
|
||||
if reject {
|
||||
let event = match rejection {
|
||||
Rejection::AfterModelMetadata => Some(serde_json::json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {"openai-model": "test-model"}
|
||||
})),
|
||||
Rejection::AfterRateLimits => Some(serde_json::json!({
|
||||
"type": "codex.rate_limits",
|
||||
"rate_limits": {"primary": {"used_percent": 1.0}}
|
||||
})),
|
||||
Rejection::AfterResponseCreated => Some(ev_response_created("started")),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(event) = event {
|
||||
socket.send(Message::text(event.to_string())).await?;
|
||||
}
|
||||
let code = if rejection == Rejection::GoingAway {
|
||||
CloseCode::Away
|
||||
} else {
|
||||
CloseCode::Size
|
||||
};
|
||||
socket
|
||||
.close(Some(CloseFrame {
|
||||
code,
|
||||
reason: "".into(),
|
||||
}))
|
||||
.await?;
|
||||
break;
|
||||
}
|
||||
let id = if warmup { "warmup" } else { "generation" };
|
||||
for event in [ev_response_created(id), ev_completed(id)] {
|
||||
socket.send(Message::text(event.to_string())).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -46,6 +46,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str {
|
||||
ApiError::Transport(TransportError::ResponseTooLarge { .. }) => "response_too_large",
|
||||
ApiError::Transport(TransportError::Policy(_)) => "network_policy_denied",
|
||||
ApiError::Stream(_) | ApiError::ContentFilter => "stream_error",
|
||||
ApiError::WebsocketMessageTooLarge => "websocket_message_too_large",
|
||||
ApiError::ResponseIncomplete(_) => "incomplete_response",
|
||||
ApiError::ContextWindowExceeded => "context_window_exceeded",
|
||||
ApiError::QuotaExceeded => "quota_exceeded",
|
||||
|
||||
@@ -114,6 +114,7 @@ impl SamplingExecution {
|
||||
ApiError::Retryable { .. }
|
||||
| ApiError::RateLimitExceeded { .. }
|
||||
| ApiError::Stream(_)
|
||||
| ApiError::WebsocketMessageTooLarge
|
||||
| ApiError::ContentFilter
|
||||
| ApiError::ServerOverloaded { .. }
|
||||
| ApiError::PolicyCapacityUnavailable { .. }
|
||||
|
||||
@@ -81,6 +81,13 @@ pub struct CodexErr {
|
||||
agent_context: Option<AgentErrorContext>,
|
||||
}
|
||||
|
||||
enum RetryPolicy {
|
||||
Never,
|
||||
ServerDirected,
|
||||
Backoff,
|
||||
FallBackToHttp,
|
||||
}
|
||||
|
||||
/// A terminal response that stopped before the model finished, with any reported usage.
|
||||
/// Content-filter retries and explicit interruptions have separate handling.
|
||||
#[derive(derive_more::Debug, Error, PartialEq, Eq)]
|
||||
@@ -114,6 +121,9 @@ pub enum CodexErrorDetails {
|
||||
/// The Session loop treats this as a transient error and will automatically retry the turn.
|
||||
#[error("stream disconnected before completion: {0}")]
|
||||
Stream(String),
|
||||
/// Retry the rejected request over HTTP instead of sending it on another socket.
|
||||
#[error("websocket request exceeds the peer's message size limit (close code 1009)")]
|
||||
WebsocketMessageTooLarge,
|
||||
/// A response stopped by the content filter. Sampling retries need developer guidance.
|
||||
#[error(
|
||||
"stream disconnected before completion: Incomplete response returned, reason: content_filter"
|
||||
@@ -438,11 +448,27 @@ impl CodexErr {
|
||||
&self.details
|
||||
}
|
||||
|
||||
/// Whether retrying this request requires switching from WebSocket to HTTP.
|
||||
pub fn requires_http_fallback(&self) -> bool {
|
||||
matches!(self.retry_policy(), RetryPolicy::FallBackToHttp)
|
||||
}
|
||||
|
||||
/// Returns the delay before the given retry attempt, or `None` for a terminal error.
|
||||
///
|
||||
/// The first retry is attempt one. Retryable errors use server advice when available and
|
||||
/// otherwise use exponential backoff with jitter. Callers enforce their own retry budgets.
|
||||
pub fn retry_delay(&self, retry_count: u64) -> Option<Duration> {
|
||||
match self.retry_policy() {
|
||||
RetryPolicy::Never => None,
|
||||
RetryPolicy::ServerDirected => self.server_retry_delay(),
|
||||
RetryPolicy::Backoff | RetryPolicy::FallBackToHttp => Some(
|
||||
self.server_retry_delay()
|
||||
.unwrap_or_else(|| backoff(retry_count)),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_policy(&self) -> RetryPolicy {
|
||||
match self.details() {
|
||||
CodexErrorDetails::UnexpectedStatus(error)
|
||||
if is_runtime_auth_denial(
|
||||
@@ -451,7 +477,7 @@ impl CodexErr {
|
||||
error.identity_error_code.as_deref(),
|
||||
) =>
|
||||
{
|
||||
None
|
||||
RetryPolicy::Never
|
||||
}
|
||||
CodexErrorDetails::TurnAborted
|
||||
| CodexErrorDetails::SessionBudgetExceeded
|
||||
@@ -485,10 +511,11 @@ impl CodexErr {
|
||||
| CodexErrorDetails::BioPolicy { .. }
|
||||
| CodexErrorDetails::WorkspaceAccessDenied(_)
|
||||
| CodexErrorDetails::AccessProgramNotEnabled(_)
|
||||
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => None,
|
||||
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => RetryPolicy::Never,
|
||||
CodexErrorDetails::ServerOverloaded
|
||||
| CodexErrorDetails::PolicyCapacityUnavailable
|
||||
| CodexErrorDetails::RetryLimit(_) => self.server_retry_delay(),
|
||||
| CodexErrorDetails::RetryLimit(_) => RetryPolicy::ServerDirected,
|
||||
CodexErrorDetails::WebsocketMessageTooLarge => RetryPolicy::FallBackToHttp,
|
||||
CodexErrorDetails::Stream(..)
|
||||
| CodexErrorDetails::ContentFilter
|
||||
| CodexErrorDetails::RateLimitExceeded(_)
|
||||
@@ -501,12 +528,11 @@ impl CodexErr {
|
||||
| CodexErrorDetails::InternalAgentDied
|
||||
| CodexErrorDetails::Io(_)
|
||||
| CodexErrorDetails::Json(_)
|
||||
| CodexErrorDetails::TokioJoin(_) => Some(
|
||||
self.server_retry_delay()
|
||||
.unwrap_or_else(|| backoff(retry_count)),
|
||||
),
|
||||
| CodexErrorDetails::TokioJoin(_) => RetryPolicy::Backoff,
|
||||
#[cfg(target_os = "linux")]
|
||||
CodexErrorDetails::LandlockRuleset(_) | CodexErrorDetails::LandlockPathFd(_) => None,
|
||||
CodexErrorDetails::LandlockRuleset(_) | CodexErrorDetails::LandlockPathFd(_) => {
|
||||
RetryPolicy::Never
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
|
||||
ApiError::Transport(transport) => telemetry_transport_error_message(transport),
|
||||
ApiError::Api { status, .. } => format!("api error {}", status.as_u16()),
|
||||
ApiError::Stream(err) => err.to_string(),
|
||||
ApiError::WebsocketMessageTooLarge => error.to_string(),
|
||||
ApiError::ContentFilter => {
|
||||
"Incomplete response returned, reason: content_filter".to_string()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user