Recover server replies after Responses WebSocket write failures (#62749)

## Why

A server can reject an oversized request after reading just the first
frame header. If the upload then fails with a broken pipe, the write
error can hide the server's message-too-large close.

## What changed

After a write fails partway through an upload, drain immediately
readable replies within the send deadline and a bounded read budget.
Use those replies to surface server errors, including
`WebsocketMessageTooLarge`. Treat a close seen before the first frame
as belonging to the idle connection, and retire the socket after a
failed send.

## Testing

Add regression tests for size-limit closes on an idle socket, just
before the first frame, and after a partial first-frame write fails
with pings and pongs ahead of the close.

GitOrigin-RevId: 2feeb1961992b425dbec06129015dbe4c44ff3f2
This commit is contained in:
Charlie Marsh
2026-11-20 12:22:56 +00:00
committed by copyberry
parent 1c297f8997
commit 9f2b87495b
2 changed files with 360 additions and 108 deletions
@@ -83,12 +83,43 @@ struct WsRequestText {
}
struct WsCommand {
deadline: Instant,
request: WsRequestText,
tx_result: oneshot::Sender<Result<(), WsError>>,
tx_result: oneshot::Sender<WsSendOutcome>,
}
enum WsSendOutcome {
Flushed,
NotStarted(ApiError),
Interrupted(ApiError),
}
impl WsSendOutcome {
async fn receive(result: &mut oneshot::Receiver<Self>) -> Result<Self, ApiError> {
let outcome = result
.await
.unwrap_or_else(|_| Self::NotStarted(map_ws_stream_error(WsError::ConnectionClosed)));
match outcome {
Self::NotStarted(error) | Self::Interrupted(error)
if matches!(error, ApiError::Transport(TransportError::Policy(_))) =>
{
Err(error)
}
outcome => Ok(outcome),
}
}
fn into_result(self) -> Result<(), ApiError> {
match self {
Self::Flushed => Ok(()),
Self::NotStarted(error) | Self::Interrupted(error) => Err(error),
}
}
}
const FRAGMENT_SIZE: usize = 1024 * 1024;
const RECEIVE_BATCH_SIZE: usize = 32;
const UPLOAD_READ_AHEAD_BYTES: usize = 1024 * 1024;
fn request_frames(text: Utf8Bytes) -> impl Iterator<Item = Message> {
let (message, mut data) = if text.len() <= FRAGMENT_SIZE {
@@ -113,10 +144,11 @@ fn request_frames(text: Utf8Bytes) -> impl Iterator<Item = Message> {
}))
}
/// Owns the socket and forwards incoming messages while waiting for a request.
/// Owns the socket and recovers peer replies after a failed upload.
struct WsPump<S> {
socket: S,
messages: mpsc::UnboundedSender<Result<Message, WsError>>,
read_budget: Option<usize>,
queued_pongs: usize,
}
@@ -128,28 +160,51 @@ where
Self {
socket,
messages,
read_budget: None,
queued_pongs: 0,
}
}
fn poll_incoming(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), WsError>> {
if self.queued_pongs != 0 && self.socket.poll_flush_unpin(cx)?.is_ready() {
fn poll_incoming(
&mut self,
cx: &mut Context<'_>,
recovering: bool,
) -> Poll<Result<(), WsError>> {
if self.queued_pongs != 0 && !recovering && self.socket.poll_flush_unpin(cx)?.is_ready() {
self.queued_pongs = 0;
}
for _ in 0..RECEIVE_BATCH_SIZE {
if self.queued_pongs >= RECEIVE_BATCH_SIZE {
if self.read_budget == Some(0) {
return Poll::Ready(Ok(()));
}
if !recovering && self.queued_pongs >= RECEIVE_BATCH_SIZE {
return Poll::Pending;
}
let Some(message) = ready!(self.socket.poll_next_unpin(cx)).transpose()? else {
let next = self.socket.poll_next_unpin(cx);
if recovering && next.is_pending() {
return Poll::Ready(Ok(()));
}
let Some(message) = ready!(next).transpose()? else {
return Poll::Ready(Err(WsError::ConnectionClosed));
};
if let Some(budget) = &mut self.read_budget
&& (recovering || !matches!(message, Message::Ping(_) | Message::Pong(_)))
{
*budget = budget.saturating_sub(message.len() + std::mem::size_of::<Message>());
}
match message {
Message::Ping(_) => {
// Queue the automatic Pong before starting the next data frame.
match self.socket.poll_flush_unpin(cx) {
Poll::Ready(Ok(())) => self.queued_pongs = 0,
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
Poll::Pending => self.queued_pongs += 1,
Poll::Ready(Err(error))
if !recovering
|| codex_websocket_client::network_policy_denial(&error)
.is_some() =>
{
return Poll::Ready(Err(error));
}
Poll::Pending | Poll::Ready(Err(_)) => self.queued_pongs += 1,
}
}
Message::Pong(_) => {}
@@ -166,20 +221,78 @@ where
Poll::Pending
}
async fn send_frames(&mut self, text: Utf8Bytes) -> Result<(), WsError> {
for frame in request_frames(text) {
self.socket.send(frame).await?;
async fn recover(&mut self, error: WsError, deadline: Instant) -> WsError {
if matches!(error, WsError::ConnectionClosed)
|| codex_websocket_client::network_policy_denial(&error).is_some()
{
return error;
}
Ok(())
// Drain immediately readable replies until the socket would block.
let replies = poll_fn(|cx| self.poll_incoming(cx, /*recovering*/ true));
if let Ok(Err(read_error)) = tokio::time::timeout_at(deadline, replies).await
&& codex_websocket_client::network_policy_denial(&read_error).is_some()
{
return read_error;
}
error
}
async fn send_frames(&mut self, text: Utf8Bytes, deadline: Instant) -> WsSendOutcome {
self.read_budget = Some(UPLOAD_READ_AHEAD_BYTES);
let mut check_idle = true;
let mut started = false;
for frame in request_frames(text) {
let mut frame = Some(frame);
let write = poll_fn(|cx| {
// A close already readable before the first frame belongs to the idle socket.
if std::mem::take(&mut check_idle) {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
let socket = &mut self.socket;
if frame.is_some() {
ready!(socket.poll_ready_unpin(cx))?;
}
if let Some(frame) = frame.take() {
// Even an I/O failure can follow a partially written frame header.
let result = socket.start_send_unpin(frame);
started |= matches!(result, Ok(()) | Err(WsError::Io(_)));
result?;
}
socket.poll_flush_unpin(cx)
});
let error = match tokio::time::timeout_at(deadline, write).await {
Ok(Ok(())) => continue,
Ok(Err(error)) if started => {
map_ws_stream_error(self.recover(error, deadline).await)
}
Ok(Err(error)) => map_ws_stream_error(error),
Err(_) => ApiError::Stream("idle timeout sending websocket request".into()),
};
return if started {
WsSendOutcome::Interrupted(error)
} else {
WsSendOutcome::NotStarted(error)
};
}
self.read_budget = None;
WsSendOutcome::Flushed
}
async fn run(mut self, mut commands: mpsc::Receiver<WsCommand>) {
loop {
let command = tokio::select! {
command = commands.recv() => Ok(command),
incoming = poll_fn(|cx| self.poll_incoming(cx)) => incoming.map(|()| None),
};
let WsCommand { request, tx_result } = match command {
let command = poll_fn(|cx| {
let command = commands.poll_recv(cx);
if command.is_pending() {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
command.map(Ok)
})
.await;
let WsCommand {
request,
deadline,
tx_result,
} = match command {
Ok(Some(command)) => command,
Ok(None) | Err(WsError::ConnectionClosed) => break,
Err(error) => {
@@ -187,16 +300,16 @@ where
break;
}
};
let send = self.send_frames(request.text);
let send = self.send_frames(request.text, deadline);
// Tungstenite traces frame payloads from inside this future.
let result = if request.private {
let outcome = if request.private {
send.with_subscriber(tracing::Dispatch::none()).await
} else {
send.await
};
let should_break = result.is_err();
let _ = tx_result.send(result);
if should_break {
let flushed = matches!(outcome, WsSendOutcome::Flushed);
let _ = tx_result.send(outcome);
if !flushed {
break;
}
}
@@ -232,21 +345,62 @@ impl WsStream {
}
}
async fn send(&mut self, request: WsRequestText) -> Result<(), WsError> {
let (tx_result, rx_result) = oneshot::channel();
/// Queue a write with a deadline.
async fn queue_request(
&mut self,
request: WsRequestText,
idle_timeout: Duration,
) -> Result<oneshot::Receiver<WsSendOutcome>, ApiError> {
if self
.tx_command
.send(WsCommand { request, tx_result })
.await
.is_ok()
&& let Ok(result) = rx_result.await
.max_message_bytes
.is_some_and(|limit| request.text.len() > limit)
{
return result;
return Err(ApiError::WebsocketMessageTooLarge);
}
while let Some(message) = self.rx_message.recv().await {
message?;
let deadline = tokio::time::sleep(idle_timeout).deadline();
let (tx_result, rx_result) = oneshot::channel();
let _ = self
.tx_command
.send(WsCommand {
request,
deadline,
tx_result,
})
.await;
Ok(rx_result)
}
async fn send_create(
&mut self,
request: WsRequestText,
idle_timeout: Duration,
telemetry: Option<&dyn WebsocketTelemetry>,
connection_reused: bool,
) -> Result<WsSendOutcome, ApiError> {
let start = Instant::now();
let mut send = self.queue_request(request, idle_timeout).await?;
let result = match WsSendOutcome::receive(&mut send).await {
Ok(WsSendOutcome::NotStarted(mut error)) => {
// Replies preceding the create are stale. Recover any idle transport error.
while let Some(message) = self.next().await {
if let Err(idle_error) = message {
error = map_ws_stream_error(idle_error);
break;
}
}
Err(error)
}
outcome => outcome,
};
if let Some(t) = telemetry {
let send_error = match &result {
Ok(WsSendOutcome::Flushed) => None,
Ok(WsSendOutcome::NotStarted(error) | WsSendOutcome::Interrupted(error))
| Err(error) => Some(error),
};
t.on_ws_request(start.elapsed(), send_error, connection_reused);
}
Err(WsError::ConnectionClosed)
result
}
async fn next(&mut self) -> Option<Result<Message, WsError>> {
@@ -277,6 +431,7 @@ const THREAD_ID_CLIENT_METADATA_KEY: &str = "thread_id";
const TURN_ID_CLIENT_METADATA_KEY: &str = "turn_id";
const WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY: &str = "x-codex-ws-stream-request-start-ms";
#[cfg_attr(test, derive(Default))]
struct ResponsesWebsocketTimingLogContext {
model: String,
session_id: Option<String>,
@@ -917,14 +1072,14 @@ async fn run_websocket_response_stream(
// No request reached the backend, so this socket is still reusable.
return Ok(());
}
send_websocket_request(
ws_stream,
request_text,
idle_timeout,
telemetry.as_ref(),
timing_log_context.connection_reused,
)
.await?;
let mut send_outcome = ws_stream
.send_create(
request_text,
idle_timeout,
telemetry.as_deref(),
timing_log_context.connection_reused,
)
.await?;
// A response owns its interrupt, and create must be sent before interrupt.
let mut interrupt_request = interrupt.clone().fuse();
@@ -938,27 +1093,31 @@ async fn run_websocket_response_stream(
response = tokio::time::timeout(idle_timeout, ws_stream.next()) => {
response.map_err(|_| ApiError::Stream("idle timeout waiting for websocket".into()))
}
request = &mut interrupt_request, if response_id.is_some() => {
interruption = Some(request);
send_websocket_request(
ws_stream,
WsRequestText {
text: serde_json::json!({
"type": "response.interrupt",
"response_id": response_id,
"mode": "discard_partial_items",
}).to_string().into(),
private: false,
},
idle_timeout,
/*telemetry*/ None,
timing_log_context.connection_reused,
)
.await?;
awaiting_interrupt_ack = matches!(request, ResponseInterrupt::Cancel { .. });
request = &mut interrupt_request, if response_id.is_some() && matches!(send_outcome, WsSendOutcome::Flushed) => {
let command = WsRequestText {
text: serde_json::json!({
"type": "response.interrupt",
"response_id": response_id,
"mode": "discard_partial_items",
}).to_string().into(),
private: false,
};
let mut send = ws_stream.queue_request(command, idle_timeout).await?;
send_outcome = WsSendOutcome::receive(&mut send).await?;
if !matches!(send_outcome, WsSendOutcome::NotStarted(_)) {
interruption = Some(request);
awaiting_interrupt_ack = matches!(request, ResponseInterrupt::Cancel { .. });
}
continue;
}
};
// A failed send retires the pump. Process replies until its queue runs dry.
if matches!(response, Ok(None))
&& let WsSendOutcome::NotStarted(error) | WsSendOutcome::Interrupted(error) =
send_outcome
{
return Err(error);
}
let poll_duration = poll_start.elapsed();
if let Some(telemetry) = &telemetry
&& is_websocket_receive_failure(&response)
@@ -1098,22 +1257,7 @@ async fn run_websocket_response_stream(
return Err(error.into_api_error());
}
}
if completion.is_some() {
if awaiting_interrupt_ack {
continue;
}
if matches!(
interrupt.clone().now_or_never(),
Some(ResponseInterrupt::Cancel { .. })
) {
break;
}
// Ordinary responses deliver completion before releasing the socket.
// Preserve it if cancellation wins while delivery is backpressured.
let permit = tx_event.reserve().await;
if let (Ok(permit), Some(event)) = (permit, completion.take()) {
permit.send(Ok(event));
}
if completion.is_some() && !awaiting_interrupt_ack {
break;
}
}
@@ -1133,6 +1277,19 @@ async fn run_websocket_response_stream(
}
}
send_outcome.into_result()?;
if matches!(
interrupt.clone().now_or_never(),
Some(ResponseInterrupt::Cancel { .. })
) {
return Ok(());
}
// Ordinary responses deliver completion before releasing the socket.
// Preserve it if cancellation wins while delivery is backpressured.
let permit = tx_event.reserve().await;
if let (Ok(permit), Some(event)) = (permit, completion.take()) {
permit.send(Ok(event));
}
Ok(())
}
@@ -1178,38 +1335,6 @@ fn safety_buffering_for_event(
event.safety_buffering(treatment)
}
async fn send_websocket_request(
ws_stream: &mut WsStream,
request_text: WsRequestText,
idle_timeout: Duration,
telemetry: Option<&Arc<dyn WebsocketTelemetry>>,
connection_reused: bool,
) -> Result<(), ApiError> {
if ws_stream
.max_message_bytes
.is_some_and(|limit| request_text.text.len() > limit)
{
return Err(ApiError::WebsocketMessageTooLarge);
}
let request_start = Instant::now();
let result = tokio::time::timeout(idle_timeout, ws_stream.send(request_text))
.await
.map_err(|_| ApiError::Stream("idle timeout sending websocket request".into()))
.and_then(|result| result.map_err(map_ws_stream_error));
if let Some(t) = telemetry.as_ref() {
t.on_ws_request(
request_start.elapsed(),
result.as_ref().err(),
connection_reused,
);
}
result?;
Ok(())
}
fn serialize_websocket_request(request: &ResponsesWsRequest<'_>) -> Result<String, ApiError> {
serde_json::to_string(request)
.map_err(|err| ApiError::Stream(format!("failed to encode websocket request: {err}")))
@@ -1,20 +1,38 @@
#![cfg(test)]
//! Verify request framing and heartbeat delivery on idle WebSockets.
//! Verify heartbeat delivery and peer responses on backpressured WebSockets.
use super::FRAGMENT_SIZE;
use super::ResponsesWebsocketTimingLogContext;
use super::WsPump;
use super::WsRequestText;
use super::WsSendOutcome;
use super::WsStream;
use super::request_frames;
use super::run_websocket_response_stream;
use crate::common::ResponseEvent;
use crate::common::ResponseInterrupt;
use crate::error::ApiError;
use crate::telemetry::WebsocketTelemetry;
use assert_matches::assert_matches;
use bytes::Bytes;
use futures::SinkExt;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use std::io;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufStream;
use tokio::io::DuplexStream;
use tokio::sync::mpsc;
use tokio::time::Instant;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Error as WsError;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::Utf8Bytes;
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
use tokio_tungstenite::tungstenite::protocol::Role;
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::protocol::frame::coding::Data;
use tokio_tungstenite::tungstenite::protocol::frame::coding::OpCode;
@@ -35,6 +53,63 @@ async fn backpressured_websockets(
(client, server)
}
async fn scripted_client(io: tokio_test::io::Mock) -> WsStream {
let socket = WebSocketStream::from_raw_socket(io, Role::Client, None).await;
WsStream::with_socket(
socket, /*max_message_bytes*/ None, /*routing_affinity*/ None,
)
}
fn large_request_text() -> Utf8Bytes {
Utf8Bytes::from(
serde_json::json!({"type": "response.create", "input": "🌱".repeat(300_000)}).to_string(),
)
}
fn size_close() -> Message {
Message::Close(Some(CloseFrame {
code: CloseCode::Size,
reason: "message too big".into(),
}))
}
async fn upload(
client: WsStream,
telemetry: Option<Arc<dyn WebsocketTelemetry>>,
) -> Result<ResponseEvent, ApiError> {
upload_with_interrupt(client, telemetry, std::future::pending()).await
}
async fn upload_with_interrupt(
mut client: WsStream,
telemetry: Option<Arc<dyn WebsocketTelemetry>>,
interrupt: impl Future<Output = ResponseInterrupt> + Clone + Unpin,
) -> Result<ResponseEvent, ApiError> {
let (tx, mut events) = mpsc::channel(1);
let mut completion = None;
let result = run_websocket_response_stream(
&mut client,
tx,
WsRequestText {
text: large_request_text(),
private: false,
},
TIMEOUT,
telemetry,
/*turn_state*/ None,
&ResponsesWebsocketTimingLogContext::default(),
interrupt,
&mut completion,
)
.await;
// The connection owner retires a failed socket and still delivers its saved completion.
if let Some(completed) = completion {
return Ok(completed);
}
result?;
events.try_recv().expect("expected a response event")
}
#[test]
fn large_turn_is_one_text_message_in_bounded_frames() {
let at_limit = Utf8Bytes::from("a".repeat(FRAGMENT_SIZE));
@@ -67,6 +142,58 @@ fn large_turn_is_one_text_message_in_bounded_frames() {
assert!(frames.next().is_none());
}
#[tokio::test(start_paused = true)]
async fn idle_size_close_is_not_an_upload_rejection() {
let (mut client, mut server) = backpressured_websockets(/*capacity*/ 1024).await;
server.send(size_close()).await.unwrap();
tokio::time::timeout(TIMEOUT, &mut client.pump_task)
.await
.expect("idle pump should observe the close")
.unwrap();
let result = upload(client, /*telemetry*/ None).await;
assert_matches!(result, Err(ApiError::Stream(message)) => {
assert_eq!(message, WsError::ConnectionClosed.to_string());
});
assert!(server.get_mut().fill_buf().await.unwrap().is_empty());
}
#[tokio::test(start_paused = true)]
async fn size_close_before_first_frame_is_not_an_upload_rejection() {
let io = tokio_test::io::Builder::new()
.read(&[0x88, 0x02, 0x03, 0xf1]) // Close(1009)
.build();
let mut socket = WebSocketStream::from_raw_socket(io, Role::Client, None).await;
let (tx, _rx) = mpsc::unbounded_channel();
// The pump has dequeued a command and is about to write its first frame.
assert!(matches!(
WsPump::new(&mut socket, tx)
.send_frames("request".into(), Instant::now() + TIMEOUT)
.await,
WsSendOutcome::NotStarted(_)
));
}
#[tokio::test(start_paused = true)]
async fn size_close_after_partial_first_frame_write_error_rejects_upload() {
// The peer learns the length of the masked text fragment before its payload arrives.
let mut header = vec![0x01, 0xff];
header.extend_from_slice(&(FRAGMENT_SIZE as u64).to_be_bytes());
let mut pings_and_close = [0x89, 0x00].repeat(64);
pings_and_close.extend_from_slice(&[0x88, 0x02, 0x03, 0xf1]); // Close(1009)
let io = tokio_test::io::Builder::new()
.write(&header)
.write_error(io::ErrorKind::BrokenPipe.into())
.read(&[0x8a, 0x00].repeat(64)) // More than one batch of Pongs.
.read(&pings_and_close)
.write_error(io::ErrorKind::BrokenPipe.into())
.wait(TIMEOUT) // Subsequent Pong writes stay blocked during recovery.
.build();
let result = upload(scripted_client(io).await, /*telemetry*/ None).await;
assert_matches!(result, Err(ApiError::WebsocketMessageTooLarge));
}
#[tokio::test(start_paused = true)]
async fn idle_ping_finishes_pong_after_write_backpressure() {
// The Pong cannot fit in the buffer and the peer sends no more messages or commands.