From 9f2b87495bf979f6c1953cf00c98143981664914 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 20 Nov 2026 11:50:44 +0000 Subject: [PATCH] Recover server replies after Responses WebSocket write failures (#62749) ## Why A server can reject an oversized request after reading just the first frame header. If the upload then fails with a broken pipe, the write error can hide the server's message-too-large close. ## What changed After a write fails partway through an upload, drain immediately readable replies within the send deadline and a bounded read budget. Use those replies to surface server errors, including `WebsocketMessageTooLarge`. Treat a close seen before the first frame as belonging to the idle connection, and retire the socket after a failed send. ## Testing Add regression tests for size-limit closes on an idle socket, just before the first frame, and after a partial first-frame write fails with pings and pongs ahead of the close. GitOrigin-RevId: 2feeb1961992b425dbec06129015dbe4c44ff3f2 --- .../src/endpoint/responses_websocket.rs | 339 ++++++++++++------ .../responses_websocket_upload_tests.rs | 129 ++++++- 2 files changed, 360 insertions(+), 108 deletions(-) diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs index 68113097cf..c799a52e2a 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs @@ -83,12 +83,43 @@ struct WsRequestText { } struct WsCommand { + deadline: Instant, request: WsRequestText, - tx_result: oneshot::Sender>, + tx_result: oneshot::Sender, +} + +enum WsSendOutcome { + Flushed, + NotStarted(ApiError), + Interrupted(ApiError), +} + +impl WsSendOutcome { + async fn receive(result: &mut oneshot::Receiver) -> Result { + let outcome = result + .await + .unwrap_or_else(|_| Self::NotStarted(map_ws_stream_error(WsError::ConnectionClosed))); + match outcome { + Self::NotStarted(error) | Self::Interrupted(error) + if matches!(error, ApiError::Transport(TransportError::Policy(_))) => + { + Err(error) + } + outcome => Ok(outcome), + } + } + + fn into_result(self) -> Result<(), ApiError> { + match self { + Self::Flushed => Ok(()), + Self::NotStarted(error) | Self::Interrupted(error) => Err(error), + } + } } const FRAGMENT_SIZE: usize = 1024 * 1024; const RECEIVE_BATCH_SIZE: usize = 32; +const UPLOAD_READ_AHEAD_BYTES: usize = 1024 * 1024; fn request_frames(text: Utf8Bytes) -> impl Iterator { let (message, mut data) = if text.len() <= FRAGMENT_SIZE { @@ -113,10 +144,11 @@ fn request_frames(text: Utf8Bytes) -> impl Iterator { })) } -/// Owns the socket and forwards incoming messages while waiting for a request. +/// Owns the socket and recovers peer replies after a failed upload. struct WsPump { socket: S, messages: mpsc::UnboundedSender>, + read_budget: Option, queued_pongs: usize, } @@ -128,28 +160,51 @@ where Self { socket, messages, + read_budget: None, queued_pongs: 0, } } - fn poll_incoming(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.queued_pongs != 0 && self.socket.poll_flush_unpin(cx)?.is_ready() { + fn poll_incoming( + &mut self, + cx: &mut Context<'_>, + recovering: bool, + ) -> Poll> { + if self.queued_pongs != 0 && !recovering && self.socket.poll_flush_unpin(cx)?.is_ready() { self.queued_pongs = 0; } for _ in 0..RECEIVE_BATCH_SIZE { - if self.queued_pongs >= RECEIVE_BATCH_SIZE { + if self.read_budget == Some(0) { + return Poll::Ready(Ok(())); + } + if !recovering && self.queued_pongs >= RECEIVE_BATCH_SIZE { return Poll::Pending; } - let Some(message) = ready!(self.socket.poll_next_unpin(cx)).transpose()? else { + let next = self.socket.poll_next_unpin(cx); + if recovering && next.is_pending() { + return Poll::Ready(Ok(())); + } + let Some(message) = ready!(next).transpose()? else { return Poll::Ready(Err(WsError::ConnectionClosed)); }; + if let Some(budget) = &mut self.read_budget + && (recovering || !matches!(message, Message::Ping(_) | Message::Pong(_))) + { + *budget = budget.saturating_sub(message.len() + std::mem::size_of::()); + } match message { Message::Ping(_) => { // Queue the automatic Pong before starting the next data frame. match self.socket.poll_flush_unpin(cx) { Poll::Ready(Ok(())) => self.queued_pongs = 0, - Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), - Poll::Pending => self.queued_pongs += 1, + Poll::Ready(Err(error)) + if !recovering + || codex_websocket_client::network_policy_denial(&error) + .is_some() => + { + return Poll::Ready(Err(error)); + } + Poll::Pending | Poll::Ready(Err(_)) => self.queued_pongs += 1, } } Message::Pong(_) => {} @@ -166,20 +221,78 @@ where Poll::Pending } - async fn send_frames(&mut self, text: Utf8Bytes) -> Result<(), WsError> { - for frame in request_frames(text) { - self.socket.send(frame).await?; + async fn recover(&mut self, error: WsError, deadline: Instant) -> WsError { + if matches!(error, WsError::ConnectionClosed) + || codex_websocket_client::network_policy_denial(&error).is_some() + { + return error; } - Ok(()) + // Drain immediately readable replies until the socket would block. + let replies = poll_fn(|cx| self.poll_incoming(cx, /*recovering*/ true)); + if let Ok(Err(read_error)) = tokio::time::timeout_at(deadline, replies).await + && codex_websocket_client::network_policy_denial(&read_error).is_some() + { + return read_error; + } + error + } + + async fn send_frames(&mut self, text: Utf8Bytes, deadline: Instant) -> WsSendOutcome { + self.read_budget = Some(UPLOAD_READ_AHEAD_BYTES); + let mut check_idle = true; + let mut started = false; + for frame in request_frames(text) { + let mut frame = Some(frame); + let write = poll_fn(|cx| { + // A close already readable before the first frame belongs to the idle socket. + if std::mem::take(&mut check_idle) { + let _ = self.poll_incoming(cx, /*recovering*/ false)?; + } + let socket = &mut self.socket; + if frame.is_some() { + ready!(socket.poll_ready_unpin(cx))?; + } + if let Some(frame) = frame.take() { + // Even an I/O failure can follow a partially written frame header. + let result = socket.start_send_unpin(frame); + started |= matches!(result, Ok(()) | Err(WsError::Io(_))); + result?; + } + socket.poll_flush_unpin(cx) + }); + let error = match tokio::time::timeout_at(deadline, write).await { + Ok(Ok(())) => continue, + Ok(Err(error)) if started => { + map_ws_stream_error(self.recover(error, deadline).await) + } + Ok(Err(error)) => map_ws_stream_error(error), + Err(_) => ApiError::Stream("idle timeout sending websocket request".into()), + }; + return if started { + WsSendOutcome::Interrupted(error) + } else { + WsSendOutcome::NotStarted(error) + }; + } + self.read_budget = None; + WsSendOutcome::Flushed } async fn run(mut self, mut commands: mpsc::Receiver) { loop { - let command = tokio::select! { - command = commands.recv() => Ok(command), - incoming = poll_fn(|cx| self.poll_incoming(cx)) => incoming.map(|()| None), - }; - let WsCommand { request, tx_result } = match command { + let command = poll_fn(|cx| { + let command = commands.poll_recv(cx); + if command.is_pending() { + let _ = self.poll_incoming(cx, /*recovering*/ false)?; + } + command.map(Ok) + }) + .await; + let WsCommand { + request, + deadline, + tx_result, + } = match command { Ok(Some(command)) => command, Ok(None) | Err(WsError::ConnectionClosed) => break, Err(error) => { @@ -187,16 +300,16 @@ where break; } }; - let send = self.send_frames(request.text); + let send = self.send_frames(request.text, deadline); // Tungstenite traces frame payloads from inside this future. - let result = if request.private { + let outcome = if request.private { send.with_subscriber(tracing::Dispatch::none()).await } else { send.await }; - let should_break = result.is_err(); - let _ = tx_result.send(result); - if should_break { + let flushed = matches!(outcome, WsSendOutcome::Flushed); + let _ = tx_result.send(outcome); + if !flushed { break; } } @@ -232,21 +345,62 @@ impl WsStream { } } - async fn send(&mut self, request: WsRequestText) -> Result<(), WsError> { - let (tx_result, rx_result) = oneshot::channel(); + /// Queue a write with a deadline. + async fn queue_request( + &mut self, + request: WsRequestText, + idle_timeout: Duration, + ) -> Result, ApiError> { if self - .tx_command - .send(WsCommand { request, tx_result }) - .await - .is_ok() - && let Ok(result) = rx_result.await + .max_message_bytes + .is_some_and(|limit| request.text.len() > limit) { - return result; + return Err(ApiError::WebsocketMessageTooLarge); } - while let Some(message) = self.rx_message.recv().await { - message?; + let deadline = tokio::time::sleep(idle_timeout).deadline(); + let (tx_result, rx_result) = oneshot::channel(); + let _ = self + .tx_command + .send(WsCommand { + request, + deadline, + tx_result, + }) + .await; + Ok(rx_result) + } + + async fn send_create( + &mut self, + request: WsRequestText, + idle_timeout: Duration, + telemetry: Option<&dyn WebsocketTelemetry>, + connection_reused: bool, + ) -> Result { + let start = Instant::now(); + let mut send = self.queue_request(request, idle_timeout).await?; + let result = match WsSendOutcome::receive(&mut send).await { + Ok(WsSendOutcome::NotStarted(mut error)) => { + // Replies preceding the create are stale. Recover any idle transport error. + while let Some(message) = self.next().await { + if let Err(idle_error) = message { + error = map_ws_stream_error(idle_error); + break; + } + } + Err(error) + } + outcome => outcome, + }; + if let Some(t) = telemetry { + let send_error = match &result { + Ok(WsSendOutcome::Flushed) => None, + Ok(WsSendOutcome::NotStarted(error) | WsSendOutcome::Interrupted(error)) + | Err(error) => Some(error), + }; + t.on_ws_request(start.elapsed(), send_error, connection_reused); } - Err(WsError::ConnectionClosed) + result } async fn next(&mut self) -> Option> { @@ -277,6 +431,7 @@ const THREAD_ID_CLIENT_METADATA_KEY: &str = "thread_id"; const TURN_ID_CLIENT_METADATA_KEY: &str = "turn_id"; const WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY: &str = "x-codex-ws-stream-request-start-ms"; +#[cfg_attr(test, derive(Default))] struct ResponsesWebsocketTimingLogContext { model: String, session_id: Option, @@ -917,14 +1072,14 @@ async fn run_websocket_response_stream( // No request reached the backend, so this socket is still reusable. return Ok(()); } - send_websocket_request( - ws_stream, - request_text, - idle_timeout, - telemetry.as_ref(), - timing_log_context.connection_reused, - ) - .await?; + let mut send_outcome = ws_stream + .send_create( + request_text, + idle_timeout, + telemetry.as_deref(), + timing_log_context.connection_reused, + ) + .await?; // A response owns its interrupt, and create must be sent before interrupt. let mut interrupt_request = interrupt.clone().fuse(); @@ -938,27 +1093,31 @@ async fn run_websocket_response_stream( response = tokio::time::timeout(idle_timeout, ws_stream.next()) => { response.map_err(|_| ApiError::Stream("idle timeout waiting for websocket".into())) } - request = &mut interrupt_request, if response_id.is_some() => { - interruption = Some(request); - send_websocket_request( - ws_stream, - WsRequestText { - text: serde_json::json!({ - "type": "response.interrupt", - "response_id": response_id, - "mode": "discard_partial_items", - }).to_string().into(), - private: false, - }, - idle_timeout, - /*telemetry*/ None, - timing_log_context.connection_reused, - ) - .await?; - awaiting_interrupt_ack = matches!(request, ResponseInterrupt::Cancel { .. }); + request = &mut interrupt_request, if response_id.is_some() && matches!(send_outcome, WsSendOutcome::Flushed) => { + let command = WsRequestText { + text: serde_json::json!({ + "type": "response.interrupt", + "response_id": response_id, + "mode": "discard_partial_items", + }).to_string().into(), + private: false, + }; + let mut send = ws_stream.queue_request(command, idle_timeout).await?; + send_outcome = WsSendOutcome::receive(&mut send).await?; + if !matches!(send_outcome, WsSendOutcome::NotStarted(_)) { + interruption = Some(request); + awaiting_interrupt_ack = matches!(request, ResponseInterrupt::Cancel { .. }); + } continue; } }; + // A failed send retires the pump. Process replies until its queue runs dry. + if matches!(response, Ok(None)) + && let WsSendOutcome::NotStarted(error) | WsSendOutcome::Interrupted(error) = + send_outcome + { + return Err(error); + } let poll_duration = poll_start.elapsed(); if let Some(telemetry) = &telemetry && is_websocket_receive_failure(&response) @@ -1098,22 +1257,7 @@ async fn run_websocket_response_stream( return Err(error.into_api_error()); } } - if completion.is_some() { - if awaiting_interrupt_ack { - continue; - } - if matches!( - interrupt.clone().now_or_never(), - Some(ResponseInterrupt::Cancel { .. }) - ) { - break; - } - // Ordinary responses deliver completion before releasing the socket. - // Preserve it if cancellation wins while delivery is backpressured. - let permit = tx_event.reserve().await; - if let (Ok(permit), Some(event)) = (permit, completion.take()) { - permit.send(Ok(event)); - } + if completion.is_some() && !awaiting_interrupt_ack { break; } } @@ -1133,6 +1277,19 @@ async fn run_websocket_response_stream( } } + send_outcome.into_result()?; + if matches!( + interrupt.clone().now_or_never(), + Some(ResponseInterrupt::Cancel { .. }) + ) { + return Ok(()); + } + // Ordinary responses deliver completion before releasing the socket. + // Preserve it if cancellation wins while delivery is backpressured. + let permit = tx_event.reserve().await; + if let (Ok(permit), Some(event)) = (permit, completion.take()) { + permit.send(Ok(event)); + } Ok(()) } @@ -1178,38 +1335,6 @@ fn safety_buffering_for_event( event.safety_buffering(treatment) } -async fn send_websocket_request( - ws_stream: &mut WsStream, - request_text: WsRequestText, - idle_timeout: Duration, - telemetry: Option<&Arc>, - connection_reused: bool, -) -> Result<(), ApiError> { - if ws_stream - .max_message_bytes - .is_some_and(|limit| request_text.text.len() > limit) - { - return Err(ApiError::WebsocketMessageTooLarge); - } - let request_start = Instant::now(); - let result = tokio::time::timeout(idle_timeout, ws_stream.send(request_text)) - .await - .map_err(|_| ApiError::Stream("idle timeout sending websocket request".into())) - .and_then(|result| result.map_err(map_ws_stream_error)); - - if let Some(t) = telemetry.as_ref() { - t.on_ws_request( - request_start.elapsed(), - result.as_ref().err(), - connection_reused, - ); - } - - result?; - - Ok(()) -} - fn serialize_websocket_request(request: &ResponsesWsRequest<'_>) -> Result { serde_json::to_string(request) .map_err(|err| ApiError::Stream(format!("failed to encode websocket request: {err}"))) diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket_upload_tests.rs b/codex-rs/codex-api/src/endpoint/responses_websocket_upload_tests.rs index 3870d8e80e..734108732a 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket_upload_tests.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket_upload_tests.rs @@ -1,20 +1,38 @@ #![cfg(test)] -//! Verify request framing and heartbeat delivery on idle WebSockets. +//! Verify heartbeat delivery and peer responses on backpressured WebSockets. use super::FRAGMENT_SIZE; +use super::ResponsesWebsocketTimingLogContext; +use super::WsPump; +use super::WsRequestText; +use super::WsSendOutcome; use super::WsStream; use super::request_frames; +use super::run_websocket_response_stream; +use crate::common::ResponseEvent; +use crate::common::ResponseInterrupt; +use crate::error::ApiError; +use crate::telemetry::WebsocketTelemetry; +use assert_matches::assert_matches; use bytes::Bytes; use futures::SinkExt; use futures::StreamExt; use pretty_assertions::assert_eq; +use std::io; +use std::sync::Arc; use std::time::Duration; +use tokio::io::AsyncBufReadExt; use tokio::io::BufStream; use tokio::io::DuplexStream; +use tokio::sync::mpsc; +use tokio::time::Instant; use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::tungstenite::Error as WsError; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::Utf8Bytes; +use tokio_tungstenite::tungstenite::protocol::CloseFrame; use tokio_tungstenite::tungstenite::protocol::Role; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use tokio_tungstenite::tungstenite::protocol::frame::coding::Data; use tokio_tungstenite::tungstenite::protocol::frame::coding::OpCode; @@ -35,6 +53,63 @@ async fn backpressured_websockets( (client, server) } +async fn scripted_client(io: tokio_test::io::Mock) -> WsStream { + let socket = WebSocketStream::from_raw_socket(io, Role::Client, None).await; + WsStream::with_socket( + socket, /*max_message_bytes*/ None, /*routing_affinity*/ None, + ) +} + +fn large_request_text() -> Utf8Bytes { + Utf8Bytes::from( + serde_json::json!({"type": "response.create", "input": "🌱".repeat(300_000)}).to_string(), + ) +} + +fn size_close() -> Message { + Message::Close(Some(CloseFrame { + code: CloseCode::Size, + reason: "message too big".into(), + })) +} + +async fn upload( + client: WsStream, + telemetry: Option>, +) -> Result { + upload_with_interrupt(client, telemetry, std::future::pending()).await +} + +async fn upload_with_interrupt( + mut client: WsStream, + telemetry: Option>, + interrupt: impl Future + Clone + Unpin, +) -> Result { + let (tx, mut events) = mpsc::channel(1); + let mut completion = None; + let result = run_websocket_response_stream( + &mut client, + tx, + WsRequestText { + text: large_request_text(), + private: false, + }, + TIMEOUT, + telemetry, + /*turn_state*/ None, + &ResponsesWebsocketTimingLogContext::default(), + interrupt, + &mut completion, + ) + .await; + // The connection owner retires a failed socket and still delivers its saved completion. + if let Some(completed) = completion { + return Ok(completed); + } + result?; + events.try_recv().expect("expected a response event") +} + #[test] fn large_turn_is_one_text_message_in_bounded_frames() { let at_limit = Utf8Bytes::from("a".repeat(FRAGMENT_SIZE)); @@ -67,6 +142,58 @@ fn large_turn_is_one_text_message_in_bounded_frames() { assert!(frames.next().is_none()); } +#[tokio::test(start_paused = true)] +async fn idle_size_close_is_not_an_upload_rejection() { + let (mut client, mut server) = backpressured_websockets(/*capacity*/ 1024).await; + server.send(size_close()).await.unwrap(); + tokio::time::timeout(TIMEOUT, &mut client.pump_task) + .await + .expect("idle pump should observe the close") + .unwrap(); + + let result = upload(client, /*telemetry*/ None).await; + assert_matches!(result, Err(ApiError::Stream(message)) => { + assert_eq!(message, WsError::ConnectionClosed.to_string()); + }); + assert!(server.get_mut().fill_buf().await.unwrap().is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn size_close_before_first_frame_is_not_an_upload_rejection() { + let io = tokio_test::io::Builder::new() + .read(&[0x88, 0x02, 0x03, 0xf1]) // Close(1009) + .build(); + let mut socket = WebSocketStream::from_raw_socket(io, Role::Client, None).await; + let (tx, _rx) = mpsc::unbounded_channel(); + + // The pump has dequeued a command and is about to write its first frame. + assert!(matches!( + WsPump::new(&mut socket, tx) + .send_frames("request".into(), Instant::now() + TIMEOUT) + .await, + WsSendOutcome::NotStarted(_) + )); +} + +#[tokio::test(start_paused = true)] +async fn size_close_after_partial_first_frame_write_error_rejects_upload() { + // The peer learns the length of the masked text fragment before its payload arrives. + let mut header = vec![0x01, 0xff]; + header.extend_from_slice(&(FRAGMENT_SIZE as u64).to_be_bytes()); + let mut pings_and_close = [0x89, 0x00].repeat(64); + pings_and_close.extend_from_slice(&[0x88, 0x02, 0x03, 0xf1]); // Close(1009) + let io = tokio_test::io::Builder::new() + .write(&header) + .write_error(io::ErrorKind::BrokenPipe.into()) + .read(&[0x8a, 0x00].repeat(64)) // More than one batch of Pongs. + .read(&pings_and_close) + .write_error(io::ErrorKind::BrokenPipe.into()) + .wait(TIMEOUT) // Subsequent Pong writes stay blocked during recovery. + .build(); + let result = upload(scripted_client(io).await, /*telemetry*/ None).await; + assert_matches!(result, Err(ApiError::WebsocketMessageTooLarge)); +} + #[tokio::test(start_paused = true)] async fn idle_ping_finishes_pong_after_write_backpressure() { // The Pong cannot fit in the buffer and the peer sends no more messages or commands.