Read Responses WebSocket messages during blocked uploads (#62771)

## Why

When a server stops reading a request, the client's WebSocket write can
stall. If the server sends a Ping or rejects the request while the write
is blocked, the client can leave that message unread until the write
fails or times out.

## What changed

Poll for incoming messages before each outgoing frame and while a write
is pending. On subsequent polls, give the pending write first use of
Tokio's cooperative budget and cap read-ahead across the entire upload.

## Testing

Add regression tests for Pong delivery before upload completion, early
size-limit closes, and bounded read-ahead without write starvation.
Cover error precedence for buffered peer replies, local send failures,
and network policy revocation, plus token usage when a close arrives
before an interrupt can be sent.

GitOrigin-RevId: c018c25dcdc70b64835b07935caede5e34dfa8f2
This commit is contained in:
Charlie Marsh
2026-11-20 14:26:42 +00:00
committed by copyberry
parent 9f2b87495b
commit bb5b9c6240
2 changed files with 296 additions and 16 deletions
@@ -144,7 +144,7 @@ fn request_frames(text: Utf8Bytes) -> impl Iterator<Item = Message> {
}))
}
/// Owns the socket and recovers peer replies after a failed upload.
/// Drives the socket while waiting for commands and while flushing their frames.
struct WsPump<S> {
socket: S,
messages: mpsc::UnboundedSender<Result<Message, WsError>>,
@@ -221,6 +221,27 @@ where
Poll::Pending
}
async fn drive<T>(
&mut self,
mut operation: impl FnMut(&mut S, &mut Context<'_>) -> Poll<Result<T, WsError>>,
) -> Result<T, WsError> {
let mut first = self.read_budget.is_some();
poll_fn(|cx| {
// Read before starting a frame. Once blocked, give the write the first
// use of the task's cooperative budget on each wake.
let read_first = std::mem::take(&mut first);
if read_first {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
let result = operation(&mut self.socket, cx);
if !read_first && result.is_pending() {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
result
})
.await
}
async fn recover(&mut self, error: WsError, deadline: Instant) -> WsError {
if matches!(error, WsError::ConnectionClosed)
|| codex_websocket_client::network_policy_denial(&error).is_some()
@@ -238,17 +259,12 @@ where
}
async fn send_frames(&mut self, text: Utf8Bytes, deadline: Instant) -> WsSendOutcome {
// Limit how far we read ahead of the response consumer during a send.
self.read_budget = Some(UPLOAD_READ_AHEAD_BYTES);
let mut check_idle = true;
let mut started = false;
for frame in request_frames(text) {
let mut frame = Some(frame);
let write = poll_fn(|cx| {
// A close already readable before the first frame belongs to the idle socket.
if std::mem::take(&mut check_idle) {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
let socket = &mut self.socket;
let write = self.drive(|socket, cx| {
if frame.is_some() {
ready!(socket.poll_ready_unpin(cx))?;
}
@@ -280,14 +296,7 @@ where
async fn run(mut self, mut commands: mpsc::Receiver<WsCommand>) {
loop {
let command = poll_fn(|cx| {
let command = commands.poll_recv(cx);
if command.is_pending() {
let _ = self.poll_incoming(cx, /*recovering*/ false)?;
}
command.map(Ok)
})
.await;
let command = self.drive(|_, cx| commands.poll_recv(cx).map(Ok)).await;
let WsCommand {
request,
deadline,
@@ -3,6 +3,7 @@
use super::FRAGMENT_SIZE;
use super::ResponsesWebsocketTimingLogContext;
use super::UPLOAD_READ_AHEAD_BYTES;
use super::WsPump;
use super::WsRequestText;
use super::WsSendOutcome;
@@ -15,16 +16,22 @@ use crate::error::ApiError;
use crate::telemetry::WebsocketTelemetry;
use assert_matches::assert_matches;
use bytes::Bytes;
use codex_client::TransportError;
use codex_http_client::NetworkPolicyDenied;
use futures::FutureExt;
use futures::SinkExt;
use futures::StreamExt;
use http::StatusCode;
use pretty_assertions::assert_eq;
use std::io;
use std::sync::Arc;
use std::sync::OnceLock;
use std::time::Duration;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufStream;
use tokio::io::DuplexStream;
use tokio::sync::mpsc;
use tokio::task::coop;
use tokio::time::Instant;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Error as WsError;
@@ -32,11 +39,15 @@ use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::Utf8Bytes;
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
use tokio_tungstenite::tungstenite::protocol::Role;
use tokio_tungstenite::tungstenite::protocol::frame::Frame;
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::protocol::frame::coding::Data;
use tokio_tungstenite::tungstenite::protocol::frame::coding::OpCode;
const TIMEOUT: Duration = Duration::from_secs(3);
const RATE_LIMIT: &str = r#"{"type":"error","status":429,"error":{"code":"rate_limit_exceeded"}}"#;
const CREATED: &str = r#"{"type":"response.created","response":{"id":"resp_test"}}"#;
const COMPLETED_WITH_USAGE: &str = r#"{"type":"response.completed","response":{"id":"resp_test","usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}"#;
async fn backpressured_websockets(
capacity: usize,
@@ -60,6 +71,18 @@ async fn scripted_client(io: tokio_test::io::Mock) -> WsStream {
)
}
fn encoded_text(message: &str) -> Vec<u8> {
let mut bytes = Vec::new();
Frame::message(
message.to_owned(),
OpCode::Data(Data::Text),
/*is_final*/ true,
)
.format(&mut bytes)
.unwrap();
bytes
}
fn large_request_text() -> Utf8Bytes {
Utf8Bytes::from(
serde_json::json!({"type": "response.create", "input": "🌱".repeat(300_000)}).to_string(),
@@ -142,6 +165,93 @@ fn large_turn_is_one_text_message_in_bounded_frames() {
assert!(frames.next().is_none());
}
#[tokio::test]
async fn ready_messages_do_not_starve_a_pending_write() {
// Empty text frames consume the whole cooperative budget in a batch.
// Cheap Pings exercise the batch limit independently of that budget.
for (frame, read_cost) in [([0x81, 0], 4), ([0x89, 0], 0)] {
let burst = frame.repeat(128);
let io = tokio::io::join(burst.as_slice(), tokio::io::sink());
let socket = WebSocketStream::from_raw_socket(io, Role::Client, None).await;
let mut socket = Box::pin(socket.then(|message| async move {
for _ in 0..read_cost {
coop::consume_budget().await;
}
message
}));
let (tx, _rx) = mpsc::unbounded_channel();
let result = WsPump::new(&mut socket, tx)
.send_frames("request".into(), Instant::now() + TIMEOUT)
.await;
assert!(matches!(result, WsSendOutcome::Flushed));
assert!(
matches!(socket.next().await, Some(Ok(_))),
"the receive burst drained before the send"
);
}
}
#[tokio::test(start_paused = true)]
async fn backpressured_upload_bounds_read_ahead_across_frames() {
let payload = "x".repeat(16 * 1024);
let limit = UPLOAD_READ_AHEAD_BYTES.div_ceil(payload.len());
let burst = encoded_text(&payload).repeat(2 * limit);
let (io, peer) = tokio::io::duplex(1024);
let mut socket = WebSocketStream::from_partially_read(io, burst, Role::Client, None).await;
let mut peer = WebSocketStream::from_raw_socket(peer, Role::Server, None).await;
let (tx, rx) = mpsc::unbounded_channel();
// Reads are always ready while the small transport requires many polls per outgoing frame.
let mut pump = WsPump::new(&mut socket, tx);
let send = pump.send_frames(large_request_text(), Instant::now() + TIMEOUT);
let (result, received) =
tokio::time::timeout(TIMEOUT, async { tokio::join!(send, peer.next()) })
.await
.expect("the write should continue once read-ahead stops");
assert!(matches!(result, WsSendOutcome::Flushed));
assert_matches!(received, Some(Ok(Message::Text(_))));
assert!(rx.len() <= limit, "queued {} messages", rx.len());
drop(peer);
for _ in rx.len()..2 * limit {
assert_eq!(socket.next().await.unwrap().unwrap().len(), payload.len());
}
}
#[tokio::test(start_paused = true)]
async fn large_request_answers_ping_before_upload_completes() {
let (client, mut server) = backpressured_websockets(/*capacity*/ 1024).await;
let completed = Message::text(r#"{"type":"response.completed","response":{"id":"resp_test"}}"#);
let serve_request = async {
// Wait until the client is writing, keeping the prefix for Tungstenite to decode.
// The small duplex buffer keeps the full message from being written before the Ping.
assert!(!server.get_mut().fill_buf().await.unwrap().is_empty());
let ping = Bytes::from_static(b"upload heartbeat");
server.send(Message::Ping(ping.clone())).await.unwrap();
let Some(Ok(Message::Pong(pong))) = server.next().await else {
panic!("expected the Pong before receiving the full request");
};
assert_eq!(pong, ping);
let Some(Ok(Message::Text(received))) = server.next().await else {
panic!("expected a complete text message after the Pong");
};
assert!(
received == large_request_text(),
"request text changed in transit"
);
server.send(completed).await.unwrap();
};
let (result, ()) = tokio::time::timeout(TIMEOUT, async {
tokio::join!(upload(client, /*telemetry*/ None), serve_request)
})
.await
.expect("peer should receive the Pong and complete the upload before the deadline");
assert_matches!(result, Ok(ResponseEvent::Completed { response_id, .. }) if response_id == "resp_test");
}
#[tokio::test(start_paused = true)]
async fn idle_size_close_is_not_an_upload_rejection() {
let (mut client, mut server) = backpressured_websockets(/*capacity*/ 1024).await;
@@ -194,6 +304,167 @@ async fn size_close_after_partial_first_frame_write_error_rejects_upload() {
assert_matches!(result, Err(ApiError::WebsocketMessageTooLarge));
}
async fn reject_upload(messages: impl IntoIterator<Item = Message>) -> ApiError {
let (client, mut server) = backpressured_websockets(/*capacity*/ 1024).await;
let reject_request = async {
// Wait for the first bytes, then reject the request without draining the upload.
assert!(!server.get_mut().fill_buf().await.unwrap().is_empty());
for message in messages {
server.send(message).await.unwrap();
}
};
let (result, ()) = tokio::time::timeout(TIMEOUT, async {
tokio::join!(upload(client, /*telemetry*/ None), reject_request)
})
.await
.expect("client should see the close while the upload is blocked");
result.unwrap_err()
}
#[tokio::test(start_paused = true)]
async fn upload_reports_size_close_without_waiting_for_remaining_bytes() {
let error = reject_upload([size_close()]).await;
assert_matches!(error, ApiError::WebsocketMessageTooLarge);
}
fn assert_rate_limit(error: ApiError) {
assert_matches!(
error,
ApiError::Transport(TransportError::Http {
status: StatusCode::TOO_MANY_REQUESTS,
body: Some(body),
..
}) if body == RATE_LIMIT
);
}
#[tokio::test(start_paused = true)]
async fn upload_preserves_peer_error_before_size_close() {
assert_rate_limit(reject_upload([Message::text(RATE_LIMIT), size_close()]).await);
}
#[derive(Default)]
struct UploadTelemetry {
send_error: OnceLock<String>,
}
impl WebsocketTelemetry for UploadTelemetry {
fn on_ws_request(&self, _: Duration, error: Option<&ApiError>, _: bool) {
let Some(ApiError::Stream(message)) = error else {
panic!("expected an upload error, got {error:?}");
};
self.send_error.set(message.clone()).unwrap();
}
fn on_ws_receive_failure(&self, _: Duration) {
panic!("upload failure counted again as a receive failure");
}
fn on_ws_text_event(&self, _: &str, _: Option<&str>, _: Duration) {}
}
fn partial_upload(message: &str) -> tokio_test::io::Builder {
let mut io = tokio_test::io::Builder::new();
io.write(&[0x01]) // Start the first text fragment.
.read(&encoded_text(message))
// Wake the pending flush after the pump has read the peer response.
.wait(Duration::from_millis(1));
io
}
fn failed_upload(message: &str) -> tokio_test::io::Builder {
let mut io = partial_upload(message);
io.write_error(io::ErrorKind::BrokenPipe.into());
io
}
fn revoked() -> io::Error {
io::Error::new(
io::ErrorKind::PermissionDenied,
NetworkPolicyDenied::Revoked,
)
}
#[tokio::test(start_paused = true)]
async fn upload_reports_policy_revocation_before_buffered_peer_error() {
let scripts = [
partial_upload(RATE_LIMIT).write_error(revoked()).build(),
failed_upload(RATE_LIMIT).read_error(revoked()).build(),
failed_upload(RATE_LIMIT)
.read(&[0x89, 0x00]) // Ping during the final read.
.write_error(revoked())
.build(),
];
for io in scripts {
let result = upload(scripted_client(io).await, /*telemetry*/ None).await;
assert_matches!(
result,
Err(ApiError::Transport(TransportError::Policy(
NetworkPolicyDenied::Revoked
)))
);
}
}
fn broken_pipe_message() -> String {
WsError::Io(io::ErrorKind::BrokenPipe.into()).to_string()
}
async fn upload_with_send_error(io: tokio_test::io::Mock, send_error: &str) -> ApiError {
let client = scripted_client(io).await;
let telemetry = Arc::new(UploadTelemetry::default());
let error = upload(client, Some(telemetry.clone())).await.unwrap_err();
assert_eq!(telemetry.send_error.get().unwrap(), send_error);
error
}
#[tokio::test(start_paused = true)]
async fn upload_preserves_peer_error_when_send_times_out() {
let io = partial_upload(RATE_LIMIT).wait(TIMEOUT * 2).build();
assert_rate_limit(upload_with_send_error(io, "idle timeout sending websocket request").await);
}
#[tokio::test(start_paused = true)]
async fn upload_delivers_buffered_peer_error_before_local_write_error() {
let io = failed_upload(RATE_LIMIT).build();
assert_rate_limit(upload_with_send_error(io, &broken_pipe_message()).await);
}
#[tokio::test(start_paused = true)]
async fn upload_reports_write_error_after_incomplete_response() {
let io = failed_upload(CREATED).build();
let error = upload_with_send_error(io, &broken_pipe_message()).await;
assert_matches!(error, ApiError::Stream(message) if message == broken_pipe_message());
}
#[tokio::test(start_paused = true)]
async fn close_before_sending_interrupt_preserves_completion_usage() {
let (client, mut server) = backpressured_websockets(/*capacity*/ 1024).await;
let created = tokio::spawn(async move {
server.next().await.unwrap().unwrap();
server.send(Message::text(CREATED)).await.unwrap();
server
});
let cancel = async {
let mut server = created.await.unwrap();
// Make both frames readable just as the response loop decides to interrupt.
server
.feed(Message::text(COMPLETED_WITH_USAGE))
.await
.unwrap();
server.send(Message::Close(None)).await.unwrap();
ResponseInterrupt::Cancel {
deadline: Instant::now() + TIMEOUT,
}
}
.shared();
let event = upload_with_interrupt(client, /*telemetry*/ None, cancel)
.await
.unwrap();
assert_matches!(event, ResponseEvent::Completed { token_usage: Some(usage), .. } if usage.total_tokens == 3);
}
#[tokio::test(start_paused = true)]
async fn idle_ping_finishes_pong_after_write_backpressure() {
// The Pong cannot fit in the buffer and the peer sends no more messages or commands.