Flush idle Responses WebSocket pongs under backpressure (#62742)

## Why

An idle Responses WebSocket still needs to answer server pings when the
write buffer is full. The pong must finish flushing once the peer drains
the socket, even if no further messages or requests arrive.

## What changed

Have the WebSocket pump flush automatically queued pongs and retry pending
flushes as the socket becomes writable. Bound incoming polling and the
number of pongs tracked while a flush is pending.

## Testing

Add an in-memory WebSocket test that forces a partial pong write and
checks that the peer receives the full heartbeat reply without further
traffic.

GitOrigin-RevId: 12865842c80a8a25a5e107be690f9c3962ce9775
This commit is contained in:
Charlie Marsh
2026-11-20 12:06:32 +00:00
committed by copyberry
parent 076ac7856b
commit 1c297f8997
2 changed files with 219 additions and 123 deletions
@@ -26,7 +26,9 @@ use codex_http_client::RetryAfter;
use codex_websocket_client::WebSocketConnection;
use codex_websocket_client::WebSocketConnector;
use futures::FutureExt;
use futures::Sink;
use futures::SinkExt;
use futures::Stream;
use futures::StreamExt;
use http::HeaderMap;
use http::StatusCode;
@@ -34,8 +36,12 @@ use serde::Deserialize;
use serde_json::Value;
use serde_json::map::Map as JsonMap;
use std::future::Future;
use std::future::poll_fn;
use std::sync::Arc;
use std::sync::OnceLock;
use std::task::Context;
use std::task::Poll;
use std::task::ready;
use std::time::Duration;
use tokio::sync::Mutex;
use tokio::sync::mpsc;
@@ -76,30 +82,25 @@ struct WsRequestText {
private: bool,
}
enum WsCommand {
Send {
request: WsRequestText,
tx_result: oneshot::Sender<Result<(), WsError>>,
},
struct WsCommand {
request: WsRequestText,
tx_result: oneshot::Sender<Result<(), WsError>>,
}
/// Send large Responses turns as one text message with backpressure between frames.
async fn send_message(
socket: &mut (impl futures::Sink<Message, Error = WsError> + Unpin),
message: Message,
) -> Result<(), WsError> {
const FRAGMENT_SIZE: usize = 1024 * 1024;
const FRAGMENT_SIZE: usize = 1024 * 1024;
const RECEIVE_BATCH_SIZE: usize = 32;
let Message::Text(text) = message else {
return socket.send(message).await;
fn request_frames(text: Utf8Bytes) -> impl Iterator<Item = Message> {
let (message, mut data) = if text.len() <= FRAGMENT_SIZE {
(Some(Message::Text(text)), Bytes::new())
} else {
(None, Bytes::from(text))
};
if text.len() <= FRAGMENT_SIZE {
return socket.send(Message::Text(text)).await;
}
let mut data = Bytes::from(text);
let mut opcode = Data::Text;
while !data.is_empty() {
message.into_iter().chain(std::iter::from_fn(move || {
if data.is_empty() {
return None;
}
let payload = data.split_to(data.len().min(FRAGMENT_SIZE));
let fragment = if data.is_empty() {
Fragment::Final
@@ -107,76 +108,120 @@ async fn send_message(
Fragment::More
};
let frame = Frame::message_fragment(payload, opcode, fragment);
socket.send(Message::Frame(frame)).await?;
opcode = Data::Continue;
Some(Message::Frame(frame))
}))
}
/// Owns the socket and forwards incoming messages while waiting for a request.
struct WsPump<S> {
socket: S,
messages: mpsc::UnboundedSender<Result<Message, WsError>>,
queued_pongs: usize,
}
impl<S> WsPump<S>
where
S: Stream<Item = Result<Message, WsError>> + Sink<Message, Error = WsError> + Unpin,
{
fn new(socket: S, messages: mpsc::UnboundedSender<Result<Message, WsError>>) -> Self {
Self {
socket,
messages,
queued_pongs: 0,
}
}
fn poll_incoming(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), WsError>> {
if self.queued_pongs != 0 && self.socket.poll_flush_unpin(cx)?.is_ready() {
self.queued_pongs = 0;
}
for _ in 0..RECEIVE_BATCH_SIZE {
if self.queued_pongs >= RECEIVE_BATCH_SIZE {
return Poll::Pending;
}
let Some(message) = ready!(self.socket.poll_next_unpin(cx)).transpose()? else {
return Poll::Ready(Err(WsError::ConnectionClosed));
};
match message {
Message::Ping(_) => {
// Queue the automatic Pong before starting the next data frame.
match self.socket.poll_flush_unpin(cx) {
Poll::Ready(Ok(())) => self.queued_pongs = 0,
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
Poll::Pending => self.queued_pongs += 1,
}
}
Message::Pong(_) => {}
Message::Text(_) | Message::Binary(_) | Message::Close(_) | Message::Frame(_) => {
let closed = matches!(message, Message::Close(_));
if self.messages.send(Ok(message)).is_err() || closed {
return Poll::Ready(Err(WsError::ConnectionClosed));
}
}
}
}
// The last read was ready, so arrange another poll after the batch.
cx.waker().wake_by_ref();
Poll::Pending
}
async fn send_frames(&mut self, text: Utf8Bytes) -> Result<(), WsError> {
for frame in request_frames(text) {
self.socket.send(frame).await?;
}
Ok(())
}
async fn run(mut self, mut commands: mpsc::Receiver<WsCommand>) {
loop {
let command = tokio::select! {
command = commands.recv() => Ok(command),
incoming = poll_fn(|cx| self.poll_incoming(cx)) => incoming.map(|()| None),
};
let WsCommand { request, tx_result } = match command {
Ok(Some(command)) => command,
Ok(None) | Err(WsError::ConnectionClosed) => break,
Err(error) => {
let _ = self.messages.send(Err(error));
break;
}
};
let send = self.send_frames(request.text);
// Tungstenite traces frame payloads from inside this future.
let result = if request.private {
send.with_subscriber(tracing::Dispatch::none()).await
} else {
send.await
};
let should_break = result.is_err();
let _ = tx_result.send(result);
if should_break {
break;
}
}
}
Ok(())
}
impl WsStream {
fn new(inner: WebSocketConnection, max_message_bytes: Option<usize>) -> Self {
let routing_affinity = inner.routing_affinity().cloned();
let (tx_command, mut rx_command) = mpsc::channel::<WsCommand>(32);
let (tx_message, rx_message) = mpsc::unbounded_channel::<Result<Message, WsError>>();
Self::with_socket(inner, max_message_bytes, routing_affinity)
}
let pump_task = tokio::spawn(async move {
let mut inner = inner;
loop {
tokio::select! {
command = rx_command.recv() => {
let Some(command) = command else {
break;
};
match command {
WsCommand::Send { request, tx_result } => {
let send = send_message(&mut inner, Message::Text(request.text));
// Tungstenite traces frame payloads from inside this future.
let result = if request.private {
send.with_subscriber(tracing::Dispatch::none()).await
} else {
send.await
};
let should_break = result.is_err();
let _ = tx_result.send(result);
if should_break {
break;
}
}
}
}
message = inner.next() => {
let Some(message) = message else {
break;
};
match message {
Ok(Message::Ping(payload)) => {
if let Err(err) = inner.send(Message::Pong(payload)).await {
let _ = tx_message.send(Err(err));
break;
}
}
Ok(Message::Pong(_)) => {}
Ok(message @ (Message::Text(_)
| Message::Binary(_)
| Message::Close(_)
| Message::Frame(_))) => {
let is_close = matches!(message, Message::Close(_));
if tx_message.send(Ok(message)).is_err() {
break;
}
if is_close {
break;
}
}
Err(err) => {
let _ = tx_message.send(Err(err));
break;
}
}
}
}
}
});
fn with_socket(
socket: impl Stream<Item = Result<Message, WsError>>
+ Sink<Message, Error = WsError>
+ Send
+ Unpin
+ 'static,
max_message_bytes: Option<usize>,
routing_affinity: Option<codex_http_client::ChatGptRoutingAffinity>,
) -> Self {
let (tx_command, rx_command) = mpsc::channel::<WsCommand>(32);
let (tx_message, rx_message) = mpsc::unbounded_channel();
let pump_task = tokio::spawn(WsPump::new(socket, tx_message).run(rx_command));
Self {
max_message_bytes,
@@ -187,12 +232,13 @@ impl WsStream {
}
}
async fn request(
&mut self,
make_command: impl FnOnce(oneshot::Sender<Result<(), WsError>>) -> WsCommand,
) -> Result<(), WsError> {
async fn send(&mut self, request: WsRequestText) -> Result<(), WsError> {
let (tx_result, rx_result) = oneshot::channel();
if self.tx_command.send(make_command(tx_result)).await.is_ok()
if self
.tx_command
.send(WsCommand { request, tx_result })
.await
.is_ok()
&& let Ok(result) = rx_result.await
{
return result;
@@ -203,11 +249,6 @@ impl WsStream {
Err(WsError::ConnectionClosed)
}
async fn send(&mut self, request: WsRequestText) -> Result<(), WsError> {
self.request(|tx_result| WsCommand::Send { request, tx_result })
.await
}
async fn next(&mut self) -> Option<Result<Message, WsError>> {
self.rx_message.recv().await
}
@@ -1174,6 +1215,10 @@ fn serialize_websocket_request(request: &ResponsesWsRequest<'_>) -> Result<Strin
.map_err(|err| ApiError::Stream(format!("failed to encode websocket request: {err}")))
}
#[cfg(test)]
#[path = "responses_websocket_upload_tests.rs"]
mod upload_tests;
#[cfg(test)]
mod tests {
use super::*;
@@ -1282,38 +1327,6 @@ mod tests {
assert_eq!(wire_payload, expected_payload);
}
#[tokio::test]
async fn large_turn_is_one_text_message_in_bounded_frames() {
const MIB: usize = 1024 * 1024;
let (tx, mut frames) = futures::channel::mpsc::unbounded();
let mut socket = tx.sink_map_err(|_| WsError::ConnectionClosed);
let at_limit = Message::text("a".repeat(MIB));
send_message(&mut socket, at_limit.clone()).await.unwrap();
assert_eq!(frames.next().await, Some(at_limit));
let mut text = "a".repeat(MIB - 1);
text.push('🌱');
text.push_str(&"b".repeat(MIB));
send_message(&mut socket, Message::text(text.clone()))
.await
.unwrap();
drop(socket);
for (i, expected) in text.as_bytes().chunks(MIB).enumerate() {
let Some(Message::Frame(frame)) = frames.next().await else {
panic!("expected a data frame");
};
let opcode = if i == 0 { Data::Text } else { Data::Continue };
assert_eq!(
frame.header().opcode,
tokio_tungstenite::tungstenite::protocol::frame::coding::OpCode::Data(opcode)
);
assert_eq!(frame.header().is_final, (i + 1) * MIB >= text.len());
assert_eq!(frame.payload(), expected);
}
assert!(frames.next().await.is_none());
}
#[test]
fn websocket_config_enables_permessage_deflate() {
let config = websocket_config();
@@ -0,0 +1,83 @@
#![cfg(test)]
//! Verify request framing and heartbeat delivery on idle WebSockets.
use super::FRAGMENT_SIZE;
use super::WsStream;
use super::request_frames;
use bytes::Bytes;
use futures::SinkExt;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use std::time::Duration;
use tokio::io::BufStream;
use tokio::io::DuplexStream;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::Utf8Bytes;
use tokio_tungstenite::tungstenite::protocol::Role;
use tokio_tungstenite::tungstenite::protocol::frame::coding::Data;
use tokio_tungstenite::tungstenite::protocol::frame::coding::OpCode;
const TIMEOUT: Duration = Duration::from_secs(3);
async fn backpressured_websockets(
capacity: usize,
) -> (WsStream, WebSocketStream<BufStream<DuplexStream>>) {
let (client_io, server_io) = tokio::io::duplex(capacity);
let socket = WebSocketStream::from_raw_socket(client_io, Role::Client, None).await;
let client = WsStream::with_socket(
socket,
/*max_message_bytes*/ Some(16 * 1024 * 1024),
/*routing_affinity*/ None,
);
let server =
WebSocketStream::from_raw_socket(BufStream::new(server_io), Role::Server, None).await;
(client, server)
}
#[test]
fn large_turn_is_one_text_message_in_bounded_frames() {
let at_limit = Utf8Bytes::from("a".repeat(FRAGMENT_SIZE));
assert_eq!(
request_frames(at_limit.clone()).collect::<Vec<_>>(),
[Message::Text(at_limit)]
);
let mut text = "a".repeat(FRAGMENT_SIZE - 1);
text.push('🌱');
text.push_str(&"b".repeat(FRAGMENT_SIZE));
let mut frames = request_frames(text.clone().into());
for (index, expected) in text.as_bytes().chunks(FRAGMENT_SIZE).enumerate() {
let Some(Message::Frame(frame)) = frames.next() else {
panic!("expected a data frame");
};
let opcode = if index == 0 {
Data::Text
} else {
Data::Continue
};
assert_eq!(frame.header().opcode, OpCode::Data(opcode));
assert_eq!(
frame.header().is_final,
(index + 1) * FRAGMENT_SIZE >= text.len()
);
assert_eq!(frame.payload(), expected);
}
assert!(frames.next().is_none());
}
#[tokio::test(start_paused = true)]
async fn idle_ping_finishes_pong_after_write_backpressure() {
// The Pong cannot fit in the buffer and the peer sends no more messages or commands.
let (_client, mut server) = backpressured_websockets(/*capacity*/ 8).await;
let payload = Bytes::from_static(b"idle heartbeat under write pressure");
let reply = tokio::time::timeout(TIMEOUT, async {
server.send(Message::Ping(payload.clone())).await.unwrap();
server.next().await
})
.await
.expect("the idle pump should finish flushing the Pong when the peer drains the socket");
assert_eq!(reply.unwrap().unwrap(), Message::Pong(payload));
}