mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Prevent remote app-server shutdown from hanging on a full queue (#62547)
## Why A stalled WebSocket write can leave the remote client's command queue full. Shutdown can then wait indefinitely to enqueue its command, before it reaches the timeout that allows it to abort the worker. ## What changed Apply `SHUTDOWN_TIMEOUT` to enqueuing the shutdown command so the client can reach the existing worker abort fallback even if the queue stays full. ## Testing Add a paused-time regression test that keeps the queue full with a stalled worker and verifies that shutdown returns and the worker has finished. GitOrigin-RevId: d7c1a90442cd2619893144b20e43f6d87d5caa94
This commit is contained in:
@@ -614,10 +614,13 @@ impl RemoteAppServerClient {
|
||||
} = self;
|
||||
let mut worker_handle = worker_handle;
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
if command_tx
|
||||
.send(RemoteClientCommand::Shutdown { response_tx })
|
||||
.await
|
||||
.is_ok()
|
||||
// A worker stalled on a socket write may leave the command queue full.
|
||||
// Bound admission too, so shutdown can reach the worker abort below.
|
||||
if let Ok(Ok(())) = timeout(
|
||||
SHUTDOWN_TIMEOUT,
|
||||
command_tx.send(RemoteClientCommand::Shutdown { response_tx }),
|
||||
)
|
||||
.await
|
||||
&& let Ok(Ok(close_result)) = timeout(SHUTDOWN_TIMEOUT, response_rx).await
|
||||
{
|
||||
close_result?;
|
||||
@@ -1073,6 +1076,37 @@ fn websocket_close_error_is_already_closed(err: &TungsteniteError) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_aborts_worker_when_command_queue_is_full() {
|
||||
let (command_tx, command_rx) = mpsc::channel(1);
|
||||
let (_event_tx, event_rx) = mpsc::unbounded_channel::<AppServerEvent>();
|
||||
let (response_tx, _response_rx) = oneshot::channel();
|
||||
assert!(
|
||||
command_tx
|
||||
.try_send(RemoteClientCommand::Shutdown { response_tx })
|
||||
.is_ok()
|
||||
);
|
||||
let worker_handle = tokio::spawn(async move {
|
||||
// Retain the full queue while simulating a stalled socket write.
|
||||
std::future::pending::<()>().await;
|
||||
drop(command_rx);
|
||||
});
|
||||
let worker_abort_handle = worker_handle.abort_handle();
|
||||
let client = RemoteAppServerClient {
|
||||
command_tx,
|
||||
event_rx,
|
||||
pending_events: VecDeque::new(),
|
||||
metadata: RemoteServerMetadata::default(),
|
||||
worker_handle,
|
||||
};
|
||||
|
||||
timeout(SHUTDOWN_TIMEOUT * 3, client.shutdown())
|
||||
.await
|
||||
.expect("shutdown must not wait indefinitely for command queue space")
|
||||
.expect("shutdown should abort a stalled worker");
|
||||
assert!(worker_abort_handle.is_finished());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_tolerates_worker_exit_after_command_is_queued() {
|
||||
let (command_tx, mut command_rx) = mpsc::channel(1);
|
||||
|
||||
Reference in New Issue
Block a user