Apply backpressure to app-server request replay on thread resume (#62609)

## Why

When a client resumes a thread with many pending server requests, replay
can fill the connection's bounded outbound queue if the client reads
slowly. Replaying inline can also stall the thread listener.

## What changed

Replay pending requests in a background task after sending the
`thread/resume` response. Allow one outstanding replay write per
connection and wait for its write receipt to complete or be dropped
before sending the next request. This extends the write-receipt
backpressure used by `command/exec`.
https://github.com/openai/codex/pull/15547

A new resume replaces the previous replay for the same thread and
connection while honoring any outstanding write receipt. Unsubscribing
or disconnecting stops the replay. Before sending each request, check
that it is still pending and apply the human approval eligibility check
where required.

## Testing

Add a regression test with more pending requests than the outbound queue
can hold. It checks that a second resume succeeds while a replay write is
stalled, dropped write receipts let replay advance, and unsubscribe and
disconnect stop replay.

GitOrigin-RevId: 77fc415be0e7b50744f5ccfcaa09857359dd5c55
This commit is contained in:
ztanner-oai
2026-11-19 22:15:23 +00:00
committed by copyberry
parent efcc0cf917
commit 6633c6e09a
7 changed files with 386 additions and 48 deletions
@@ -881,6 +881,7 @@ impl MessageProcessor {
connection_id: ConnectionId,
request_attestation: bool,
) {
self.outgoing.connection_initialized(connection_id).await;
self.account_processor
.notify_workspace_routing_to_connection(connection_id);
self.thread_processor
@@ -943,6 +944,7 @@ impl MessageProcessor {
session_state: &ConnectionSessionState,
) {
session_state.disconnect.cancel();
self.outgoing.stop_connection_replays(connection_id).await;
session_state.rpc_gate.close().await;
self.account_processor
.gateway_connection_closed(connection_id);
+124 -21
View File
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::sync::OnceLock;
use std::sync::atomic::AtomicI64;
use std::sync::atomic::Ordering;
@@ -24,9 +25,14 @@ use codex_otel::span_w3c_trace_context;
use codex_protocol::ThreadId;
use codex_protocol::protocol::W3cTraceContext;
use codex_protocol::request_permissions::RequestPermissionsResponse;
use futures::FutureExt;
use futures::future::Shared;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
use tokio::sync::SemaphorePermit;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
use tracing::Instrument;
use tracing::Span;
use tracing::warn;
@@ -147,6 +153,7 @@ pub(crate) struct OutgoingMessageSender {
verification_auth: OnceLock<Arc<codex_login::AuthManager>>,
verification_connections: Mutex<HashSet<ConnectionId>>,
human_approval_connections: Mutex<HashSet<ConnectionId>>,
replay_connections: Mutex<HashMap<ConnectionId, Arc<ReplayConnection>>>,
next_server_request_id: AtomicI64,
sender: mpsc::Sender<OutgoingEnvelope>,
process_reply_sender: mpsc::Sender<OutgoingEnvelope>,
@@ -158,6 +165,48 @@ pub(crate) struct OutgoingMessageSender {
analytics_events_client: AnalyticsEventsClient,
}
struct ReplayConnection {
// Keep the last receipt on the connection, rather than the replay task: a
// replacement listener must not bypass a write its predecessor queued.
pending: StdMutex<Option<Shared<oneshot::Receiver<()>>>>,
turn: Semaphore,
disconnect: CancellationToken,
}
impl ReplayConnection {
async fn acquire_turn(&self) -> Option<SemaphorePermit<'_>> {
tokio::select! {
biased;
_ = self.disconnect.cancelled() => None,
permit = self.turn.acquire() => permit.ok(),
}
}
async fn wait_pending(&self) -> bool {
let receipt = self
.pending
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let Some(receipt) = receipt else {
return !self.disconnect.is_cancelled();
};
tokio::select! {
biased;
_ = self.disconnect.cancelled() => {}
_ = receipt => {}
};
self.pending
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
// A dropped physical-write receipt can mean either encoding rejection
// or a message retained for reconnect. In either case it has left the
// bounded connection queue; only logical disconnect ends the replay.
!self.disconnect.is_cancelled()
}
}
#[derive(Clone)]
pub(crate) struct ThreadScopedOutgoingMessageSender {
outgoing: Arc<OutgoingMessageSender>,
@@ -283,6 +332,7 @@ impl OutgoingMessageSender {
verification_auth: OnceLock::new(),
verification_connections: Mutex::new(HashSet::new()),
human_approval_connections: Mutex::new(HashSet::new()),
replay_connections: Mutex::new(HashMap::new()),
next_server_request_id: AtomicI64::new(0),
process_reply_sender: sender.clone(),
sender,
@@ -354,7 +404,26 @@ impl OutgoingMessageSender {
.insert(connection_id);
}
pub(crate) async fn connection_initialized(&self, connection_id: ConnectionId) {
self.replay_connections.lock().await.insert(
connection_id,
Arc::new(ReplayConnection {
pending: StdMutex::new(None),
turn: Semaphore::new(1),
disconnect: CancellationToken::new(),
}),
);
}
pub(crate) async fn stop_connection_replays(&self, connection_id: ConnectionId) {
let replay = self.replay_connections.lock().await.remove(&connection_id);
if let Some(replay) = replay {
replay.disconnect.cancel();
}
}
pub(crate) async fn connection_closed(&self, connection_id: ConnectionId) {
self.stop_connection_replays(connection_id).await;
self.human_approval_connections
.lock()
.await
@@ -562,40 +631,74 @@ impl OutgoingMessageSender {
(outgoing_message_id, rx_approve)
}
pub(crate) async fn replay_requests_to_connection_for_thread(
pub(crate) async fn replay_requests_to_connection(
&self,
connection_id: ConnectionId,
thread_id: ThreadId,
requests: Vec<ServerRequest>,
) {
let requests = self.pending_requests_for_thread(thread_id).await;
let human_approval = self
.human_approval_connections
let Some(replay) = self
.replay_connections
.lock()
.await
.contains(&connection_id);
.get(&connection_id)
.cloned()
else {
return;
};
for request in requests {
if requires_human_approval(&request) {
if !human_approval {
let Some(_turn) = replay.acquire_turn().await else {
return;
};
if !replay.wait_pending().await {
return;
}
let permit = tokio::select! {
biased;
_ = replay.disconnect.cancelled() => return,
permit = self.sender.reserve() => match permit {
Ok(permit) => permit,
Err(_) => return,
},
};
let approval = requires_human_approval(&request);
if approval {
let eligible = tokio::select! {
biased;
_ = replay.disconnect.cancelled() => return,
eligible = self.human_approval_connections.lock() => eligible,
};
if !eligible.contains(&connection_id) {
continue;
}
// Register the replay recipient before sending so its response
// cannot race ahead of callback authorization.
let mut callbacks = self.request_id_to_callback.lock().await;
}
{
let mut callbacks = tokio::select! {
biased;
_ = replay.disconnect.cancelled() => return,
callbacks = self.request_id_to_callback.lock() => callbacks,
};
let Some(entry) = callbacks.get_mut(request.id()) else {
continue;
};
entry.human_approval_recipients.insert(connection_id);
}
if let Err(err) = self
.sender
.send(OutgoingEnvelope::ToConnection {
if replay.disconnect.is_cancelled() {
return;
}
if approval {
entry.human_approval_recipients.insert(connection_id);
}
let (tx, receipt) = oneshot::channel();
permit.send(OutgoingEnvelope::ToConnection {
connection_id,
message: OutgoingMessage::Request(request),
write_complete_tx: None,
})
.await
{
warn!("failed to resend request to client: {err:?}");
write_complete_tx: Some(tx),
});
*replay
.pending
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(receipt.shared());
}
if !replay.wait_pending().await {
return;
}
}
}
@@ -14,11 +14,16 @@ use codex_protocol::config_types::MultiAgentMode;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::SessionSource;
use std::borrow::Cow;
use tokio::task::JoinSet;
#[cfg(test)]
#[path = "thread_listener_shutdown_tests.rs"]
mod listener_shutdown_tests;
#[cfg(test)]
#[path = "thread_replay_tests.rs"]
mod replay_tests;
#[derive(Clone)]
pub(super) struct ListenerTaskContext {
pub(super) thread_manager: Arc<ThreadManager>,
@@ -507,6 +512,7 @@ async fn start_listener(
} = listener_task_context;
let outgoing_for_task = Arc::clone(&outgoing);
tokio::spawn(async move {
let mut replays = JoinSet::new();
let shutdown_complete = loop {
tokio::select! {
biased;
@@ -527,10 +533,16 @@ async fn start_listener(
&thread_watch_manager,
&outgoing_for_task,
&pending_thread_unloads,
Some(&mut replays),
listener_command,
)
.await;
}
Some(result) = replays.join_next(), if !replays.is_empty() => {
if let Err(error) = result {
warn!("thread resume replay task failed: {error}");
}
}
event = conversation.next_event() => {
let event = match event {
Ok(event) => event,
@@ -748,6 +760,8 @@ async fn start_listener(
}
}
};
// Dropping the set aborts replay workers before this listener retires.
drop(replays);
// The thread owns its event sender, so waiting for channel closure retains
// the runtime. Keep only the notification relay after ShutdownComplete.
drop(conversation);
@@ -786,6 +800,7 @@ async fn start_listener(
&thread_watch_manager,
&outgoing_for_task,
&pending_thread_unloads,
/*replays*/ None,
command,
)
};
@@ -943,7 +958,7 @@ pub(super) async fn unload_thread_without_subscribers(
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn handle_thread_listener_command(
async fn handle_thread_listener_command(
conversation_id: ThreadId,
conversation: Option<&Arc<CodexThread>>,
codex_home: &Path,
@@ -952,6 +967,7 @@ pub(super) async fn handle_thread_listener_command(
thread_watch_manager: &ThreadWatchManager,
outgoing: &Arc<OutgoingMessageSender>,
pending_thread_unloads: &PendingThreadUnloads,
replays: Option<&mut JoinSet<()>>,
listener_command: ThreadListenerCommand,
) {
match listener_command {
@@ -959,7 +975,7 @@ pub(super) async fn handle_thread_listener_command(
request: resume_request,
completion_tx,
} => {
let Some(conversation) = conversation else {
let (Some(conversation), Some(replays)) = (conversation, replays) else {
// Dropping the completion sender returns the closing error.
return;
};
@@ -972,9 +988,11 @@ pub(super) async fn handle_thread_listener_command(
thread_watch_manager,
outgoing,
pending_thread_unloads,
replays,
*resume_request,
)
.await;
// The resume response is complete even if its replay is later cancelled.
let _ = completion_tx.send(());
}
ThreadListenerCommand::EmitThreadGoalUpdated { turn_id, goal } => {
@@ -1066,7 +1084,7 @@ pub(super) async fn handle_thread_listener_command(
clippy::await_holding_invalid_type,
reason = "running-thread resume subscription must be serialized against pending unloads"
)]
pub(super) async fn handle_pending_thread_resume_request(
async fn handle_pending_thread_resume_request(
conversation_id: ThreadId,
conversation: &Arc<CodexThread>,
_codex_home: &Path,
@@ -1075,6 +1093,7 @@ pub(super) async fn handle_pending_thread_resume_request(
thread_watch_manager: &ThreadWatchManager,
outgoing: &Arc<OutgoingMessageSender>,
pending_thread_unloads: &PendingThreadUnloads,
replays: &mut JoinSet<()>,
mut pending: crate::thread_state::PendingThreadResumeRequest,
) {
let (active_turn_metadata, mut active_turn, active_page_params) = {
@@ -1307,6 +1326,11 @@ pub(super) async fn handle_pending_thread_resume_request(
turns_backwards_cursor,
items_backwards_cursor,
};
// Retire the previous replay before the new response can reach this client.
// The connection retains the receipt for any write already queued.
let stopped = thread_state_manager
.begin_resume_replay(conversation_id, connection_id)
.await;
outgoing
.send_response_with_thread_originator(request_id, response, originator)
.await;
@@ -1334,11 +1358,18 @@ pub(super) async fn handle_pending_thread_resume_request(
);
}
}
outgoing
.replay_requests_to_connection_for_thread(connection_id, conversation_id)
.await;
// App-server owns resume response and snapshot ordering, so wait until
// replay completes before letting extensions react to the idle thread.
let requests = outgoing.pending_requests_for_thread(conversation_id).await;
if let Some(mut stopped) = stopped.filter(|_| !requests.is_empty()) {
let outgoing = Arc::clone(outgoing);
replays.spawn(async move {
tokio::select! {
biased;
_ = stopped.changed() => {}
_ = outgoing.replay_requests_to_connection(connection_id, requests) => {}
}
});
}
// Give queued input a chance to start before releasing the next same-thread RPC.
conversation
.emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed)
.await;
@@ -39,7 +39,7 @@ use std::collections::HashSet;
use test_case::test_case;
use tokio::sync::mpsc;
async fn test_processor(
pub(super) async fn test_processor(
test: &TestCodex,
) -> Result<(ThreadRequestProcessor, mpsc::Receiver<OutgoingEnvelope>)> {
let config = Arc::new(test.config.clone());
@@ -0,0 +1,174 @@
#![cfg(test)]
//! Checks stalled thread resume replay against the bounded outbound router.
use super::super::thread_processor::ThreadResumeTarget;
use super::listener_shutdown_tests::test_processor;
use super::*;
use crate::outgoing_message::OutgoingMessage;
use crate::thread_state::ConnectionCapabilities;
use crate::transport::CHANNEL_CAPACITY;
use crate::transport::OutboundConnectionState;
use crate::transport::QueuedOutgoingMessage;
use crate::transport::route_outgoing_envelope;
use anyhow::Result;
use codex_app_server_protocol::DynamicToolCallParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerRequestPayload;
use codex_app_server_protocol::ThreadResumeParams;
use codex_thread_store::PersistContext;
use core_test_support::test_codex::test_codex;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::HashSet;
use std::sync::RwLock;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use tokio::sync::mpsc;
use tokio::time::timeout;
async fn next_message(writer: &mut mpsc::Receiver<QueuedOutgoingMessage>) -> QueuedOutgoingMessage {
timeout(Duration::from_secs(5), async {
loop {
let queued = writer.recv().await.expect("connection closed");
if !matches!(queued.message, OutgoingMessage::AppServerNotification(_)) {
return queued;
}
}
})
.await
.expect("expected resume response or replayed request")
}
#[tokio::test]
async fn stalled_replay_can_be_replaced_or_unsubscribed() -> Result<()> {
let server = wiremock::MockServer::start().await;
let test = test_codex().build_with_auto_env(&server).await?;
let thread_id = test.session_configured.thread_id;
test.thread_store
.persist_thread(thread_id, PersistContext::Standard)
.await?;
let (processor, mut outgoing_rx) = test_processor(&test).await?;
let connection_id = ConnectionId(1);
processor
.connection_initialized(connection_id, ConnectionCapabilities::default())
.await;
let outgoing = &processor.outgoing;
outgoing.connection_initialized(connection_id).await;
let mut expected = Vec::new();
for index in 0..CHANNEL_CAPACITY + 32 {
let (id, _callback) = outgoing
.send_request_to_connections(
Some(&[]),
ServerRequestPayload::DynamicToolCall(DynamicToolCallParams {
thread_id: thread_id.to_string(),
turn_id: "turn".into(),
call_id: index.to_string(),
namespace: None,
tool: "update_up_next".into(),
arguments: json!({}),
}),
Some(thread_id),
)
.await;
expected.push(id);
}
let (writer_tx, mut writer_rx) = mpsc::channel(CHANNEL_CAPACITY);
let disconnect = CancellationToken::new();
let router = tokio::spawn({
let disconnect = disconnect.clone();
async move {
let mut connections = HashMap::from([(
connection_id,
OutboundConnectionState::new(
writer_tx,
Arc::new(AtomicBool::new(true)),
Arc::new(AtomicBool::new(true)),
Arc::new(RwLock::new(HashSet::new())),
Some(disconnect),
),
)]);
while let Some(envelope) = outgoing_rx.recv().await {
route_outgoing_envelope(&mut connections, envelope).await;
}
}
});
let resume = async |id, writer: &mut mpsc::Receiver<QueuedOutgoingMessage>| {
timeout(
Duration::from_secs(10),
processor.thread_resume(
ThreadResumeTarget::Client(ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(id),
}),
ThreadResumeParams {
thread_id: thread_id.to_string(),
exclude_turns: true,
..Default::default()
},
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
/*client_mcp_extensions*/ Default::default(),
),
)
.await
.unwrap()
.unwrap();
let response = next_message(writer).await;
assert!(
matches!(response.message, OutgoingMessage::Response(r) if r.id == RequestId::Integer(id))
);
};
resume(1, &mut writer_rx).await;
let first = next_message(&mut writer_rx).await;
let OutgoingMessage::Request(request) = first.message else {
panic!("expected replay")
};
assert_eq!(request.id(), &expected[0]);
// Resume must succeed while the first replay is still waiting on this receipt.
resume(2, &mut writer_rx).await;
let stall = Duration::from_millis(50);
assert!(timeout(stall, next_message(&mut writer_rx)).await.is_err());
first.write_complete_tx.unwrap().send(()).unwrap();
for id in &expected {
let queued = next_message(&mut writer_rx).await;
let OutgoingMessage::Request(request) = queued.message else {
panic!("expected replay")
};
assert_eq!(request.id(), id);
// Losing a socket write receipt during reconnect must also let replay advance.
drop(queued.write_complete_tx.unwrap());
}
assert!(!disconnect.is_cancelled());
resume(3, &mut writer_rx).await;
let first = next_message(&mut writer_rx).await;
assert!(
processor
.thread_state_manager
.unsubscribe_connection_from_thread(thread_id, connection_id)
.await
);
drop(first.write_complete_tx.unwrap());
assert!(timeout(stall, next_message(&mut writer_rx)).await.is_err());
let replay = tokio::spawn({
let outgoing = Arc::clone(outgoing);
async move {
outgoing
.replay_requests_to_connection(
connection_id,
outgoing.pending_requests_for_thread(thread_id).await,
)
.await
}
});
let held = next_message(&mut writer_rx).await;
outgoing.stop_connection_replays(connection_id).await;
timeout(Duration::from_secs(5), replay)
.await
.unwrap()
.unwrap();
drop(held);
router.abort();
Ok(())
}
+41 -17
View File
@@ -421,9 +421,17 @@ mod tests {
}
}
async fn stop_replay(stop: Option<watch::Sender<()>>) {
if let Some(stop) = stop {
let _ = stop.send(());
// The worker drops its receiver once it can no longer enqueue requests.
stop.closed().await;
}
}
struct ThreadEntry {
state: Arc<Mutex<ThreadState>>,
connection_ids: HashSet<ConnectionId>,
subscriptions: HashMap<ConnectionId, Option<watch::Sender<()>>>,
has_connections_watcher: watch::Sender<bool>,
}
@@ -437,7 +445,7 @@ impl ThreadEntry {
fn new(state: Arc<Mutex<ThreadState>>) -> Self {
Self {
state,
connection_ids: HashSet::new(),
subscriptions: HashMap::new(),
has_connections_watcher: watch::channel(false).0,
}
}
@@ -445,7 +453,7 @@ impl ThreadEntry {
fn update_has_connections(&self) {
let _ = self.has_connections_watcher.send_if_modified(|current| {
let prev = *current;
*current = !self.connection_ids.is_empty();
*current = !self.subscriptions.is_empty();
prev != *current
});
}
@@ -533,8 +541,8 @@ impl ThreadStateManager {
state
.threads
.get(&thread_id)?
.connection_ids
.iter()
.subscriptions
.keys()
.filter_map(|connection_id| {
state
.live_connections
@@ -567,7 +575,7 @@ impl ThreadStateManager {
state
.threads
.get(&thread_id)
.map(|thread_entry| thread_entry.connection_ids.iter().copied().collect())
.map(|thread_entry| thread_entry.subscriptions.keys().copied().collect())
.unwrap_or_default()
}
@@ -660,7 +668,7 @@ impl ThreadStateManager {
let mut state = self.state.lock().await;
if thread_state.listener_generation != generation
|| !state.threads.get(&thread_id).is_some_and(|entry| {
Arc::ptr_eq(&entry.state, expected) && entry.connection_ids.is_empty()
Arc::ptr_eq(&entry.state, expected) && entry.subscriptions.is_empty()
})
{
return false;
@@ -695,12 +703,27 @@ impl ThreadStateManager {
}
}
pub(crate) async fn begin_resume_replay(
&self,
thread_id: ThreadId,
connection_id: ConnectionId,
) -> Option<watch::Receiver<()>> {
let (stop, stopped) = watch::channel(());
let previous = {
let mut state = self.state.lock().await;
let entry = state.threads.get_mut(&thread_id)?;
entry.subscriptions.get_mut(&connection_id)?.replace(stop)
};
stop_replay(previous).await;
Some(stopped)
}
pub(crate) async fn unsubscribe_connection_from_thread(
&self,
thread_id: ThreadId,
connection_id: ConnectionId,
) -> bool {
{
let stop = {
let mut state = self.state.lock().await;
if !state
.thread_ids_by_connection
@@ -716,12 +739,13 @@ impl ThreadStateManager {
state.thread_ids_by_connection.remove(&connection_id);
}
}
if let Some(thread_entry) = state.threads.get_mut(&thread_id) {
thread_entry.connection_ids.remove(&connection_id);
state.threads.get_mut(&thread_id).and_then(|thread_entry| {
let stop = thread_entry.subscriptions.remove(&connection_id);
thread_entry.update_has_connections();
}
stop.flatten()
})
};
stop_replay(stop).await;
true
}
@@ -731,7 +755,7 @@ impl ThreadStateManager {
.await
.threads
.get(&thread_id)
.is_some_and(|thread_entry| !thread_entry.connection_ids.is_empty())
.is_some_and(|thread_entry| !thread_entry.subscriptions.is_empty())
}
/// Publishes a new state only when listener installation succeeds, or reuses a current state.
@@ -766,7 +790,7 @@ impl ThreadStateManager {
.entry(connection_id)
.or_default()
.insert(thread_id);
entry.connection_ids.insert(connection_id);
entry.subscriptions.entry(connection_id).or_default();
}
}
entry.update_has_connections();
@@ -792,7 +816,7 @@ impl ThreadStateManager {
.or_default()
.insert(thread_id);
let thread_entry = state.threads.entry(thread_id).or_default();
thread_entry.connection_ids.insert(connection_id);
thread_entry.subscriptions.entry(connection_id).or_default();
thread_entry.update_has_connections();
true
}
@@ -807,7 +831,7 @@ impl ThreadStateManager {
.unwrap_or_default();
for thread_id in &thread_ids {
if let Some(thread_entry) = state.threads.get_mut(thread_id) {
thread_entry.connection_ids.remove(&connection_id);
thread_entry.subscriptions.remove(&connection_id);
thread_entry.update_has_connections();
}
}
@@ -817,7 +841,7 @@ impl ThreadStateManager {
state
.threads
.get(thread_id)
.is_some_and(|thread_entry| thread_entry.connection_ids.is_empty())
.is_some_and(|thread_entry| thread_entry.subscriptions.is_empty())
})
.collect::<Vec<_>>()
}
@@ -64,8 +64,12 @@ async fn verification_is_delivered_to_one_owner_and_other_connections_cannot_res
.await
.contains_key(&id)
);
outgoing.connection_initialized(ConnectionId(2)).await;
outgoing
.replay_requests_to_connection_for_thread(ConnectionId(2), thread_id)
.replay_requests_to_connection(
ConnectionId(2),
outgoing.pending_requests_for_thread(thread_id).await,
)
.await;
assert!(messages.try_recv().is_err());
outgoing