Gate new app-server work during graceful shutdown (#43959)

## Why

New client requests and automatic continuations can start more work while the app-server is draining. Shutdown also needs to account for requests still preparing or submitting work before they appear as running turns.

## What changed

- Close a shared admission gate when shutdown begins, rejecting new turn work and thread lifecycle changes with the server-draining error.
- Apply the gate to automatic turn starts, including queued work and goal continuations, and recheck serialized turn requests before execution.
- Wait for admitted requests and running turns to finish, while preserving forced shutdown. Keep reads and `turn/interrupt` available during drain.

## Testing

Add coverage for admission permit tracking, discarded queued requests, forced shutdown, and WebSocket drain behavior, including request rejection, interruption, and suppression of automatic continuations.

GitOrigin-RevId: 1bc108aa00a427cfa41974fa9ec60754c2b968c2
This commit is contained in:
Eric Traut
2026-09-09 01:03:42 +00:00
committed by copyberry
parent 4e09b0c1f1
commit c3eeaae9a3
8 changed files with 448 additions and 33 deletions
@@ -78,6 +78,7 @@ use codex_core::ThreadManager;
use codex_core::config::Config;
use codex_core::config::ThreadStoreConfig;
use codex_exec_server::EnvironmentManager;
use codex_extension_api::TurnStartAdmission;
use codex_feedback::CodexFeedback;
use codex_goal_extension::GoalService;
use codex_home::CodexHomeUserInstructionsProvider;
@@ -101,6 +102,7 @@ use tokio_util::sync::CancellationToken;
use tracing::Instrument;
use crate::models_refresh_worker::ModelsRefreshWorker;
use crate::turn_admission::TurnAdmission;
const CONNECTION_RPC_DRAIN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30);
@@ -136,6 +138,7 @@ fn reject_removed_permission_profile(request: &JSONRPCRequest) -> Result<(), JSO
}
pub(crate) struct MessageProcessor {
pub(crate) turn_admission: TurnAdmission,
user_verification: Arc<crate::user_verification::Service>,
outgoing: Arc<OutgoingMessageSender>,
models_refresh_worker: ModelsRefreshWorker,
@@ -311,6 +314,8 @@ impl MessageProcessor {
),
);
let goal_service = Arc::new(GoalService::new());
let turn_admission = TurnAdmission::default();
let turn_start_admission: Arc<dyn TurnStartAdmission> = Arc::new(turn_admission.clone());
let extension_event_sink =
app_server_extension_event_sink(outgoing.clone(), thread_state_manager.clone());
let mut queue_service = None;
@@ -343,6 +348,7 @@ impl MessageProcessor {
git_attribution_base_url: config.chatgpt_base_url.clone(),
http_client_factory: config.http_client_factory(),
queue_service: queue_service.clone(),
turn_start_admission: Some(Arc::clone(&turn_start_admission)),
},
),
Arc::new(CodexHomeUserInstructionsProvider::new(
@@ -570,6 +576,7 @@ impl MessageProcessor {
);
Self {
turn_admission,
user_verification,
outgoing,
models_refresh_worker,
@@ -939,6 +946,24 @@ impl MessageProcessor {
&codex_request,
);
let (turn_admission, recheck_turn_admission) = match &codex_request {
ClientRequest::ThreadStart { .. }
| ClientRequest::ThreadFork { .. }
| ClientRequest::ThreadResume { .. }
| ClientRequest::ThreadRollback { .. }
| ClientRequest::ThreadRevert { .. } => (Some(self.turn_admission.admit()?), false),
ClientRequest::TurnStart { .. }
| ClientRequest::TurnSteer { .. }
| ClientRequest::ReviewStart { .. }
| ClientRequest::ThreadCompactStart { .. }
| ClientRequest::ThreadShellCommand { .. }
| ClientRequest::ThreadQueueStart { .. }
| ClientRequest::ThreadRealtimeStart { .. } => {
(Some(self.turn_admission.admit()?), true)
}
_ => (None, false),
};
let event_stream_ready = match &codex_request {
ClientRequest::McpServerEventStreamStart { params, .. } => Some(
session
@@ -956,6 +981,13 @@ impl MessageProcessor {
let request = QueuedInitializedRequest::new(
rpc_gate,
async move {
let _turn_admission = turn_admission;
// Runtime changes already admitted before drain finish normally. Turn work
// still waiting in serialization must observe the newly closed gate.
if recheck_turn_admission && let Err(error) = processor.turn_admission.admit() {
processor.outgoing.send_error(error_request_id, error).await;
return;
}
let processor_for_request = Arc::clone(&processor);
let result = processor_for_request
.handle_initialized_client_request(