mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
[app-server] Reap orphaned idle threads on websocket disconnect [ci changed_files]
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -3272,9 +3272,14 @@ impl CodexMessageProcessor {
|
||||
self.command_exec_manager
|
||||
.connection_closed(connection_id)
|
||||
.await;
|
||||
self.thread_state_manager
|
||||
let orphaned_thread_ids = self
|
||||
.thread_state_manager
|
||||
.remove_connection(connection_id)
|
||||
.await;
|
||||
for thread_id in orphaned_thread_ids {
|
||||
self.maybe_unload_orphaned_thread_after_disconnect(thread_id)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn subscribe_running_assistant_turn_count(&self) -> watch::Receiver<usize> {
|
||||
@@ -4834,6 +4839,93 @@ impl CodexMessageProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
fn should_unload_orphaned_thread_after_disconnect(
|
||||
rollout_path: Option<&Path>,
|
||||
agent_status: AgentStatus,
|
||||
loaded_status: ThreadStatus,
|
||||
) -> bool {
|
||||
rollout_path.is_some_and(std::path::Path::exists)
|
||||
&& !matches!(agent_status, AgentStatus::Running)
|
||||
&& matches!(
|
||||
loaded_status,
|
||||
ThreadStatus::Idle | ThreadStatus::SystemError
|
||||
)
|
||||
}
|
||||
|
||||
async fn begin_thread_shutdown(&self, thread_id: ThreadId, thread: Arc<CodexThread>) {
|
||||
info!("thread {thread_id} has no subscribers; shutting down");
|
||||
self.pending_thread_unloads.lock().await.insert(thread_id);
|
||||
// Any pending app-server -> client requests for this thread can no longer be
|
||||
// answered; cancel their callbacks before shutdown/unload.
|
||||
self.outgoing
|
||||
.cancel_requests_for_thread(thread_id, None)
|
||||
.await;
|
||||
self.thread_state_manager
|
||||
.remove_thread_state(thread_id)
|
||||
.await;
|
||||
|
||||
let outgoing = self.outgoing.clone();
|
||||
let pending_thread_unloads = self.pending_thread_unloads.clone();
|
||||
let thread_manager = self.thread_manager.clone();
|
||||
let thread_watch_manager = self.thread_watch_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
match Self::wait_for_thread_shutdown(&thread).await {
|
||||
ThreadShutdownResult::Complete => {
|
||||
if thread_manager.remove_thread(&thread_id).await.is_none() {
|
||||
info!("thread {thread_id} was already removed before shutdown finalized");
|
||||
thread_watch_manager
|
||||
.remove_thread(&thread_id.to_string())
|
||||
.await;
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
return;
|
||||
}
|
||||
thread_watch_manager
|
||||
.remove_thread(&thread_id.to_string())
|
||||
.await;
|
||||
let notification = ThreadClosedNotification {
|
||||
thread_id: thread_id.to_string(),
|
||||
};
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::ThreadClosed(notification))
|
||||
.await;
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
}
|
||||
ThreadShutdownResult::SubmitFailed => {
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
warn!("failed to submit Shutdown to thread {thread_id}");
|
||||
}
|
||||
ThreadShutdownResult::TimedOut => {
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
warn!("thread {thread_id} shutdown timed out; leaving thread loaded");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn maybe_unload_orphaned_thread_after_disconnect(&self, thread_id: ThreadId) {
|
||||
let Ok(thread) = self.thread_manager.get_thread(thread_id).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let loaded_status = self
|
||||
.thread_watch_manager
|
||||
.loaded_status_for_thread(&thread_id.to_string())
|
||||
.await;
|
||||
let agent_status = thread.agent_status().await;
|
||||
if !Self::should_unload_orphaned_thread_after_disconnect(
|
||||
thread.rollout_path().as_deref(),
|
||||
agent_status,
|
||||
loaded_status,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if self.thread_state_manager.has_subscribers(thread_id).await {
|
||||
return;
|
||||
}
|
||||
|
||||
self.begin_thread_shutdown(thread_id, thread).await;
|
||||
}
|
||||
|
||||
async fn finalize_thread_teardown(&mut self, thread_id: ThreadId) {
|
||||
self.pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
self.outgoing
|
||||
@@ -4895,57 +4987,7 @@ impl CodexMessageProcessor {
|
||||
|
||||
if !self.thread_state_manager.has_subscribers(thread_id).await {
|
||||
// This connection was the last subscriber. Only now do we unload the thread.
|
||||
info!("thread {thread_id} has no subscribers; shutting down");
|
||||
self.pending_thread_unloads.lock().await.insert(thread_id);
|
||||
// Any pending app-server -> client requests for this thread can no longer be
|
||||
// answered; cancel their callbacks before shutdown/unload.
|
||||
self.outgoing
|
||||
.cancel_requests_for_thread(thread_id, /*error*/ None)
|
||||
.await;
|
||||
self.thread_state_manager
|
||||
.remove_thread_state(thread_id)
|
||||
.await;
|
||||
|
||||
let outgoing = self.outgoing.clone();
|
||||
let pending_thread_unloads = self.pending_thread_unloads.clone();
|
||||
let thread_manager = self.thread_manager.clone();
|
||||
let thread_watch_manager = self.thread_watch_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
match Self::wait_for_thread_shutdown(&thread).await {
|
||||
ThreadShutdownResult::Complete => {
|
||||
if thread_manager.remove_thread(&thread_id).await.is_none() {
|
||||
info!(
|
||||
"thread {thread_id} was already removed before unsubscribe finalized"
|
||||
);
|
||||
thread_watch_manager
|
||||
.remove_thread(&thread_id.to_string())
|
||||
.await;
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
return;
|
||||
}
|
||||
thread_watch_manager
|
||||
.remove_thread(&thread_id.to_string())
|
||||
.await;
|
||||
let notification = ThreadClosedNotification {
|
||||
thread_id: thread_id.to_string(),
|
||||
};
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::ThreadClosed(
|
||||
notification,
|
||||
))
|
||||
.await;
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
}
|
||||
ThreadShutdownResult::SubmitFailed => {
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
warn!("failed to submit Shutdown to thread {thread_id}");
|
||||
}
|
||||
ThreadShutdownResult::TimedOut => {
|
||||
pending_thread_unloads.lock().await.remove(&thread_id);
|
||||
warn!("thread {thread_id} shutdown timed out; leaving thread loaded");
|
||||
}
|
||||
}
|
||||
});
|
||||
self.begin_thread_shutdown(thread_id, thread).await;
|
||||
}
|
||||
|
||||
self.outgoing
|
||||
@@ -8598,12 +8640,13 @@ mod tests {
|
||||
state.lock().await.cancel_tx = Some(cancel_tx);
|
||||
}
|
||||
|
||||
manager.remove_connection(connection_a).await;
|
||||
let orphaned_thread_ids = manager.remove_connection(connection_a).await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut cancel_rx)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(orphaned_thread_ids.is_empty());
|
||||
|
||||
assert_eq!(
|
||||
manager.subscribed_connection_ids(thread_id).await,
|
||||
@@ -8612,6 +8655,25 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_last_connection_reports_orphaned_thread() -> Result<()> {
|
||||
let manager = ThreadStateManager::new();
|
||||
let thread_id = ThreadId::from_string("ad7f0408-99b8-4f6e-a46f-bd0eec433370")?;
|
||||
let connection = ConnectionId(1);
|
||||
|
||||
manager.connection_initialized(connection).await;
|
||||
manager
|
||||
.try_ensure_connection_subscribed(thread_id, connection, false)
|
||||
.await
|
||||
.expect("connection should be live");
|
||||
|
||||
let orphaned_thread_ids = manager.remove_connection(connection).await;
|
||||
|
||||
assert_eq!(orphaned_thread_ids, vec![thread_id]);
|
||||
assert!(!manager.has_subscribers(thread_id).await);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_connection_cannot_be_reintroduced_by_auto_subscribe() -> Result<()> {
|
||||
let manager = ThreadStateManager::new();
|
||||
@@ -8619,7 +8681,8 @@ mod tests {
|
||||
let connection = ConnectionId(1);
|
||||
|
||||
manager.connection_initialized(connection).await;
|
||||
manager.remove_connection(connection).await;
|
||||
let orphaned_thread_ids = manager.remove_connection(connection).await;
|
||||
assert!(orphaned_thread_ids.is_empty());
|
||||
|
||||
assert!(
|
||||
manager
|
||||
|
||||
@@ -313,8 +313,8 @@ impl ThreadStateManager {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_connection(&self, connection_id: ConnectionId) {
|
||||
let thread_states = {
|
||||
pub(crate) async fn remove_connection(&self, connection_id: ConnectionId) -> Vec<ThreadId> {
|
||||
let orphaned_threads = {
|
||||
let mut state = self.state.lock().await;
|
||||
state.live_connections.remove(&connection_id);
|
||||
let thread_ids = state
|
||||
@@ -344,7 +344,9 @@ impl ThreadStateManager {
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for (thread_id, no_subscribers, thread_state) in thread_states {
|
||||
let mut orphaned_thread_ids = Vec::new();
|
||||
|
||||
for (thread_id, no_subscribers, thread_state) in orphaned_threads {
|
||||
if !no_subscribers {
|
||||
continue;
|
||||
}
|
||||
@@ -358,6 +360,9 @@ impl ThreadStateManager {
|
||||
listener_generation,
|
||||
"retaining thread listener after connection disconnect left zero subscribers"
|
||||
);
|
||||
orphaned_thread_ids.push(thread_id);
|
||||
}
|
||||
|
||||
orphaned_thread_ids
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user