use super::thread_processor::THREAD_ITEMS_MAX_LIMIT; use super::thread_processor::thread_turns_page_size; use super::*; use crate::bespoke_event_handling::complete_command_execution_item; use crate::extensions::send_thread_warning; use crate::thread_state::ListenerStateCandidate; use codex_app_server_protocol::CommandExecutionStatus; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadQueueChangedNotification; use codex_core::ThreadEvictionOutcome; use codex_core::ThreadRegistration; use codex_extension_api::ThreadIdleCause; use codex_protocol::config_types::MultiAgentMode; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::SessionSource; use std::borrow::Cow; #[cfg(test)] #[path = "thread_listener_shutdown_tests.rs"] mod listener_shutdown_tests; #[derive(Clone)] pub(super) struct ListenerTaskContext { pub(super) thread_manager: Arc, pub(super) thread_state_manager: ThreadStateManager, pub(super) outgoing: Arc, pub(super) pending_thread_unloads: PendingThreadUnloads, pub(super) thread_watch_manager: ThreadWatchManager, pub(super) codex_home: PathBuf, pub(super) skills_watcher: Arc, pub(super) turn_cost_worker: Option, } struct UnloadingState { delay: Duration, has_subscribers_rx: watch::Receiver, has_subscribers: (bool, Instant), thread_status_rx: watch::Receiver, is_active: (bool, Instant), } impl UnloadingState { fn new( delay: Duration, has_subscribers_rx: watch::Receiver, thread_status_rx: watch::Receiver, ) -> Self { let has_subscribers = (*has_subscribers_rx.borrow(), Instant::now()); let is_active = ( matches!(*thread_status_rx.borrow(), ThreadStatus::Active { .. }), Instant::now(), ); Self { delay, has_subscribers_rx, has_subscribers, thread_status_rx, is_active, } } fn unloading_target(&self) -> Option { match (self.has_subscribers, self.is_active) { ((false, has_no_subscribers_since), (false, is_inactive_since)) => { std::cmp::max(has_no_subscribers_since, is_inactive_since).checked_add(self.delay) } _ => None, } } fn sync_receiver_values(&mut self) { let has_subscribers = *self.has_subscribers_rx.borrow(); if self.has_subscribers.0 != has_subscribers { self.has_subscribers = (has_subscribers, Instant::now()); } let is_active = matches!(*self.thread_status_rx.borrow(), ThreadStatus::Active { .. }); if self.is_active.0 != is_active { self.is_active = (is_active, Instant::now()); } } fn should_unload_now(&mut self) -> bool { self.sync_receiver_values(); self.unloading_target() .is_some_and(|target| target <= Instant::now()) } fn note_thread_activity_observed(&mut self) { if !self.is_active.0 { self.is_active = (false, Instant::now()); } } async fn wait_for_unloading_trigger(&mut self) -> bool { loop { self.sync_receiver_values(); let unloading_target = self.unloading_target(); if let Some(target) = unloading_target && target <= Instant::now() { tokio::task::yield_now().await; return true; } let unloading_sleep = async { if let Some(target) = unloading_target { tokio::time::sleep_until(target.into()).await; } else { futures::future::pending::<()>().await; } }; tokio::select! { _ = unloading_sleep => return true, changed = self.has_subscribers_rx.changed() => { if changed.is_err() { return false; } self.sync_receiver_values(); }, changed = self.thread_status_rx.changed() => { if changed.is_err() { return false; } self.sync_receiver_values(); }, } } } } pub(super) enum ThreadShutdownResult { Complete, SubmitFailed, TimedOut, } pub(super) enum EnsureConversationListenerResult { Attached, ConnectionClosed, } pub(super) fn thread_closing_for_resume(thread_id: impl std::fmt::Display) -> JSONRPCErrorError { invalid_request(format!( "thread {thread_id} is closing; retry thread/resume after the thread is closed" )) } /// Marks this exact runtime after listener startup, even after its app-server state is removed. struct ListenerStarted; enum ListenerStartup<'a> { Active, Subscribe { connection_id: ConnectionId, raw_events_enabled: bool, }, Created { connection_ids: &'a [ConnectionId], raw_events_enabled: bool, }, } enum ListenerStartOutcome { Started { attached: bool }, Stale, Rejected, } impl ListenerStartOutcome { fn into_subscription( self, thread_id: ThreadId, ) -> Result { match self { Self::Started { attached: true } => Ok(EnsureConversationListenerResult::Attached), Self::Started { attached: false } => { Ok(EnsureConversationListenerResult::ConnectionClosed) } Self::Stale => Err(invalid_request(format!( "thread {thread_id} is closing; retry after the thread is closed" ))), Self::Rejected => Err(thread_closing_for_resume(thread_id)), } } } #[expect( clippy::await_holding_invalid_type, reason = "listener state creation must be serialized against pending unloads" )] pub(super) async fn attach_created_thread_listener( listener_task_context: ListenerTaskContext, thread_id: ThreadId, thread: Arc, connection_ids: Vec, ) { let config_snapshot = thread.config_snapshot().await; let raw_events_enabled = if let Some(parent_thread_id) = config_snapshot.parent_thread_id { !connection_ids.is_empty() && listener_task_context .thread_state_manager .thread_state(parent_thread_id) .await .lock() .await .experimental_raw_events } else { false }; loop { let pending = listener_task_context.pending_thread_unloads.lock().await; if let Some(unload) = pending.get(&thread_id) { let mut completion = unload.subscribe(); drop(pending); let _ = completion.changed().await; continue; } if connection_ids.is_empty() && (thread.multi_agent_version() != Some(MultiAgentVersion::V2) || config_snapshot.ephemeral) { return; } let listener_state = listener_task_context .thread_state_manager .created_listener_state(thread_id) .await; drop(pending); match start_listener( listener_task_context.clone(), thread_id, Arc::clone(&thread), listener_state, ListenerStartup::Created { connection_ids: &connection_ids, raw_events_enabled, }, ) .await { ListenerStartOutcome::Started { .. } | ListenerStartOutcome::Rejected => return, ListenerStartOutcome::Stale => continue, } } } pub(super) async fn ensure_conversation_listener( listener_task_context: ListenerTaskContext, conversation_id: ThreadId, connection_id: ConnectionId, raw_events_enabled: bool, ) -> Result { let conversation = match listener_task_context .thread_manager .get_thread(conversation_id) .await { Ok(conv) => conv, Err(_) => { return Err(invalid_request(format!( "thread not found: {conversation_id}" ))); } }; if conversation.has_shutdown_status() { return Err(thread_closing_for_resume(conversation_id)); } let state = listener_task_context .thread_state_manager .thread_state(conversation_id) .await; start_listener( listener_task_context, conversation_id, conversation, ListenerStateCandidate::Existing(state), ListenerStartup::Subscribe { connection_id, raw_events_enabled, }, ) .await .into_subscription(conversation_id) } pub(super) fn log_listener_attach_result( result: Result, thread_id: ThreadId, connection_id: ConnectionId, thread_kind: &'static str, ) { match result { Ok(EnsureConversationListenerResult::Attached) => {} Ok(EnsureConversationListenerResult::ConnectionClosed) => { tracing::debug!( thread_id = %thread_id, connection_id = ?connection_id, "skipping auto-attach for closed connection" ); } Err(err) => { tracing::warn!( "failed to attach listener for {thread_kind} {thread_id}: {message}", message = err.message ); } } } pub(super) async fn ensure_listener_task_running( listener_task_context: ListenerTaskContext, conversation_id: ThreadId, conversation: Arc, thread_state: Arc>, ) -> Result<(), JSONRPCErrorError> { start_listener( listener_task_context, conversation_id, conversation, ListenerStateCandidate::Existing(thread_state), ListenerStartup::Active, ) .await .into_subscription(conversation_id) .map(|_| ()) } #[expect( clippy::await_holding_invalid_type, reason = "listener installation and removal must be serialized against subscriptions" )] async fn start_listener( listener_task_context: ListenerTaskContext, conversation_id: ThreadId, conversation: Arc, listener_state: ListenerStateCandidate, startup: ListenerStartup<'_>, ) -> ListenerStartOutcome { let (cancel_tx, mut cancel_rx) = oneshot::channel(); let (drained_tx, drained_rx) = watch::channel(false); let config_snapshot = conversation.config_snapshot().await; let config = conversation.config().await; let environments = conversation.environment_selections().await; let watch_registration = listener_task_context .skills_watcher .register_thread_config( config.as_ref(), listener_task_context.thread_manager.as_ref(), &environments, ) .await; let is_v2_child = conversation.multi_agent_version() == Some(MultiAgentVersion::V2) && matches!(config_snapshot.session_source, SessionSource::SubAgent(_)); let thread_settings_baseline = thread_settings_from_config_snapshot(&config_snapshot); let thread_state = Arc::clone(listener_state.state()); let (mut listener_command_rx, listener_generation, mut unloading_state, attached) = { let (created, connection_ids, raw_events_enabled) = match &startup { ListenerStartup::Active => (false, &[][..], false), ListenerStartup::Subscribe { connection_id, raw_events_enabled, } => ( false, std::slice::from_ref(connection_id), *raw_events_enabled, ), ListenerStartup::Created { connection_ids, raw_events_enabled, } => (true, *connection_ids, *raw_events_enabled), }; // Publish the drain receiver before a cold resume can start a replacement. let _admission_guard = if created { Some(ThreadState::lock_goal_resume(&thread_state).await) } else { None }; let (pending, mut state) = loop { let pending = if created || !connection_ids.is_empty() { let pending = listener_task_context.pending_thread_unloads.lock().await; if pending.contains_key(&conversation_id) { return ListenerStartOutcome::Stale; } Some(pending) } else { None }; let state = thread_state.lock().await; if !state.listener_matches(&conversation) && let Some(mut drain) = state.pending_shutdown_drain() { drop(state); drop(pending); let _ = drain.wait_for(|drained| *drained).await; continue; } break (pending, state); }; if created || !connection_ids.is_empty() { match listener_task_context .thread_manager .thread_registration(conversation_id, &conversation) .await { ThreadRegistration::Current => {} // Core can remove a closed child while listener setup awaits I/O. Drain its // captured runtime only if it cannot supersede a newer listener. ThreadRegistration::Absent if state.cancel_tx.is_some() && !state.listener_matches(&conversation) => { return ListenerStartOutcome::Rejected; } ThreadRegistration::Absent if created && conversation.has_shutdown_status() => {} ThreadRegistration::Absent | ThreadRegistration::Replaced => { return ListenerStartOutcome::Rejected; } } } let installed = listener_task_context .thread_state_manager .subscribe_and_install_listener( conversation_id, &listener_state, connection_ids, |attached| { let listener_matches = state.listener_matches(&conversation); if !created && conversation.has_shutdown_status() { return Err(()); } // Each runtime can have only one event reader. A created runtime may // have shut down before its first reader drains the queued events. if !listener_matches && !conversation .thread_extension_data() .insert_if(ListenerStarted, |existing| existing.is_none()) { return Err(()); } if attached && raw_events_enabled { state.set_experimental_raw_events(/*enabled*/ true); } if listener_matches { return Ok(None); } let (listener_command_rx, listener_generation) = state.set_listener( cancel_tx, drained_rx, &conversation, watch_registration, thread_settings_baseline, config_snapshot.history_mode, ); let Some(listener_command_tx) = state.listener_command_tx() else { tracing::warn!( "thread listener command sender missing immediately after listener registration" ); return Ok(None); }; listener_task_context .thread_state_manager .register_listener_command_tx(conversation_id, listener_command_tx); Ok(Some((listener_command_rx, listener_generation))) }, ) .await; let ((listener_command_rx, listener_generation), has_subscribers_rx, attached) = match installed { Ok(Some((Some(listener), has_subscribers_rx, attached))) => { (listener, has_subscribers_rx, attached) } Ok(Some((None, _, attached))) => return ListenerStartOutcome::Started { attached }, Ok(None) => return ListenerStartOutcome::Stale, Err(()) => return ListenerStartOutcome::Rejected, }; drop(pending); // A cold V2 resume can install the child's listener before its creation callback runs. if created || is_v2_child { listener_task_context .thread_watch_manager .upsert_thread(&conversation_id.to_string()) .await; } let thread_status_rx = listener_task_context .thread_watch_manager .subscribe(conversation_id) .await; ( listener_command_rx, listener_generation, UnloadingState::new( config.thread_unload_delay, has_subscribers_rx, thread_status_rx, ), attached, ) }; let ListenerTaskContext { outgoing, thread_manager, thread_state_manager, pending_thread_unloads, thread_watch_manager, codex_home, turn_cost_worker, .. } = listener_task_context; let outgoing_for_task = Arc::clone(&outgoing); tokio::spawn(async move { let shutdown_complete = loop { tokio::select! { biased; _ = &mut cancel_rx => { // Listener was superseded or the thread is being torn down. break false; } listener_command = listener_command_rx.recv() => { let Some(listener_command) = listener_command else { break false; }; handle_thread_listener_command( conversation_id, Some(&conversation), codex_home.as_path(), &thread_state_manager, &thread_state, &thread_watch_manager, &outgoing_for_task, &pending_thread_unloads, listener_command, ) .await; } event = conversation.next_event() => { let event = match event { Ok(event) => event, Err(err) => { tracing::warn!("thread.next_event() failed with: {err}"); break false; } }; if let Some(worker) = &turn_cost_worker { worker.observe_event( conversation_id, config.as_ref(), &event, || conversation.session_telemetry(), ); } // Track the event before emitting any typed translations // so thread-local state such as raw event opt-in stays // synchronized with the conversation. let (raw_events_enabled, terminal_summary) = { let mut thread_state = thread_state.lock().await; thread_state.track_current_turn_event(&event.id, &event.msg); let terminal = matches!(event.msg, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_)) .then(|| thread_state.turn_summaries.get(&event.id).cloned().unwrap_or_default()); (thread_state.experimental_raw_events, terminal) }; if matches!( &event.msg, EventMsg::RawResponseItem(_) | EventMsg::RawResponseCompleted(_) | EventMsg::PrivateSessionEvent(_) ) && !raw_events_enabled { continue; } let subscribed_connection_ids = thread_state_manager .subscribed_connection_ids(conversation_id) .await; let thread_outgoing = ThreadScopedOutgoingMessageSender::new( outgoing_for_task.clone(), subscribed_connection_ids, conversation_id, ); // Publish the committed receipt before turn/completed so completion // remains the last notification for this terminal result. if let Some(summary) = terminal_summary && let Some(db) = super::thread_read_state::publish( &event, conversation_id, &conversation, &summary, ).await { super::thread_read_state::notify( &db, &thread_state_manager, &outgoing_for_task, conversation_id, ).await; } let is_shutdown_complete = matches!(event.msg, EventMsg::ShutdownComplete); apply_bespoke_event_handling( event, conversation_id, conversation.clone(), thread_manager.clone(), thread_outgoing, thread_state.clone(), thread_watch_manager.clone(), ) .await; if is_shutdown_complete { break true; } } idle_guard = async { if unloading_state.wait_for_unloading_trigger().await { // Keep serving listener commands while a cold resume attaches its subscriber. Some(ThreadState::lock_goal_resume(&thread_state).await) } else { None } } => { let Some(_idle_guard) = idle_guard else { break false; }; if !unloading_state.should_unload_now() { continue; } if matches!(conversation.agent_status().await, AgentStatus::Running) { unloading_state.note_thread_activity_observed(); continue; } { let mut pending_thread_unloads = pending_thread_unloads.lock().await; if pending_thread_unloads.contains_key(&conversation_id) { continue; } if !unloading_state.should_unload_now() { continue; } pending_thread_unloads.insert(conversation_id, watch::channel(()).0); } if is_v2_child { let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation)); tokio::pin!(eviction); let result = match tokio::time::timeout( Duration::from_secs(/*secs*/ 10), &mut eviction, ).await { Ok(result) => result, Err(_) => { warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait"); eviction.await } }; let outcome = match result { Ok(ThreadEvictionOutcome::Busy) => { pending_thread_unloads.lock().await.remove(&conversation_id); unloading_state.note_thread_activity_observed(); continue; } Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome, Err(err) => { warn!("failed to evict thread {conversation_id}: {err}"); pending_thread_unloads.lock().await.remove(&conversation_id); unloading_state.note_thread_activity_observed(); continue; } }; { let mut pending = pending_thread_unloads.lock().await; if !thread_state_manager.remove_unsubscribed_listener( conversation_id, &thread_state, listener_generation, ).await { pending.remove(&conversation_id); break false; } } outgoing_for_task .cancel_requests_for_thread(conversation_id, /*error*/ None) .await; // A runtime removed by someone else may already have a replacement. if outcome == ThreadEvictionOutcome::Evicted { thread_watch_manager .remove_thread(&conversation_id.to_string()) .await; outgoing_for_task .send_server_notification(ServerNotification::ThreadClosed( ThreadClosedNotification { thread_id: conversation_id.to_string() }, )) .await; } pending_thread_unloads.lock().await.remove(&conversation_id); return; } if is_v2_child { let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation)); tokio::pin!(eviction); let result = match tokio::time::timeout( Duration::from_secs(/*secs*/ 10), &mut eviction, ).await { Ok(result) => result, Err(_) => { warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait"); eviction.await } }; let outcome = match result { Ok(ThreadEvictionOutcome::Busy) => { pending_thread_unloads.lock().await.remove(&conversation_id); unloading_state.note_thread_activity_observed(); continue; } Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome, Err(err) => { warn!("failed to evict thread {conversation_id}: {err}"); pending_thread_unloads.lock().await.remove(&conversation_id); unloading_state.note_thread_activity_observed(); continue; } }; { let mut pending = pending_thread_unloads.lock().await; if !thread_state_manager.remove_unsubscribed_listener( conversation_id, &thread_state, listener_generation, ).await { pending.remove(&conversation_id); break false; } } outgoing_for_task .cancel_requests_for_thread(conversation_id, /*error*/ None) .await; // A runtime removed by someone else may already have a replacement. if outcome == ThreadEvictionOutcome::Evicted { thread_watch_manager .remove_thread(&conversation_id.to_string()) .await; outgoing_for_task .send_server_notification(ServerNotification::ThreadClosed( ThreadClosedNotification { thread_id: conversation_id.to_string() }, )) .await; } pending_thread_unloads.lock().await.remove(&conversation_id); return; } unload_thread_without_subscribers( thread_manager.clone(), outgoing_for_task.clone(), pending_thread_unloads.clone(), thread_state_manager.clone(), thread_watch_manager.clone(), conversation_id, conversation.clone(), ) .await; break false; } } }; // The thread owns its event sender, so waiting for channel closure retains // the runtime. Keep only the notification relay after ShutdownComplete. drop(conversation); drop(config); drop(thread_manager); drop(turn_cost_worker); if shutdown_complete { drained_tx.send_replace(true); let relay_command = |command| { handle_thread_listener_command( conversation_id, /*conversation*/ None, codex_home.as_path(), &thread_state_manager, &thread_state, &thread_watch_manager, &outgoing_for_task, &pending_thread_unloads, command, ) }; // Preserve the existing queue for late request resolutions and ordered // notifications until normal listener cancellation or replacement. loop { tokio::select! { biased; _ = &mut cancel_rx => break, listener_command = listener_command_rx.recv() => { let Some(listener_command) = listener_command else { break; }; relay_command(listener_command).await; } // Keep relaying commands while waiting for a cold resume or goal mutation. resume_guard = async { if unloading_state.wait_for_unloading_trigger().await { Some(ThreadState::lock_goal_resume(&thread_state).await) } else { None } } => { { let Some(resume_guard) = resume_guard else { break; }; // A goal mutation may have queued a command between the select's // receive poll and the lock acquisition. Deliver it before cleanup. if !listener_command_rx.is_empty() { continue; } let mut pending = pending_thread_unloads.lock().await; if !unloading_state.should_unload_now() { continue; } if let Some(existing) = pending.get(&conversation_id) { let mut completion = existing.subscribe(); drop(pending); drop(resume_guard); tokio::select! { _ = &mut cancel_rx => break, _ = completion.changed() => continue, } } if !thread_state_manager.remove_unsubscribed_listener( conversation_id, &thread_state, listener_generation, ).await { break; } // Fence this thread while cleanup awaits without blocking other threads. pending.insert(conversation_id, watch::channel(()).0); } // A producer may still hold the retired sender. Deliver every command // it queued successfully and let later sends take their fallback. listener_command_rx.close(); while let Some(command) = listener_command_rx.recv().await { relay_command(command).await; } outgoing_for_task .cancel_requests_for_thread(conversation_id, /*error*/ None) .await; thread_watch_manager .remove_thread(&conversation_id.to_string()) .await; pending_thread_unloads.lock().await.remove(&conversation_id); return; } } } } let mut thread_state = thread_state.lock().await; if thread_state.listener_generation == listener_generation { thread_state_manager.unregister_listener_command_tx(conversation_id); thread_state.clear_listener(); } }); ListenerStartOutcome::Started { attached } } pub(super) async fn wait_for_thread_shutdown(thread: &Arc) -> ThreadShutdownResult { match tokio::time::timeout(Duration::from_secs(10), thread.shutdown_and_wait()).await { Ok(Ok(())) => ThreadShutdownResult::Complete, Ok(Err(_)) => ThreadShutdownResult::SubmitFailed, Err(_) => ThreadShutdownResult::TimedOut, } } pub(super) async fn unload_thread_without_subscribers( thread_manager: Arc, outgoing: Arc, pending_thread_unloads: PendingThreadUnloads, thread_state_manager: ThreadStateManager, thread_watch_manager: ThreadWatchManager, thread_id: ThreadId, thread: Arc, ) { info!("thread {thread_id} has no subscribers and is idle; shutting down"); // Any pending app-server -> client requests for this thread can no longer be // answered; cancel their callbacks before shutdown/unload. outgoing .cancel_requests_for_thread(thread_id, /*error*/ None) .await; thread_state_manager.remove_thread_state(thread_id).await; tokio::spawn(async move { // The deadline bounds our warning, not background cleanup. Keep polling the // same future so even delayed shutdown submission can eventually finish. let shutdown = thread.shutdown_and_wait(); tokio::pin!(shutdown); let result = match tokio::time::timeout(Duration::from_secs(/*secs*/ 10), &mut shutdown) .await { Ok(result) => result, Err(_) => { warn!( event.name = "codex.app_server.thread_shutdown_slow", "thread {thread_id} shutdown is taking longer than expected; continuing to wait" ); shutdown.await } }; match result { Ok(()) => { // A delayed unload can finish after thread/revert replaces this runtime under // the same thread ID. Only the runtime that scheduled this unload may remove it. if thread_manager .remove_thread_if_matches(&thread_id, &thread) .await .is_none() { info!("thread {thread_id} was replaced or removed before teardown finalized"); pending_thread_unloads.lock().await.remove(&thread_id); return; } thread_watch_manager .remove_thread(&thread_id.to_string()) .await; let notification = ThreadClosedNotification { thread_id: thread_id.to_string(), }; outgoing .send_server_notification(ServerNotification::ThreadClosed(notification)) .await; pending_thread_unloads.lock().await.remove(&thread_id); } Err(_) => { pending_thread_unloads.lock().await.remove(&thread_id); warn!("failed to submit Shutdown to thread {thread_id}"); } } }); } #[allow(clippy::too_many_arguments)] pub(super) async fn handle_thread_listener_command( conversation_id: ThreadId, conversation: Option<&Arc>, codex_home: &Path, thread_state_manager: &ThreadStateManager, thread_state: &Arc>, thread_watch_manager: &ThreadWatchManager, outgoing: &Arc, pending_thread_unloads: &PendingThreadUnloads, listener_command: ThreadListenerCommand, ) { match listener_command { ThreadListenerCommand::SendThreadResumeResponse { request: resume_request, completion_tx, } => { let Some(conversation) = conversation else { // Dropping the completion sender returns the closing error. return; }; handle_pending_thread_resume_request( conversation_id, conversation, codex_home, thread_state_manager, thread_state, thread_watch_manager, outgoing, pending_thread_unloads, *resume_request, ) .await; let _ = completion_tx.send(()); } ThreadListenerCommand::EmitThreadGoalUpdated { turn_id, goal } => { outgoing .send_server_notification(ServerNotification::ThreadGoalUpdated( ThreadGoalUpdatedNotification { thread_id: conversation_id.to_string(), turn_id, goal, }, )) .await; } ThreadListenerCommand::EmitThreadQueueChanged => { let subscribed_connection_ids = thread_state_manager .subscribed_connection_ids(conversation_id) .await; let outgoing = ThreadScopedOutgoingMessageSender::new( Arc::clone(outgoing), subscribed_connection_ids, conversation_id, ); outgoing .send_server_notification(ServerNotification::ThreadQueueChanged( ThreadQueueChangedNotification { thread_id: conversation_id.to_string(), }, )) .await; } ThreadListenerCommand::EmitWarning { message } => { send_thread_warning(outgoing, thread_state_manager, conversation_id, message).await; } ThreadListenerCommand::EmitThreadGoalCleared => { outgoing .send_server_notification(ServerNotification::ThreadGoalCleared( ThreadGoalClearedNotification { thread_id: conversation_id.to_string(), }, )) .await; } ThreadListenerCommand::EmitThreadGoalSnapshot { state_db } => { send_thread_goal_snapshot_notification(outgoing, conversation_id, &state_db).await; } ThreadListenerCommand::CompleteLateCommandApproval { turn_id, item_id, completion_item, } => { let subscribers = thread_state_manager .subscribed_connection_ids(conversation_id) .await; let outgoing = ThreadScopedOutgoingMessageSender::new( Arc::clone(outgoing), subscribers, conversation_id, ); complete_command_execution_item( &conversation_id, turn_id, item_id, *completion_item, CommandExecutionStatus::Declined, /*completed_at_ms*/ None, &outgoing, thread_state, ) .await; } ThreadListenerCommand::ResolveServerRequest { request_id, completion_tx, } => { resolve_pending_server_request( conversation_id, thread_state_manager, outgoing, request_id, ) .await; let _ = completion_tx.send(()); } } } #[allow(clippy::too_many_arguments)] #[expect( clippy::await_holding_invalid_type, reason = "running-thread resume subscription must be serialized against pending unloads" )] pub(super) async fn handle_pending_thread_resume_request( conversation_id: ThreadId, conversation: &Arc, _codex_home: &Path, thread_state_manager: &ThreadStateManager, thread_state: &Arc>, thread_watch_manager: &ThreadWatchManager, outgoing: &Arc, pending_thread_unloads: &PendingThreadUnloads, mut pending: crate::thread_state::PendingThreadResumeRequest, ) { let (active_turn_metadata, mut active_turn, active_page_params) = { let state = thread_state.lock().await; let active_page_params = pending.initial_turns_page.as_ref().filter(|params| { pending .paginated_initial_turns_page .as_ref() .is_none_or(|page| { params.sort_direction != Some(SortDirection::Asc) || (page.next_cursor.is_none() && page.data.len() < thread_turns_page_size(params.limit)) || state.active_turn_id().is_some_and(|active_turn_id| { page.data.iter().any(|turn| turn.id == active_turn_id) }) }) }); let items_view = if pending.include_turns { Some(TurnItemsView::Full) } else { active_page_params.map(|params| params.items_view.unwrap_or(TurnItemsView::Summary)) }; let active_turn = items_view.and_then(|view| state.active_turn_snapshot_with_items_view(view)); ( state.active_turn_metadata_snapshot(), active_turn, active_page_params, ) }; if let Some(store) = pending.resume_cursor_store.as_deref() && let Some(turn) = &mut active_turn && turn.items_view == TurnItemsView::Full { let stored = pending.paginated_turns.iter_mut().flatten().chain( pending .paginated_initial_turns_page .iter_mut() .flat_map(|page| &mut page.data), ); if let Err(error) = restore_evicted_agent_text(store, conversation_id, turn, stored).await { outgoing.send_error(pending.request_id, error).await; return; } } tracing::debug!( thread_id = %conversation_id, request_id = ?pending.request_id, active_turn_present = active_turn_metadata.is_some(), active_turn_id = ?active_turn_metadata.as_ref().map(|turn| turn.turn_id.as_str()), active_turn_status = ?active_turn_metadata.as_ref().map(|turn| &turn.status), "composing running thread resume response" ); let has_live_in_progress_turn = matches!(conversation.agent_status().await, AgentStatus::Running) || active_turn_metadata .as_ref() .is_some_and(|turn| matches!(turn.status, TurnStatus::InProgress)); let request_id = pending.request_id; let connection_id = request_id.connection_id; let mut thread = pending.thread_summary; if pending.include_turns { if let Some(turns) = pending.paginated_turns.take() { thread.turns = turns; } else { populate_thread_turns_from_history( &mut thread, &pending.history_items, /*active_turn*/ None, ); } if let Some(active_turn) = active_turn.as_ref() { merge_turn_history_with_active_turn(&mut thread.turns, active_turn.clone()); } } let thread_status = thread_watch_manager .loaded_status_for_thread(&thread.id) .await; set_thread_status_and_interrupt_stale_turns( &mut thread, thread_status.clone(), has_live_in_progress_turn, ); let mut initial_turns_page = if let Some(mut page) = pending.paginated_initial_turns_page.take() { if let (Some(active_turn), Some(params)) = (active_turn, active_page_params) { let sort_direction = params.sort_direction.unwrap_or(SortDirection::Desc); let active_turn_is_in_page = page.data.iter().any(|turn| turn.id == active_turn.id); if matches!(sort_direction, SortDirection::Desc) && !active_turn_is_in_page && let Some(page_with_active_slot) = pending.paginated_initial_turns_page_with_active_slot.take() { page = page_with_active_slot; } merge_active_turn_into_page(&mut page, active_turn, params); } super::thread_processor::normalize_thread_turns_status( &mut page.data, thread_status, has_live_in_progress_turn, ); Some(page) } else if let Some(params) = pending.initial_turns_page.as_ref() { match super::thread_processor::build_thread_resume_initial_turns_page( &pending.history_items, thread.status.clone(), has_live_in_progress_turn, active_turn, params, ) { Ok(page) => Some(page), Err(error) => { outgoing.send_error(request_id, error).await; return; } } } else { None }; let token_usage_turn_id = pending.cold_resume_token_usage_turn_id.or_else(|| { pending .include_turns .then(|| restored_token_usage_turn_id(&pending.history_items, thread.turns.as_slice())) }); if pending.initial_turns_page.is_none() { initial_turns_page = None; } if pending.redact_resume_payloads { redact_thread_resume_payloads(&mut thread.turns); if let Some(initial_turns_page) = initial_turns_page.as_mut() { redact_thread_resume_payloads(&mut initial_turns_page.data); } } { let pending_thread_unloads = pending_thread_unloads.lock().await; if pending_thread_unloads.contains_key(&conversation_id) || matches!(conversation.agent_status().await, AgentStatus::Shutdown) { drop(pending_thread_unloads); outgoing .send_error(request_id, thread_closing_for_resume(conversation_id)) .await; return; } if !thread_state_manager .try_add_connection_to_thread(conversation_id, connection_id) .await { tracing::debug!( thread_id = %conversation_id, connection_id = ?connection_id, "skipping running thread resume for closed connection" ); return; } } let (turns_backwards_cursor, items_backwards_cursor) = if let Some(thread_store) = pending.resume_cursor_store.as_ref() { match super::thread_processor::ThreadRequestProcessor::paginated_resume_backwards_cursors( thread_store.as_ref(), conversation_id, ) .await { Ok(cursors) => cursors, Err(error) => { outgoing.send_error(request_id, error).await; return; } } } else { (None, None) }; let config_snapshot = pending.config_snapshot; let sandbox = config_snapshot.sandbox_policy().into(); let cwd = config_snapshot.cwd().clone(); let ThreadConfigSnapshot { model, disabled_plugin_ids, model_provider_id, service_tier, approval_policy, approvals_reviewer, active_permission_profile, workspace_roots, reasoning_effort, collaboration_mode, originator, .. } = config_snapshot; let instruction_sources = pending.instruction_sources; let active_permission_profile = thread_response_active_permission_profile(active_permission_profile); let session_id = conversation.startup_metadata().session_id.to_string(); thread.session_id = session_id; let response = ThreadResumeResponse { thread, disabled_plugin_ids, model, model_provider: model_provider_id, service_tier, cwd, runtime_workspace_roots: workspace_roots, instruction_sources, approval_policy: approval_policy.into(), approvals_reviewer: approvals_reviewer.into(), sandbox, active_permission_profile, reasoning_effort, collaboration_mode: Some(collaboration_mode), multi_agent_mode: MultiAgentMode::ExplicitRequestOnly, initial_turns_page, turns_backwards_cursor, items_backwards_cursor, }; outgoing .send_response_with_thread_originator(request_id, response, originator) .await; // Warm metadata-only resumes skip history reconstruction. Cold paginated children can // replay usage using attribution captured before the listener was attached. if let Some(token_usage_turn_id) = token_usage_turn_id { // Rejoining a loaded thread has the same UI contract as a cold resume, but // uses the live conversation state instead of reconstructing a new session. send_thread_token_usage_update_to_connection( outgoing, connection_id, conversation_id, conversation.as_ref(), token_usage_turn_id, ) .await; } if pending.emit_thread_goal_update { if let Some(state_db) = pending.thread_goal_state_db { send_thread_goal_snapshot_notification(outgoing, conversation_id, &state_db).await; } else { tracing::warn!( thread_id = %conversation_id, "state db unavailable when reading thread goal for running thread resume" ); } } outgoing .replay_requests_to_connection_for_thread(connection_id, conversation_id) .await; // App-server owns resume response and snapshot ordering, so wait until // replay completes before letting extensions react to the idle thread. conversation .emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed) .await; } pub(super) async fn send_thread_goal_snapshot_notification( outgoing: &Arc, thread_id: ThreadId, state_db: &StateDbHandle, ) { match state_db.thread_goals().get_thread_goal(thread_id).await { Ok(Some(goal)) => { outgoing .send_server_notification(ServerNotification::ThreadGoalUpdated( ThreadGoalUpdatedNotification { thread_id: thread_id.to_string(), turn_id: None, goal: api_thread_goal_from_state(goal), }, )) .await; } Ok(None) => { outgoing .send_server_notification(ServerNotification::ThreadGoalCleared( ThreadGoalClearedNotification { thread_id: thread_id.to_string(), }, )) .await; } Err(err) => { tracing::warn!( thread_id = %thread_id, "failed to read thread goal for resume snapshot: {err}" ); } } } pub(crate) fn populate_thread_turns_from_history( thread: &mut Thread, items: &[RolloutItem], active_turn: Option<&Turn>, ) { let mut turns = build_legacy_api_turns_from_rollout_items(items); if let Some(active_turn) = active_turn { merge_turn_history_with_active_turn(&mut turns, active_turn.clone()); } thread.turns = turns; } pub(super) async fn resolve_pending_server_request( conversation_id: ThreadId, thread_state_manager: &ThreadStateManager, outgoing: &Arc, request_id: RequestId, ) { let subscribed_connection_ids = thread_state_manager .subscribed_connection_ids(conversation_id) .await; let outgoing = ThreadScopedOutgoingMessageSender::new( outgoing.clone(), subscribed_connection_ids, conversation_id, ); outgoing.resolve_server_request(request_id).await; } #[derive(Clone, Eq, Hash, PartialEq)] struct HistoryItemId<'a>(Cow<'a, str>); async fn restore_evicted_agent_text<'a>( store: &dyn ThreadStore, thread_id: ThreadId, turn: &mut Turn, stored: impl Iterator, ) -> Result<(), JSONRPCErrorError> { // Live history skips empty assistant completions, so every empty slot was evicted. let mut missing: HashMap<_, _> = turn .items .iter_mut() .filter_map(|item| match &mut item.payload { ThreadItemPayload::AgentMessage { id, text, .. } if text.is_empty() => { Some((HistoryItemId(Cow::Borrowed(id)), text)) } _ => None, }) .collect(); if missing.is_empty() { return Ok(()); } for stored in stored.filter(|stored| stored.id == turn.id) { fill_agent_text(&mut missing, std::mem::take(&mut stored.items)); if missing.is_empty() { return Ok(()); } } // The prepared page can predate some of the listener's items. Fetch only those gaps. let ids: Vec<_> = missing.keys().cloned().collect(); for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) { let params = ThreadItemsReadParams { thread_id: thread_id.to_string(), turn_id: turn.id.clone(), item_ids: chunk.iter().map(|id| id.0.to_string()).collect(), }; let saved = ThreadRequestProcessor::read_thread_items(store, params).await?; fill_agent_text(&mut missing, saved.data.into_iter().map(|entry| entry.item)); } if missing.is_empty() { Ok(()) } else { Err(internal_error(format!( "saved history is missing completed assistant items for turn {}", turn.id ))) } } fn fill_agent_text( missing: &mut HashMap, &mut String>, items: impl IntoIterator, ) { for item in items { if let ThreadItemPayload::AgentMessage { id, text, .. } = item.payload && !text.is_empty() && let Some(slot) = missing.remove(&HistoryItemId(Cow::Owned(id))) { *slot = text; } } } pub(super) fn merge_turn_history_with_active_turn(turns: &mut Vec, active_turn: Turn) { turns.retain(|turn| turn.id != active_turn.id); turns.push(active_turn); } fn merge_active_turn_into_page( page: &mut codex_app_server_protocol::TurnsPage, mut active_turn: Turn, params: &codex_app_server_protocol::ThreadResumeInitialTurnsPageParams, ) { super::thread_processor::apply_thread_turns_items_view( std::slice::from_mut(&mut active_turn), params.items_view.unwrap_or(TurnItemsView::Summary), ); let sort_direction = params.sort_direction.unwrap_or(SortDirection::Desc); page.data.retain(|turn| turn.id != active_turn.id); match sort_direction { SortDirection::Asc => page.data.push(active_turn), SortDirection::Desc => page.data.insert(0, active_turn), } } pub(super) fn set_thread_status_and_interrupt_stale_turns( thread: &mut Thread, loaded_status: ThreadStatus, has_live_in_progress_turn: bool, ) { let status = resolve_thread_status(loaded_status, has_live_in_progress_turn); if !matches!(status, ThreadStatus::Active { .. }) { for turn in &mut thread.turns { if matches!(turn.status, TurnStatus::InProgress) { turn.status = TurnStatus::Interrupted; } } } thread.status = status; }