mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
## Why Live paginated history retains completed assistant text even when summaries only need the latest message. Release older text for threads with persisted history while preserving full resume responses. ## What changed - Evict superseded assistant text from live turn state while preserving corrected items and the latest assistant message. - Restore evicted text for full resume responses from prepared history pages, fetching remaining items by ID in bounded batches. Return an error if saved text is missing. - Only load the active turn when it belongs in the requested initial page, and use the item ID index for targeted history reads. ## Testing Add reducer coverage for text eviction and corrected messages. Extend running-thread resume coverage to compare full and summary items with live notifications, exercise restoration when the prepared page lacks the active turn, and check missing-text errors. GitOrigin-RevId: 3229c0cea820f117dc6331312f1e6056a8d1d52c
1458 lines
57 KiB
Rust
1458 lines
57 KiB
Rust
use super::thread_processor::THREAD_ITEMS_MAX_LIMIT;
|
|
use super::thread_processor::thread_turns_page_size;
|
|
use super::*;
|
|
use crate::extensions::send_thread_warning;
|
|
use crate::thread_state::ListenerStateCandidate;
|
|
use codex_app_server_protocol::ThreadItem;
|
|
use codex_app_server_protocol::ThreadQueueChangedNotification;
|
|
use codex_core::ThreadEvictionOutcome;
|
|
use codex_core::ThreadRegistration;
|
|
use codex_extension_api::ThreadIdleCause;
|
|
use codex_protocol::config_types::MultiAgentMode;
|
|
use codex_protocol::protocol::MultiAgentVersion;
|
|
use codex_protocol::protocol::SessionSource;
|
|
use std::borrow::Cow;
|
|
|
|
#[cfg(test)]
|
|
#[path = "thread_listener_shutdown_tests.rs"]
|
|
mod listener_shutdown_tests;
|
|
|
|
#[derive(Clone)]
|
|
pub(super) struct ListenerTaskContext {
|
|
pub(super) thread_manager: Arc<ThreadManager>,
|
|
pub(super) thread_state_manager: ThreadStateManager,
|
|
pub(super) outgoing: Arc<OutgoingMessageSender>,
|
|
pub(super) pending_thread_unloads: PendingThreadUnloads,
|
|
pub(super) thread_watch_manager: ThreadWatchManager,
|
|
pub(super) codex_home: PathBuf,
|
|
pub(super) skills_watcher: Arc<SkillsWatcher>,
|
|
pub(super) turn_cost_worker: Option<crate::turn_cost_worker::TurnCostWorkerHandle>,
|
|
}
|
|
|
|
struct UnloadingState {
|
|
delay: Duration,
|
|
has_subscribers_rx: watch::Receiver<bool>,
|
|
has_subscribers: (bool, Instant),
|
|
thread_status_rx: watch::Receiver<ThreadStatus>,
|
|
is_active: (bool, Instant),
|
|
}
|
|
|
|
impl UnloadingState {
|
|
fn new(
|
|
delay: Duration,
|
|
has_subscribers_rx: watch::Receiver<bool>,
|
|
thread_status_rx: watch::Receiver<ThreadStatus>,
|
|
) -> Self {
|
|
let has_subscribers = (*has_subscribers_rx.borrow(), Instant::now());
|
|
let is_active = (
|
|
matches!(*thread_status_rx.borrow(), ThreadStatus::Active { .. }),
|
|
Instant::now(),
|
|
);
|
|
Self {
|
|
delay,
|
|
has_subscribers_rx,
|
|
has_subscribers,
|
|
thread_status_rx,
|
|
is_active,
|
|
}
|
|
}
|
|
|
|
fn unloading_target(&self) -> Option<Instant> {
|
|
match (self.has_subscribers, self.is_active) {
|
|
((false, has_no_subscribers_since), (false, is_inactive_since)) => {
|
|
std::cmp::max(has_no_subscribers_since, is_inactive_since).checked_add(self.delay)
|
|
}
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn sync_receiver_values(&mut self) {
|
|
let has_subscribers = *self.has_subscribers_rx.borrow();
|
|
if self.has_subscribers.0 != has_subscribers {
|
|
self.has_subscribers = (has_subscribers, Instant::now());
|
|
}
|
|
|
|
let is_active = matches!(*self.thread_status_rx.borrow(), ThreadStatus::Active { .. });
|
|
if self.is_active.0 != is_active {
|
|
self.is_active = (is_active, Instant::now());
|
|
}
|
|
}
|
|
|
|
fn should_unload_now(&mut self) -> bool {
|
|
self.sync_receiver_values();
|
|
self.unloading_target()
|
|
.is_some_and(|target| target <= Instant::now())
|
|
}
|
|
|
|
fn note_thread_activity_observed(&mut self) {
|
|
if !self.is_active.0 {
|
|
self.is_active = (false, Instant::now());
|
|
}
|
|
}
|
|
|
|
async fn wait_for_unloading_trigger(&mut self) -> bool {
|
|
loop {
|
|
self.sync_receiver_values();
|
|
let unloading_target = self.unloading_target();
|
|
if let Some(target) = unloading_target
|
|
&& target <= Instant::now()
|
|
{
|
|
tokio::task::yield_now().await;
|
|
return true;
|
|
}
|
|
let unloading_sleep = async {
|
|
if let Some(target) = unloading_target {
|
|
tokio::time::sleep_until(target.into()).await;
|
|
} else {
|
|
futures::future::pending::<()>().await;
|
|
}
|
|
};
|
|
tokio::select! {
|
|
_ = unloading_sleep => return true,
|
|
changed = self.has_subscribers_rx.changed() => {
|
|
if changed.is_err() {
|
|
return false;
|
|
}
|
|
self.sync_receiver_values();
|
|
},
|
|
changed = self.thread_status_rx.changed() => {
|
|
if changed.is_err() {
|
|
return false;
|
|
}
|
|
self.sync_receiver_values();
|
|
},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(super) enum ThreadShutdownResult {
|
|
Complete,
|
|
SubmitFailed,
|
|
TimedOut,
|
|
}
|
|
|
|
pub(super) enum EnsureConversationListenerResult {
|
|
Attached,
|
|
ConnectionClosed,
|
|
}
|
|
|
|
pub(super) fn thread_closing_for_resume(thread_id: impl std::fmt::Display) -> JSONRPCErrorError {
|
|
invalid_request(format!(
|
|
"thread {thread_id} is closing; retry thread/resume after the thread is closed"
|
|
))
|
|
}
|
|
|
|
/// Marks this exact runtime after listener startup, even after its app-server state is removed.
|
|
struct ListenerStarted;
|
|
|
|
enum ListenerStartup<'a> {
|
|
Active,
|
|
Subscribe {
|
|
connection_id: ConnectionId,
|
|
raw_events_enabled: bool,
|
|
},
|
|
Created {
|
|
connection_ids: &'a [ConnectionId],
|
|
raw_events_enabled: bool,
|
|
},
|
|
}
|
|
|
|
enum ListenerStartOutcome {
|
|
Started { attached: bool },
|
|
Stale,
|
|
Rejected,
|
|
}
|
|
|
|
impl ListenerStartOutcome {
|
|
fn into_subscription(
|
|
self,
|
|
thread_id: ThreadId,
|
|
) -> Result<EnsureConversationListenerResult, JSONRPCErrorError> {
|
|
match self {
|
|
Self::Started { attached: true } => Ok(EnsureConversationListenerResult::Attached),
|
|
Self::Started { attached: false } => {
|
|
Ok(EnsureConversationListenerResult::ConnectionClosed)
|
|
}
|
|
Self::Stale => Err(invalid_request(format!(
|
|
"thread {thread_id} is closing; retry after the thread is closed"
|
|
))),
|
|
Self::Rejected => Err(thread_closing_for_resume(thread_id)),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[expect(
|
|
clippy::await_holding_invalid_type,
|
|
reason = "listener state creation must be serialized against pending unloads"
|
|
)]
|
|
pub(super) async fn attach_created_thread_listener(
|
|
listener_task_context: ListenerTaskContext,
|
|
thread_id: ThreadId,
|
|
thread: Arc<CodexThread>,
|
|
connection_ids: Vec<ConnectionId>,
|
|
) {
|
|
let config_snapshot = thread.config_snapshot().await;
|
|
let raw_events_enabled = if let Some(parent_thread_id) = config_snapshot.parent_thread_id {
|
|
!connection_ids.is_empty()
|
|
&& listener_task_context
|
|
.thread_state_manager
|
|
.thread_state(parent_thread_id)
|
|
.await
|
|
.lock()
|
|
.await
|
|
.experimental_raw_events
|
|
} else {
|
|
false
|
|
};
|
|
loop {
|
|
let pending = listener_task_context.pending_thread_unloads.lock().await;
|
|
if let Some(unload) = pending.get(&thread_id) {
|
|
let mut completion = unload.subscribe();
|
|
drop(pending);
|
|
let _ = completion.changed().await;
|
|
continue;
|
|
}
|
|
if connection_ids.is_empty()
|
|
&& (thread.multi_agent_version() != Some(MultiAgentVersion::V2)
|
|
|| config_snapshot.ephemeral)
|
|
{
|
|
return;
|
|
}
|
|
let listener_state = listener_task_context
|
|
.thread_state_manager
|
|
.created_listener_state(thread_id)
|
|
.await;
|
|
drop(pending);
|
|
match start_listener(
|
|
listener_task_context.clone(),
|
|
thread_id,
|
|
Arc::clone(&thread),
|
|
listener_state,
|
|
ListenerStartup::Created {
|
|
connection_ids: &connection_ids,
|
|
raw_events_enabled,
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
ListenerStartOutcome::Started { .. } | ListenerStartOutcome::Rejected => return,
|
|
ListenerStartOutcome::Stale => continue,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(super) async fn ensure_conversation_listener(
|
|
listener_task_context: ListenerTaskContext,
|
|
conversation_id: ThreadId,
|
|
connection_id: ConnectionId,
|
|
raw_events_enabled: bool,
|
|
) -> Result<EnsureConversationListenerResult, JSONRPCErrorError> {
|
|
let conversation = match listener_task_context
|
|
.thread_manager
|
|
.get_thread(conversation_id)
|
|
.await
|
|
{
|
|
Ok(conv) => conv,
|
|
Err(_) => {
|
|
return Err(invalid_request(format!(
|
|
"thread not found: {conversation_id}"
|
|
)));
|
|
}
|
|
};
|
|
if conversation.has_shutdown_status() {
|
|
return Err(thread_closing_for_resume(conversation_id));
|
|
}
|
|
let state = listener_task_context
|
|
.thread_state_manager
|
|
.thread_state(conversation_id)
|
|
.await;
|
|
start_listener(
|
|
listener_task_context,
|
|
conversation_id,
|
|
conversation,
|
|
ListenerStateCandidate::Existing(state),
|
|
ListenerStartup::Subscribe {
|
|
connection_id,
|
|
raw_events_enabled,
|
|
},
|
|
)
|
|
.await
|
|
.into_subscription(conversation_id)
|
|
}
|
|
|
|
pub(super) fn log_listener_attach_result(
|
|
result: Result<EnsureConversationListenerResult, JSONRPCErrorError>,
|
|
thread_id: ThreadId,
|
|
connection_id: ConnectionId,
|
|
thread_kind: &'static str,
|
|
) {
|
|
match result {
|
|
Ok(EnsureConversationListenerResult::Attached) => {}
|
|
Ok(EnsureConversationListenerResult::ConnectionClosed) => {
|
|
tracing::debug!(
|
|
thread_id = %thread_id,
|
|
connection_id = ?connection_id,
|
|
"skipping auto-attach for closed connection"
|
|
);
|
|
}
|
|
Err(err) => {
|
|
tracing::warn!(
|
|
"failed to attach listener for {thread_kind} {thread_id}: {message}",
|
|
message = err.message
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(super) async fn ensure_listener_task_running(
|
|
listener_task_context: ListenerTaskContext,
|
|
conversation_id: ThreadId,
|
|
conversation: Arc<CodexThread>,
|
|
thread_state: Arc<Mutex<ThreadState>>,
|
|
) -> Result<(), JSONRPCErrorError> {
|
|
start_listener(
|
|
listener_task_context,
|
|
conversation_id,
|
|
conversation,
|
|
ListenerStateCandidate::Existing(thread_state),
|
|
ListenerStartup::Active,
|
|
)
|
|
.await
|
|
.into_subscription(conversation_id)
|
|
.map(|_| ())
|
|
}
|
|
|
|
#[expect(
|
|
clippy::await_holding_invalid_type,
|
|
reason = "listener installation and removal must be serialized against subscriptions"
|
|
)]
|
|
async fn start_listener(
|
|
listener_task_context: ListenerTaskContext,
|
|
conversation_id: ThreadId,
|
|
conversation: Arc<CodexThread>,
|
|
listener_state: ListenerStateCandidate,
|
|
startup: ListenerStartup<'_>,
|
|
) -> ListenerStartOutcome {
|
|
let (cancel_tx, mut cancel_rx) = oneshot::channel();
|
|
let (drained_tx, drained_rx) = watch::channel(false);
|
|
let config_snapshot = conversation.config_snapshot().await;
|
|
let config = conversation.config().await;
|
|
let environments = conversation.environment_selections().await;
|
|
let watch_registration = listener_task_context
|
|
.skills_watcher
|
|
.register_thread_config(
|
|
config.as_ref(),
|
|
listener_task_context.thread_manager.as_ref(),
|
|
&environments,
|
|
)
|
|
.await;
|
|
let is_v2_child = conversation.multi_agent_version() == Some(MultiAgentVersion::V2)
|
|
&& matches!(config_snapshot.session_source, SessionSource::SubAgent(_));
|
|
let thread_settings_baseline = thread_settings_from_config_snapshot(&config_snapshot);
|
|
let thread_state = Arc::clone(listener_state.state());
|
|
let (mut listener_command_rx, listener_generation, mut unloading_state, attached) = {
|
|
let (created, connection_ids, raw_events_enabled) = match &startup {
|
|
ListenerStartup::Active => (false, &[][..], false),
|
|
ListenerStartup::Subscribe {
|
|
connection_id,
|
|
raw_events_enabled,
|
|
} => (
|
|
false,
|
|
std::slice::from_ref(connection_id),
|
|
*raw_events_enabled,
|
|
),
|
|
ListenerStartup::Created {
|
|
connection_ids,
|
|
raw_events_enabled,
|
|
} => (true, *connection_ids, *raw_events_enabled),
|
|
};
|
|
// Publish the drain receiver before a cold resume can start a replacement.
|
|
let _admission_guard = if created {
|
|
Some(ThreadState::lock_goal_resume(&thread_state).await)
|
|
} else {
|
|
None
|
|
};
|
|
let (pending, mut state) = loop {
|
|
let pending = if created || !connection_ids.is_empty() {
|
|
let pending = listener_task_context.pending_thread_unloads.lock().await;
|
|
if pending.contains_key(&conversation_id) {
|
|
return ListenerStartOutcome::Stale;
|
|
}
|
|
Some(pending)
|
|
} else {
|
|
None
|
|
};
|
|
let state = thread_state.lock().await;
|
|
if !state.listener_matches(&conversation)
|
|
&& let Some(mut drain) = state.pending_shutdown_drain()
|
|
{
|
|
drop(state);
|
|
drop(pending);
|
|
let _ = drain.wait_for(|drained| *drained).await;
|
|
continue;
|
|
}
|
|
break (pending, state);
|
|
};
|
|
if created || !connection_ids.is_empty() {
|
|
match listener_task_context
|
|
.thread_manager
|
|
.thread_registration(conversation_id, &conversation)
|
|
.await
|
|
{
|
|
ThreadRegistration::Current => {}
|
|
// Core can remove a closed child while listener setup awaits I/O. Drain its
|
|
// captured runtime only if it cannot supersede a newer listener.
|
|
ThreadRegistration::Absent
|
|
if state.cancel_tx.is_some() && !state.listener_matches(&conversation) =>
|
|
{
|
|
return ListenerStartOutcome::Rejected;
|
|
}
|
|
ThreadRegistration::Absent if created && conversation.has_shutdown_status() => {}
|
|
ThreadRegistration::Absent | ThreadRegistration::Replaced => {
|
|
return ListenerStartOutcome::Rejected;
|
|
}
|
|
}
|
|
}
|
|
let installed = listener_task_context
|
|
.thread_state_manager
|
|
.subscribe_and_install_listener(
|
|
conversation_id,
|
|
&listener_state,
|
|
connection_ids,
|
|
|attached| {
|
|
let listener_matches = state.listener_matches(&conversation);
|
|
if !created && conversation.has_shutdown_status() {
|
|
return Err(());
|
|
}
|
|
// Each runtime can have only one event reader. A created runtime may
|
|
// have shut down before its first reader drains the queued events.
|
|
if !listener_matches
|
|
&& !conversation
|
|
.thread_extension_data()
|
|
.insert_if(ListenerStarted, |existing| existing.is_none())
|
|
{
|
|
return Err(());
|
|
}
|
|
if attached && raw_events_enabled {
|
|
state.set_experimental_raw_events(/*enabled*/ true);
|
|
}
|
|
if listener_matches {
|
|
return Ok(None);
|
|
}
|
|
let (listener_command_rx, listener_generation) = state.set_listener(
|
|
cancel_tx,
|
|
drained_rx,
|
|
&conversation,
|
|
watch_registration,
|
|
thread_settings_baseline,
|
|
config_snapshot.history_mode,
|
|
);
|
|
let Some(listener_command_tx) = state.listener_command_tx() else {
|
|
tracing::warn!(
|
|
"thread listener command sender missing immediately after listener registration"
|
|
);
|
|
return Ok(None);
|
|
};
|
|
listener_task_context
|
|
.thread_state_manager
|
|
.register_listener_command_tx(conversation_id, listener_command_tx);
|
|
Ok(Some((listener_command_rx, listener_generation)))
|
|
},
|
|
)
|
|
.await;
|
|
let ((listener_command_rx, listener_generation), has_subscribers_rx, attached) =
|
|
match installed {
|
|
Ok(Some((Some(listener), has_subscribers_rx, attached))) => {
|
|
(listener, has_subscribers_rx, attached)
|
|
}
|
|
Ok(Some((None, _, attached))) => return ListenerStartOutcome::Started { attached },
|
|
Ok(None) => return ListenerStartOutcome::Stale,
|
|
Err(()) => return ListenerStartOutcome::Rejected,
|
|
};
|
|
drop(pending);
|
|
// A cold V2 resume can install the child's listener before its creation callback runs.
|
|
if created || is_v2_child {
|
|
listener_task_context
|
|
.thread_watch_manager
|
|
.upsert_thread(&conversation_id.to_string())
|
|
.await;
|
|
}
|
|
let thread_status_rx = listener_task_context
|
|
.thread_watch_manager
|
|
.subscribe(conversation_id)
|
|
.await;
|
|
(
|
|
listener_command_rx,
|
|
listener_generation,
|
|
UnloadingState::new(
|
|
config.thread_unload_delay,
|
|
has_subscribers_rx,
|
|
thread_status_rx,
|
|
),
|
|
attached,
|
|
)
|
|
};
|
|
let ListenerTaskContext {
|
|
outgoing,
|
|
thread_manager,
|
|
thread_state_manager,
|
|
pending_thread_unloads,
|
|
thread_watch_manager,
|
|
codex_home,
|
|
turn_cost_worker,
|
|
..
|
|
} = listener_task_context;
|
|
let outgoing_for_task = Arc::clone(&outgoing);
|
|
tokio::spawn(async move {
|
|
let shutdown_complete = loop {
|
|
tokio::select! {
|
|
biased;
|
|
_ = &mut cancel_rx => {
|
|
// Listener was superseded or the thread is being torn down.
|
|
break false;
|
|
}
|
|
listener_command = listener_command_rx.recv() => {
|
|
let Some(listener_command) = listener_command else {
|
|
break false;
|
|
};
|
|
handle_thread_listener_command(
|
|
conversation_id,
|
|
Some(&conversation),
|
|
codex_home.as_path(),
|
|
&thread_state_manager,
|
|
&thread_state,
|
|
&thread_watch_manager,
|
|
&outgoing_for_task,
|
|
&pending_thread_unloads,
|
|
listener_command,
|
|
)
|
|
.await;
|
|
}
|
|
event = conversation.next_event() => {
|
|
let event = match event {
|
|
Ok(event) => event,
|
|
Err(err) => {
|
|
tracing::warn!("thread.next_event() failed with: {err}");
|
|
break false;
|
|
}
|
|
};
|
|
|
|
if let Some(worker) = &turn_cost_worker {
|
|
worker.observe_event(
|
|
conversation_id,
|
|
config.as_ref(),
|
|
&event,
|
|
|| conversation.session_telemetry(),
|
|
);
|
|
}
|
|
|
|
// Track the event before emitting any typed translations
|
|
// so thread-local state such as raw event opt-in stays
|
|
// synchronized with the conversation.
|
|
let (raw_events_enabled, terminal_summary) = {
|
|
let mut thread_state = thread_state.lock().await;
|
|
thread_state.track_current_turn_event(&event.id, &event.msg);
|
|
let terminal = matches!(event.msg, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_))
|
|
.then(|| thread_state.turn_summary.clone());
|
|
(thread_state.experimental_raw_events, terminal)
|
|
};
|
|
if matches!(
|
|
&event.msg,
|
|
EventMsg::RawResponseItem(_) | EventMsg::RawResponseCompleted(_) | EventMsg::PrivateSessionEvent(_)
|
|
) && !raw_events_enabled
|
|
{
|
|
continue;
|
|
}
|
|
let subscribed_connection_ids = thread_state_manager
|
|
.subscribed_connection_ids(conversation_id)
|
|
.await;
|
|
let thread_outgoing = ThreadScopedOutgoingMessageSender::new(
|
|
outgoing_for_task.clone(),
|
|
subscribed_connection_ids,
|
|
conversation_id,
|
|
);
|
|
|
|
// Publish the committed receipt before turn/completed so completion
|
|
// remains the last notification for this terminal result.
|
|
if let Some(summary) = terminal_summary
|
|
&& let Some(db) = super::thread_read_state::publish(
|
|
&event, conversation_id, &conversation, &summary,
|
|
).await
|
|
{
|
|
super::thread_read_state::notify(
|
|
&db, &thread_state_manager,
|
|
&outgoing_for_task, conversation_id,
|
|
).await;
|
|
}
|
|
let is_shutdown_complete = matches!(event.msg, EventMsg::ShutdownComplete);
|
|
apply_bespoke_event_handling(
|
|
event,
|
|
conversation_id,
|
|
conversation.clone(),
|
|
thread_manager.clone(),
|
|
thread_outgoing,
|
|
thread_state.clone(),
|
|
thread_watch_manager.clone(),
|
|
)
|
|
.await;
|
|
if is_shutdown_complete {
|
|
break true;
|
|
}
|
|
}
|
|
idle_guard = async {
|
|
if unloading_state.wait_for_unloading_trigger().await {
|
|
// Keep serving listener commands while a cold resume attaches its subscriber.
|
|
Some(ThreadState::lock_goal_resume(&thread_state).await)
|
|
} else {
|
|
None
|
|
}
|
|
} => {
|
|
let Some(_idle_guard) = idle_guard else {
|
|
break false;
|
|
};
|
|
if !unloading_state.should_unload_now() {
|
|
continue;
|
|
}
|
|
if matches!(conversation.agent_status().await, AgentStatus::Running) {
|
|
unloading_state.note_thread_activity_observed();
|
|
continue;
|
|
}
|
|
{
|
|
let mut pending_thread_unloads = pending_thread_unloads.lock().await;
|
|
if pending_thread_unloads.contains_key(&conversation_id) {
|
|
continue;
|
|
}
|
|
if !unloading_state.should_unload_now() {
|
|
continue;
|
|
}
|
|
pending_thread_unloads.insert(conversation_id, watch::channel(()).0);
|
|
}
|
|
if is_v2_child {
|
|
let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation));
|
|
tokio::pin!(eviction);
|
|
let result = match tokio::time::timeout(
|
|
Duration::from_secs(/*secs*/ 10), &mut eviction,
|
|
).await {
|
|
Ok(result) => result,
|
|
Err(_) => {
|
|
warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait");
|
|
eviction.await
|
|
}
|
|
};
|
|
let outcome = match result {
|
|
Ok(ThreadEvictionOutcome::Busy) => {
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
unloading_state.note_thread_activity_observed();
|
|
continue;
|
|
}
|
|
Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome,
|
|
Err(err) => {
|
|
warn!("failed to evict thread {conversation_id}: {err}");
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
unloading_state.note_thread_activity_observed();
|
|
continue;
|
|
}
|
|
};
|
|
{
|
|
let mut pending = pending_thread_unloads.lock().await;
|
|
if !thread_state_manager.remove_unsubscribed_listener(
|
|
conversation_id, &thread_state, listener_generation,
|
|
).await {
|
|
pending.remove(&conversation_id);
|
|
break false;
|
|
}
|
|
}
|
|
outgoing_for_task
|
|
.cancel_requests_for_thread(conversation_id, /*error*/ None)
|
|
.await;
|
|
// A runtime removed by someone else may already have a replacement.
|
|
if outcome == ThreadEvictionOutcome::Evicted {
|
|
thread_watch_manager
|
|
.remove_thread(&conversation_id.to_string())
|
|
.await;
|
|
outgoing_for_task
|
|
.send_server_notification(ServerNotification::ThreadClosed(
|
|
ThreadClosedNotification { thread_id: conversation_id.to_string() },
|
|
))
|
|
.await;
|
|
}
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
return;
|
|
}
|
|
if is_v2_child {
|
|
let eviction = thread_manager.try_evict_v2_thread(Arc::clone(&conversation));
|
|
tokio::pin!(eviction);
|
|
let result = match tokio::time::timeout(
|
|
Duration::from_secs(/*secs*/ 10), &mut eviction,
|
|
).await {
|
|
Ok(result) => result,
|
|
Err(_) => {
|
|
warn!(event.name = "codex.app_server.thread_shutdown_slow", "thread {conversation_id} shutdown is taking longer than expected; continuing to wait");
|
|
eviction.await
|
|
}
|
|
};
|
|
let outcome = match result {
|
|
Ok(ThreadEvictionOutcome::Busy) => {
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
unloading_state.note_thread_activity_observed();
|
|
continue;
|
|
}
|
|
Ok(outcome @ (ThreadEvictionOutcome::Evicted | ThreadEvictionOutcome::NotCurrent)) => outcome,
|
|
Err(err) => {
|
|
warn!("failed to evict thread {conversation_id}: {err}");
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
unloading_state.note_thread_activity_observed();
|
|
continue;
|
|
}
|
|
};
|
|
{
|
|
let mut pending = pending_thread_unloads.lock().await;
|
|
if !thread_state_manager.remove_unsubscribed_listener(
|
|
conversation_id, &thread_state, listener_generation,
|
|
).await {
|
|
pending.remove(&conversation_id);
|
|
break false;
|
|
}
|
|
}
|
|
outgoing_for_task
|
|
.cancel_requests_for_thread(conversation_id, /*error*/ None)
|
|
.await;
|
|
// A runtime removed by someone else may already have a replacement.
|
|
if outcome == ThreadEvictionOutcome::Evicted {
|
|
thread_watch_manager
|
|
.remove_thread(&conversation_id.to_string())
|
|
.await;
|
|
outgoing_for_task
|
|
.send_server_notification(ServerNotification::ThreadClosed(
|
|
ThreadClosedNotification { thread_id: conversation_id.to_string() },
|
|
))
|
|
.await;
|
|
}
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
return;
|
|
}
|
|
unload_thread_without_subscribers(
|
|
thread_manager.clone(),
|
|
outgoing_for_task.clone(),
|
|
pending_thread_unloads.clone(),
|
|
thread_state_manager.clone(),
|
|
thread_watch_manager.clone(),
|
|
conversation_id,
|
|
conversation.clone(),
|
|
)
|
|
.await;
|
|
break false;
|
|
}
|
|
}
|
|
};
|
|
// The thread owns its event sender, so waiting for channel closure retains
|
|
// the runtime. Keep only the notification relay after ShutdownComplete.
|
|
drop(conversation);
|
|
drop(config);
|
|
drop(thread_manager);
|
|
drop(turn_cost_worker);
|
|
|
|
if shutdown_complete {
|
|
drained_tx.send_replace(true);
|
|
let relay_command = |command| {
|
|
handle_thread_listener_command(
|
|
conversation_id,
|
|
/*conversation*/ None,
|
|
codex_home.as_path(),
|
|
&thread_state_manager,
|
|
&thread_state,
|
|
&thread_watch_manager,
|
|
&outgoing_for_task,
|
|
&pending_thread_unloads,
|
|
command,
|
|
)
|
|
};
|
|
// Preserve the existing queue for late request resolutions and ordered
|
|
// notifications until normal listener cancellation or replacement.
|
|
loop {
|
|
tokio::select! {
|
|
biased;
|
|
_ = &mut cancel_rx => break,
|
|
listener_command = listener_command_rx.recv() => {
|
|
let Some(listener_command) = listener_command else {
|
|
break;
|
|
};
|
|
relay_command(listener_command).await;
|
|
}
|
|
// Keep relaying commands while waiting for a cold resume or goal mutation.
|
|
resume_guard = async {
|
|
if unloading_state.wait_for_unloading_trigger().await {
|
|
Some(ThreadState::lock_goal_resume(&thread_state).await)
|
|
} else {
|
|
None
|
|
}
|
|
} => {
|
|
{
|
|
let Some(resume_guard) = resume_guard else {
|
|
break;
|
|
};
|
|
// A goal mutation may have queued a command between the select's
|
|
// receive poll and the lock acquisition. Deliver it before cleanup.
|
|
if !listener_command_rx.is_empty() {
|
|
continue;
|
|
}
|
|
let mut pending = pending_thread_unloads.lock().await;
|
|
if !unloading_state.should_unload_now() {
|
|
continue;
|
|
}
|
|
if let Some(existing) = pending.get(&conversation_id) {
|
|
let mut completion = existing.subscribe();
|
|
drop(pending);
|
|
drop(resume_guard);
|
|
tokio::select! {
|
|
_ = &mut cancel_rx => break,
|
|
_ = completion.changed() => continue,
|
|
}
|
|
}
|
|
if !thread_state_manager.remove_unsubscribed_listener(
|
|
conversation_id, &thread_state, listener_generation,
|
|
).await {
|
|
break;
|
|
}
|
|
// Fence this thread while cleanup awaits without blocking other threads.
|
|
pending.insert(conversation_id, watch::channel(()).0);
|
|
}
|
|
// A producer may still hold the retired sender. Deliver every command
|
|
// it queued successfully and let later sends take their fallback.
|
|
listener_command_rx.close();
|
|
while let Some(command) = listener_command_rx.recv().await {
|
|
relay_command(command).await;
|
|
}
|
|
outgoing_for_task
|
|
.cancel_requests_for_thread(conversation_id, /*error*/ None)
|
|
.await;
|
|
thread_watch_manager
|
|
.remove_thread(&conversation_id.to_string())
|
|
.await;
|
|
pending_thread_unloads.lock().await.remove(&conversation_id);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut thread_state = thread_state.lock().await;
|
|
if thread_state.listener_generation == listener_generation {
|
|
thread_state_manager.unregister_listener_command_tx(conversation_id);
|
|
thread_state.clear_listener();
|
|
}
|
|
});
|
|
ListenerStartOutcome::Started { attached }
|
|
}
|
|
|
|
pub(super) async fn wait_for_thread_shutdown(thread: &Arc<CodexThread>) -> ThreadShutdownResult {
|
|
match tokio::time::timeout(Duration::from_secs(10), thread.shutdown_and_wait()).await {
|
|
Ok(Ok(())) => ThreadShutdownResult::Complete,
|
|
Ok(Err(_)) => ThreadShutdownResult::SubmitFailed,
|
|
Err(_) => ThreadShutdownResult::TimedOut,
|
|
}
|
|
}
|
|
|
|
pub(super) async fn unload_thread_without_subscribers(
|
|
thread_manager: Arc<ThreadManager>,
|
|
outgoing: Arc<OutgoingMessageSender>,
|
|
pending_thread_unloads: PendingThreadUnloads,
|
|
thread_state_manager: ThreadStateManager,
|
|
thread_watch_manager: ThreadWatchManager,
|
|
thread_id: ThreadId,
|
|
thread: Arc<CodexThread>,
|
|
) {
|
|
info!("thread {thread_id} has no subscribers and is idle; shutting down");
|
|
|
|
// Any pending app-server -> client requests for this thread can no longer be
|
|
// answered; cancel their callbacks before shutdown/unload.
|
|
outgoing
|
|
.cancel_requests_for_thread(thread_id, /*error*/ None)
|
|
.await;
|
|
thread_state_manager.remove_thread_state(thread_id).await;
|
|
|
|
tokio::spawn(async move {
|
|
// The deadline bounds our warning, not background cleanup. Keep polling the
|
|
// same future so even delayed shutdown submission can eventually finish.
|
|
let shutdown = thread.shutdown_and_wait();
|
|
tokio::pin!(shutdown);
|
|
let result = match tokio::time::timeout(Duration::from_secs(/*secs*/ 10), &mut shutdown)
|
|
.await
|
|
{
|
|
Ok(result) => result,
|
|
Err(_) => {
|
|
warn!(
|
|
event.name = "codex.app_server.thread_shutdown_slow",
|
|
"thread {thread_id} shutdown is taking longer than expected; continuing to wait"
|
|
);
|
|
shutdown.await
|
|
}
|
|
};
|
|
match result {
|
|
Ok(()) => {
|
|
// A delayed unload can finish after thread/revert replaces this runtime under
|
|
// the same thread ID. Only the runtime that scheduled this unload may remove it.
|
|
if thread_manager
|
|
.remove_thread_if_matches(&thread_id, &thread)
|
|
.await
|
|
.is_none()
|
|
{
|
|
info!("thread {thread_id} was replaced or removed before teardown finalized");
|
|
pending_thread_unloads.lock().await.remove(&thread_id);
|
|
return;
|
|
}
|
|
thread_watch_manager
|
|
.remove_thread(&thread_id.to_string())
|
|
.await;
|
|
let notification = ThreadClosedNotification {
|
|
thread_id: thread_id.to_string(),
|
|
};
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadClosed(notification))
|
|
.await;
|
|
pending_thread_unloads.lock().await.remove(&thread_id);
|
|
}
|
|
Err(_) => {
|
|
pending_thread_unloads.lock().await.remove(&thread_id);
|
|
warn!("failed to submit Shutdown to thread {thread_id}");
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(super) async fn handle_thread_listener_command(
|
|
conversation_id: ThreadId,
|
|
conversation: Option<&Arc<CodexThread>>,
|
|
codex_home: &Path,
|
|
thread_state_manager: &ThreadStateManager,
|
|
thread_state: &Arc<Mutex<ThreadState>>,
|
|
thread_watch_manager: &ThreadWatchManager,
|
|
outgoing: &Arc<OutgoingMessageSender>,
|
|
pending_thread_unloads: &PendingThreadUnloads,
|
|
listener_command: ThreadListenerCommand,
|
|
) {
|
|
match listener_command {
|
|
ThreadListenerCommand::SendThreadResumeResponse {
|
|
request: resume_request,
|
|
completion_tx,
|
|
} => {
|
|
let Some(conversation) = conversation else {
|
|
// Dropping the completion sender returns the closing error.
|
|
return;
|
|
};
|
|
handle_pending_thread_resume_request(
|
|
conversation_id,
|
|
conversation,
|
|
codex_home,
|
|
thread_state_manager,
|
|
thread_state,
|
|
thread_watch_manager,
|
|
outgoing,
|
|
pending_thread_unloads,
|
|
*resume_request,
|
|
)
|
|
.await;
|
|
let _ = completion_tx.send(());
|
|
}
|
|
ThreadListenerCommand::EmitThreadGoalUpdated { turn_id, goal } => {
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadGoalUpdated(
|
|
ThreadGoalUpdatedNotification {
|
|
thread_id: conversation_id.to_string(),
|
|
turn_id,
|
|
goal,
|
|
},
|
|
))
|
|
.await;
|
|
}
|
|
ThreadListenerCommand::EmitThreadQueueChanged => {
|
|
let subscribed_connection_ids = thread_state_manager
|
|
.subscribed_connection_ids(conversation_id)
|
|
.await;
|
|
let outgoing = ThreadScopedOutgoingMessageSender::new(
|
|
Arc::clone(outgoing),
|
|
subscribed_connection_ids,
|
|
conversation_id,
|
|
);
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadQueueChanged(
|
|
ThreadQueueChangedNotification {
|
|
thread_id: conversation_id.to_string(),
|
|
},
|
|
))
|
|
.await;
|
|
}
|
|
ThreadListenerCommand::EmitWarning { message } => {
|
|
send_thread_warning(outgoing, thread_state_manager, conversation_id, message).await;
|
|
}
|
|
ThreadListenerCommand::EmitThreadGoalCleared => {
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadGoalCleared(
|
|
ThreadGoalClearedNotification {
|
|
thread_id: conversation_id.to_string(),
|
|
},
|
|
))
|
|
.await;
|
|
}
|
|
ThreadListenerCommand::EmitThreadGoalSnapshot { state_db } => {
|
|
send_thread_goal_snapshot_notification(outgoing, conversation_id, &state_db).await;
|
|
}
|
|
ThreadListenerCommand::ResolveServerRequest {
|
|
request_id,
|
|
completion_tx,
|
|
} => {
|
|
resolve_pending_server_request(
|
|
conversation_id,
|
|
thread_state_manager,
|
|
outgoing,
|
|
request_id,
|
|
)
|
|
.await;
|
|
let _ = completion_tx.send(());
|
|
}
|
|
}
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[expect(
|
|
clippy::await_holding_invalid_type,
|
|
reason = "running-thread resume subscription must be serialized against pending unloads"
|
|
)]
|
|
pub(super) async fn handle_pending_thread_resume_request(
|
|
conversation_id: ThreadId,
|
|
conversation: &Arc<CodexThread>,
|
|
_codex_home: &Path,
|
|
thread_state_manager: &ThreadStateManager,
|
|
thread_state: &Arc<Mutex<ThreadState>>,
|
|
thread_watch_manager: &ThreadWatchManager,
|
|
outgoing: &Arc<OutgoingMessageSender>,
|
|
pending_thread_unloads: &PendingThreadUnloads,
|
|
mut pending: crate::thread_state::PendingThreadResumeRequest,
|
|
) {
|
|
let (active_turn_metadata, mut active_turn, active_page_params) = {
|
|
let state = thread_state.lock().await;
|
|
let active_page_params = pending.initial_turns_page.as_ref().filter(|params| {
|
|
pending
|
|
.paginated_initial_turns_page
|
|
.as_ref()
|
|
.is_none_or(|page| {
|
|
params.sort_direction != Some(SortDirection::Asc)
|
|
|| (page.next_cursor.is_none()
|
|
&& page.data.len() < thread_turns_page_size(params.limit))
|
|
|| state.active_turn_id().is_some_and(|active_turn_id| {
|
|
page.data.iter().any(|turn| turn.id == active_turn_id)
|
|
})
|
|
})
|
|
});
|
|
let items_view = if pending.include_turns {
|
|
Some(TurnItemsView::Full)
|
|
} else {
|
|
active_page_params.map(|params| params.items_view.unwrap_or(TurnItemsView::Summary))
|
|
};
|
|
let active_turn =
|
|
items_view.and_then(|view| state.active_turn_snapshot_with_items_view(view));
|
|
(
|
|
state.active_turn_metadata_snapshot(),
|
|
active_turn,
|
|
active_page_params,
|
|
)
|
|
};
|
|
if let Some(store) = pending.resume_cursor_store.as_deref()
|
|
&& let Some(turn) = &mut active_turn
|
|
&& turn.items_view == TurnItemsView::Full
|
|
{
|
|
let stored = pending.paginated_turns.iter_mut().flatten().chain(
|
|
pending
|
|
.paginated_initial_turns_page
|
|
.iter_mut()
|
|
.flat_map(|page| &mut page.data),
|
|
);
|
|
if let Err(error) = restore_evicted_agent_text(store, conversation_id, turn, stored).await {
|
|
outgoing.send_error(pending.request_id, error).await;
|
|
return;
|
|
}
|
|
}
|
|
tracing::debug!(
|
|
thread_id = %conversation_id,
|
|
request_id = ?pending.request_id,
|
|
active_turn_present = active_turn_metadata.is_some(),
|
|
active_turn_id = ?active_turn_metadata.as_ref().map(|turn| turn.turn_id.as_str()),
|
|
active_turn_status = ?active_turn_metadata.as_ref().map(|turn| &turn.status),
|
|
"composing running thread resume response"
|
|
);
|
|
let has_live_in_progress_turn =
|
|
matches!(conversation.agent_status().await, AgentStatus::Running)
|
|
|| active_turn_metadata
|
|
.as_ref()
|
|
.is_some_and(|turn| matches!(turn.status, TurnStatus::InProgress));
|
|
|
|
let request_id = pending.request_id;
|
|
let connection_id = request_id.connection_id;
|
|
let mut thread = pending.thread_summary;
|
|
if pending.include_turns {
|
|
if let Some(turns) = pending.paginated_turns.take() {
|
|
thread.turns = turns;
|
|
} else {
|
|
populate_thread_turns_from_history(
|
|
&mut thread,
|
|
&pending.history_items,
|
|
/*active_turn*/ None,
|
|
);
|
|
}
|
|
if let Some(active_turn) = active_turn.as_ref() {
|
|
merge_turn_history_with_active_turn(&mut thread.turns, active_turn.clone());
|
|
}
|
|
}
|
|
|
|
let thread_status = thread_watch_manager
|
|
.loaded_status_for_thread(&thread.id)
|
|
.await;
|
|
|
|
set_thread_status_and_interrupt_stale_turns(
|
|
&mut thread,
|
|
thread_status.clone(),
|
|
has_live_in_progress_turn,
|
|
);
|
|
let mut initial_turns_page = if let Some(mut page) = pending.paginated_initial_turns_page.take()
|
|
{
|
|
if let (Some(active_turn), Some(params)) = (active_turn, active_page_params) {
|
|
let sort_direction = params.sort_direction.unwrap_or(SortDirection::Desc);
|
|
let active_turn_is_in_page = page.data.iter().any(|turn| turn.id == active_turn.id);
|
|
if matches!(sort_direction, SortDirection::Desc)
|
|
&& !active_turn_is_in_page
|
|
&& let Some(page_with_active_slot) =
|
|
pending.paginated_initial_turns_page_with_active_slot.take()
|
|
{
|
|
page = page_with_active_slot;
|
|
}
|
|
merge_active_turn_into_page(&mut page, active_turn, params);
|
|
}
|
|
super::thread_processor::normalize_thread_turns_status(
|
|
&mut page.data,
|
|
thread_status,
|
|
has_live_in_progress_turn,
|
|
);
|
|
Some(page)
|
|
} else if let Some(params) = pending.initial_turns_page.as_ref() {
|
|
match super::thread_processor::build_thread_resume_initial_turns_page(
|
|
&pending.history_items,
|
|
thread.status.clone(),
|
|
has_live_in_progress_turn,
|
|
active_turn,
|
|
params,
|
|
) {
|
|
Ok(page) => Some(page),
|
|
Err(error) => {
|
|
outgoing.send_error(request_id, error).await;
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
let token_usage_turn_id = pending.cold_resume_token_usage_turn_id.or_else(|| {
|
|
pending
|
|
.include_turns
|
|
.then(|| restored_token_usage_turn_id(&pending.history_items, thread.turns.as_slice()))
|
|
});
|
|
if pending.initial_turns_page.is_none() {
|
|
initial_turns_page = None;
|
|
}
|
|
if pending.redact_resume_payloads {
|
|
redact_thread_resume_payloads(&mut thread.turns);
|
|
if let Some(initial_turns_page) = initial_turns_page.as_mut() {
|
|
redact_thread_resume_payloads(&mut initial_turns_page.data);
|
|
}
|
|
}
|
|
|
|
{
|
|
let pending_thread_unloads = pending_thread_unloads.lock().await;
|
|
if pending_thread_unloads.contains_key(&conversation_id)
|
|
|| matches!(conversation.agent_status().await, AgentStatus::Shutdown)
|
|
{
|
|
drop(pending_thread_unloads);
|
|
outgoing
|
|
.send_error(request_id, thread_closing_for_resume(conversation_id))
|
|
.await;
|
|
return;
|
|
}
|
|
if !thread_state_manager
|
|
.try_add_connection_to_thread(conversation_id, connection_id)
|
|
.await
|
|
{
|
|
tracing::debug!(
|
|
thread_id = %conversation_id,
|
|
connection_id = ?connection_id,
|
|
"skipping running thread resume for closed connection"
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
|
|
let (turns_backwards_cursor, items_backwards_cursor) = if let Some(thread_store) =
|
|
pending.resume_cursor_store.as_ref()
|
|
{
|
|
match super::thread_processor::ThreadRequestProcessor::paginated_resume_backwards_cursors(
|
|
thread_store.as_ref(),
|
|
conversation_id,
|
|
)
|
|
.await
|
|
{
|
|
Ok(cursors) => cursors,
|
|
Err(error) => {
|
|
outgoing.send_error(request_id, error).await;
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
(None, None)
|
|
};
|
|
|
|
let config_snapshot = pending.config_snapshot;
|
|
let sandbox = config_snapshot.sandbox_policy().into();
|
|
let cwd = config_snapshot.cwd().clone();
|
|
let ThreadConfigSnapshot {
|
|
model,
|
|
disabled_plugin_ids,
|
|
model_provider_id,
|
|
service_tier,
|
|
approval_policy,
|
|
approvals_reviewer,
|
|
active_permission_profile,
|
|
workspace_roots,
|
|
reasoning_effort,
|
|
collaboration_mode,
|
|
originator,
|
|
..
|
|
} = config_snapshot;
|
|
let instruction_sources = pending.instruction_sources;
|
|
let active_permission_profile =
|
|
thread_response_active_permission_profile(active_permission_profile);
|
|
let session_id = conversation.startup_metadata().session_id.to_string();
|
|
thread.session_id = session_id;
|
|
|
|
let response = ThreadResumeResponse {
|
|
thread,
|
|
disabled_plugin_ids,
|
|
model,
|
|
model_provider: model_provider_id,
|
|
service_tier,
|
|
cwd,
|
|
runtime_workspace_roots: workspace_roots,
|
|
instruction_sources,
|
|
approval_policy: approval_policy.into(),
|
|
approvals_reviewer: approvals_reviewer.into(),
|
|
sandbox,
|
|
active_permission_profile,
|
|
reasoning_effort,
|
|
collaboration_mode: Some(collaboration_mode),
|
|
multi_agent_mode: MultiAgentMode::ExplicitRequestOnly,
|
|
initial_turns_page,
|
|
turns_backwards_cursor,
|
|
items_backwards_cursor,
|
|
};
|
|
outgoing
|
|
.send_response_with_thread_originator(request_id, response, originator)
|
|
.await;
|
|
// Warm metadata-only resumes skip history reconstruction. Cold paginated children can
|
|
// replay usage using attribution captured before the listener was attached.
|
|
if let Some(token_usage_turn_id) = token_usage_turn_id {
|
|
// Rejoining a loaded thread has the same UI contract as a cold resume, but
|
|
// uses the live conversation state instead of reconstructing a new session.
|
|
send_thread_token_usage_update_to_connection(
|
|
outgoing,
|
|
connection_id,
|
|
conversation_id,
|
|
conversation.as_ref(),
|
|
token_usage_turn_id,
|
|
)
|
|
.await;
|
|
}
|
|
if pending.emit_thread_goal_update {
|
|
if let Some(state_db) = pending.thread_goal_state_db {
|
|
send_thread_goal_snapshot_notification(outgoing, conversation_id, &state_db).await;
|
|
} else {
|
|
tracing::warn!(
|
|
thread_id = %conversation_id,
|
|
"state db unavailable when reading thread goal for running thread resume"
|
|
);
|
|
}
|
|
}
|
|
outgoing
|
|
.replay_requests_to_connection_for_thread(connection_id, conversation_id)
|
|
.await;
|
|
// App-server owns resume response and snapshot ordering, so wait until
|
|
// replay completes before letting extensions react to the idle thread.
|
|
conversation
|
|
.emit_thread_idle_lifecycle_if_idle(ThreadIdleCause::Completed)
|
|
.await;
|
|
}
|
|
|
|
pub(super) async fn send_thread_goal_snapshot_notification(
|
|
outgoing: &Arc<OutgoingMessageSender>,
|
|
thread_id: ThreadId,
|
|
state_db: &StateDbHandle,
|
|
) {
|
|
match state_db.thread_goals().get_thread_goal(thread_id).await {
|
|
Ok(Some(goal)) => {
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadGoalUpdated(
|
|
ThreadGoalUpdatedNotification {
|
|
thread_id: thread_id.to_string(),
|
|
turn_id: None,
|
|
goal: api_thread_goal_from_state(goal),
|
|
},
|
|
))
|
|
.await;
|
|
}
|
|
Ok(None) => {
|
|
outgoing
|
|
.send_server_notification(ServerNotification::ThreadGoalCleared(
|
|
ThreadGoalClearedNotification {
|
|
thread_id: thread_id.to_string(),
|
|
},
|
|
))
|
|
.await;
|
|
}
|
|
Err(err) => {
|
|
tracing::warn!(
|
|
thread_id = %thread_id,
|
|
"failed to read thread goal for resume snapshot: {err}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(crate) fn populate_thread_turns_from_history(
|
|
thread: &mut Thread,
|
|
items: &[RolloutItem],
|
|
active_turn: Option<&Turn>,
|
|
) {
|
|
let mut turns = build_legacy_api_turns_from_rollout_items(items);
|
|
if let Some(active_turn) = active_turn {
|
|
merge_turn_history_with_active_turn(&mut turns, active_turn.clone());
|
|
}
|
|
thread.turns = turns;
|
|
}
|
|
|
|
pub(super) async fn resolve_pending_server_request(
|
|
conversation_id: ThreadId,
|
|
thread_state_manager: &ThreadStateManager,
|
|
outgoing: &Arc<OutgoingMessageSender>,
|
|
request_id: RequestId,
|
|
) {
|
|
let subscribed_connection_ids = thread_state_manager
|
|
.subscribed_connection_ids(conversation_id)
|
|
.await;
|
|
let outgoing = ThreadScopedOutgoingMessageSender::new(
|
|
outgoing.clone(),
|
|
subscribed_connection_ids,
|
|
conversation_id,
|
|
);
|
|
outgoing.resolve_server_request(request_id).await;
|
|
}
|
|
|
|
#[derive(Clone, Eq, Hash, PartialEq)]
|
|
struct HistoryItemId<'a>(Cow<'a, str>);
|
|
|
|
async fn restore_evicted_agent_text<'a>(
|
|
store: &dyn ThreadStore,
|
|
thread_id: ThreadId,
|
|
turn: &mut Turn,
|
|
stored: impl Iterator<Item = &'a mut Turn>,
|
|
) -> Result<(), JSONRPCErrorError> {
|
|
// Live history skips empty assistant completions, so every empty slot was evicted.
|
|
let mut missing: HashMap<_, _> = turn
|
|
.items
|
|
.iter_mut()
|
|
.filter_map(|item| match &mut item.payload {
|
|
ThreadItemPayload::AgentMessage { id, text, .. } if text.is_empty() => {
|
|
Some((HistoryItemId(Cow::Borrowed(id)), text))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
if missing.is_empty() {
|
|
return Ok(());
|
|
}
|
|
for stored in stored.filter(|stored| stored.id == turn.id) {
|
|
fill_agent_text(&mut missing, std::mem::take(&mut stored.items));
|
|
if missing.is_empty() {
|
|
return Ok(());
|
|
}
|
|
}
|
|
// The prepared page can predate some of the listener's items. Fetch only those gaps.
|
|
let ids: Vec<_> = missing.keys().cloned().collect();
|
|
for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) {
|
|
let params = ThreadItemsReadParams {
|
|
thread_id: thread_id.to_string(),
|
|
turn_id: turn.id.clone(),
|
|
item_ids: chunk.iter().map(|id| id.0.to_string()).collect(),
|
|
};
|
|
let saved = ThreadRequestProcessor::read_thread_items(store, params).await?;
|
|
fill_agent_text(&mut missing, saved.data.into_iter().map(|entry| entry.item));
|
|
}
|
|
if missing.is_empty() {
|
|
Ok(())
|
|
} else {
|
|
Err(internal_error(format!(
|
|
"saved history is missing completed assistant items for turn {}",
|
|
turn.id
|
|
)))
|
|
}
|
|
}
|
|
|
|
fn fill_agent_text(
|
|
missing: &mut HashMap<HistoryItemId<'_>, &mut String>,
|
|
items: impl IntoIterator<Item = ThreadItem>,
|
|
) {
|
|
for item in items {
|
|
if let ThreadItemPayload::AgentMessage { id, text, .. } = item.payload
|
|
&& !text.is_empty()
|
|
&& let Some(slot) = missing.remove(&HistoryItemId(Cow::Owned(id)))
|
|
{
|
|
*slot = text;
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(super) fn merge_turn_history_with_active_turn(turns: &mut Vec<Turn>, active_turn: Turn) {
|
|
turns.retain(|turn| turn.id != active_turn.id);
|
|
turns.push(active_turn);
|
|
}
|
|
|
|
fn merge_active_turn_into_page(
|
|
page: &mut codex_app_server_protocol::TurnsPage,
|
|
mut active_turn: Turn,
|
|
params: &codex_app_server_protocol::ThreadResumeInitialTurnsPageParams,
|
|
) {
|
|
super::thread_processor::apply_thread_turns_items_view(
|
|
std::slice::from_mut(&mut active_turn),
|
|
params.items_view.unwrap_or(TurnItemsView::Summary),
|
|
);
|
|
let sort_direction = params.sort_direction.unwrap_or(SortDirection::Desc);
|
|
page.data.retain(|turn| turn.id != active_turn.id);
|
|
match sort_direction {
|
|
SortDirection::Asc => page.data.push(active_turn),
|
|
SortDirection::Desc => page.data.insert(0, active_turn),
|
|
}
|
|
}
|
|
|
|
pub(super) fn set_thread_status_and_interrupt_stale_turns(
|
|
thread: &mut Thread,
|
|
loaded_status: ThreadStatus,
|
|
has_live_in_progress_turn: bool,
|
|
) {
|
|
let status = resolve_thread_status(loaded_status, has_live_in_progress_turn);
|
|
if !matches!(status, ThreadStatus::Active { .. }) {
|
|
for turn in &mut thread.turns {
|
|
if matches!(turn.status, TurnStatus::InProgress) {
|
|
turn.status = TurnStatus::Interrupted;
|
|
}
|
|
}
|
|
}
|
|
thread.status = status;
|
|
}
|