From 05eb6e5e8b57cdd7c732baa6782a3ea26a3cfac3 Mon Sep 17 00:00:00 2001 From: Anton Panasenko Date: Sun, 5 Jul 2026 01:46:05 -0700 Subject: [PATCH] fix terminal thread persistence shutdown --- codex-rs/core/src/session/handlers.rs | 61 ++++++-- codex-rs/core/src/session/tests.rs | 143 ++++++++++++++++++ codex-rs/rollout/src/recorder.rs | 48 +++++- .../thread-store/src/local/live_writer.rs | 71 ++++++--- codex-rs/thread-store/src/local/mod.rs | 143 +++++++++++++++++- codex-rs/thread-store/src/store.rs | 6 +- 6 files changed, 434 insertions(+), 38 deletions(-) diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 924750bc34..4c7400ccb1 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -617,6 +617,47 @@ async fn emit_thread_stop_lifecycle(sess: &Session) { } } +enum ThreadPersistenceShutdown { + Closed, + DiscardedAfterError, +} + +async fn shutdown_thread_persistence( + sess: &Session, + failure_context: &str, +) -> ThreadPersistenceShutdown { + let Some(live_thread) = sess.live_thread().cloned() else { + return ThreadPersistenceShutdown::Closed; + }; + let failure_context = failure_context.to_owned(); + // Terminal cleanup must outlive the submission loop. If the loop is canceled while a + // graceful shutdown is still waiting on I/O, the same task still reaches the discard + // fallback and releases the live-writer lease. + tokio::spawn(async move { + if let Err(err) = live_thread.shutdown().await { + warn!("{failure_context}: {err}"); + // A terminal session has no remaining owner that can retry a buffered writer. + // Release the writer lease so a later resume can recover the durable prefix + // instead of failing forever with a duplicate live writer. + if let Err(discard_err) = live_thread.discard().await { + warn!( + "thread persistence discard reported an error after terminal release: {discard_err}" + ); + } + warn!( + "discarded thread persistence after shutdown error; buffered rollout items may be lost" + ); + return ThreadPersistenceShutdown::DiscardedAfterError; + } + ThreadPersistenceShutdown::Closed + }) + .await + .unwrap_or_else(|err| { + warn!("terminal thread persistence cleanup task failed: {err}"); + ThreadPersistenceShutdown::DiscardedAfterError + }) +} + pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { shutdown_session_runtime(sess).await; info!("Shutting down Codex instance"); @@ -634,12 +675,10 @@ pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { emit_thread_stop_lifecycle(sess.as_ref()).await; - // Gracefully flush and shutdown thread persistence on session end so tests - // that inspect durable state do not race with the background writer. - if let Some(live_thread) = sess.live_thread() - && let Err(e) = live_thread.shutdown().await - { - warn!("failed to shutdown thread persistence: {e}"); + if matches!( + shutdown_thread_persistence(sess.as_ref(), "failed to shutdown thread persistence").await, + ThreadPersistenceShutdown::DiscardedAfterError + ) { let event = Event { id: sub_id.clone(), msg: EventMsg::Error(ErrorEvent { @@ -851,11 +890,11 @@ pub(super) async fn submission_loop( if !shutdown_received { shutdown_session_runtime(&sess).await; emit_thread_stop_lifecycle(sess.as_ref()).await; - if let Some(live_thread) = sess.live_thread() - && let Err(err) = live_thread.shutdown().await - { - warn!("failed to shutdown thread persistence after submission channel closed: {err}"); - } + let _ = shutdown_thread_persistence( + sess.as_ref(), + "failed to shutdown thread persistence after submission channel closed", + ) + .await; } debug!("Agent loop exited"); } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 1ff469cf4c..fc69955ada 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -6991,6 +6991,149 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() { ); } +async fn make_session_with_unflushable_thread_persistence() -> ( + tempfile::TempDir, + Session, + Arc, + async_channel::Receiver, +) { + let codex_home = tempfile::tempdir().expect("create codex home"); + let (session, _turn_context, rx_event) = make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + |_config| {}, + ) + .await; + let mut session = Arc::into_inner(session).expect("test session should be uniquely owned"); + let config = session.get_config().await; + let store = Arc::new(codex_thread_store::LocalThreadStore::new( + codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()), + /*state_db*/ None, + )); + let thread_store: Arc = store.clone(); + let live_thread = LiveThread::create( + Arc::clone(&thread_store), + CreateThreadParams { + session_id: session.session_id(), + thread_id: session.thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Exec, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode: Default::default(), + initial_window_id: Uuid::now_v7().to_string(), + metadata: ThreadPersistenceMetadata { + cwd: Some(config.cwd.to_path_buf()), + model_provider: config.model_provider_id.clone(), + memory_mode: if config.memories.generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }, + }, + }, + ) + .await + .expect("create thread persistence"); + let sessions_blocker_path = config.codex_home.join("sessions"); + std::fs::File::create(&sessions_blocker_path).expect("block sessions directory"); + live_thread + .append_items(&[RolloutItem::EventMsg(EventMsg::UserMessage( + UserMessageEvent { + client_id: None, + message: "buffered before failed shutdown".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + ..Default::default() + }, + ))]) + .await + .expect_err("blocked sessions directory should fail append"); + session.services.thread_store = thread_store; + session.services.live_thread = Some(live_thread); + (codex_home, session, store, rx_event) +} + +#[tokio::test] +async fn shutdown_discards_unflushable_thread_persistence_before_completing() { + let (_codex_home, session, store, rx_event) = + make_session_with_unflushable_thread_persistence().await; + let thread_id = session.thread_id; + let config = session.get_config().await; + let session = Arc::new(session); + let (tx_sub, rx_sub) = async_channel::bounded(4); + let (_agent_status_tx, agent_status) = watch::channel(AgentStatus::PendingInit); + let session_for_loop = Arc::clone(&session); + let session_loop_handle = tokio::spawn(async move { + submission_loop(session_for_loop, config, rx_sub).await; + }); + let codex = Codex { + tx_sub, + rx_event, + agent_status, + session, + session_loop_termination: session_loop_termination_from_handle(session_loop_handle), + }; + + codex + .submit(Op::Shutdown) + .await + .expect("submit terminal shutdown"); + + let mut saw_persistence_error = false; + loop { + let event = timeout(StdDuration::from_secs(2), codex.next_event()) + .await + .expect("timeout waiting for shutdown event") + .expect("receive shutdown event"); + match event.msg { + EventMsg::Error(_) => saw_persistence_error = true, + EventMsg::ShutdownComplete => break, + _ => {} + } + } + assert!( + saw_persistence_error, + "failed graceful shutdown should report its persistence error" + ); + let err = store + .live_rollout_path(thread_id) + .await + .expect_err("terminal shutdown should release the live writer"); + assert!( + matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id) + ); + codex.session_loop_termination.clone().await; +} + +#[tokio::test] +async fn submission_loop_channel_close_discards_unflushable_thread_persistence() { + let (_codex_home, session, store, _rx_event) = + make_session_with_unflushable_thread_persistence().await; + let thread_id = session.thread_id; + let config = session.get_config().await; + let (tx_sub, rx_sub) = async_channel::bounded(1); + drop(tx_sub); + + submission_loop(Arc::new(session), config, rx_sub).await; + + let err = store + .live_rollout_path(thread_id) + .await + .expect_err("channel close should release the live writer"); + assert!( + matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id) + ); +} + #[tokio::test] async fn submission_loop_channel_close_runs_full_thread_teardown() { struct SessionStopMarker; diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 87e97997a2..005cdf3794 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -115,6 +115,9 @@ enum RolloutCmd { Shutdown { ack: oneshot::Sender>, }, + Discard { + ack: oneshot::Sender<()>, + }, } /// Observable state for the background rollout writer task. @@ -1038,6 +1041,23 @@ impl RolloutRecorder { }; Ok(()) } + + /// Stop the writer without draining buffered items. + /// + /// This returns only after the writer has closed its command receiver, so stale recorder + /// clones cannot enqueue writes after the caller releases its live-writer lease. + pub async fn discard(&self) { + let (tx_done, rx_done) = oneshot::channel(); + if self + .tx + .send(RolloutCmd::Discard { ack: tx_done }) + .await + .is_err() + { + return; + } + let _ = rx_done.await; + } } pub(crate) fn reject_unknown_thread_history_mode(value: &Value) -> std::io::Result<()> { @@ -1723,10 +1743,18 @@ async fn rollout_writer( cwd: PathBuf, rollout_path: PathBuf, ) -> std::io::Result<()> { + enum TerminalAck { + Shutdown(oneshot::Sender>), + Discard(oneshot::Sender<()>), + } + let mut state = RolloutWriterState::new(file, deferred_log_file_info, meta, cwd, rollout_path); // Process rollout commands - while let Some(cmd) = rx.recv().await { + let terminal_ack = loop { + let Some(cmd) = rx.recv().await else { + break None; + }; match cmd { RolloutCmd::AddItems(items) => { state.add_items(items); @@ -1740,14 +1768,28 @@ async fn rollout_writer( } RolloutCmd::Shutdown { ack } => match state.shutdown().await { Ok(()) => { - let _ = ack.send(Ok(())); - break; + break Some(TerminalAck::Shutdown(ack)); } Err(err) => { let _ = ack.send(Err(err)); } }, + RolloutCmd::Discard { ack } => { + break Some(TerminalAck::Discard(ack)); + } } + }; + + drop(state); + drop(rx); + match terminal_ack { + Some(TerminalAck::Shutdown(ack)) => { + let _ = ack.send(Ok(())); + } + Some(TerminalAck::Discard(ack)) => { + let _ = ack.send(()); + } + None => {} } Ok(()) diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index 74e707bb58..1b91b83ac5 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use codex_protocol::ThreadId; use codex_protocol::protocol::ThreadMemoryMode; @@ -160,30 +161,66 @@ pub(super) async fn shutdown_thread( thread_id: ThreadId, ) -> ThreadStoreResult<()> { let recorder = store.live_recorder(thread_id).await?; - let rollout_path = recorder.rollout_path().to_path_buf(); - recorder.shutdown().await.map_err(thread_store_io_error)?; - sync_materialized_rollout_path(store, thread_id).await?; - if let Some(metrics) = codex_otel::global() - && let Ok(metadata) = tokio::fs::metadata(rollout_path).await - { - let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX); - let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]); - } - store.live_recorders.lock().await.remove(&thread_id); - Ok(()) + let store = store.clone(); + tokio::spawn(async move { + let rollout_path = recorder.rollout_path().to_path_buf(); + recorder.shutdown().await.map_err(thread_store_io_error)?; + sync_materialized_rollout_path(&store, thread_id).await?; + if let Some(metrics) = codex_otel::global() + && let Ok(metadata) = tokio::fs::metadata(rollout_path).await + { + let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX); + let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]); + } + remove_live_recorder_if_current(&store, thread_id, &recorder).await; + Ok(()) + }) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"), + })? } pub(super) async fn discard_thread( store: &LocalThreadStore, thread_id: ThreadId, ) -> ThreadStoreResult<()> { - store - .live_recorders - .lock() + let recorder = match store.live_recorder(thread_id).await { + Ok(recorder) => recorder, + Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()), + Err(err) => return Err(err), + }; + spawn_discard_cleanup(store.clone(), thread_id, recorder) .await - .remove(&thread_id) - .map(|_| ()) - .ok_or(ThreadStoreError::ThreadNotFound { thread_id }) + .map_err(|err| ThreadStoreError::Internal { + message: format!("discard cleanup task failed for thread {thread_id}: {err}"), + })?; + Ok(()) +} + +pub(super) fn spawn_discard_cleanup( + store: LocalThreadStore, + thread_id: ThreadId, + recorder: Arc, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + recorder.discard().await; + remove_live_recorder_if_current(&store, thread_id, &recorder).await; + }) +} + +async fn remove_live_recorder_if_current( + store: &LocalThreadStore, + thread_id: ThreadId, + recorder: &Arc, +) { + let mut live_recorders = store.live_recorders.lock().await; + if live_recorders + .get(&thread_id) + .is_some_and(|entry| Arc::ptr_eq(&entry.recorder, recorder)) + { + live_recorders.remove(&thread_id); + } } pub(super) async fn rollout_path( diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index 5939ea606e..e4e80260d6 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -63,7 +63,7 @@ pub struct LocalThreadStore { } struct LiveRecorderEntry { - recorder: RolloutRecorder, + recorder: Arc, // Local rollout files are materialized lazily, but metadata updates can arrive before the // canonical SessionMeta is durable. Retain the mode captured when live persistence was opened // so missing SQLite rows can still be seeded. @@ -139,12 +139,12 @@ impl LocalThreadStore { pub(super) async fn live_recorder( &self, thread_id: ThreadId, - ) -> ThreadStoreResult { + ) -> ThreadStoreResult> { self.live_recorders .lock() .await .get(&thread_id) - .map(|entry| entry.recorder.clone()) + .map(|entry| Arc::clone(&entry.recorder)) .ok_or(ThreadStoreError::ThreadNotFound { thread_id }) } @@ -172,7 +172,7 @@ impl LocalThreadStore { }), Entry::Vacant(entry) => { entry.insert(LiveRecorderEntry { - recorder, + recorder: Arc::new(recorder), history_mode, }); Ok(()) @@ -805,7 +805,7 @@ mod tests { } #[tokio::test] - async fn discard_thread_drops_unmaterialized_live_writer() { + async fn discard_thread_stops_cloned_recorder_and_drops_unmaterialized_live_writer() { let home = TempDir::new().expect("temp dir"); let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); let thread_id = ThreadId::default(); @@ -818,11 +818,23 @@ mod tests { .live_rollout_path(thread_id) .await .expect("load rollout path"); + let stale_recorder = store + .live_recorder(thread_id) + .await + .expect("clone live recorder"); store .discard_thread(thread_id) .await .expect("discard live thread"); + store + .discard_thread(thread_id) + .await + .expect("discard should be idempotent"); + stale_recorder + .record_canonical_items(&[user_message_item("write through stale recorder")]) + .await + .expect_err("discard should stop stale recorder clones"); assert!( !tokio::fs::try_exists(rollout_path.as_path()) .await @@ -840,6 +852,127 @@ mod tests { ); } + #[tokio::test] + async fn discard_thread_preserves_durable_prefix_for_resume() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let thread_id = ThreadId::default(); + + let first_store = LocalThreadStore::new(config.clone(), /*state_db*/ None); + first_store + .create_thread(create_thread_params(thread_id)) + .await + .expect("create initial thread"); + first_store + .append_items(AppendThreadItemsParams { + thread_id, + items: vec![user_message_item("before discard")], + }) + .await + .expect("append durable prefix"); + first_store + .persist_thread(thread_id) + .await + .expect("persist durable prefix"); + first_store + .flush_thread(thread_id) + .await + .expect("flush durable prefix"); + let rollout_path = first_store + .live_rollout_path(thread_id) + .await + .expect("load rollout path"); + first_store + .discard_thread(thread_id) + .await + .expect("discard live thread"); + + let resumed_store = LocalThreadStore::new(config, /*state_db*/ None); + resumed_store + .resume_thread(ResumeThreadParams { + thread_id, + rollout_path: None, + history: None, + include_archived: true, + metadata: thread_metadata(), + }) + .await + .expect("resume live thread"); + resumed_store + .append_items(AppendThreadItemsParams { + thread_id, + items: vec![user_message_item("after resume")], + }) + .await + .expect("append resumed item"); + resumed_store + .flush_thread(thread_id) + .await + .expect("flush resumed thread"); + + assert_rollout_contains_message(rollout_path.as_path(), "before discard").await; + assert_rollout_contains_message(rollout_path.as_path(), "after resume").await; + } + + #[tokio::test] + async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let thread_id = ThreadId::default(); + + store + .create_thread(create_thread_params(thread_id)) + .await + .expect("create live thread"); + let recorder = store + .live_recorder(thread_id) + .await + .expect("clone live recorder"); + let stale_recorder = Arc::clone(&recorder); + let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await; + let (release_lock, lock_released) = std::sync::mpsc::channel(); + let lock_holder = tokio::task::spawn_blocking(move || { + let _live_recorders = live_recorders; + lock_released + .recv() + .expect("receive live recorder lock release"); + }); + let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if stale_recorder + .record_canonical_items(&[user_message_item("write through stale recorder")]) + .await + .is_err() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("discard should close the stale recorder"); + drop(cleanup); + release_lock.send(()).expect("release live recorder lock"); + lock_holder.await.expect("join live recorder lock holder"); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if matches!( + store.live_rollout_path(thread_id).await, + Err(ThreadStoreError::ThreadNotFound { thread_id: missing }) + if missing == thread_id + ) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("detached discard cleanup should release the live writer"); + } + #[tokio::test] async fn resume_thread_reopens_live_writer_and_appends() { let home = TempDir::new().expect("temp dir"); diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 67ee603f49..cf2a867e7e 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -66,8 +66,10 @@ pub trait ThreadStore: Any + Send + Sync { /// Discards the live thread writer without forcing pending in-memory items to become durable. /// /// Core calls this when session initialization fails after a live writer has been created. - /// Implementations should release any live writer resources for the thread while preserving - /// already-durable thread data. + /// This is an idempotent terminal release: implementations must preserve already-durable + /// thread data, finish releasing the lease after cleanup starts even if the returned future + /// is canceled or reports a diagnostic error, and prevent previously obtained writer handles + /// from appending once it returns. fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>; /// Loads persisted history for resume, fork, rollback, and memory jobs.