diff --git a/codex-rs/thread-store/src/in_memory.rs b/codex-rs/thread-store/src/in_memory.rs index cb3f198ff3..3f3c84149b 100644 --- a/codex-rs/thread-store/src/in_memory.rs +++ b/codex-rs/thread-store/src/in_memory.rs @@ -33,6 +33,7 @@ use crate::ThreadMetadataPatch; use crate::ThreadPage; use crate::ThreadRelationFilter; use crate::ThreadStore; +use crate::ThreadStoreCleanup; use crate::ThreadStoreError; use crate::ThreadStoreFuture; use crate::ThreadStoreResult; @@ -385,7 +386,7 @@ pub struct InMemoryThreadStoreCalls { /// service. #[derive(Default)] pub struct InMemoryThreadStore { - state: tokio::sync::Mutex, + state: Arc>, } #[derive(Default)] @@ -634,16 +635,18 @@ impl ThreadStore for InMemoryThreadStore { }) } - fn shutdown_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> { - Box::pin(async move { - self.state.lock().await.calls.shutdown_thread += 1; + fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup { + let state = Arc::clone(&self.state); + ThreadStoreCleanup::spawn("shutdown", thread_id, async move { + state.lock().await.calls.shutdown_thread += 1; Ok(()) }) } - fn discard_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> { - Box::pin(async move { - self.state.lock().await.calls.discard_thread += 1; + fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup { + let state = Arc::clone(&self.state); + ThreadStoreCleanup::spawn("discard", thread_id, async move { + state.lock().await.calls.discard_thread += 1; Ok(()) }) } diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 34a0ab2ba1..8d16c9b7b2 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -21,6 +21,7 @@ pub use live_thread::LiveThreadInitGuard; pub use local::LocalThreadStore; pub use local::LocalThreadStoreConfig; pub use store::ThreadStore; +pub use store::ThreadStoreCleanup; pub use store::ThreadStoreFuture; pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index df9ad2df7c..4b974e26de 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -15,6 +15,7 @@ use crate::AppendThreadItemsParams; use crate::CreateThreadParams; use crate::ReadThreadParams; use crate::ResumeThreadParams; +use crate::ThreadStoreCleanup; use crate::ThreadStoreError; use crate::ThreadStoreResult; use crate::error::reject_paginated_history_mode; @@ -156,62 +157,57 @@ pub(super) async fn flush_thread( sync_materialized_rollout_path(store, thread_id).await } -pub(super) async fn shutdown_thread( - store: &LocalThreadStore, - thread_id: ThreadId, -) -> ThreadStoreResult<()> { - let recorder = store.live_recorder(thread_id).await?; - let store = store.clone(); - // The writer can close before this future removes its live-recorder entry. Keep those steps - // in one detached task so canceling an arbitrary ThreadStore caller cannot strand a closed - // recorder that blocks a later resume. - tokio::spawn(async move { - let rollout_path = recorder.rollout_path().to_path_buf(); - recorder.shutdown().await.map_err(thread_store_io_error)?; - sync_materialized_rollout_path(&store, thread_id).await?; - if let Some(metrics) = codex_otel::global() - && let Ok(metadata) = tokio::fs::metadata(rollout_path).await - { - let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX); - let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]); - } - remove_live_recorder_if_current(&store, thread_id, &recorder).await; - Ok(()) +pub(super) fn shutdown_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup { + let generation_cutoff = store.live_recorder_generation_cutoff(); + ThreadStoreCleanup::spawn("shutdown", thread_id, async move { + let recorder = store + .live_recorder_before_generation(thread_id, generation_cutoff) + .await?; + shutdown_cleanup(store, thread_id, recorder).await }) - .await - .map_err(|err| ThreadStoreError::Internal { - message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"), - })? } -pub(super) async fn discard_thread( - store: &LocalThreadStore, - thread_id: ThreadId, -) -> ThreadStoreResult<()> { - let recorder = match store.live_recorder(thread_id).await { - Ok(recorder) => recorder, - Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()), - Err(err) => return Err(err), - }; - // Discard closes stale recorder clones before removing the lease. Keep that sequence alive if - // an arbitrary ThreadStore caller is canceled between the two steps. - spawn_discard_cleanup(store.clone(), thread_id, recorder) - .await - .map_err(|err| ThreadStoreError::Internal { - message: format!("discard cleanup task failed for thread {thread_id}: {err}"), - })?; - Ok(()) +pub(super) fn discard_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup { + let generation_cutoff = store.live_recorder_generation_cutoff(); + ThreadStoreCleanup::spawn("discard", thread_id, async move { + let recorder = match store + .live_recorder_before_generation(thread_id, generation_cutoff) + .await + { + Ok(recorder) => recorder, + Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()), + Err(err) => return Err(err), + }; + discard_cleanup(store, thread_id, recorder).await + }) } -pub(super) fn spawn_discard_cleanup( +pub(super) async fn shutdown_cleanup( store: LocalThreadStore, thread_id: ThreadId, recorder: Arc, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - recorder.discard().await; - remove_live_recorder_if_current(&store, thread_id, &recorder).await; - }) +) -> ThreadStoreResult<()> { + let rollout_path = recorder.rollout_path().to_path_buf(); + recorder.shutdown().await.map_err(thread_store_io_error)?; + sync_materialized_rollout_path(&store, thread_id).await?; + if let Some(metrics) = codex_otel::global() + && let Ok(metadata) = tokio::fs::metadata(rollout_path).await + { + let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX); + let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]); + } + remove_live_recorder_if_current(&store, thread_id, &recorder).await; + Ok(()) +} + +pub(super) async fn discard_cleanup( + store: LocalThreadStore, + thread_id: ThreadId, + recorder: Arc, +) -> ThreadStoreResult<()> { + recorder.discard().await; + remove_live_recorder_if_current(&store, thread_id, &recorder).await; + Ok(()) } async fn remove_live_recorder_if_current( diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index e4e80260d6..f2d06aa8e3 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -20,6 +20,8 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::path::PathBuf; use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; use tokio::sync::Mutex; use crate::AppendThreadItemsParams; @@ -37,6 +39,7 @@ use crate::StoredThreadHistory; use crate::ThreadPage; use crate::ThreadSearchPage; use crate::ThreadStore; +use crate::ThreadStoreCleanup; use crate::ThreadStoreError; use crate::ThreadStoreFuture; use crate::ThreadStoreResult; @@ -59,11 +62,15 @@ use crate::UpdateThreadMetadataParams; pub struct LocalThreadStore { pub(super) config: LocalThreadStoreConfig, live_recorders: Arc>>, + next_live_recorder_generation: Arc, state_db: Option, } struct LiveRecorderEntry { recorder: Arc, + // Detached terminal cleanup captures an exclusive cutoff before it spawns, so a delayed task + // cannot bind itself to a writer inserted after the cleanup request. + generation: u64, // Local rollout files are materialized lazily, but metadata updates can arrive before the // canonical SessionMeta is durable. Retain the mode captured when live persistence was opened // so missing SQLite rows can still be seeded. @@ -106,6 +113,7 @@ impl LocalThreadStore { Self { config, live_recorders: Arc::new(Mutex::new(HashMap::new())), + next_live_recorder_generation: Arc::new(AtomicU64::new(0)), state_db, } } @@ -148,6 +156,25 @@ impl LocalThreadStore { .ok_or(ThreadStoreError::ThreadNotFound { thread_id }) } + /// Returns the exclusive generation bound for a terminal cleanup requested now. + pub(super) fn live_recorder_generation_cutoff(&self) -> u64 { + self.next_live_recorder_generation.load(Ordering::SeqCst) + } + + pub(super) async fn live_recorder_before_generation( + &self, + thread_id: ThreadId, + generation_cutoff: u64, + ) -> ThreadStoreResult> { + self.live_recorders + .lock() + .await + .get(&thread_id) + .filter(|entry| entry.generation < generation_cutoff) + .map(|entry| Arc::clone(&entry.recorder)) + .ok_or(ThreadStoreError::ThreadNotFound { thread_id }) + } + pub(super) async fn ensure_live_recorder_absent( &self, thread_id: ThreadId, @@ -171,8 +198,12 @@ impl LocalThreadStore { message: format!("thread {} already has a live local writer", entry.key()), }), Entry::Vacant(entry) => { + let generation = self + .next_live_recorder_generation + .fetch_add(1, Ordering::SeqCst); entry.insert(LiveRecorderEntry { recorder: Arc::new(recorder), + generation, history_mode, }); Ok(()) @@ -262,12 +293,12 @@ impl ThreadStore for LocalThreadStore { Box::pin(async move { live_writer::flush_thread(self, thread_id).await }) } - fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> { - Box::pin(async move { live_writer::shutdown_thread(self, thread_id).await }) + fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup { + live_writer::shutdown_thread(self.clone(), thread_id) } - fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> { - Box::pin(async move { live_writer::discard_thread(self, thread_id).await }) + fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup { + live_writer::discard_thread(self.clone(), thread_id) } fn load_history( @@ -915,7 +946,7 @@ mod tests { } #[tokio::test] - async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() { + async fn discard_cleanup_handle_removes_closed_writer_after_caller_cancels() { let home = TempDir::new().expect("temp dir"); let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); let thread_id = ThreadId::default(); @@ -937,7 +968,11 @@ mod tests { .recv() .expect("receive live recorder lock release"); }); - let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder); + let cleanup = ThreadStoreCleanup::spawn( + "discard", + thread_id, + live_writer::discard_cleanup(store.clone(), thread_id, recorder), + ); tokio::time::timeout(std::time::Duration::from_secs(1), async { loop { @@ -970,7 +1005,118 @@ mod tests { } }) .await - .expect("detached discard cleanup should release the live writer"); + .expect("discard cleanup handle should release the live writer"); + } + + #[tokio::test] + async fn shutdown_cleanup_handle_removes_closed_writer_after_caller_cancels() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let thread_id = ThreadId::default(); + + store + .create_thread(create_thread_params(thread_id)) + .await + .expect("create live thread"); + let recorder = store + .live_recorder(thread_id) + .await + .expect("clone live recorder"); + let stale_recorder = Arc::clone(&recorder); + let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await; + let (release_lock, lock_released) = std::sync::mpsc::channel(); + let lock_holder = tokio::task::spawn_blocking(move || { + let _live_recorders = live_recorders; + lock_released + .recv() + .expect("receive live recorder lock release"); + }); + let cleanup = ThreadStoreCleanup::spawn( + "shutdown", + thread_id, + live_writer::shutdown_cleanup(store.clone(), thread_id, recorder), + ); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if stale_recorder + .record_canonical_items(&[user_message_item("write through stale recorder")]) + .await + .is_err() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown should close the stale recorder"); + drop(cleanup); + release_lock.send(()).expect("release live recorder lock"); + lock_holder.await.expect("join live recorder lock holder"); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if matches!( + store.live_rollout_path(thread_id).await, + Err(ThreadStoreError::ThreadNotFound { thread_id: missing }) + if missing == thread_id + ) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown cleanup handle should release the live writer"); + } + + #[tokio::test] + async fn stale_cleanup_generation_does_not_discard_replacement_writer() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let thread_id = ThreadId::default(); + + store + .create_thread(create_thread_params(thread_id)) + .await + .expect("create initial live thread"); + let stale_generation_cutoff = store.live_recorder_generation_cutoff(); + store + .discard_thread(thread_id) + .await + .expect("discard initial live thread"); + store + .create_thread(create_thread_params(thread_id)) + .await + .expect("create replacement live thread"); + + let delayed_store = store.clone(); + ThreadStoreCleanup::spawn("discard", thread_id, async move { + let recorder = match delayed_store + .live_recorder_before_generation(thread_id, stale_generation_cutoff) + .await + { + Ok(recorder) => recorder, + Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()), + Err(err) => return Err(err), + }; + live_writer::discard_cleanup(delayed_store, thread_id, recorder).await + }) + .await + .expect("stale cleanup should not touch the replacement writer"); + + store + .append_items(AppendThreadItemsParams { + thread_id, + items: vec![user_message_item("write through replacement writer")], + }) + .await + .expect("replacement writer should remain live"); + store + .discard_thread(thread_id) + .await + .expect("discard replacement live thread"); } #[tokio::test] diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index cf2a867e7e..b6da79b7db 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -3,6 +3,9 @@ use codex_protocol::protocol::ThreadHistoryMode; use std::any::Any; use std::future::Future; use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use tokio::task::JoinHandle; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; @@ -29,6 +32,54 @@ use crate::UpdateThreadMetadataParams; /// Future returned by [`ThreadStore`] operations. pub type ThreadStoreFuture<'a, T> = Pin> + Send + 'a>>; +/// Handle for eagerly started terminal thread cleanup. +/// +/// Awaiting the handle joins cleanup and reports its result. Dropping it detaches the underlying +/// task instead of canceling cleanup, so terminal cleanup keeps running after its caller is +/// canceled. +#[must_use = "await this handle to observe terminal cleanup failures"] +pub struct ThreadStoreCleanup { + operation: &'static str, + thread_id: ThreadId, + task: JoinHandle>, +} + +impl ThreadStoreCleanup { + /// Starts terminal cleanup immediately on the current Tokio runtime. + pub fn spawn( + operation: &'static str, + thread_id: ThreadId, + cleanup: impl Future> + Send + 'static, + ) -> Self { + Self { + operation, + thread_id, + task: tokio::spawn(cleanup), + } + } +} + +impl Future for ThreadStoreCleanup { + type Output = ThreadStoreResult<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match Pin::new(&mut this.task).poll(cx) { + Poll::Ready(Ok(result)) => Poll::Ready(result), + Poll::Ready(Err(err)) => { + let operation = this.operation; + let thread_id = &this.thread_id; + Poll::Ready(Err(ThreadStoreError::Internal { + message: format!( + "{operation} cleanup task failed for thread {thread_id}: {err}" + ), + })) + } + Poll::Pending => Poll::Pending, + } + } +} + /// Storage-neutral thread persistence boundary. pub trait ThreadStore: Any + Send + Sync { /// Return this store as [`Any`] for implementation-owned escape hatches. @@ -60,17 +111,21 @@ pub trait ThreadStore: Any + Send + Sync { /// Flushes all queued items and returns once they are durable/readable. fn flush_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>; - /// Flushes pending items and closes the live thread writer. - fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>; + /// Starts terminal cleanup that flushes pending items and closes the live thread writer. + /// + /// This starts terminal cleanup before returning. Await the handle to observe completion; if + /// the caller is canceled, dropping the handle detaches the terminal task instead of + /// canceling it. + fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup; /// Discards the live thread writer without forcing pending in-memory items to become durable. /// /// Core calls this when session initialization fails after a live writer has been created. /// This is an idempotent terminal release: implementations must preserve already-durable - /// thread data, finish releasing the lease after cleanup starts even if the returned future - /// is canceled or reports a diagnostic error, and prevent previously obtained writer handles - /// from appending once it returns. - fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>; + /// thread data, keep releasing the lease after cleanup starts even if the returned handle is + /// dropped, and prevent previously obtained writer handles from appending after cleanup + /// completes. + fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup; /// Loads persisted history for resume, fork, rollback, and memory jobs. fn load_history(