refactor: encode terminal cleanup ownership

This commit is contained in:
Anton Panasenko
2026-07-06 20:47:12 -07:00
parent fbf5bfd1a9
commit 7e96bab9b3
5 changed files with 269 additions and 68 deletions
+10 -7
View File
@@ -33,6 +33,7 @@ use crate::ThreadMetadataPatch;
use crate::ThreadPage;
use crate::ThreadRelationFilter;
use crate::ThreadStore;
use crate::ThreadStoreCleanup;
use crate::ThreadStoreError;
use crate::ThreadStoreFuture;
use crate::ThreadStoreResult;
@@ -385,7 +386,7 @@ pub struct InMemoryThreadStoreCalls {
/// service.
#[derive(Default)]
pub struct InMemoryThreadStore {
state: tokio::sync::Mutex<InMemoryThreadStoreState>,
state: Arc<tokio::sync::Mutex<InMemoryThreadStoreState>>,
}
#[derive(Default)]
@@ -634,16 +635,18 @@ impl ThreadStore for InMemoryThreadStore {
})
}
fn shutdown_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move {
self.state.lock().await.calls.shutdown_thread += 1;
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
let state = Arc::clone(&self.state);
ThreadStoreCleanup::spawn("shutdown", thread_id, async move {
state.lock().await.calls.shutdown_thread += 1;
Ok(())
})
}
fn discard_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move {
self.state.lock().await.calls.discard_thread += 1;
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
let state = Arc::clone(&self.state);
ThreadStoreCleanup::spawn("discard", thread_id, async move {
state.lock().await.calls.discard_thread += 1;
Ok(())
})
}
+1
View File
@@ -21,6 +21,7 @@ pub use live_thread::LiveThreadInitGuard;
pub use local::LocalThreadStore;
pub use local::LocalThreadStoreConfig;
pub use store::ThreadStore;
pub use store::ThreadStoreCleanup;
pub use store::ThreadStoreFuture;
pub use types::AppendThreadItemsParams;
pub use types::ArchiveThreadParams;
+44 -48
View File
@@ -15,6 +15,7 @@ use crate::AppendThreadItemsParams;
use crate::CreateThreadParams;
use crate::ReadThreadParams;
use crate::ResumeThreadParams;
use crate::ThreadStoreCleanup;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
use crate::error::reject_paginated_history_mode;
@@ -156,62 +157,57 @@ pub(super) async fn flush_thread(
sync_materialized_rollout_path(store, thread_id).await
}
pub(super) async fn shutdown_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
let recorder = store.live_recorder(thread_id).await?;
let store = store.clone();
// The writer can close before this future removes its live-recorder entry. Keep those steps
// in one detached task so canceling an arbitrary ThreadStore caller cannot strand a closed
// recorder that blocks a later resume.
tokio::spawn(async move {
let rollout_path = recorder.rollout_path().to_path_buf();
recorder.shutdown().await.map_err(thread_store_io_error)?;
sync_materialized_rollout_path(&store, thread_id).await?;
if let Some(metrics) = codex_otel::global()
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
{
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
}
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
Ok(())
pub(super) fn shutdown_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup {
let generation_cutoff = store.live_recorder_generation_cutoff();
ThreadStoreCleanup::spawn("shutdown", thread_id, async move {
let recorder = store
.live_recorder_before_generation(thread_id, generation_cutoff)
.await?;
shutdown_cleanup(store, thread_id, recorder).await
})
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"),
})?
}
pub(super) async fn discard_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
let recorder = match store.live_recorder(thread_id).await {
Ok(recorder) => recorder,
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
Err(err) => return Err(err),
};
// Discard closes stale recorder clones before removing the lease. Keep that sequence alive if
// an arbitrary ThreadStore caller is canceled between the two steps.
spawn_discard_cleanup(store.clone(), thread_id, recorder)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("discard cleanup task failed for thread {thread_id}: {err}"),
})?;
Ok(())
pub(super) fn discard_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup {
let generation_cutoff = store.live_recorder_generation_cutoff();
ThreadStoreCleanup::spawn("discard", thread_id, async move {
let recorder = match store
.live_recorder_before_generation(thread_id, generation_cutoff)
.await
{
Ok(recorder) => recorder,
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
Err(err) => return Err(err),
};
discard_cleanup(store, thread_id, recorder).await
})
}
pub(super) fn spawn_discard_cleanup(
pub(super) async fn shutdown_cleanup(
store: LocalThreadStore,
thread_id: ThreadId,
recorder: Arc<RolloutRecorder>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
recorder.discard().await;
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
})
) -> ThreadStoreResult<()> {
let rollout_path = recorder.rollout_path().to_path_buf();
recorder.shutdown().await.map_err(thread_store_io_error)?;
sync_materialized_rollout_path(&store, thread_id).await?;
if let Some(metrics) = codex_otel::global()
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
{
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
}
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
Ok(())
}
pub(super) async fn discard_cleanup(
store: LocalThreadStore,
thread_id: ThreadId,
recorder: Arc<RolloutRecorder>,
) -> ThreadStoreResult<()> {
recorder.discard().await;
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
Ok(())
}
async fn remove_live_recorder_if_current(
+153 -7
View File
@@ -20,6 +20,8 @@ use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use tokio::sync::Mutex;
use crate::AppendThreadItemsParams;
@@ -37,6 +39,7 @@ use crate::StoredThreadHistory;
use crate::ThreadPage;
use crate::ThreadSearchPage;
use crate::ThreadStore;
use crate::ThreadStoreCleanup;
use crate::ThreadStoreError;
use crate::ThreadStoreFuture;
use crate::ThreadStoreResult;
@@ -59,11 +62,15 @@ use crate::UpdateThreadMetadataParams;
pub struct LocalThreadStore {
pub(super) config: LocalThreadStoreConfig,
live_recorders: Arc<Mutex<HashMap<ThreadId, LiveRecorderEntry>>>,
next_live_recorder_generation: Arc<AtomicU64>,
state_db: Option<StateDbHandle>,
}
struct LiveRecorderEntry {
recorder: Arc<RolloutRecorder>,
// Detached terminal cleanup captures an exclusive cutoff before it spawns, so a delayed task
// cannot bind itself to a writer inserted after the cleanup request.
generation: u64,
// Local rollout files are materialized lazily, but metadata updates can arrive before the
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
// so missing SQLite rows can still be seeded.
@@ -106,6 +113,7 @@ impl LocalThreadStore {
Self {
config,
live_recorders: Arc::new(Mutex::new(HashMap::new())),
next_live_recorder_generation: Arc::new(AtomicU64::new(0)),
state_db,
}
}
@@ -148,6 +156,25 @@ impl LocalThreadStore {
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
}
/// Returns the exclusive generation bound for a terminal cleanup requested now.
pub(super) fn live_recorder_generation_cutoff(&self) -> u64 {
self.next_live_recorder_generation.load(Ordering::SeqCst)
}
pub(super) async fn live_recorder_before_generation(
&self,
thread_id: ThreadId,
generation_cutoff: u64,
) -> ThreadStoreResult<Arc<RolloutRecorder>> {
self.live_recorders
.lock()
.await
.get(&thread_id)
.filter(|entry| entry.generation < generation_cutoff)
.map(|entry| Arc::clone(&entry.recorder))
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
}
pub(super) async fn ensure_live_recorder_absent(
&self,
thread_id: ThreadId,
@@ -171,8 +198,12 @@ impl LocalThreadStore {
message: format!("thread {} already has a live local writer", entry.key()),
}),
Entry::Vacant(entry) => {
let generation = self
.next_live_recorder_generation
.fetch_add(1, Ordering::SeqCst);
entry.insert(LiveRecorderEntry {
recorder: Arc::new(recorder),
generation,
history_mode,
});
Ok(())
@@ -262,12 +293,12 @@ impl ThreadStore for LocalThreadStore {
Box::pin(async move { live_writer::flush_thread(self, thread_id).await })
}
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move { live_writer::shutdown_thread(self, thread_id).await })
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
live_writer::shutdown_thread(self.clone(), thread_id)
}
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move { live_writer::discard_thread(self, thread_id).await })
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
live_writer::discard_thread(self.clone(), thread_id)
}
fn load_history(
@@ -915,7 +946,7 @@ mod tests {
}
#[tokio::test]
async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() {
async fn discard_cleanup_handle_removes_closed_writer_after_caller_cancels() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
@@ -937,7 +968,11 @@ mod tests {
.recv()
.expect("receive live recorder lock release");
});
let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder);
let cleanup = ThreadStoreCleanup::spawn(
"discard",
thread_id,
live_writer::discard_cleanup(store.clone(), thread_id, recorder),
);
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
@@ -970,7 +1005,118 @@ mod tests {
}
})
.await
.expect("detached discard cleanup should release the live writer");
.expect("discard cleanup handle should release the live writer");
}
#[tokio::test]
async fn shutdown_cleanup_handle_removes_closed_writer_after_caller_cancels() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
store
.create_thread(create_thread_params(thread_id))
.await
.expect("create live thread");
let recorder = store
.live_recorder(thread_id)
.await
.expect("clone live recorder");
let stale_recorder = Arc::clone(&recorder);
let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await;
let (release_lock, lock_released) = std::sync::mpsc::channel();
let lock_holder = tokio::task::spawn_blocking(move || {
let _live_recorders = live_recorders;
lock_released
.recv()
.expect("receive live recorder lock release");
});
let cleanup = ThreadStoreCleanup::spawn(
"shutdown",
thread_id,
live_writer::shutdown_cleanup(store.clone(), thread_id, recorder),
);
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
if stale_recorder
.record_canonical_items(&[user_message_item("write through stale recorder")])
.await
.is_err()
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("shutdown should close the stale recorder");
drop(cleanup);
release_lock.send(()).expect("release live recorder lock");
lock_holder.await.expect("join live recorder lock holder");
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
if matches!(
store.live_rollout_path(thread_id).await,
Err(ThreadStoreError::ThreadNotFound { thread_id: missing })
if missing == thread_id
) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("shutdown cleanup handle should release the live writer");
}
#[tokio::test]
async fn stale_cleanup_generation_does_not_discard_replacement_writer() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
store
.create_thread(create_thread_params(thread_id))
.await
.expect("create initial live thread");
let stale_generation_cutoff = store.live_recorder_generation_cutoff();
store
.discard_thread(thread_id)
.await
.expect("discard initial live thread");
store
.create_thread(create_thread_params(thread_id))
.await
.expect("create replacement live thread");
let delayed_store = store.clone();
ThreadStoreCleanup::spawn("discard", thread_id, async move {
let recorder = match delayed_store
.live_recorder_before_generation(thread_id, stale_generation_cutoff)
.await
{
Ok(recorder) => recorder,
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
Err(err) => return Err(err),
};
live_writer::discard_cleanup(delayed_store, thread_id, recorder).await
})
.await
.expect("stale cleanup should not touch the replacement writer");
store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![user_message_item("write through replacement writer")],
})
.await
.expect("replacement writer should remain live");
store
.discard_thread(thread_id)
.await
.expect("discard replacement live thread");
}
#[tokio::test]
+61 -6
View File
@@ -3,6 +3,9 @@ use codex_protocol::protocol::ThreadHistoryMode;
use std::any::Any;
use std::future::Future;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use tokio::task::JoinHandle;
use crate::AppendThreadItemsParams;
use crate::ArchiveThreadParams;
@@ -29,6 +32,54 @@ use crate::UpdateThreadMetadataParams;
/// Future returned by [`ThreadStore`] operations.
pub type ThreadStoreFuture<'a, T> = Pin<Box<dyn Future<Output = ThreadStoreResult<T>> + Send + 'a>>;
/// Handle for eagerly started terminal thread cleanup.
///
/// Awaiting the handle joins cleanup and reports its result. Dropping it detaches the underlying
/// task instead of canceling cleanup, so terminal cleanup keeps running after its caller is
/// canceled.
#[must_use = "await this handle to observe terminal cleanup failures"]
pub struct ThreadStoreCleanup {
operation: &'static str,
thread_id: ThreadId,
task: JoinHandle<ThreadStoreResult<()>>,
}
impl ThreadStoreCleanup {
/// Starts terminal cleanup immediately on the current Tokio runtime.
pub fn spawn(
operation: &'static str,
thread_id: ThreadId,
cleanup: impl Future<Output = ThreadStoreResult<()>> + Send + 'static,
) -> Self {
Self {
operation,
thread_id,
task: tokio::spawn(cleanup),
}
}
}
impl Future for ThreadStoreCleanup {
type Output = ThreadStoreResult<()>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match Pin::new(&mut this.task).poll(cx) {
Poll::Ready(Ok(result)) => Poll::Ready(result),
Poll::Ready(Err(err)) => {
let operation = this.operation;
let thread_id = &this.thread_id;
Poll::Ready(Err(ThreadStoreError::Internal {
message: format!(
"{operation} cleanup task failed for thread {thread_id}: {err}"
),
}))
}
Poll::Pending => Poll::Pending,
}
}
}
/// Storage-neutral thread persistence boundary.
pub trait ThreadStore: Any + Send + Sync {
/// Return this store as [`Any`] for implementation-owned escape hatches.
@@ -60,17 +111,21 @@ pub trait ThreadStore: Any + Send + Sync {
/// Flushes all queued items and returns once they are durable/readable.
fn flush_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
/// Flushes pending items and closes the live thread writer.
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
/// Starts terminal cleanup that flushes pending items and closes the live thread writer.
///
/// This starts terminal cleanup before returning. Await the handle to observe completion; if
/// the caller is canceled, dropping the handle detaches the terminal task instead of
/// canceling it.
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup;
/// Discards the live thread writer without forcing pending in-memory items to become durable.
///
/// Core calls this when session initialization fails after a live writer has been created.
/// This is an idempotent terminal release: implementations must preserve already-durable
/// thread data, finish releasing the lease after cleanup starts even if the returned future
/// is canceled or reports a diagnostic error, and prevent previously obtained writer handles
/// from appending once it returns.
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
/// thread data, keep releasing the lease after cleanup starts even if the returned handle is
/// dropped, and prevent previously obtained writer handles from appending after cleanup
/// completes.
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup;
/// Loads persisted history for resume, fork, rollback, and memory jobs.
fn load_history(