mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
refactor: encode terminal cleanup ownership
This commit is contained in:
@@ -33,6 +33,7 @@ use crate::ThreadMetadataPatch;
|
||||
use crate::ThreadPage;
|
||||
use crate::ThreadRelationFilter;
|
||||
use crate::ThreadStore;
|
||||
use crate::ThreadStoreCleanup;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreFuture;
|
||||
use crate::ThreadStoreResult;
|
||||
@@ -385,7 +386,7 @@ pub struct InMemoryThreadStoreCalls {
|
||||
/// service.
|
||||
#[derive(Default)]
|
||||
pub struct InMemoryThreadStore {
|
||||
state: tokio::sync::Mutex<InMemoryThreadStoreState>,
|
||||
state: Arc<tokio::sync::Mutex<InMemoryThreadStoreState>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -634,16 +635,18 @@ impl ThreadStore for InMemoryThreadStore {
|
||||
})
|
||||
}
|
||||
|
||||
fn shutdown_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.state.lock().await.calls.shutdown_thread += 1;
|
||||
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
let state = Arc::clone(&self.state);
|
||||
ThreadStoreCleanup::spawn("shutdown", thread_id, async move {
|
||||
state.lock().await.calls.shutdown_thread += 1;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn discard_thread(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.state.lock().await.calls.discard_thread += 1;
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
let state = Arc::clone(&self.state);
|
||||
ThreadStoreCleanup::spawn("discard", thread_id, async move {
|
||||
state.lock().await.calls.discard_thread += 1;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ pub use live_thread::LiveThreadInitGuard;
|
||||
pub use local::LocalThreadStore;
|
||||
pub use local::LocalThreadStoreConfig;
|
||||
pub use store::ThreadStore;
|
||||
pub use store::ThreadStoreCleanup;
|
||||
pub use store::ThreadStoreFuture;
|
||||
pub use types::AppendThreadItemsParams;
|
||||
pub use types::ArchiveThreadParams;
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::AppendThreadItemsParams;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::ReadThreadParams;
|
||||
use crate::ResumeThreadParams;
|
||||
use crate::ThreadStoreCleanup;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
use crate::error::reject_paginated_history_mode;
|
||||
@@ -156,62 +157,57 @@ pub(super) async fn flush_thread(
|
||||
sync_materialized_rollout_path(store, thread_id).await
|
||||
}
|
||||
|
||||
pub(super) async fn shutdown_thread(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let recorder = store.live_recorder(thread_id).await?;
|
||||
let store = store.clone();
|
||||
// The writer can close before this future removes its live-recorder entry. Keep those steps
|
||||
// in one detached task so canceling an arbitrary ThreadStore caller cannot strand a closed
|
||||
// recorder that blocks a later resume.
|
||||
tokio::spawn(async move {
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
recorder.shutdown().await.map_err(thread_store_io_error)?;
|
||||
sync_materialized_rollout_path(&store, thread_id).await?;
|
||||
if let Some(metrics) = codex_otel::global()
|
||||
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
|
||||
{
|
||||
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
|
||||
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
|
||||
}
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
Ok(())
|
||||
pub(super) fn shutdown_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
let generation_cutoff = store.live_recorder_generation_cutoff();
|
||||
ThreadStoreCleanup::spawn("shutdown", thread_id, async move {
|
||||
let recorder = store
|
||||
.live_recorder_before_generation(thread_id, generation_cutoff)
|
||||
.await?;
|
||||
shutdown_cleanup(store, thread_id, recorder).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"),
|
||||
})?
|
||||
}
|
||||
|
||||
pub(super) async fn discard_thread(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let recorder = match store.live_recorder(thread_id).await {
|
||||
Ok(recorder) => recorder,
|
||||
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
// Discard closes stale recorder clones before removing the lease. Keep that sequence alive if
|
||||
// an arbitrary ThreadStore caller is canceled between the two steps.
|
||||
spawn_discard_cleanup(store.clone(), thread_id, recorder)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("discard cleanup task failed for thread {thread_id}: {err}"),
|
||||
})?;
|
||||
Ok(())
|
||||
pub(super) fn discard_thread(store: LocalThreadStore, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
let generation_cutoff = store.live_recorder_generation_cutoff();
|
||||
ThreadStoreCleanup::spawn("discard", thread_id, async move {
|
||||
let recorder = match store
|
||||
.live_recorder_before_generation(thread_id, generation_cutoff)
|
||||
.await
|
||||
{
|
||||
Ok(recorder) => recorder,
|
||||
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
discard_cleanup(store, thread_id, recorder).await
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn spawn_discard_cleanup(
|
||||
pub(super) async fn shutdown_cleanup(
|
||||
store: LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
recorder: Arc<RolloutRecorder>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
recorder.discard().await;
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
})
|
||||
) -> ThreadStoreResult<()> {
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
recorder.shutdown().await.map_err(thread_store_io_error)?;
|
||||
sync_materialized_rollout_path(&store, thread_id).await?;
|
||||
if let Some(metrics) = codex_otel::global()
|
||||
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
|
||||
{
|
||||
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
|
||||
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
|
||||
}
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn discard_cleanup(
|
||||
store: LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
recorder: Arc<RolloutRecorder>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
recorder.discard().await;
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn remove_live_recorder_if_current(
|
||||
|
||||
@@ -20,6 +20,8 @@ use std::collections::HashMap;
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::AppendThreadItemsParams;
|
||||
@@ -37,6 +39,7 @@ use crate::StoredThreadHistory;
|
||||
use crate::ThreadPage;
|
||||
use crate::ThreadSearchPage;
|
||||
use crate::ThreadStore;
|
||||
use crate::ThreadStoreCleanup;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreFuture;
|
||||
use crate::ThreadStoreResult;
|
||||
@@ -59,11 +62,15 @@ use crate::UpdateThreadMetadataParams;
|
||||
pub struct LocalThreadStore {
|
||||
pub(super) config: LocalThreadStoreConfig,
|
||||
live_recorders: Arc<Mutex<HashMap<ThreadId, LiveRecorderEntry>>>,
|
||||
next_live_recorder_generation: Arc<AtomicU64>,
|
||||
state_db: Option<StateDbHandle>,
|
||||
}
|
||||
|
||||
struct LiveRecorderEntry {
|
||||
recorder: Arc<RolloutRecorder>,
|
||||
// Detached terminal cleanup captures an exclusive cutoff before it spawns, so a delayed task
|
||||
// cannot bind itself to a writer inserted after the cleanup request.
|
||||
generation: u64,
|
||||
// Local rollout files are materialized lazily, but metadata updates can arrive before the
|
||||
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
|
||||
// so missing SQLite rows can still be seeded.
|
||||
@@ -106,6 +113,7 @@ impl LocalThreadStore {
|
||||
Self {
|
||||
config,
|
||||
live_recorders: Arc::new(Mutex::new(HashMap::new())),
|
||||
next_live_recorder_generation: Arc::new(AtomicU64::new(0)),
|
||||
state_db,
|
||||
}
|
||||
}
|
||||
@@ -148,6 +156,25 @@ impl LocalThreadStore {
|
||||
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
|
||||
}
|
||||
|
||||
/// Returns the exclusive generation bound for a terminal cleanup requested now.
|
||||
pub(super) fn live_recorder_generation_cutoff(&self) -> u64 {
|
||||
self.next_live_recorder_generation.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(super) async fn live_recorder_before_generation(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
generation_cutoff: u64,
|
||||
) -> ThreadStoreResult<Arc<RolloutRecorder>> {
|
||||
self.live_recorders
|
||||
.lock()
|
||||
.await
|
||||
.get(&thread_id)
|
||||
.filter(|entry| entry.generation < generation_cutoff)
|
||||
.map(|entry| Arc::clone(&entry.recorder))
|
||||
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
|
||||
}
|
||||
|
||||
pub(super) async fn ensure_live_recorder_absent(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
@@ -171,8 +198,12 @@ impl LocalThreadStore {
|
||||
message: format!("thread {} already has a live local writer", entry.key()),
|
||||
}),
|
||||
Entry::Vacant(entry) => {
|
||||
let generation = self
|
||||
.next_live_recorder_generation
|
||||
.fetch_add(1, Ordering::SeqCst);
|
||||
entry.insert(LiveRecorderEntry {
|
||||
recorder: Arc::new(recorder),
|
||||
generation,
|
||||
history_mode,
|
||||
});
|
||||
Ok(())
|
||||
@@ -262,12 +293,12 @@ impl ThreadStore for LocalThreadStore {
|
||||
Box::pin(async move { live_writer::flush_thread(self, thread_id).await })
|
||||
}
|
||||
|
||||
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
|
||||
Box::pin(async move { live_writer::shutdown_thread(self, thread_id).await })
|
||||
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
live_writer::shutdown_thread(self.clone(), thread_id)
|
||||
}
|
||||
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()> {
|
||||
Box::pin(async move { live_writer::discard_thread(self, thread_id).await })
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup {
|
||||
live_writer::discard_thread(self.clone(), thread_id)
|
||||
}
|
||||
|
||||
fn load_history(
|
||||
@@ -915,7 +946,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() {
|
||||
async fn discard_cleanup_handle_removes_closed_writer_after_caller_cancels() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
@@ -937,7 +968,11 @@ mod tests {
|
||||
.recv()
|
||||
.expect("receive live recorder lock release");
|
||||
});
|
||||
let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder);
|
||||
let cleanup = ThreadStoreCleanup::spawn(
|
||||
"discard",
|
||||
thread_id,
|
||||
live_writer::discard_cleanup(store.clone(), thread_id, recorder),
|
||||
);
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
@@ -970,7 +1005,118 @@ mod tests {
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("detached discard cleanup should release the live writer");
|
||||
.expect("discard cleanup handle should release the live writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cleanup_handle_removes_closed_writer_after_caller_cancels() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
store
|
||||
.create_thread(create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create live thread");
|
||||
let recorder = store
|
||||
.live_recorder(thread_id)
|
||||
.await
|
||||
.expect("clone live recorder");
|
||||
let stale_recorder = Arc::clone(&recorder);
|
||||
let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await;
|
||||
let (release_lock, lock_released) = std::sync::mpsc::channel();
|
||||
let lock_holder = tokio::task::spawn_blocking(move || {
|
||||
let _live_recorders = live_recorders;
|
||||
lock_released
|
||||
.recv()
|
||||
.expect("receive live recorder lock release");
|
||||
});
|
||||
let cleanup = ThreadStoreCleanup::spawn(
|
||||
"shutdown",
|
||||
thread_id,
|
||||
live_writer::shutdown_cleanup(store.clone(), thread_id, recorder),
|
||||
);
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if stale_recorder
|
||||
.record_canonical_items(&[user_message_item("write through stale recorder")])
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("shutdown should close the stale recorder");
|
||||
drop(cleanup);
|
||||
release_lock.send(()).expect("release live recorder lock");
|
||||
lock_holder.await.expect("join live recorder lock holder");
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if matches!(
|
||||
store.live_rollout_path(thread_id).await,
|
||||
Err(ThreadStoreError::ThreadNotFound { thread_id: missing })
|
||||
if missing == thread_id
|
||||
) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("shutdown cleanup handle should release the live writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_cleanup_generation_does_not_discard_replacement_writer() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
store
|
||||
.create_thread(create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create initial live thread");
|
||||
let stale_generation_cutoff = store.live_recorder_generation_cutoff();
|
||||
store
|
||||
.discard_thread(thread_id)
|
||||
.await
|
||||
.expect("discard initial live thread");
|
||||
store
|
||||
.create_thread(create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create replacement live thread");
|
||||
|
||||
let delayed_store = store.clone();
|
||||
ThreadStoreCleanup::spawn("discard", thread_id, async move {
|
||||
let recorder = match delayed_store
|
||||
.live_recorder_before_generation(thread_id, stale_generation_cutoff)
|
||||
.await
|
||||
{
|
||||
Ok(recorder) => recorder,
|
||||
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
live_writer::discard_cleanup(delayed_store, thread_id, recorder).await
|
||||
})
|
||||
.await
|
||||
.expect("stale cleanup should not touch the replacement writer");
|
||||
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
items: vec![user_message_item("write through replacement writer")],
|
||||
})
|
||||
.await
|
||||
.expect("replacement writer should remain live");
|
||||
store
|
||||
.discard_thread(thread_id)
|
||||
.await
|
||||
.expect("discard replacement live thread");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -3,6 +3,9 @@ use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use std::any::Any;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::AppendThreadItemsParams;
|
||||
use crate::ArchiveThreadParams;
|
||||
@@ -29,6 +32,54 @@ use crate::UpdateThreadMetadataParams;
|
||||
/// Future returned by [`ThreadStore`] operations.
|
||||
pub type ThreadStoreFuture<'a, T> = Pin<Box<dyn Future<Output = ThreadStoreResult<T>> + Send + 'a>>;
|
||||
|
||||
/// Handle for eagerly started terminal thread cleanup.
|
||||
///
|
||||
/// Awaiting the handle joins cleanup and reports its result. Dropping it detaches the underlying
|
||||
/// task instead of canceling cleanup, so terminal cleanup keeps running after its caller is
|
||||
/// canceled.
|
||||
#[must_use = "await this handle to observe terminal cleanup failures"]
|
||||
pub struct ThreadStoreCleanup {
|
||||
operation: &'static str,
|
||||
thread_id: ThreadId,
|
||||
task: JoinHandle<ThreadStoreResult<()>>,
|
||||
}
|
||||
|
||||
impl ThreadStoreCleanup {
|
||||
/// Starts terminal cleanup immediately on the current Tokio runtime.
|
||||
pub fn spawn(
|
||||
operation: &'static str,
|
||||
thread_id: ThreadId,
|
||||
cleanup: impl Future<Output = ThreadStoreResult<()>> + Send + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
operation,
|
||||
thread_id,
|
||||
task: tokio::spawn(cleanup),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Future for ThreadStoreCleanup {
|
||||
type Output = ThreadStoreResult<()>;
|
||||
|
||||
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
let this = self.get_mut();
|
||||
match Pin::new(&mut this.task).poll(cx) {
|
||||
Poll::Ready(Ok(result)) => Poll::Ready(result),
|
||||
Poll::Ready(Err(err)) => {
|
||||
let operation = this.operation;
|
||||
let thread_id = &this.thread_id;
|
||||
Poll::Ready(Err(ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"{operation} cleanup task failed for thread {thread_id}: {err}"
|
||||
),
|
||||
}))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Storage-neutral thread persistence boundary.
|
||||
pub trait ThreadStore: Any + Send + Sync {
|
||||
/// Return this store as [`Any`] for implementation-owned escape hatches.
|
||||
@@ -60,17 +111,21 @@ pub trait ThreadStore: Any + Send + Sync {
|
||||
/// Flushes all queued items and returns once they are durable/readable.
|
||||
fn flush_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
|
||||
|
||||
/// Flushes pending items and closes the live thread writer.
|
||||
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
|
||||
/// Starts terminal cleanup that flushes pending items and closes the live thread writer.
|
||||
///
|
||||
/// This starts terminal cleanup before returning. Await the handle to observe completion; if
|
||||
/// the caller is canceled, dropping the handle detaches the terminal task instead of
|
||||
/// canceling it.
|
||||
fn shutdown_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup;
|
||||
|
||||
/// Discards the live thread writer without forcing pending in-memory items to become durable.
|
||||
///
|
||||
/// Core calls this when session initialization fails after a live writer has been created.
|
||||
/// This is an idempotent terminal release: implementations must preserve already-durable
|
||||
/// thread data, finish releasing the lease after cleanup starts even if the returned future
|
||||
/// is canceled or reports a diagnostic error, and prevent previously obtained writer handles
|
||||
/// from appending once it returns.
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
|
||||
/// thread data, keep releasing the lease after cleanup starts even if the returned handle is
|
||||
/// dropped, and prevent previously obtained writer handles from appending after cleanup
|
||||
/// completes.
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreCleanup;
|
||||
|
||||
/// Loads persisted history for resume, fork, rollback, and memory jobs.
|
||||
fn load_history(
|
||||
|
||||
Reference in New Issue
Block a user