fix terminal thread persistence shutdown

This commit is contained in:
Anton Panasenko
2026-07-06 20:47:12 -07:00
parent b9b934e99b
commit 05eb6e5e8b
6 changed files with 434 additions and 38 deletions
+50 -11
View File
@@ -617,6 +617,47 @@ async fn emit_thread_stop_lifecycle(sess: &Session) {
}
}
enum ThreadPersistenceShutdown {
Closed,
DiscardedAfterError,
}
async fn shutdown_thread_persistence(
sess: &Session,
failure_context: &str,
) -> ThreadPersistenceShutdown {
let Some(live_thread) = sess.live_thread().cloned() else {
return ThreadPersistenceShutdown::Closed;
};
let failure_context = failure_context.to_owned();
// Terminal cleanup must outlive the submission loop. If the loop is canceled while a
// graceful shutdown is still waiting on I/O, the same task still reaches the discard
// fallback and releases the live-writer lease.
tokio::spawn(async move {
if let Err(err) = live_thread.shutdown().await {
warn!("{failure_context}: {err}");
// A terminal session has no remaining owner that can retry a buffered writer.
// Release the writer lease so a later resume can recover the durable prefix
// instead of failing forever with a duplicate live writer.
if let Err(discard_err) = live_thread.discard().await {
warn!(
"thread persistence discard reported an error after terminal release: {discard_err}"
);
}
warn!(
"discarded thread persistence after shutdown error; buffered rollout items may be lost"
);
return ThreadPersistenceShutdown::DiscardedAfterError;
}
ThreadPersistenceShutdown::Closed
})
.await
.unwrap_or_else(|err| {
warn!("terminal thread persistence cleanup task failed: {err}");
ThreadPersistenceShutdown::DiscardedAfterError
})
}
pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
shutdown_session_runtime(sess).await;
info!("Shutting down Codex instance");
@@ -634,12 +675,10 @@ pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
emit_thread_stop_lifecycle(sess.as_ref()).await;
// Gracefully flush and shutdown thread persistence on session end so tests
// that inspect durable state do not race with the background writer.
if let Some(live_thread) = sess.live_thread()
&& let Err(e) = live_thread.shutdown().await
{
warn!("failed to shutdown thread persistence: {e}");
if matches!(
shutdown_thread_persistence(sess.as_ref(), "failed to shutdown thread persistence").await,
ThreadPersistenceShutdown::DiscardedAfterError
) {
let event = Event {
id: sub_id.clone(),
msg: EventMsg::Error(ErrorEvent {
@@ -851,11 +890,11 @@ pub(super) async fn submission_loop(
if !shutdown_received {
shutdown_session_runtime(&sess).await;
emit_thread_stop_lifecycle(sess.as_ref()).await;
if let Some(live_thread) = sess.live_thread()
&& let Err(err) = live_thread.shutdown().await
{
warn!("failed to shutdown thread persistence after submission channel closed: {err}");
}
let _ = shutdown_thread_persistence(
sess.as_ref(),
"failed to shutdown thread persistence after submission channel closed",
)
.await;
}
debug!("Agent loop exited");
}
+143
View File
@@ -6991,6 +6991,149 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() {
);
}
async fn make_session_with_unflushable_thread_persistence() -> (
tempfile::TempDir,
Session,
Arc<codex_thread_store::LocalThreadStore>,
async_channel::Receiver<Event>,
) {
let codex_home = tempfile::tempdir().expect("create codex home");
let (session, _turn_context, rx_event) = make_session_and_context_with_auth_config_home_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
codex_home.path(),
|_config| {},
)
.await;
let mut session = Arc::into_inner(session).expect("test session should be uniquely owned");
let config = session.get_config().await;
let store = Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
/*state_db*/ None,
));
let thread_store: Arc<dyn codex_thread_store::ThreadStore> = store.clone();
let live_thread = LiveThread::create(
Arc::clone(&thread_store),
CreateThreadParams {
session_id: session.session_id(),
thread_id: session.thread_id,
extra_config: None,
forked_from_id: None,
parent_thread_id: None,
source: SessionSource::Exec,
thread_source: None,
originator: "test_originator".to_string(),
base_instructions: BaseInstructions::default(),
dynamic_tools: Vec::new(),
selected_capability_roots: Vec::new(),
multi_agent_version: None,
history_mode: Default::default(),
initial_window_id: Uuid::now_v7().to_string(),
metadata: ThreadPersistenceMetadata {
cwd: Some(config.cwd.to_path_buf()),
model_provider: config.model_provider_id.clone(),
memory_mode: if config.memories.generate_memories {
ThreadMemoryMode::Enabled
} else {
ThreadMemoryMode::Disabled
},
},
},
)
.await
.expect("create thread persistence");
let sessions_blocker_path = config.codex_home.join("sessions");
std::fs::File::create(&sessions_blocker_path).expect("block sessions directory");
live_thread
.append_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
UserMessageEvent {
client_id: None,
message: "buffered before failed shutdown".to_string(),
images: None,
local_images: Vec::new(),
text_elements: Vec::new(),
..Default::default()
},
))])
.await
.expect_err("blocked sessions directory should fail append");
session.services.thread_store = thread_store;
session.services.live_thread = Some(live_thread);
(codex_home, session, store, rx_event)
}
#[tokio::test]
async fn shutdown_discards_unflushable_thread_persistence_before_completing() {
let (_codex_home, session, store, rx_event) =
make_session_with_unflushable_thread_persistence().await;
let thread_id = session.thread_id;
let config = session.get_config().await;
let session = Arc::new(session);
let (tx_sub, rx_sub) = async_channel::bounded(4);
let (_agent_status_tx, agent_status) = watch::channel(AgentStatus::PendingInit);
let session_for_loop = Arc::clone(&session);
let session_loop_handle = tokio::spawn(async move {
submission_loop(session_for_loop, config, rx_sub).await;
});
let codex = Codex {
tx_sub,
rx_event,
agent_status,
session,
session_loop_termination: session_loop_termination_from_handle(session_loop_handle),
};
codex
.submit(Op::Shutdown)
.await
.expect("submit terminal shutdown");
let mut saw_persistence_error = false;
loop {
let event = timeout(StdDuration::from_secs(2), codex.next_event())
.await
.expect("timeout waiting for shutdown event")
.expect("receive shutdown event");
match event.msg {
EventMsg::Error(_) => saw_persistence_error = true,
EventMsg::ShutdownComplete => break,
_ => {}
}
}
assert!(
saw_persistence_error,
"failed graceful shutdown should report its persistence error"
);
let err = store
.live_rollout_path(thread_id)
.await
.expect_err("terminal shutdown should release the live writer");
assert!(
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
);
codex.session_loop_termination.clone().await;
}
#[tokio::test]
async fn submission_loop_channel_close_discards_unflushable_thread_persistence() {
let (_codex_home, session, store, _rx_event) =
make_session_with_unflushable_thread_persistence().await;
let thread_id = session.thread_id;
let config = session.get_config().await;
let (tx_sub, rx_sub) = async_channel::bounded(1);
drop(tx_sub);
submission_loop(Arc::new(session), config, rx_sub).await;
let err = store
.live_rollout_path(thread_id)
.await
.expect_err("channel close should release the live writer");
assert!(
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
);
}
#[tokio::test]
async fn submission_loop_channel_close_runs_full_thread_teardown() {
struct SessionStopMarker;
+45 -3
View File
@@ -115,6 +115,9 @@ enum RolloutCmd {
Shutdown {
ack: oneshot::Sender<std::io::Result<()>>,
},
Discard {
ack: oneshot::Sender<()>,
},
}
/// Observable state for the background rollout writer task.
@@ -1038,6 +1041,23 @@ impl RolloutRecorder {
};
Ok(())
}
/// Stop the writer without draining buffered items.
///
/// This returns only after the writer has closed its command receiver, so stale recorder
/// clones cannot enqueue writes after the caller releases its live-writer lease.
pub async fn discard(&self) {
let (tx_done, rx_done) = oneshot::channel();
if self
.tx
.send(RolloutCmd::Discard { ack: tx_done })
.await
.is_err()
{
return;
}
let _ = rx_done.await;
}
}
pub(crate) fn reject_unknown_thread_history_mode(value: &Value) -> std::io::Result<()> {
@@ -1723,10 +1743,18 @@ async fn rollout_writer(
cwd: PathBuf,
rollout_path: PathBuf,
) -> std::io::Result<()> {
enum TerminalAck {
Shutdown(oneshot::Sender<std::io::Result<()>>),
Discard(oneshot::Sender<()>),
}
let mut state = RolloutWriterState::new(file, deferred_log_file_info, meta, cwd, rollout_path);
// Process rollout commands
while let Some(cmd) = rx.recv().await {
let terminal_ack = loop {
let Some(cmd) = rx.recv().await else {
break None;
};
match cmd {
RolloutCmd::AddItems(items) => {
state.add_items(items);
@@ -1740,14 +1768,28 @@ async fn rollout_writer(
}
RolloutCmd::Shutdown { ack } => match state.shutdown().await {
Ok(()) => {
let _ = ack.send(Ok(()));
break;
break Some(TerminalAck::Shutdown(ack));
}
Err(err) => {
let _ = ack.send(Err(err));
}
},
RolloutCmd::Discard { ack } => {
break Some(TerminalAck::Discard(ack));
}
}
};
drop(state);
drop(rx);
match terminal_ack {
Some(TerminalAck::Shutdown(ack)) => {
let _ = ack.send(Ok(()));
}
Some(TerminalAck::Discard(ack)) => {
let _ = ack.send(());
}
None => {}
}
Ok(())
+54 -17
View File
@@ -1,4 +1,5 @@
use std::path::PathBuf;
use std::sync::Arc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::ThreadMemoryMode;
@@ -160,30 +161,66 @@ pub(super) async fn shutdown_thread(
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
let recorder = store.live_recorder(thread_id).await?;
let rollout_path = recorder.rollout_path().to_path_buf();
recorder.shutdown().await.map_err(thread_store_io_error)?;
sync_materialized_rollout_path(store, thread_id).await?;
if let Some(metrics) = codex_otel::global()
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
{
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
}
store.live_recorders.lock().await.remove(&thread_id);
Ok(())
let store = store.clone();
tokio::spawn(async move {
let rollout_path = recorder.rollout_path().to_path_buf();
recorder.shutdown().await.map_err(thread_store_io_error)?;
sync_materialized_rollout_path(&store, thread_id).await?;
if let Some(metrics) = codex_otel::global()
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
{
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
}
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
Ok(())
})
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"),
})?
}
pub(super) async fn discard_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
store
.live_recorders
.lock()
let recorder = match store.live_recorder(thread_id).await {
Ok(recorder) => recorder,
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
Err(err) => return Err(err),
};
spawn_discard_cleanup(store.clone(), thread_id, recorder)
.await
.remove(&thread_id)
.map(|_| ())
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
.map_err(|err| ThreadStoreError::Internal {
message: format!("discard cleanup task failed for thread {thread_id}: {err}"),
})?;
Ok(())
}
pub(super) fn spawn_discard_cleanup(
store: LocalThreadStore,
thread_id: ThreadId,
recorder: Arc<RolloutRecorder>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
recorder.discard().await;
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
})
}
async fn remove_live_recorder_if_current(
store: &LocalThreadStore,
thread_id: ThreadId,
recorder: &Arc<RolloutRecorder>,
) {
let mut live_recorders = store.live_recorders.lock().await;
if live_recorders
.get(&thread_id)
.is_some_and(|entry| Arc::ptr_eq(&entry.recorder, recorder))
{
live_recorders.remove(&thread_id);
}
}
pub(super) async fn rollout_path(
+138 -5
View File
@@ -63,7 +63,7 @@ pub struct LocalThreadStore {
}
struct LiveRecorderEntry {
recorder: RolloutRecorder,
recorder: Arc<RolloutRecorder>,
// Local rollout files are materialized lazily, but metadata updates can arrive before the
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
// so missing SQLite rows can still be seeded.
@@ -139,12 +139,12 @@ impl LocalThreadStore {
pub(super) async fn live_recorder(
&self,
thread_id: ThreadId,
) -> ThreadStoreResult<RolloutRecorder> {
) -> ThreadStoreResult<Arc<RolloutRecorder>> {
self.live_recorders
.lock()
.await
.get(&thread_id)
.map(|entry| entry.recorder.clone())
.map(|entry| Arc::clone(&entry.recorder))
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
}
@@ -172,7 +172,7 @@ impl LocalThreadStore {
}),
Entry::Vacant(entry) => {
entry.insert(LiveRecorderEntry {
recorder,
recorder: Arc::new(recorder),
history_mode,
});
Ok(())
@@ -805,7 +805,7 @@ mod tests {
}
#[tokio::test]
async fn discard_thread_drops_unmaterialized_live_writer() {
async fn discard_thread_stops_cloned_recorder_and_drops_unmaterialized_live_writer() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
@@ -818,11 +818,23 @@ mod tests {
.live_rollout_path(thread_id)
.await
.expect("load rollout path");
let stale_recorder = store
.live_recorder(thread_id)
.await
.expect("clone live recorder");
store
.discard_thread(thread_id)
.await
.expect("discard live thread");
store
.discard_thread(thread_id)
.await
.expect("discard should be idempotent");
stale_recorder
.record_canonical_items(&[user_message_item("write through stale recorder")])
.await
.expect_err("discard should stop stale recorder clones");
assert!(
!tokio::fs::try_exists(rollout_path.as_path())
.await
@@ -840,6 +852,127 @@ mod tests {
);
}
#[tokio::test]
async fn discard_thread_preserves_durable_prefix_for_resume() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let thread_id = ThreadId::default();
let first_store = LocalThreadStore::new(config.clone(), /*state_db*/ None);
first_store
.create_thread(create_thread_params(thread_id))
.await
.expect("create initial thread");
first_store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![user_message_item("before discard")],
})
.await
.expect("append durable prefix");
first_store
.persist_thread(thread_id)
.await
.expect("persist durable prefix");
first_store
.flush_thread(thread_id)
.await
.expect("flush durable prefix");
let rollout_path = first_store
.live_rollout_path(thread_id)
.await
.expect("load rollout path");
first_store
.discard_thread(thread_id)
.await
.expect("discard live thread");
let resumed_store = LocalThreadStore::new(config, /*state_db*/ None);
resumed_store
.resume_thread(ResumeThreadParams {
thread_id,
rollout_path: None,
history: None,
include_archived: true,
metadata: thread_metadata(),
})
.await
.expect("resume live thread");
resumed_store
.append_items(AppendThreadItemsParams {
thread_id,
items: vec![user_message_item("after resume")],
})
.await
.expect("append resumed item");
resumed_store
.flush_thread(thread_id)
.await
.expect("flush resumed thread");
assert_rollout_contains_message(rollout_path.as_path(), "before discard").await;
assert_rollout_contains_message(rollout_path.as_path(), "after resume").await;
}
#[tokio::test]
async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
store
.create_thread(create_thread_params(thread_id))
.await
.expect("create live thread");
let recorder = store
.live_recorder(thread_id)
.await
.expect("clone live recorder");
let stale_recorder = Arc::clone(&recorder);
let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await;
let (release_lock, lock_released) = std::sync::mpsc::channel();
let lock_holder = tokio::task::spawn_blocking(move || {
let _live_recorders = live_recorders;
lock_released
.recv()
.expect("receive live recorder lock release");
});
let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder);
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
if stale_recorder
.record_canonical_items(&[user_message_item("write through stale recorder")])
.await
.is_err()
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("discard should close the stale recorder");
drop(cleanup);
release_lock.send(()).expect("release live recorder lock");
lock_holder.await.expect("join live recorder lock holder");
tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
if matches!(
store.live_rollout_path(thread_id).await,
Err(ThreadStoreError::ThreadNotFound { thread_id: missing })
if missing == thread_id
) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("detached discard cleanup should release the live writer");
}
#[tokio::test]
async fn resume_thread_reopens_live_writer_and_appends() {
let home = TempDir::new().expect("temp dir");
+4 -2
View File
@@ -66,8 +66,10 @@ pub trait ThreadStore: Any + Send + Sync {
/// Discards the live thread writer without forcing pending in-memory items to become durable.
///
/// Core calls this when session initialization fails after a live writer has been created.
/// Implementations should release any live writer resources for the thread while preserving
/// already-durable thread data.
/// This is an idempotent terminal release: implementations must preserve already-durable
/// thread data, finish releasing the lease after cleanup starts even if the returned future
/// is canceled or reports a diagnostic error, and prevent previously obtained writer handles
/// from appending once it returns.
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
/// Loads persisted history for resume, fork, rollback, and memory jobs.