mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
fix terminal thread persistence shutdown
This commit is contained in:
@@ -617,6 +617,47 @@ async fn emit_thread_stop_lifecycle(sess: &Session) {
|
||||
}
|
||||
}
|
||||
|
||||
enum ThreadPersistenceShutdown {
|
||||
Closed,
|
||||
DiscardedAfterError,
|
||||
}
|
||||
|
||||
async fn shutdown_thread_persistence(
|
||||
sess: &Session,
|
||||
failure_context: &str,
|
||||
) -> ThreadPersistenceShutdown {
|
||||
let Some(live_thread) = sess.live_thread().cloned() else {
|
||||
return ThreadPersistenceShutdown::Closed;
|
||||
};
|
||||
let failure_context = failure_context.to_owned();
|
||||
// Terminal cleanup must outlive the submission loop. If the loop is canceled while a
|
||||
// graceful shutdown is still waiting on I/O, the same task still reaches the discard
|
||||
// fallback and releases the live-writer lease.
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = live_thread.shutdown().await {
|
||||
warn!("{failure_context}: {err}");
|
||||
// A terminal session has no remaining owner that can retry a buffered writer.
|
||||
// Release the writer lease so a later resume can recover the durable prefix
|
||||
// instead of failing forever with a duplicate live writer.
|
||||
if let Err(discard_err) = live_thread.discard().await {
|
||||
warn!(
|
||||
"thread persistence discard reported an error after terminal release: {discard_err}"
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
"discarded thread persistence after shutdown error; buffered rollout items may be lost"
|
||||
);
|
||||
return ThreadPersistenceShutdown::DiscardedAfterError;
|
||||
}
|
||||
ThreadPersistenceShutdown::Closed
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("terminal thread persistence cleanup task failed: {err}");
|
||||
ThreadPersistenceShutdown::DiscardedAfterError
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
|
||||
shutdown_session_runtime(sess).await;
|
||||
info!("Shutting down Codex instance");
|
||||
@@ -634,12 +675,10 @@ pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
|
||||
|
||||
emit_thread_stop_lifecycle(sess.as_ref()).await;
|
||||
|
||||
// Gracefully flush and shutdown thread persistence on session end so tests
|
||||
// that inspect durable state do not race with the background writer.
|
||||
if let Some(live_thread) = sess.live_thread()
|
||||
&& let Err(e) = live_thread.shutdown().await
|
||||
{
|
||||
warn!("failed to shutdown thread persistence: {e}");
|
||||
if matches!(
|
||||
shutdown_thread_persistence(sess.as_ref(), "failed to shutdown thread persistence").await,
|
||||
ThreadPersistenceShutdown::DiscardedAfterError
|
||||
) {
|
||||
let event = Event {
|
||||
id: sub_id.clone(),
|
||||
msg: EventMsg::Error(ErrorEvent {
|
||||
@@ -851,11 +890,11 @@ pub(super) async fn submission_loop(
|
||||
if !shutdown_received {
|
||||
shutdown_session_runtime(&sess).await;
|
||||
emit_thread_stop_lifecycle(sess.as_ref()).await;
|
||||
if let Some(live_thread) = sess.live_thread()
|
||||
&& let Err(err) = live_thread.shutdown().await
|
||||
{
|
||||
warn!("failed to shutdown thread persistence after submission channel closed: {err}");
|
||||
}
|
||||
let _ = shutdown_thread_persistence(
|
||||
sess.as_ref(),
|
||||
"failed to shutdown thread persistence after submission channel closed",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
debug!("Agent loop exited");
|
||||
}
|
||||
|
||||
@@ -6991,6 +6991,149 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() {
|
||||
);
|
||||
}
|
||||
|
||||
async fn make_session_with_unflushable_thread_persistence() -> (
|
||||
tempfile::TempDir,
|
||||
Session,
|
||||
Arc<codex_thread_store::LocalThreadStore>,
|
||||
async_channel::Receiver<Event>,
|
||||
) {
|
||||
let codex_home = tempfile::tempdir().expect("create codex home");
|
||||
let (session, _turn_context, rx_event) = make_session_and_context_with_auth_config_home_and_rx(
|
||||
CodexAuth::from_api_key("Test API Key"),
|
||||
Vec::new(),
|
||||
codex_home.path(),
|
||||
|_config| {},
|
||||
)
|
||||
.await;
|
||||
let mut session = Arc::into_inner(session).expect("test session should be uniquely owned");
|
||||
let config = session.get_config().await;
|
||||
let store = Arc::new(codex_thread_store::LocalThreadStore::new(
|
||||
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
|
||||
/*state_db*/ None,
|
||||
));
|
||||
let thread_store: Arc<dyn codex_thread_store::ThreadStore> = store.clone();
|
||||
let live_thread = LiveThread::create(
|
||||
Arc::clone(&thread_store),
|
||||
CreateThreadParams {
|
||||
session_id: session.session_id(),
|
||||
thread_id: session.thread_id,
|
||||
extra_config: None,
|
||||
forked_from_id: None,
|
||||
parent_thread_id: None,
|
||||
source: SessionSource::Exec,
|
||||
thread_source: None,
|
||||
originator: "test_originator".to_string(),
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
model_provider: config.model_provider_id.clone(),
|
||||
memory_mode: if config.memories.generate_memories {
|
||||
ThreadMemoryMode::Enabled
|
||||
} else {
|
||||
ThreadMemoryMode::Disabled
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create thread persistence");
|
||||
let sessions_blocker_path = config.codex_home.join("sessions");
|
||||
std::fs::File::create(&sessions_blocker_path).expect("block sessions directory");
|
||||
live_thread
|
||||
.append_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
client_id: None,
|
||||
message: "buffered before failed shutdown".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
text_elements: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
))])
|
||||
.await
|
||||
.expect_err("blocked sessions directory should fail append");
|
||||
session.services.thread_store = thread_store;
|
||||
session.services.live_thread = Some(live_thread);
|
||||
(codex_home, session, store, rx_event)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_discards_unflushable_thread_persistence_before_completing() {
|
||||
let (_codex_home, session, store, rx_event) =
|
||||
make_session_with_unflushable_thread_persistence().await;
|
||||
let thread_id = session.thread_id;
|
||||
let config = session.get_config().await;
|
||||
let session = Arc::new(session);
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(4);
|
||||
let (_agent_status_tx, agent_status) = watch::channel(AgentStatus::PendingInit);
|
||||
let session_for_loop = Arc::clone(&session);
|
||||
let session_loop_handle = tokio::spawn(async move {
|
||||
submission_loop(session_for_loop, config, rx_sub).await;
|
||||
});
|
||||
let codex = Codex {
|
||||
tx_sub,
|
||||
rx_event,
|
||||
agent_status,
|
||||
session,
|
||||
session_loop_termination: session_loop_termination_from_handle(session_loop_handle),
|
||||
};
|
||||
|
||||
codex
|
||||
.submit(Op::Shutdown)
|
||||
.await
|
||||
.expect("submit terminal shutdown");
|
||||
|
||||
let mut saw_persistence_error = false;
|
||||
loop {
|
||||
let event = timeout(StdDuration::from_secs(2), codex.next_event())
|
||||
.await
|
||||
.expect("timeout waiting for shutdown event")
|
||||
.expect("receive shutdown event");
|
||||
match event.msg {
|
||||
EventMsg::Error(_) => saw_persistence_error = true,
|
||||
EventMsg::ShutdownComplete => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_persistence_error,
|
||||
"failed graceful shutdown should report its persistence error"
|
||||
);
|
||||
let err = store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect_err("terminal shutdown should release the live writer");
|
||||
assert!(
|
||||
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
|
||||
);
|
||||
codex.session_loop_termination.clone().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submission_loop_channel_close_discards_unflushable_thread_persistence() {
|
||||
let (_codex_home, session, store, _rx_event) =
|
||||
make_session_with_unflushable_thread_persistence().await;
|
||||
let thread_id = session.thread_id;
|
||||
let config = session.get_config().await;
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(1);
|
||||
drop(tx_sub);
|
||||
|
||||
submission_loop(Arc::new(session), config, rx_sub).await;
|
||||
|
||||
let err = store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect_err("channel close should release the live writer");
|
||||
assert!(
|
||||
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submission_loop_channel_close_runs_full_thread_teardown() {
|
||||
struct SessionStopMarker;
|
||||
|
||||
@@ -115,6 +115,9 @@ enum RolloutCmd {
|
||||
Shutdown {
|
||||
ack: oneshot::Sender<std::io::Result<()>>,
|
||||
},
|
||||
Discard {
|
||||
ack: oneshot::Sender<()>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Observable state for the background rollout writer task.
|
||||
@@ -1038,6 +1041,23 @@ impl RolloutRecorder {
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop the writer without draining buffered items.
|
||||
///
|
||||
/// This returns only after the writer has closed its command receiver, so stale recorder
|
||||
/// clones cannot enqueue writes after the caller releases its live-writer lease.
|
||||
pub async fn discard(&self) {
|
||||
let (tx_done, rx_done) = oneshot::channel();
|
||||
if self
|
||||
.tx
|
||||
.send(RolloutCmd::Discard { ack: tx_done })
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let _ = rx_done.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reject_unknown_thread_history_mode(value: &Value) -> std::io::Result<()> {
|
||||
@@ -1723,10 +1743,18 @@ async fn rollout_writer(
|
||||
cwd: PathBuf,
|
||||
rollout_path: PathBuf,
|
||||
) -> std::io::Result<()> {
|
||||
enum TerminalAck {
|
||||
Shutdown(oneshot::Sender<std::io::Result<()>>),
|
||||
Discard(oneshot::Sender<()>),
|
||||
}
|
||||
|
||||
let mut state = RolloutWriterState::new(file, deferred_log_file_info, meta, cwd, rollout_path);
|
||||
|
||||
// Process rollout commands
|
||||
while let Some(cmd) = rx.recv().await {
|
||||
let terminal_ack = loop {
|
||||
let Some(cmd) = rx.recv().await else {
|
||||
break None;
|
||||
};
|
||||
match cmd {
|
||||
RolloutCmd::AddItems(items) => {
|
||||
state.add_items(items);
|
||||
@@ -1740,14 +1768,28 @@ async fn rollout_writer(
|
||||
}
|
||||
RolloutCmd::Shutdown { ack } => match state.shutdown().await {
|
||||
Ok(()) => {
|
||||
let _ = ack.send(Ok(()));
|
||||
break;
|
||||
break Some(TerminalAck::Shutdown(ack));
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = ack.send(Err(err));
|
||||
}
|
||||
},
|
||||
RolloutCmd::Discard { ack } => {
|
||||
break Some(TerminalAck::Discard(ack));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
drop(state);
|
||||
drop(rx);
|
||||
match terminal_ack {
|
||||
Some(TerminalAck::Shutdown(ack)) => {
|
||||
let _ = ack.send(Ok(()));
|
||||
}
|
||||
Some(TerminalAck::Discard(ack)) => {
|
||||
let _ = ack.send(());
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
@@ -160,30 +161,66 @@ pub(super) async fn shutdown_thread(
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let recorder = store.live_recorder(thread_id).await?;
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
recorder.shutdown().await.map_err(thread_store_io_error)?;
|
||||
sync_materialized_rollout_path(store, thread_id).await?;
|
||||
if let Some(metrics) = codex_otel::global()
|
||||
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
|
||||
{
|
||||
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
|
||||
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
|
||||
}
|
||||
store.live_recorders.lock().await.remove(&thread_id);
|
||||
Ok(())
|
||||
let store = store.clone();
|
||||
tokio::spawn(async move {
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
recorder.shutdown().await.map_err(thread_store_io_error)?;
|
||||
sync_materialized_rollout_path(&store, thread_id).await?;
|
||||
if let Some(metrics) = codex_otel::global()
|
||||
&& let Ok(metadata) = tokio::fs::metadata(rollout_path).await
|
||||
{
|
||||
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
|
||||
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
|
||||
}
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("shutdown cleanup task failed for thread {thread_id}: {err}"),
|
||||
})?
|
||||
}
|
||||
|
||||
pub(super) async fn discard_thread(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<()> {
|
||||
store
|
||||
.live_recorders
|
||||
.lock()
|
||||
let recorder = match store.live_recorder(thread_id).await {
|
||||
Ok(recorder) => recorder,
|
||||
Err(ThreadStoreError::ThreadNotFound { .. }) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
spawn_discard_cleanup(store.clone(), thread_id, recorder)
|
||||
.await
|
||||
.remove(&thread_id)
|
||||
.map(|_| ())
|
||||
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("discard cleanup task failed for thread {thread_id}: {err}"),
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn spawn_discard_cleanup(
|
||||
store: LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
recorder: Arc<RolloutRecorder>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
recorder.discard().await;
|
||||
remove_live_recorder_if_current(&store, thread_id, &recorder).await;
|
||||
})
|
||||
}
|
||||
|
||||
async fn remove_live_recorder_if_current(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
recorder: &Arc<RolloutRecorder>,
|
||||
) {
|
||||
let mut live_recorders = store.live_recorders.lock().await;
|
||||
if live_recorders
|
||||
.get(&thread_id)
|
||||
.is_some_and(|entry| Arc::ptr_eq(&entry.recorder, recorder))
|
||||
{
|
||||
live_recorders.remove(&thread_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn rollout_path(
|
||||
|
||||
@@ -63,7 +63,7 @@ pub struct LocalThreadStore {
|
||||
}
|
||||
|
||||
struct LiveRecorderEntry {
|
||||
recorder: RolloutRecorder,
|
||||
recorder: Arc<RolloutRecorder>,
|
||||
// Local rollout files are materialized lazily, but metadata updates can arrive before the
|
||||
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
|
||||
// so missing SQLite rows can still be seeded.
|
||||
@@ -139,12 +139,12 @@ impl LocalThreadStore {
|
||||
pub(super) async fn live_recorder(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<RolloutRecorder> {
|
||||
) -> ThreadStoreResult<Arc<RolloutRecorder>> {
|
||||
self.live_recorders
|
||||
.lock()
|
||||
.await
|
||||
.get(&thread_id)
|
||||
.map(|entry| entry.recorder.clone())
|
||||
.map(|entry| Arc::clone(&entry.recorder))
|
||||
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ impl LocalThreadStore {
|
||||
}),
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(LiveRecorderEntry {
|
||||
recorder,
|
||||
recorder: Arc::new(recorder),
|
||||
history_mode,
|
||||
});
|
||||
Ok(())
|
||||
@@ -805,7 +805,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discard_thread_drops_unmaterialized_live_writer() {
|
||||
async fn discard_thread_stops_cloned_recorder_and_drops_unmaterialized_live_writer() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
@@ -818,11 +818,23 @@ mod tests {
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect("load rollout path");
|
||||
let stale_recorder = store
|
||||
.live_recorder(thread_id)
|
||||
.await
|
||||
.expect("clone live recorder");
|
||||
store
|
||||
.discard_thread(thread_id)
|
||||
.await
|
||||
.expect("discard live thread");
|
||||
store
|
||||
.discard_thread(thread_id)
|
||||
.await
|
||||
.expect("discard should be idempotent");
|
||||
|
||||
stale_recorder
|
||||
.record_canonical_items(&[user_message_item("write through stale recorder")])
|
||||
.await
|
||||
.expect_err("discard should stop stale recorder clones");
|
||||
assert!(
|
||||
!tokio::fs::try_exists(rollout_path.as_path())
|
||||
.await
|
||||
@@ -840,6 +852,127 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discard_thread_preserves_durable_prefix_for_resume() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
let first_store = LocalThreadStore::new(config.clone(), /*state_db*/ None);
|
||||
first_store
|
||||
.create_thread(create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create initial thread");
|
||||
first_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
items: vec![user_message_item("before discard")],
|
||||
})
|
||||
.await
|
||||
.expect("append durable prefix");
|
||||
first_store
|
||||
.persist_thread(thread_id)
|
||||
.await
|
||||
.expect("persist durable prefix");
|
||||
first_store
|
||||
.flush_thread(thread_id)
|
||||
.await
|
||||
.expect("flush durable prefix");
|
||||
let rollout_path = first_store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect("load rollout path");
|
||||
first_store
|
||||
.discard_thread(thread_id)
|
||||
.await
|
||||
.expect("discard live thread");
|
||||
|
||||
let resumed_store = LocalThreadStore::new(config, /*state_db*/ None);
|
||||
resumed_store
|
||||
.resume_thread(ResumeThreadParams {
|
||||
thread_id,
|
||||
rollout_path: None,
|
||||
history: None,
|
||||
include_archived: true,
|
||||
metadata: thread_metadata(),
|
||||
})
|
||||
.await
|
||||
.expect("resume live thread");
|
||||
resumed_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
items: vec![user_message_item("after resume")],
|
||||
})
|
||||
.await
|
||||
.expect("append resumed item");
|
||||
resumed_store
|
||||
.flush_thread(thread_id)
|
||||
.await
|
||||
.expect("flush resumed thread");
|
||||
|
||||
assert_rollout_contains_message(rollout_path.as_path(), "before discard").await;
|
||||
assert_rollout_contains_message(rollout_path.as_path(), "after resume").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detached_discard_cleanup_removes_closed_writer_after_caller_cancels() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
store
|
||||
.create_thread(create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create live thread");
|
||||
let recorder = store
|
||||
.live_recorder(thread_id)
|
||||
.await
|
||||
.expect("clone live recorder");
|
||||
let stale_recorder = Arc::clone(&recorder);
|
||||
let live_recorders = Arc::clone(&store.live_recorders).lock_owned().await;
|
||||
let (release_lock, lock_released) = std::sync::mpsc::channel();
|
||||
let lock_holder = tokio::task::spawn_blocking(move || {
|
||||
let _live_recorders = live_recorders;
|
||||
lock_released
|
||||
.recv()
|
||||
.expect("receive live recorder lock release");
|
||||
});
|
||||
let cleanup = live_writer::spawn_discard_cleanup(store.clone(), thread_id, recorder);
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if stale_recorder
|
||||
.record_canonical_items(&[user_message_item("write through stale recorder")])
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("discard should close the stale recorder");
|
||||
drop(cleanup);
|
||||
release_lock.send(()).expect("release live recorder lock");
|
||||
lock_holder.await.expect("join live recorder lock holder");
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if matches!(
|
||||
store.live_rollout_path(thread_id).await,
|
||||
Err(ThreadStoreError::ThreadNotFound { thread_id: missing })
|
||||
if missing == thread_id
|
||||
) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("detached discard cleanup should release the live writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resume_thread_reopens_live_writer_and_appends() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
@@ -66,8 +66,10 @@ pub trait ThreadStore: Any + Send + Sync {
|
||||
/// Discards the live thread writer without forcing pending in-memory items to become durable.
|
||||
///
|
||||
/// Core calls this when session initialization fails after a live writer has been created.
|
||||
/// Implementations should release any live writer resources for the thread while preserving
|
||||
/// already-durable thread data.
|
||||
/// This is an idempotent terminal release: implementations must preserve already-durable
|
||||
/// thread data, finish releasing the lease after cleanup starts even if the returned future
|
||||
/// is canceled or reports a diagnostic error, and prevent previously obtained writer handles
|
||||
/// from appending once it returns.
|
||||
fn discard_thread(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, ()>;
|
||||
|
||||
/// Loads persisted history for resume, fork, rollback, and memory jobs.
|
||||
|
||||
Reference in New Issue
Block a user