mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
fix terminal thread persistence shutdown
This commit is contained in:
@@ -617,6 +617,47 @@ async fn emit_thread_stop_lifecycle(sess: &Session) {
|
||||
}
|
||||
}
|
||||
|
||||
enum ThreadPersistenceShutdown {
|
||||
Closed,
|
||||
DiscardedAfterError,
|
||||
}
|
||||
|
||||
async fn shutdown_thread_persistence(
|
||||
sess: &Session,
|
||||
failure_context: &str,
|
||||
) -> ThreadPersistenceShutdown {
|
||||
let Some(live_thread) = sess.live_thread().cloned() else {
|
||||
return ThreadPersistenceShutdown::Closed;
|
||||
};
|
||||
let failure_context = failure_context.to_owned();
|
||||
// Terminal cleanup must outlive the submission loop. If the loop is canceled while a
|
||||
// graceful shutdown is still waiting on I/O, the same task still reaches the discard
|
||||
// fallback and releases the live-writer lease.
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = live_thread.shutdown().await {
|
||||
warn!("{failure_context}: {err}");
|
||||
// A terminal session has no remaining owner that can retry a buffered writer.
|
||||
// Release the writer lease so a later resume can recover the durable prefix
|
||||
// instead of failing forever with a duplicate live writer.
|
||||
if let Err(discard_err) = live_thread.discard().await {
|
||||
warn!(
|
||||
"thread persistence discard reported an error after terminal release: {discard_err}"
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
"discarded thread persistence after shutdown error; buffered rollout items may be lost"
|
||||
);
|
||||
return ThreadPersistenceShutdown::DiscardedAfterError;
|
||||
}
|
||||
ThreadPersistenceShutdown::Closed
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("terminal thread persistence cleanup task failed: {err}");
|
||||
ThreadPersistenceShutdown::DiscardedAfterError
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
|
||||
shutdown_session_runtime(sess).await;
|
||||
info!("Shutting down Codex instance");
|
||||
@@ -634,12 +675,10 @@ pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
|
||||
|
||||
emit_thread_stop_lifecycle(sess.as_ref()).await;
|
||||
|
||||
// Gracefully flush and shutdown thread persistence on session end so tests
|
||||
// that inspect durable state do not race with the background writer.
|
||||
if let Some(live_thread) = sess.live_thread()
|
||||
&& let Err(e) = live_thread.shutdown().await
|
||||
{
|
||||
warn!("failed to shutdown thread persistence: {e}");
|
||||
if matches!(
|
||||
shutdown_thread_persistence(sess.as_ref(), "failed to shutdown thread persistence").await,
|
||||
ThreadPersistenceShutdown::DiscardedAfterError
|
||||
) {
|
||||
let event = Event {
|
||||
id: sub_id.clone(),
|
||||
msg: EventMsg::Error(ErrorEvent {
|
||||
@@ -851,11 +890,11 @@ pub(super) async fn submission_loop(
|
||||
if !shutdown_received {
|
||||
shutdown_session_runtime(&sess).await;
|
||||
emit_thread_stop_lifecycle(sess.as_ref()).await;
|
||||
if let Some(live_thread) = sess.live_thread()
|
||||
&& let Err(err) = live_thread.shutdown().await
|
||||
{
|
||||
warn!("failed to shutdown thread persistence after submission channel closed: {err}");
|
||||
}
|
||||
let _ = shutdown_thread_persistence(
|
||||
sess.as_ref(),
|
||||
"failed to shutdown thread persistence after submission channel closed",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
debug!("Agent loop exited");
|
||||
}
|
||||
|
||||
@@ -6991,6 +6991,149 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() {
|
||||
);
|
||||
}
|
||||
|
||||
async fn make_session_with_unflushable_thread_persistence() -> (
|
||||
tempfile::TempDir,
|
||||
Session,
|
||||
Arc<codex_thread_store::LocalThreadStore>,
|
||||
async_channel::Receiver<Event>,
|
||||
) {
|
||||
let codex_home = tempfile::tempdir().expect("create codex home");
|
||||
let (session, _turn_context, rx_event) = make_session_and_context_with_auth_config_home_and_rx(
|
||||
CodexAuth::from_api_key("Test API Key"),
|
||||
Vec::new(),
|
||||
codex_home.path(),
|
||||
|_config| {},
|
||||
)
|
||||
.await;
|
||||
let mut session = Arc::into_inner(session).expect("test session should be uniquely owned");
|
||||
let config = session.get_config().await;
|
||||
let store = Arc::new(codex_thread_store::LocalThreadStore::new(
|
||||
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
|
||||
/*state_db*/ None,
|
||||
));
|
||||
let thread_store: Arc<dyn codex_thread_store::ThreadStore> = store.clone();
|
||||
let live_thread = LiveThread::create(
|
||||
Arc::clone(&thread_store),
|
||||
CreateThreadParams {
|
||||
session_id: session.session_id(),
|
||||
thread_id: session.thread_id,
|
||||
extra_config: None,
|
||||
forked_from_id: None,
|
||||
parent_thread_id: None,
|
||||
source: SessionSource::Exec,
|
||||
thread_source: None,
|
||||
originator: "test_originator".to_string(),
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
model_provider: config.model_provider_id.clone(),
|
||||
memory_mode: if config.memories.generate_memories {
|
||||
ThreadMemoryMode::Enabled
|
||||
} else {
|
||||
ThreadMemoryMode::Disabled
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create thread persistence");
|
||||
let sessions_blocker_path = config.codex_home.join("sessions");
|
||||
std::fs::File::create(&sessions_blocker_path).expect("block sessions directory");
|
||||
live_thread
|
||||
.append_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
client_id: None,
|
||||
message: "buffered before failed shutdown".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
text_elements: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
))])
|
||||
.await
|
||||
.expect_err("blocked sessions directory should fail append");
|
||||
session.services.thread_store = thread_store;
|
||||
session.services.live_thread = Some(live_thread);
|
||||
(codex_home, session, store, rx_event)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_discards_unflushable_thread_persistence_before_completing() {
|
||||
let (_codex_home, session, store, rx_event) =
|
||||
make_session_with_unflushable_thread_persistence().await;
|
||||
let thread_id = session.thread_id;
|
||||
let config = session.get_config().await;
|
||||
let session = Arc::new(session);
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(4);
|
||||
let (_agent_status_tx, agent_status) = watch::channel(AgentStatus::PendingInit);
|
||||
let session_for_loop = Arc::clone(&session);
|
||||
let session_loop_handle = tokio::spawn(async move {
|
||||
submission_loop(session_for_loop, config, rx_sub).await;
|
||||
});
|
||||
let codex = Codex {
|
||||
tx_sub,
|
||||
rx_event,
|
||||
agent_status,
|
||||
session,
|
||||
session_loop_termination: session_loop_termination_from_handle(session_loop_handle),
|
||||
};
|
||||
|
||||
codex
|
||||
.submit(Op::Shutdown)
|
||||
.await
|
||||
.expect("submit terminal shutdown");
|
||||
|
||||
let mut saw_persistence_error = false;
|
||||
loop {
|
||||
let event = timeout(StdDuration::from_secs(2), codex.next_event())
|
||||
.await
|
||||
.expect("timeout waiting for shutdown event")
|
||||
.expect("receive shutdown event");
|
||||
match event.msg {
|
||||
EventMsg::Error(_) => saw_persistence_error = true,
|
||||
EventMsg::ShutdownComplete => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_persistence_error,
|
||||
"failed graceful shutdown should report its persistence error"
|
||||
);
|
||||
let err = store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect_err("terminal shutdown should release the live writer");
|
||||
assert!(
|
||||
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
|
||||
);
|
||||
codex.session_loop_termination.clone().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submission_loop_channel_close_discards_unflushable_thread_persistence() {
|
||||
let (_codex_home, session, store, _rx_event) =
|
||||
make_session_with_unflushable_thread_persistence().await;
|
||||
let thread_id = session.thread_id;
|
||||
let config = session.get_config().await;
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(1);
|
||||
drop(tx_sub);
|
||||
|
||||
submission_loop(Arc::new(session), config, rx_sub).await;
|
||||
|
||||
let err = store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect_err("channel close should release the live writer");
|
||||
assert!(
|
||||
matches!(err, codex_thread_store::ThreadStoreError::ThreadNotFound { thread_id: missing } if missing == thread_id)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submission_loop_channel_close_runs_full_thread_teardown() {
|
||||
struct SessionStopMarker;
|
||||
|
||||
Reference in New Issue
Block a user