Files
codex/codex-rs/thread-store/src/local/live_writer.rs
T
Zahan Malkani bd00a8a42e Preserve pending realtime speech when a call ends (#62299)
## Why

A realtime call can end with completed speech that has yet to reach a
Codex turn. The next turn needs that context even if the call's transport
has already disconnected.

## What changed

- For V3 calls with realtime frame history enabled, save pending speech
  as context for the next turn. Restore it on thread reload and exclude
  it from the turn count used by rollback.
- Drain already received transcript events and settle pending handoffs
  before saving the call's closing marker. Report recovery required if
  shutdown cannot confirm that history was saved.
- Keep transport detach separate from ending a call. Reattaching restores
  saved pending speech, and an open call can be ended from persisted
  history after a reload. Use the call ID for TUI attachments so successive
  calls on one thread have distinct history.

## Testing

Add coverage for queued speech at hangup, reconnect and stop after
reload, retries after an uncertain write, and rollback across saved
session-end context.

GitOrigin-RevId: 657491f1d7e446a1a48e22189758810c8bdd82cb
2026-11-18 15:24:29 +00:00

592 lines
21 KiB
Rust

use std::path::PathBuf;
use std::sync::Arc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutRecorder;
use codex_rollout::RolloutRecorderParams;
use codex_rollout::into_persisted_rollout_items;
use tracing::warn;
use super::LocalThreadStore;
use super::create_thread;
use crate::AppendThreadItemsParams;
use crate::CreateThreadParams;
use crate::ResumeThreadParams;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
use crate::types::canonical_history_mode_from_rollout_items;
const ROLLOUT_SIZE_BYTES_METRIC: &str = "codex.rollout.size_bytes";
pub(super) async fn create_thread(
store: &LocalThreadStore,
params: CreateThreadParams,
) -> ThreadStoreResult<()> {
let thread_id = params.thread_id;
let inherited_cursor = if params.initial_window_id.allocation().is_some()
&& let (Some(parent), Some(base)) = (params.forked_from_id, params.history_base)
{
let lineage = store.resolve_rollout_lineage_for_reference(parent).await?;
let prefix = super::model_context::load_for_fork(lineage, Some(base)).await?;
codex_rollout::next_model_id_ordinals(&prefix).0
} else {
0
};
let attachment = if params.attached_thread {
let parent = params
.forked_from_id
.filter(|parent| *parent != thread_id)
.ok_or_else(|| ThreadStoreError::InvalidRequest {
message: "attached threads require a distinct fork source".to_string(),
})?;
let state = store
.state_db
.as_ref()
.ok_or(ThreadStoreError::Unsupported {
operation: "attached_threads",
})?;
// Keep parent deletion out until the child's canonical metadata is durable.
let source_ownership = (
store.live_writer_locks.lock(parent).await,
store.acquire_writer_locks(&[parent]).await?,
);
super::thread_rollout_resolver::resolve_current_including_archived(store, parent)
.await?
.ok_or(ThreadStoreError::ThreadNotFound { thread_id: parent })?;
Some((state, source_ownership))
} else {
None
};
let mut cursor = store.live_writer_locks.lock(thread_id).await;
let history_mode = params.history_mode;
store.ensure_live_recorder_absent(thread_id).await?;
let writer_lock = store.acquire_writer_lock(thread_id)?;
let recorder = create_thread::create_thread(store, params, writer_lock.clone()).await?;
if let Some((state, _source_ownership)) = attachment {
recorder.persist().await.map_err(thread_store_io_error)?;
if let Err(error) = index_attached_thread(store, state, recorder.rollout_path()).await {
recorder.discard().await.map_err(thread_store_io_error)?;
super::delete_thread::delete_rollout_file(store, recorder.rollout_path())?;
super::delete_thread::delete_state_rows(store, &[thread_id]).await?;
return Err(error);
}
}
store
.insert_live_recorder(thread_id, recorder, thread_id, history_mode, writer_lock)
.await?;
*cursor = inherited_cursor;
Ok(())
}
pub(super) async fn resume_thread(
store: &LocalThreadStore,
params: ResumeThreadParams,
) -> ThreadStoreResult<Arc<Vec<RolloutItem>>> {
let mut cursor = store.live_writer_locks.lock(params.thread_id).await;
store.ensure_live_recorder_absent(params.thread_id).await?;
let writer_lock = store.acquire_writer_lock(params.thread_id)?;
let rollout_path = match params.rollout_path {
Some(rollout_path) => rollout_path,
None => {
let resolved = if params.include_archived {
super::thread_rollout_resolver::resolve_current_including_archived(
store,
params.thread_id,
)
.await?
} else {
super::thread_rollout_resolver::resolve_current(store, params.thread_id).await?
};
resolved
.ok_or(ThreadStoreError::ThreadNotFound {
thread_id: params.thread_id,
})?
.path
}
};
let rollout_path =
super::read_thread::resolve_requested_rollout_path(store, rollout_path).await?;
if !params.include_archived
&& super::helpers::rollout_path_is_archived(&store.config.codex_home, &rollout_path)
{
return Err(ThreadStoreError::InvalidRequest {
message: format!("thread {} is archived", params.thread_id),
});
}
let history = match params.history {
Some(history)
if !matches!(
history.first(),
Some(RolloutItem::SessionMeta(meta)) if meta.meta.id == params.thread_id
) =>
{
history
}
Some(history)
if params.history_revision.is_some()
&& params.history_revision
== super::history_revision::read(&rollout_path).await =>
{
history
}
_ => Arc::new(
super::model_context::load_from_rollout_path(store, params.thread_id, &rollout_path)
.await?
.items,
),
};
let history_mode = canonical_history_mode_from_rollout_items(&history);
let cwd = params
.metadata
.cwd
.clone()
.ok_or_else(|| ThreadStoreError::InvalidRequest {
message: "local thread store requires a cwd".to_string(),
})?;
let config = RolloutConfig {
codex_home: store.config.codex_home.clone(),
sqlite: store.config.sqlite.clone(),
cwd,
model_provider_id: params.metadata.model_provider.clone(),
generate_memories: matches!(params.metadata.memory_mode, ThreadMemoryMode::Enabled),
};
let rollout_id = super::thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id(
rollout_path.as_path(),
params.thread_id,
history_mode,
)?;
let recorder = RolloutRecorder::new_with_writer_lock(
&config,
RolloutRecorderParams::resume(rollout_path),
writer_lock.clone(),
)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to resume local thread recorder: {err}"),
})?;
store
.insert_live_recorder(
params.thread_id,
recorder,
rollout_id,
history_mode,
writer_lock,
)
.await?;
*cursor = codex_rollout::next_model_id_ordinals(&history).0;
Ok(history)
}
#[tracing::instrument(
level = "trace",
skip_all,
fields(item_count = params.items.len())
)]
pub(super) async fn append_items(
store: &LocalThreadStore,
params: AppendThreadItemsParams,
) -> ThreadStoreResult<()> {
write_and_project(
store,
params.thread_id,
RolloutWriteOp::AppendItems(params.items),
)
.await
}
pub(super) async fn seal_realtime_items(
store: &LocalThreadStore,
thread_id: ThreadId,
items: Vec<codex_protocol::realtime::RealtimeItem>,
) -> ThreadStoreResult<()> {
let id = items
.last()
.filter(|item| {
matches!(
item.content,
codex_protocol::realtime::RealtimeItemContent::SessionInputSaved {
reason: codex_protocol::realtime::SessionInputReason::SessionEnd,
..
}
)
})
.map(|item| item.id.clone());
let records = items.into_iter().map(RolloutItem::RealtimeItem).collect();
if let Some(id) = id {
seal_realtime_records(store, thread_id, id, records).await
} else {
append_items(
store,
AppendThreadItemsParams {
thread_id,
items: records,
},
)
.await?;
flush_thread(store, thread_id).await
}
}
pub(super) async fn seal_realtime_input(
store: &LocalThreadStore,
thread_id: ThreadId,
frames: Vec<codex_protocol::realtime::RealtimeItem>,
input: codex_rollout::ResponseItemEnvelope,
) -> ThreadStoreResult<()> {
let id = input
.realtime_handoff_item()
.ok_or_else(|| ThreadStoreError::InvalidRequest {
message: "realtime input is missing handoff metadata".to_owned(),
})?
.id;
let records = frames
.into_iter()
.map(RolloutItem::RealtimeItem)
.chain(std::iter::once(RolloutItem::ResponseItem(input)))
.collect();
seal_realtime_records(store, thread_id, id, records).await
}
/// An uncertain flush keeps ownership of the already queued model-facing seal.
/// Retrying confirms that batch instead of appending it again.
async fn seal_realtime_records(
store: &LocalThreadStore,
thread_id: ThreadId,
id: String,
items: Vec<RolloutItem>,
) -> ThreadStoreResult<()> {
let mut cursor = store.live_writer_locks.lock(thread_id).await;
let (recorder, rollout_id, history_mode, seals) = {
let entries = store.live_recorders.lock().await;
let entry = entries
.get(&thread_id)
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?;
(
entry.recorder.clone(),
entry.rollout_id,
entry.history_mode,
entry.realtime_seals.clone(),
)
};
let mut seals = seals.lock_owned().await;
if let Some(pending) = seals.pending.clone() {
recorder.flush().await.map_err(thread_store_io_error)?;
seals.pending = None;
seals.committed.insert(pending);
}
if !seals.committed.contains(&id) {
let mut items = into_persisted_rollout_items(items, history_mode);
let next = codex_rollout::stamp_item_ordinals(*cursor, &mut items);
recorder
.record_canonical_items(&items)
.await
.map_err(thread_store_io_error)?;
*cursor = next;
seals.pending = Some(id.clone());
recorder.flush().await.map_err(thread_store_io_error)?;
seals.pending = None;
seals.committed.insert(id);
}
if history_mode == ThreadHistoryMode::Paginated
&& let Err(err) = super::thread_history_materialization::materialize_to_sqlite(
store,
rollout_id,
recorder.rollout_path(),
)
.await
{
warn!("failed to project durable rollout for {thread_id}: {err}");
}
sync_materialized_rollout_path(store, thread_id, recorder.rollout_path()).await
}
pub(super) async fn persist_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
write_and_project(store, thread_id, RolloutWriteOp::Persist).await
}
pub(super) async fn flush_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
write_and_project(store, thread_id, RolloutWriteOp::Flush).await
}
pub(super) async fn shutdown_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
let mut pending_metadata = store.pending_thread_metadata.lock(thread_id).await;
let _live_writer_guard = store.live_writer_locks.lock(thread_id).await;
let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?;
let rollout_path = recorder.rollout_path().to_path_buf();
if matches!(history_mode, ThreadHistoryMode::Legacy) {
recorder.shutdown().await.map_err(thread_store_io_error)?;
} else {
recorder.shutdown().await.map_err(thread_store_io_error)?;
if let Err(err) = super::thread_history_materialization::materialize_to_sqlite(
store,
rollout_id,
rollout_path.as_path(),
)
.await
{
warn!("failed to project durable rollout during shutdown for {thread_id}: {err}");
}
}
sync_materialized_rollout_path(store, thread_id, rollout_path.as_path()).await?;
if let Some(metrics) = codex_otel::global()
&& let Ok(metadata) = tokio::fs::metadata(&rollout_path).await
{
let size_bytes = i64::try_from(metadata.len()).unwrap_or(i64::MAX);
let _ = metrics.histogram(ROLLOUT_SIZE_BYTES_METRIC, size_bytes, &[]);
}
let rollout_exists = match tokio::fs::try_exists(&rollout_path).await {
Ok(rollout_exists) => rollout_exists,
Err(err) => {
warn!(
"failed to check rollout path after shutdown for {thread_id}; preserving pending metadata: {err}"
);
true
}
};
store.live_recorders.lock().await.remove(&thread_id);
drop(_live_writer_guard);
if !rollout_exists && pending_metadata.take().is_some() {
store.pending_thread_metadata.remove(thread_id).await;
}
Ok(())
}
pub(super) async fn discard_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<()> {
let mut pending_metadata = store.pending_thread_metadata.lock(thread_id).await;
let _live_writer_guard = store.live_writer_locks.lock(thread_id).await;
if pending_metadata.take().is_some() {
store.pending_thread_metadata.remove(thread_id).await;
}
let entry = store
.live_recorders
.lock()
.await
.remove(&thread_id)
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?;
entry
.recorder
.discard()
.await
.map_err(thread_store_io_error)
}
async fn index_attached_thread(
store: &LocalThreadStore,
state: &codex_state::StateRuntime,
path: &std::path::Path,
) -> ThreadStoreResult<()> {
let meta = codex_rollout::read_session_meta_line(path)
.await
.map_err(thread_store_io_error)?;
let memory_mode = meta.meta.memory_mode.clone();
let items = [RolloutItem::SessionMeta(Box::new(meta))];
let builder = codex_rollout::builder_from_items(&items, path).ok_or_else(|| {
ThreadStoreError::Internal {
message: format!("invalid attached thread metadata at {}", path.display()),
}
})?;
let mut metadata = Box::new(state.build_thread_metadata(&builder));
if super::helpers::rollout_path_is_archived(&store.config.codex_home, path) {
metadata.archived_at = Some(metadata.created_at);
}
state
.apply_rollout_items(
metadata,
&items,
memory_mode.as_deref(),
/*updated_at_override*/ None,
)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!(
"failed to index attached thread at {}: {err}",
path.display()
),
})
}
pub(super) async fn rollout_path(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<PathBuf> {
Ok(store
.live_recorders
.lock()
.await
.get(&thread_id)
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?
.recorder
.rollout_path()
.to_path_buf())
}
async fn sync_materialized_rollout_path(
store: &LocalThreadStore,
thread_id: ThreadId,
rollout_path: &std::path::Path,
) -> ThreadStoreResult<()> {
if codex_rollout::existing_rollout_path(rollout_path)
.await
.is_none()
{
return Ok(());
}
let Some(state_db) = store.state_db().await else {
return Ok(());
};
let result: ThreadStoreResult<()> = async {
let Some(mut metadata) =
state_db
.get_thread(thread_id)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to read thread metadata for {thread_id}: {err}"),
})?
else {
return Ok(());
};
if metadata.rollout_path != rollout_path {
metadata.rollout_path = rollout_path.to_path_buf();
state_db
.upsert_thread(&metadata)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to update thread metadata for {thread_id}: {err}"),
})?;
}
Ok(())
}
.await;
if let Err(err) = result {
warn!("failed to sync materialized rollout path for thread {thread_id}: {err}");
}
Ok(())
}
fn thread_store_io_error(err: std::io::Error) -> ThreadStoreError {
ThreadStoreError::Internal {
message: err.to_string(),
}
}
/// The rollout writer has three distinct lifecycle moments:
/// - `AppendItems` is normal turn/event persistence and adds new rollout records.
/// - `Persist` makes the thread durable before any turn items exist; locally this can write the
/// initial `SessionMeta`.
/// - `Flush` writes any rollout records already queued in the recorder and ensures they are
/// durably persisted.
///
/// Each can advance the rollout JSONL file on disk, so we need to make sure we materialize the
/// new data into the SQLite history tables (turns and items) as necessary.
enum RolloutWriteOp {
AppendItems(Vec<RolloutItem>),
Persist,
Flush,
}
pub(super) async fn live_writer_parts(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<(RolloutRecorder, ThreadId, ThreadHistoryMode)> {
let live_recorders = store.live_recorders.lock().await;
let entry = live_recorders
.get(&thread_id)
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?;
Ok((entry.recorder.clone(), entry.rollout_id, entry.history_mode))
}
/// Uses the writer's creation record only until its rollout has been materialized.
pub(super) async fn unpersisted_session_meta(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> Option<(PathBuf, SessionMeta)> {
let (recorder, _, _) = live_writer_parts(store, thread_id).await.ok()?;
// Materialized reads should not queue behind unrelated transcript writes.
if codex_rollout::existing_rollout_path(recorder.rollout_path())
.await
.is_some()
{
return None;
}
// A writer may finish shutdown after lookup; durable reads remain the fallback.
let meta = recorder.unpersisted_session_meta().await.ok()??;
Some((recorder.rollout_path().to_path_buf(), meta))
}
async fn write_and_project(
store: &LocalThreadStore,
thread_id: ThreadId,
write_op: RolloutWriteOp,
) -> ThreadStoreResult<()> {
// Every live write should have a recorder: create/resume installs one, while
// shutdown/discard/delete removes it. Keep the lookup defensive so late writes fail after
// teardown.
let mut cursor = store.live_writer_locks.lock(thread_id).await;
let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?;
let sync_rollout_path = matches!(&write_op, RolloutWriteOp::Persist | RolloutWriteOp::Flush);
let is_flush = matches!(&write_op, RolloutWriteOp::Flush);
match write_op {
RolloutWriteOp::AppendItems(items) => {
let mut items = into_persisted_rollout_items(items, history_mode);
if items.is_empty() {
return Ok(());
}
let next = codex_rollout::stamp_item_ordinals(*cursor, &mut items);
recorder
.record_canonical_items(&items)
.await
.map_err(thread_store_io_error)?;
// Once queued, the recorder owns the batch even if flush is cancelled or retried.
*cursor = next;
recorder.flush().await.map_err(thread_store_io_error)?;
}
RolloutWriteOp::Persist => recorder.persist().await.map_err(thread_store_io_error)?,
RolloutWriteOp::Flush => recorder.flush().await.map_err(thread_store_io_error)?,
}
if matches!(history_mode, ThreadHistoryMode::Paginated) {
// SQLite is a rebuildable view and must never get ahead of canonical history.
let rollout_path = recorder.rollout_path();
// Only the plain file's length is a decoded JSONL byte offset.
let flushed_len = if is_flush && store.state_db.is_some() {
tokio::fs::metadata(codex_rollout::plain_rollout_path(rollout_path))
.await
.ok()
.filter(std::fs::Metadata::is_file)
.map(|file| file.len())
} else {
None
};
if let Err(err) = super::thread_history_materialization::materialize_durable_suffix(
store,
rollout_id,
rollout_path,
flushed_len,
)
.await
{
warn!("failed to project durable rollout for {thread_id}: {err}");
}
}
if sync_rollout_path {
sync_materialized_rollout_path(store, thread_id, recorder.rollout_path()).await?;
}
Ok(())
}