diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index 0ffd8c7276..ed0d66a353 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -435,6 +435,7 @@ async fn write_and_project( let mut cursor = store.live_writer_locks.lock(thread_id).await; let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?; let sync_rollout_path = matches!(&write_op, RolloutWriteOp::Persist | RolloutWriteOp::Flush); + let is_flush = matches!(&write_op, RolloutWriteOp::Flush); match write_op { RolloutWriteOp::AppendItems(items) => { let mut items = into_persisted_rollout_items(items, history_mode); @@ -456,10 +457,21 @@ async fn write_and_project( if matches!(history_mode, ThreadHistoryMode::Paginated) { // SQLite is a rebuildable view and must never get ahead of canonical history. let rollout_path = recorder.rollout_path(); - if let Err(err) = super::thread_history_materialization::materialize_to_sqlite( + // Only the plain file's length is a decoded JSONL byte offset. + let flushed_len = if is_flush && store.state_db.is_some() { + tokio::fs::metadata(codex_rollout::plain_rollout_path(rollout_path)) + .await + .ok() + .filter(std::fs::Metadata::is_file) + .map(|file| file.len()) + } else { + None + }; + if let Err(err) = super::thread_history_materialization::materialize_durable_suffix( store, rollout_id, rollout_path, + flushed_len, ) .await { diff --git a/codex-rs/thread-store/src/local/thread_history_materialization.rs b/codex-rs/thread-store/src/local/thread_history_materialization.rs index a21ea56250..8367e57345 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization.rs @@ -50,15 +50,47 @@ pub(super) async fn materialize_to_sqlite_with_reads( thread_id: ThreadId, rollout_path: &Path, reads: &AncestorReads, +) -> ThreadStoreResult<()> { + materialize_projection( + store, + thread_id, + ProjectionRollout { + path: rollout_path.to_owned(), + reads: reads.clone(), + }, + /*flushed_len*/ None, + ) + .await +} + +pub(super) async fn materialize_durable_suffix( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout_path: &Path, + flushed_len: Option, +) -> ThreadStoreResult<()> { + materialize_projection( + store, + thread_id, + ProjectionRollout { + path: rollout_path.to_owned(), + reads: AncestorReads::default(), + }, + flushed_len, + ) + .await +} + +async fn materialize_projection( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout: ProjectionRollout, + flushed_len: Option, ) -> ThreadStoreResult<()> { if store.state_db.is_none() { return Ok(()); } - let rollout = ProjectionRollout { - path: rollout_path.to_owned(), - reads: reads.clone(), - }; - let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout).await; + let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout, flushed_len).await; record_projection_outcome(&result); result } @@ -67,11 +99,15 @@ async fn materialize_to_sqlite_with_state_db( store: &LocalThreadStore, thread_id: ThreadId, rollout: ProjectionRollout, + flushed_len: Option, ) -> ThreadStoreResult<()> { let projection_state = super::thread_history::projection_state(store, thread_id).await?; let start_offset = projection_state .as_ref() .map_or(0, |state| state.next_byte_offset); + if start_offset > 0 && Some(start_offset) == flushed_len { + return Ok(()); + } if rollout .reads .cutoff(&rollout.path) diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs index 7447bfc4a5..3cfa0e5030 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs @@ -2177,7 +2177,7 @@ SELECT } #[tokio::test] -async fn synchronized_catch_up_does_not_replay_old_rows() { +async fn flush_projects_only_the_durable_suffix() { let home = TempDir::new().expect("temp dir"); let store = projection_store(home.path()).await; let thread_id = ThreadId::default(); @@ -2205,7 +2205,8 @@ async fn synchronized_catch_up_does_not_replay_old_rows() { .live_rollout_path(thread_id) .await .expect("rollout path"); - super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path()) + store + .flush_thread(thread_id) .await .expect("catch up synchronized rollout"); @@ -2217,6 +2218,32 @@ async fn synchronized_catch_up_does_not_replay_old_rows() { .await .expect("read projected turn"); assert_eq!(status, "sentinel"); + + // The writer can flush without new items while the projector is behind on disk. + for (ordinal, item, compress) in [ + (2, user_message("suffix"), false), + (3, turn_completed("turn-1"), true), + ] { + append_suffix( + &rollout_path, + &format!("{}\n", rollout_line(Some(ordinal), item)), + ); + let end = fs::metadata(&rollout_path).expect("metadata").len(); + if compress { + compress_rollout(&rollout_path); + } + store + .flush_thread(thread_id) + .await + .expect("flush durable suffix"); + assert_eq!( + projection_state(&pool, thread_id).await, + ( + i64::try_from(end).unwrap(), + i64::try_from(ordinal + 1).unwrap() + ) + ); + } } /// Large records keep their byte positions and remain pending until their newline is durable.