From c08843a5bd16de5524129b2f4270490447991984 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Tue, 17 Nov 2026 15:50:07 +0000 Subject: [PATCH] Skip redundant history materialization on flush (#62085) ## Why Flushing an already projected rollout still reads its metadata and opens its history for SQLite materialization, even when no durable bytes remain to project. ## What changed For paginated thread history, compare the plain rollout file length with the saved projection offset on flush and return early when a nonzero offset matches. Use only plain file lengths because compressed sizes are not decoded JSONL byte offsets. Continue materializing any durable suffix when the projection is behind. ## Testing Extend the flush regression test to verify that an up-to-date projection preserves existing rows and that flush catches up with durable suffixes in both plain and compressed rollouts. GitOrigin-RevId: 7078fe77e2b43beed7ceee024f309501da3a1d2b --- .../thread-store/src/local/live_writer.rs | 14 +++++- .../local/thread_history_materialization.rs | 46 +++++++++++++++++-- .../thread_history_materialization_tests.rs | 31 ++++++++++++- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index 0ffd8c7276..ed0d66a353 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -435,6 +435,7 @@ async fn write_and_project( let mut cursor = store.live_writer_locks.lock(thread_id).await; let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?; let sync_rollout_path = matches!(&write_op, RolloutWriteOp::Persist | RolloutWriteOp::Flush); + let is_flush = matches!(&write_op, RolloutWriteOp::Flush); match write_op { RolloutWriteOp::AppendItems(items) => { let mut items = into_persisted_rollout_items(items, history_mode); @@ -456,10 +457,21 @@ async fn write_and_project( if matches!(history_mode, ThreadHistoryMode::Paginated) { // SQLite is a rebuildable view and must never get ahead of canonical history. let rollout_path = recorder.rollout_path(); - if let Err(err) = super::thread_history_materialization::materialize_to_sqlite( + // Only the plain file's length is a decoded JSONL byte offset. + let flushed_len = if is_flush && store.state_db.is_some() { + tokio::fs::metadata(codex_rollout::plain_rollout_path(rollout_path)) + .await + .ok() + .filter(std::fs::Metadata::is_file) + .map(|file| file.len()) + } else { + None + }; + if let Err(err) = super::thread_history_materialization::materialize_durable_suffix( store, rollout_id, rollout_path, + flushed_len, ) .await { diff --git a/codex-rs/thread-store/src/local/thread_history_materialization.rs b/codex-rs/thread-store/src/local/thread_history_materialization.rs index a21ea56250..8367e57345 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization.rs @@ -50,15 +50,47 @@ pub(super) async fn materialize_to_sqlite_with_reads( thread_id: ThreadId, rollout_path: &Path, reads: &AncestorReads, +) -> ThreadStoreResult<()> { + materialize_projection( + store, + thread_id, + ProjectionRollout { + path: rollout_path.to_owned(), + reads: reads.clone(), + }, + /*flushed_len*/ None, + ) + .await +} + +pub(super) async fn materialize_durable_suffix( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout_path: &Path, + flushed_len: Option, +) -> ThreadStoreResult<()> { + materialize_projection( + store, + thread_id, + ProjectionRollout { + path: rollout_path.to_owned(), + reads: AncestorReads::default(), + }, + flushed_len, + ) + .await +} + +async fn materialize_projection( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout: ProjectionRollout, + flushed_len: Option, ) -> ThreadStoreResult<()> { if store.state_db.is_none() { return Ok(()); } - let rollout = ProjectionRollout { - path: rollout_path.to_owned(), - reads: reads.clone(), - }; - let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout).await; + let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout, flushed_len).await; record_projection_outcome(&result); result } @@ -67,11 +99,15 @@ async fn materialize_to_sqlite_with_state_db( store: &LocalThreadStore, thread_id: ThreadId, rollout: ProjectionRollout, + flushed_len: Option, ) -> ThreadStoreResult<()> { let projection_state = super::thread_history::projection_state(store, thread_id).await?; let start_offset = projection_state .as_ref() .map_or(0, |state| state.next_byte_offset); + if start_offset > 0 && Some(start_offset) == flushed_len { + return Ok(()); + } if rollout .reads .cutoff(&rollout.path) diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs index 7447bfc4a5..3cfa0e5030 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs @@ -2177,7 +2177,7 @@ SELECT } #[tokio::test] -async fn synchronized_catch_up_does_not_replay_old_rows() { +async fn flush_projects_only_the_durable_suffix() { let home = TempDir::new().expect("temp dir"); let store = projection_store(home.path()).await; let thread_id = ThreadId::default(); @@ -2205,7 +2205,8 @@ async fn synchronized_catch_up_does_not_replay_old_rows() { .live_rollout_path(thread_id) .await .expect("rollout path"); - super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path()) + store + .flush_thread(thread_id) .await .expect("catch up synchronized rollout"); @@ -2217,6 +2218,32 @@ async fn synchronized_catch_up_does_not_replay_old_rows() { .await .expect("read projected turn"); assert_eq!(status, "sentinel"); + + // The writer can flush without new items while the projector is behind on disk. + for (ordinal, item, compress) in [ + (2, user_message("suffix"), false), + (3, turn_completed("turn-1"), true), + ] { + append_suffix( + &rollout_path, + &format!("{}\n", rollout_line(Some(ordinal), item)), + ); + let end = fs::metadata(&rollout_path).expect("metadata").len(); + if compress { + compress_rollout(&rollout_path); + } + store + .flush_thread(thread_id) + .await + .expect("flush durable suffix"); + assert_eq!( + projection_state(&pool, thread_id).await, + ( + i64::try_from(end).unwrap(), + i64::try_from(ordinal + 1).unwrap() + ) + ); + } } /// Large records keep their byte positions and remain pending until their newline is durable.