Skip redundant history materialization on flush (#62085)

## Why

Flushing an already projected rollout still reads its metadata and opens its history for SQLite materialization, even when no durable bytes remain to project.

## What changed

For paginated thread history, compare the plain rollout file length with the saved projection offset on flush and return early when a nonzero offset matches. Use only plain file lengths because compressed sizes are not decoded JSONL byte offsets. Continue materializing any durable suffix when the projection is behind.

## Testing

Extend the flush regression test to verify that an up-to-date projection preserves existing rows and that flush catches up with durable suffixes in both plain and compressed rollouts.

GitOrigin-RevId: 7078fe77e2b43beed7ceee024f309501da3a1d2b
This commit is contained in:
Charlie Marsh
2026-11-17 17:29:51 +00:00
committed by copyberry
parent 6d94d4af10
commit c08843a5bd
3 changed files with 83 additions and 8 deletions
+13 -1
View File
@@ -435,6 +435,7 @@ async fn write_and_project(
let mut cursor = store.live_writer_locks.lock(thread_id).await;
let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?;
let sync_rollout_path = matches!(&write_op, RolloutWriteOp::Persist | RolloutWriteOp::Flush);
let is_flush = matches!(&write_op, RolloutWriteOp::Flush);
match write_op {
RolloutWriteOp::AppendItems(items) => {
let mut items = into_persisted_rollout_items(items, history_mode);
@@ -456,10 +457,21 @@ async fn write_and_project(
if matches!(history_mode, ThreadHistoryMode::Paginated) {
// SQLite is a rebuildable view and must never get ahead of canonical history.
let rollout_path = recorder.rollout_path();
if let Err(err) = super::thread_history_materialization::materialize_to_sqlite(
// Only the plain file's length is a decoded JSONL byte offset.
let flushed_len = if is_flush && store.state_db.is_some() {
tokio::fs::metadata(codex_rollout::plain_rollout_path(rollout_path))
.await
.ok()
.filter(std::fs::Metadata::is_file)
.map(|file| file.len())
} else {
None
};
if let Err(err) = super::thread_history_materialization::materialize_durable_suffix(
store,
rollout_id,
rollout_path,
flushed_len,
)
.await
{
@@ -50,15 +50,47 @@ pub(super) async fn materialize_to_sqlite_with_reads(
thread_id: ThreadId,
rollout_path: &Path,
reads: &AncestorReads,
) -> ThreadStoreResult<()> {
materialize_projection(
store,
thread_id,
ProjectionRollout {
path: rollout_path.to_owned(),
reads: reads.clone(),
},
/*flushed_len*/ None,
)
.await
}
pub(super) async fn materialize_durable_suffix(
store: &LocalThreadStore,
thread_id: ThreadId,
rollout_path: &Path,
flushed_len: Option<u64>,
) -> ThreadStoreResult<()> {
materialize_projection(
store,
thread_id,
ProjectionRollout {
path: rollout_path.to_owned(),
reads: AncestorReads::default(),
},
flushed_len,
)
.await
}
async fn materialize_projection(
store: &LocalThreadStore,
thread_id: ThreadId,
rollout: ProjectionRollout,
flushed_len: Option<u64>,
) -> ThreadStoreResult<()> {
if store.state_db.is_none() {
return Ok(());
}
let rollout = ProjectionRollout {
path: rollout_path.to_owned(),
reads: reads.clone(),
};
let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout).await;
let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout, flushed_len).await;
record_projection_outcome(&result);
result
}
@@ -67,11 +99,15 @@ async fn materialize_to_sqlite_with_state_db(
store: &LocalThreadStore,
thread_id: ThreadId,
rollout: ProjectionRollout,
flushed_len: Option<u64>,
) -> ThreadStoreResult<()> {
let projection_state = super::thread_history::projection_state(store, thread_id).await?;
let start_offset = projection_state
.as_ref()
.map_or(0, |state| state.next_byte_offset);
if start_offset > 0 && Some(start_offset) == flushed_len {
return Ok(());
}
if rollout
.reads
.cutoff(&rollout.path)
@@ -2177,7 +2177,7 @@ SELECT
}
#[tokio::test]
async fn synchronized_catch_up_does_not_replay_old_rows() {
async fn flush_projects_only_the_durable_suffix() {
let home = TempDir::new().expect("temp dir");
let store = projection_store(home.path()).await;
let thread_id = ThreadId::default();
@@ -2205,7 +2205,8 @@ async fn synchronized_catch_up_does_not_replay_old_rows() {
.live_rollout_path(thread_id)
.await
.expect("rollout path");
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
store
.flush_thread(thread_id)
.await
.expect("catch up synchronized rollout");
@@ -2217,6 +2218,32 @@ async fn synchronized_catch_up_does_not_replay_old_rows() {
.await
.expect("read projected turn");
assert_eq!(status, "sentinel");
// The writer can flush without new items while the projector is behind on disk.
for (ordinal, item, compress) in [
(2, user_message("suffix"), false),
(3, turn_completed("turn-1"), true),
] {
append_suffix(
&rollout_path,
&format!("{}\n", rollout_line(Some(ordinal), item)),
);
let end = fs::metadata(&rollout_path).expect("metadata").len();
if compress {
compress_rollout(&rollout_path);
}
store
.flush_thread(thread_id)
.await
.expect("flush durable suffix");
assert_eq!(
projection_state(&pool, thread_id).await,
(
i64::try_from(end).unwrap(),
i64::try_from(ordinal + 1).unwrap()
)
);
}
}
/// Large records keep their byte positions and remain pending until their newline is durable.