mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Skip redundant history materialization on flush (#62085)
## Why Flushing an already projected rollout still reads its metadata and opens its history for SQLite materialization, even when no durable bytes remain to project. ## What changed For paginated thread history, compare the plain rollout file length with the saved projection offset on flush and return early when a nonzero offset matches. Use only plain file lengths because compressed sizes are not decoded JSONL byte offsets. Continue materializing any durable suffix when the projection is behind. ## Testing Extend the flush regression test to verify that an up-to-date projection preserves existing rows and that flush catches up with durable suffixes in both plain and compressed rollouts. GitOrigin-RevId: 7078fe77e2b43beed7ceee024f309501da3a1d2b
This commit is contained in:
@@ -435,6 +435,7 @@ async fn write_and_project(
|
||||
let mut cursor = store.live_writer_locks.lock(thread_id).await;
|
||||
let (recorder, rollout_id, history_mode) = live_writer_parts(store, thread_id).await?;
|
||||
let sync_rollout_path = matches!(&write_op, RolloutWriteOp::Persist | RolloutWriteOp::Flush);
|
||||
let is_flush = matches!(&write_op, RolloutWriteOp::Flush);
|
||||
match write_op {
|
||||
RolloutWriteOp::AppendItems(items) => {
|
||||
let mut items = into_persisted_rollout_items(items, history_mode);
|
||||
@@ -456,10 +457,21 @@ async fn write_and_project(
|
||||
if matches!(history_mode, ThreadHistoryMode::Paginated) {
|
||||
// SQLite is a rebuildable view and must never get ahead of canonical history.
|
||||
let rollout_path = recorder.rollout_path();
|
||||
if let Err(err) = super::thread_history_materialization::materialize_to_sqlite(
|
||||
// Only the plain file's length is a decoded JSONL byte offset.
|
||||
let flushed_len = if is_flush && store.state_db.is_some() {
|
||||
tokio::fs::metadata(codex_rollout::plain_rollout_path(rollout_path))
|
||||
.await
|
||||
.ok()
|
||||
.filter(std::fs::Metadata::is_file)
|
||||
.map(|file| file.len())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Err(err) = super::thread_history_materialization::materialize_durable_suffix(
|
||||
store,
|
||||
rollout_id,
|
||||
rollout_path,
|
||||
flushed_len,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -50,15 +50,47 @@ pub(super) async fn materialize_to_sqlite_with_reads(
|
||||
thread_id: ThreadId,
|
||||
rollout_path: &Path,
|
||||
reads: &AncestorReads,
|
||||
) -> ThreadStoreResult<()> {
|
||||
materialize_projection(
|
||||
store,
|
||||
thread_id,
|
||||
ProjectionRollout {
|
||||
path: rollout_path.to_owned(),
|
||||
reads: reads.clone(),
|
||||
},
|
||||
/*flushed_len*/ None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn materialize_durable_suffix(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
rollout_path: &Path,
|
||||
flushed_len: Option<u64>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
materialize_projection(
|
||||
store,
|
||||
thread_id,
|
||||
ProjectionRollout {
|
||||
path: rollout_path.to_owned(),
|
||||
reads: AncestorReads::default(),
|
||||
},
|
||||
flushed_len,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn materialize_projection(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
rollout: ProjectionRollout,
|
||||
flushed_len: Option<u64>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
if store.state_db.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let rollout = ProjectionRollout {
|
||||
path: rollout_path.to_owned(),
|
||||
reads: reads.clone(),
|
||||
};
|
||||
let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout).await;
|
||||
let result = materialize_to_sqlite_with_state_db(store, thread_id, rollout, flushed_len).await;
|
||||
record_projection_outcome(&result);
|
||||
result
|
||||
}
|
||||
@@ -67,11 +99,15 @@ async fn materialize_to_sqlite_with_state_db(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
rollout: ProjectionRollout,
|
||||
flushed_len: Option<u64>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let projection_state = super::thread_history::projection_state(store, thread_id).await?;
|
||||
let start_offset = projection_state
|
||||
.as_ref()
|
||||
.map_or(0, |state| state.next_byte_offset);
|
||||
if start_offset > 0 && Some(start_offset) == flushed_len {
|
||||
return Ok(());
|
||||
}
|
||||
if rollout
|
||||
.reads
|
||||
.cutoff(&rollout.path)
|
||||
|
||||
@@ -2177,7 +2177,7 @@ SELECT
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn synchronized_catch_up_does_not_replay_old_rows() {
|
||||
async fn flush_projects_only_the_durable_suffix() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = projection_store(home.path()).await;
|
||||
let thread_id = ThreadId::default();
|
||||
@@ -2205,7 +2205,8 @@ async fn synchronized_catch_up_does_not_replay_old_rows() {
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect("rollout path");
|
||||
super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path())
|
||||
store
|
||||
.flush_thread(thread_id)
|
||||
.await
|
||||
.expect("catch up synchronized rollout");
|
||||
|
||||
@@ -2217,6 +2218,32 @@ async fn synchronized_catch_up_does_not_replay_old_rows() {
|
||||
.await
|
||||
.expect("read projected turn");
|
||||
assert_eq!(status, "sentinel");
|
||||
|
||||
// The writer can flush without new items while the projector is behind on disk.
|
||||
for (ordinal, item, compress) in [
|
||||
(2, user_message("suffix"), false),
|
||||
(3, turn_completed("turn-1"), true),
|
||||
] {
|
||||
append_suffix(
|
||||
&rollout_path,
|
||||
&format!("{}\n", rollout_line(Some(ordinal), item)),
|
||||
);
|
||||
let end = fs::metadata(&rollout_path).expect("metadata").len();
|
||||
if compress {
|
||||
compress_rollout(&rollout_path);
|
||||
}
|
||||
store
|
||||
.flush_thread(thread_id)
|
||||
.await
|
||||
.expect("flush durable suffix");
|
||||
assert_eq!(
|
||||
projection_state(&pool, thread_id).await,
|
||||
(
|
||||
i64::try_from(end).unwrap(),
|
||||
i64::try_from(ordinal + 1).unwrap()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Large records keep their byte positions and remain pending until their newline is durable.
|
||||
|
||||
Reference in New Issue
Block a user