diff --git a/codex-rs/thread-store/src/local/rollout_lineage.rs b/codex-rs/thread-store/src/local/rollout_lineage.rs index d52c01cb38..55140b466a 100644 --- a/codex-rs/thread-store/src/local/rollout_lineage.rs +++ b/codex-rs/thread-store/src/local/rollout_lineage.rs @@ -1,6 +1,4 @@ use std::borrow::Cow; -use std::collections::HashMap; -use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; @@ -12,8 +10,11 @@ use codex_protocol::protocol::ThreadHistoryMode; use codex_rollout::RolloutItem; mod ancestor_reads; +mod walk; + pub(super) use ancestor_reads::AncestorReads; use ancestor_reads::FailedAncestor; +pub(super) use walk::LineageWalk; use super::LocalThreadStore; use super::thread_rollout_resolver; @@ -30,10 +31,7 @@ pub(super) struct RolloutLineageSegment { hides_parent: bool, } -/// Ordered rollout ranges contributing to one forked history. -/// -/// This is the only local abstraction that follows SessionMeta.history_base pointers. Readers -/// consume its bounded rollout segments without resolving or mutating fork pointers themselves. +/// The resolved ranges of a forked history, ordered from oldest to newest. #[derive(Clone, Debug)] pub(super) struct RolloutLineage { pub(super) segments: Vec, @@ -48,12 +46,14 @@ impl LocalThreadStore { requested_thread_id: ThreadId, initial_path: Option, ) -> ThreadStoreResult { - self.resolve_rollout_lineage_with_mode( + LineageWalk::new( + self, requested_thread_id, LineageResolutionMode::Existing, initial_path, - &HashMap::new(), + /*excluded*/ None, ) + .finish() .await } @@ -61,155 +61,25 @@ impl LocalThreadStore { &self, requested_thread_id: ThreadId, ) -> ThreadStoreResult { - self.resolve_rollout_lineage_with_mode( - requested_thread_id, - LineageResolutionMode::HistoryPage, - /*initial_path*/ None, - &HashMap::new(), - ) - .await + LineageWalk::for_history_page(self, requested_thread_id) + .finish() + .await } pub(super) async fn resolve_rollout_lineage_for_reference( &self, requested_thread_id: ThreadId, ) -> ThreadStoreResult { - self.resolve_rollout_lineage_with_mode( + LineageWalk::new( + self, requested_thread_id, LineageResolutionMode::PlainForReference, /*initial_path*/ None, - &HashMap::new(), + /*excluded*/ None, ) + .finish() .await } - - async fn resolve_rollout_lineage_with_mode( - &self, - requested_thread_id: ThreadId, - mode: LineageResolutionMode, - mut initial_path: Option, - excluded: &HashMap, - ) -> ThreadStoreResult { - let mut segments = Vec::new(); - let mut seen = HashSet::new(); - let mut end: Option = None; - let mut prefetched_paths = HashMap::new(); - let mut ancestor_paths = HashMap::new(); - - loop { - let coordination_id = end.map_or(requested_thread_id, |end| end.thread_id); - let _writer_guard = match mode { - LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None, - LineageResolutionMode::PlainForReference => { - Some(self.live_writer_locks.lock(coordination_id).await) - } - }; - let (rollout_id, (rollout_path, bounds)) = if let Some(end) = end { - let ancestor = resolve_ancestor_rollout( - self, - requested_thread_id, - end, - mode, - prefetched_paths.remove(&end.thread_id), - excluded.get(&end.thread_id), - ) - .await?; - (end.thread_id, ancestor) - } else { - let (rollout_id, path) = match initial_path.take() { - Some(path) => { - let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id( - &path, - requested_thread_id, - ThreadHistoryMode::Paginated, - )?; - (id, path) - } - None => { - let resolved = thread_rollout_resolver::resolve_current_including_archived( - self, - requested_thread_id, - ) - .await? - .ok_or_else(|| { - malformed_lineage(requested_thread_id, "missing source rollout") - })?; - (resolved.rollout_id, resolved.path) - } - }; - let (path, mut bounds) = mode - .read_rollout(self, path, requested_thread_id, Some(requested_thread_id)) - .await?; - if matches!(mode, LineageResolutionMode::HistoryPage) { - bounds - .read_history_page_bounds(&path, requested_thread_id) - .await?; - } - (rollout_id, (path, bounds)) - }; - if !seen.insert(rollout_id) { - return Err(malformed_lineage(requested_thread_id, "cycle detected")); - } - let rollout_path = match mode { - LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => { - rollout_path - } - LineageResolutionMode::PlainForReference - if end.is_none() && bounds.parent.is_none() => - { - // A newly shared standalone source must remain readable by older binaries. - codex_rollout::materialize_rollout_for_reference(rollout_path.as_path()) - .await - .map_err(|err| ThreadStoreError::Internal { - message: format!( - "failed to materialize referenced rollout {}: {err}", - rollout_path.display() - ), - })? - } - // Already-shared compressed history requires a compatible reader regardless of - // new forks. Read it without publishing decoded copies into ancestors' folders; - // their owners may concurrently archive or unarchive those immutable files. - LineageResolutionMode::PlainForReference => rollout_path, - }; - if let Some(end) = end - && !matches!(mode, LineageResolutionMode::HistoryPage) - { - ancestor_paths.insert(rollout_path.clone(), (end, bounds.owner)); - } - segments.push(RolloutLineageSegment { - rollout_id, - rollout_path, - start_ordinal: bounds.start_ordinal, - end, - hides_parent: bounds.hides_parent, - }); - - let Some(base) = bounds.parent else { - break; - }; - if matches!(mode, LineageResolutionMode::HistoryPage) - && let [child] = segments.as_slice() - { - prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths( - &self.config.codex_home, - &child.rollout_path, - base.thread_id, - ) - .await - .unwrap_or_default(); - } - end = Some(base); - } - - segments.reverse(); - Ok(RolloutLineage { - segments, - reads: AncestorReads::new(ancestor_paths), - requested_thread_id, - mode, - }) - } } /// A mid-turn fork starts with its header, settings, optional model guidance, and parent abort. @@ -533,12 +403,9 @@ impl RolloutLineage { } pub(super) fn segment_index_for_ordinal(&self, ordinal: u64) -> Option { - self.segments.iter().position(|segment| { - ordinal >= segment.start_ordinal() - && segment - .end_ordinal() - .is_none_or(|end_ordinal| ordinal < end_ordinal) - }) + self.segments + .iter() + .position(|segment| segment.contains(ordinal)) } pub(super) async fn truncate_at( @@ -585,6 +452,10 @@ impl RolloutLineage { } impl RolloutLineageSegment { + pub(super) fn contains(&self, ordinal: u64) -> bool { + ordinal >= self.start_ordinal && self.end_ordinal().is_none_or(|end| ordinal < end) + } + pub(super) fn rollout_id(&self) -> ThreadId { self.rollout_id } diff --git a/codex-rs/thread-store/src/local/rollout_lineage/ancestor_reads.rs b/codex-rs/thread-store/src/local/rollout_lineage/ancestor_reads.rs index f81e9953d0..0dd543014b 100644 --- a/codex-rs/thread-store/src/local/rollout_lineage/ancestor_reads.rs +++ b/codex-rs/thread-store/src/local/rollout_lineage/ancestor_reads.rs @@ -14,6 +14,7 @@ use std::sync::OnceLock; use codex_protocol::ThreadId; use codex_protocol::protocol::HistoryPosition; +use super::LineageWalk; use super::LocalThreadStore; use super::RolloutLineage; use crate::ThreadStoreError; @@ -133,15 +134,16 @@ impl RolloutLineage { error: error.clone(), }, ); - lineage = store - .resolve_rollout_lineage_with_mode( - requested_thread_id, - mode, - leaf.clone(), - &excluded, - ) - .await - .map_err(|_| error)?; + lineage = LineageWalk::new( + store, + requested_thread_id, + mode, + leaf.clone(), + Some(&excluded), + ) + .finish() + .await + .map_err(|_| error)?; } } } diff --git a/codex-rs/thread-store/src/local/rollout_lineage/walk.rs b/codex-rs/thread-store/src/local/rollout_lineage/walk.rs new file mode 100644 index 0000000000..52932f8908 --- /dev/null +++ b/codex-rs/thread-store/src/local/rollout_lineage/walk.rs @@ -0,0 +1,215 @@ +//! Walks immutable rollout ranges from the selected leaf toward its ancestors on demand. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +use codex_protocol::ThreadId; +use codex_protocol::protocol::HistoryPosition; +use codex_protocol::protocol::ThreadHistoryMode; + +use super::AncestorReads; +use super::FailedAncestor; +use super::LineageResolutionMode; +use super::LocalThreadStore; +use super::RolloutLineage; +use super::RolloutLineageSegment; +use super::malformed_lineage; +use super::resolve_ancestor_rollout; +use super::thread_rollout_resolver; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +/// The next file to open. A parent bookmark bounds the rows inherited from that file. +enum NextRollout { + Selected(Option), + Parent(HistoryPosition), +} + +pub(in crate::local) struct LineageWalk<'a> { + store: &'a LocalThreadStore, + requested_thread_id: ThreadId, + mode: LineageResolutionMode, + next: Option, + seen: HashSet, + prefetched_paths: HashMap, + excluded: Option<&'a HashMap>, + ancestor_paths: HashMap, + // Newest first, so discovering one older segment is a push. Only finish reverses this. + segments: Vec, +} + +impl<'a> LineageWalk<'a> { + pub(super) fn new( + store: &'a LocalThreadStore, + requested_thread_id: ThreadId, + mode: LineageResolutionMode, + initial_path: Option, + excluded: Option<&'a HashMap>, + ) -> Self { + Self { + store, + requested_thread_id, + mode, + next: Some(NextRollout::Selected(initial_path)), + seen: HashSet::new(), + prefetched_paths: HashMap::new(), + excluded, + ancestor_paths: HashMap::new(), + segments: Vec::new(), + } + } + + pub(in crate::local) fn for_history_page( + store: &'a LocalThreadStore, + thread_id: ThreadId, + ) -> Self { + Self::new( + store, + thread_id, + LineageResolutionMode::HistoryPage, + /*initial_path*/ None, + /*excluded*/ None, + ) + } + + /// Opens all remaining ancestors, oldest first in the resulting lineage. + pub(in crate::local) async fn finish(mut self) -> ThreadStoreResult { + self.resolve_all().await?; + self.segments.reverse(); + Ok(RolloutLineage { + segments: self.segments, + reads: AncestorReads::new(self.ancestor_paths), + requested_thread_id: self.requested_thread_id, + mode: self.mode, + }) + } + + pub(in crate::local) async fn resolve_all( + &mut self, + ) -> ThreadStoreResult<&[RolloutLineageSegment]> { + while self.next_older().await? {} + Ok(&self.segments) + } + + /// Returns the segment at the given depth from the leaf and all newer segments. + pub(in crate::local) async fn at_depth( + &mut self, + depth: usize, + ) -> ThreadStoreResult> { + while self.segments.len() <= depth { + if !self.next_older().await? { + return Ok(None); + } + } + Ok(self.segments[..=depth].split_last()) + } + + async fn next_older(&mut self) -> ThreadStoreResult { + let Some(next) = self.next.take() else { + return Ok(false); + }; + let requested_thread_id = self.requested_thread_id; + let store = self.store; + let _writer_guard = match self.mode { + LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None, + LineageResolutionMode::PlainForReference => { + let id = match &next { + NextRollout::Selected(_) => requested_thread_id, + NextRollout::Parent(end) => end.thread_id, + }; + Some(store.live_writer_locks.lock(id).await) + } + }; + let (rollout_id, rollout_path, bounds, end) = match next { + NextRollout::Selected(initial_path) => { + let (rollout_id, path) = match initial_path { + Some(path) => { + let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id( + &path, + requested_thread_id, + ThreadHistoryMode::Paginated, + )?; + (id, path) + } + None => { + let resolved = thread_rollout_resolver::resolve_current_including_archived( + store, + requested_thread_id, + ) + .await? + .ok_or_else(|| { + malformed_lineage(requested_thread_id, "missing source rollout") + })?; + (resolved.rollout_id, resolved.path) + } + }; + let (mut path, mut bounds) = self + .mode + .read_rollout(store, path, requested_thread_id, Some(requested_thread_id)) + .await?; + match self.mode { + LineageResolutionMode::HistoryPage => { + bounds + .read_history_page_bounds(&path, requested_thread_id) + .await? + } + LineageResolutionMode::PlainForReference if bounds.parent.is_none() => { + // A newly shared standalone source must remain readable by older binaries. + path = codex_rollout::materialize_rollout_for_reference(&path) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!( + "failed to materialize referenced rollout {}: {err}", + path.display() + ), + })?; + } + LineageResolutionMode::Existing | LineageResolutionMode::PlainForReference => {} + } + (rollout_id, path, bounds, None) + } + NextRollout::Parent(end) => { + // Defer the directory scan too: pages contained in the leaf never need it. + if matches!(self.mode, LineageResolutionMode::HistoryPage) + && let [child] = self.segments.as_slice() + { + self.prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths( + &store.config.codex_home, + &child.rollout_path, + end.thread_id, + ) + .await + .unwrap_or_default(); + } + let (path, bounds) = resolve_ancestor_rollout( + store, + requested_thread_id, + end, + self.mode, + self.prefetched_paths.remove(&end.thread_id), + self.excluded + .and_then(|excluded| excluded.get(&end.thread_id)), + ) + .await?; + if !matches!(self.mode, LineageResolutionMode::HistoryPage) { + self.ancestor_paths + .insert(path.clone(), (end, bounds.owner)); + } + (end.thread_id, path, bounds, Some(end)) + } + }; + if !self.seen.insert(rollout_id) { + return Err(malformed_lineage(requested_thread_id, "cycle detected")); + } + self.next = bounds.parent.map(NextRollout::Parent); + self.segments.push(RolloutLineageSegment { + rollout_id, + rollout_path, + start_ordinal: bounds.start_ordinal, + end, + hides_parent: bounds.hides_parent, + }); + Ok(true) + } +} diff --git a/codex-rs/thread-store/src/local/thread_history.rs b/codex-rs/thread-store/src/local/thread_history.rs index 6923a7333c..cff15327b0 100644 --- a/codex-rs/thread-store/src/local/thread_history.rs +++ b/codex-rs/thread-store/src/local/thread_history.rs @@ -10,6 +10,7 @@ use super::LocalThreadStore; use crate::ThreadStoreError; use crate::ThreadStoreResult; +mod page_segments; mod read; mod realtime; mod realtime_stream; diff --git a/codex-rs/thread-store/src/local/thread_history/page_segments.rs b/codex-rs/thread-store/src/local/thread_history/page_segments.rs new file mode 100644 index 0000000000..edc7c45eb8 --- /dev/null +++ b/codex-rs/thread-store/src/local/thread_history/page_segments.rs @@ -0,0 +1,67 @@ +//! Seeks a history cursor and supplies the pager with each segment and its newer overrides. + +use super::super::rollout_lineage::LineageWalk; +use super::super::rollout_lineage::RolloutLineageSegment; +use super::read::HistoryCursor; +use super::read::invalid_cursor; +use crate::SortDirection; +use crate::ThreadStoreResult; + +pub(super) struct PageSegments<'a, 'store> { + lineage: &'a mut LineageWalk<'store>, + step: isize, + next_depth: Option, + cursor: Option, +} + +pub(super) struct PageSegment<'a> { + pub segment: &'a RolloutLineageSegment, + pub newer: &'a [RolloutLineageSegment], + pub cursor: Option, +} + +impl<'a, 'store> PageSegments<'a, 'store> { + pub(super) async fn from_cursor( + lineage: &'a mut LineageWalk<'store>, + direction: SortDirection, + cursor: Option, + ) -> ThreadStoreResult { + // Ascending reads must discover the oldest segment up front. Descending reads start + // at the leaf and seek only as far as the cursor, retaining all newer shadowing turns. + let (start, step) = match direction { + SortDirection::Asc => (lineage.resolve_all().await?.len().checked_sub(1), -1), + SortDirection::Desc => (Some(0), 1), + }; + let mut next_depth = start; + if let Some(cursor) = &cursor { + loop { + let missing = || invalid_cursor("position outside thread lineage"); + let depth = next_depth.ok_or_else(missing)?; + let (segment, _) = lineage.at_depth(depth).await?.ok_or_else(missing)?; + if segment.contains(cursor.rollout_ordinal) { + break; + } + next_depth = depth.checked_add_signed(step); + } + } + Ok(Self { + lineage, + step, + next_depth, + cursor, + }) + } + + pub(super) async fn next(&mut self) -> ThreadStoreResult>> { + let Some(depth) = self.next_depth else { + return Ok(None); + }; + let segment = self.lineage.at_depth(depth).await?; + self.next_depth = depth.checked_add_signed(self.step); + Ok(segment.map(|(segment, newer)| PageSegment { + segment, + newer, + cursor: self.cursor.take(), + })) + } +} diff --git a/codex-rs/thread-store/src/local/thread_history/read.rs b/codex-rs/thread-store/src/local/thread_history/read.rs index b00acc3199..acec4fcbdc 100644 --- a/codex-rs/thread-store/src/local/thread_history/read.rs +++ b/codex-rs/thread-store/src/local/thread_history/read.rs @@ -5,12 +5,12 @@ use serde::Serialize; use sqlx::Row; use super::super::LocalThreadStore; -use super::super::rollout_lineage::RolloutLineage; +use super::super::rollout_lineage::LineageWalk; use super::segment_paging::page_item_rows; use super::segment_paging::page_turn_rows; use super::segment_paging::validate_page_size; use super::sqlite_integer; -use super::turn_lookup::find_source_turn; +use super::turn_lookup::find_turn; use crate::ItemPage; use crate::ListItemsParams; use crate::ListTurnsParams; @@ -111,14 +111,12 @@ pub(in crate::local) async fn list_turns( ) .await?; validate_page_size(params.page_size)?; - let lineage = store - .resolve_rollout_lineage_for_history_page(params.thread_id) - .await?; let pool = store.thread_history_db().await?; + let mut lineage = LineageWalk::for_history_page(store, params.thread_id); let page = page_turn_rows( pool, params.thread_id, - &lineage, + &mut lineage, params.cursor.as_deref(), params.page_size, params.sort_direction, @@ -136,7 +134,7 @@ pub(in crate::local) async fn list_turns( { // Synthetic fork-boundary rows are interrupted without local summary IDs. // Load their summary from the earliest visible source turn. - load_inherited_summary_items(pool, &lineage, &turn).await? + load_inherited_summary_items(pool, &mut lineage, &turn).await? } StoredTurnItemsView::Summary => turn.summary_items, }; @@ -188,11 +186,9 @@ pub(in crate::local) async fn list_items( params.sort_direction = crate::SortDirection::Asc; } validate_page_size(params.page_size)?; - let lineage = store - .resolve_rollout_lineage_for_history_page(params.thread_id) - .await?; let pool = store.thread_history_db().await?; - let page = page_item_rows(pool, &lineage, ¶ms).await?; + let mut lineage = LineageWalk::for_history_page(store, params.thread_id); + let page = page_item_rows(pool, &mut lineage, ¶ms).await?; Ok(ItemPage { items: page.rows.into_iter().map(|row| row.item).collect(), @@ -233,12 +229,12 @@ pub(super) async fn validate_thread_for_paginated_reads( async fn load_inherited_summary_items( pool: &sqlx::SqlitePool, - lineage: &RolloutLineage, + lineage: &mut LineageWalk<'_>, turn: &StoredTurnRow, ) -> ThreadStoreResult> { - let source = find_source_turn(pool, lineage, turn.turn_id.as_str()).await?; + let lineage = lineage.resolve_all().await?; + let source = find_turn(pool, lineage.iter().rev(), turn.turn_id.as_str()).await?; let Some(segment) = lineage - .segments() .iter() .find(|segment| segment.rollout_id() == source.rollout_id) else { diff --git a/codex-rs/thread-store/src/local/thread_history/read_tests.rs b/codex-rs/thread-store/src/local/thread_history/read_tests.rs index 2184b5a1b3..b01646791f 100644 --- a/codex-rs/thread-store/src/local/thread_history/read_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history/read_tests.rs @@ -31,6 +31,184 @@ use crate::StoredTurnError; use crate::StoredTurnStatus; use crate::local::test_support::test_config; +#[tokio::test] +async fn newest_indexed_pages_do_not_read_unused_ancestors() { + let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await; + let parent_id = ThreadId::default(); + let parent_path = rollout_path(home.path(), parent_id); + write_rollout_with_end( + &parent_path, + parent_id, + /*history_base*/ None, + /*next_ordinal*/ 5, + ); + write_rollout_with_end( + &rollout_path(home.path(), child_id), + child_id, + Some(history_position( + &parent_path, + parent_id, + /*end_ordinal_exclusive*/ 4, + )), + /*next_ordinal*/ 8, + ); + let db = history_db(&store).await; + for (turn_id, item_id, ordinal) in [("one", "a", 5), ("two", "b", 7), ("three", "c", 9)] { + insert_turn( + db, + child_id, + turn_id, + ordinal, + "completed", + /*error_json*/ None, + Some(item_id), + /*final_agent_item_id*/ None, + ) + .await; + insert_item(db, child_id, turn_id, item_id, ordinal + 1).await; + } + let turns = turn_params( + child_id, + /*cursor*/ None, + /*page_size*/ 2, + SortDirection::Desc, + StoredTurnItemsView::Summary, + ); + let items = item_params( + child_id, + /*turn_id*/ None, + /*cursor*/ None, + /*page_size*/ 2, + SortDirection::Desc, + ); + let expected_turns = store.list_turns(turns.clone()).await.unwrap(); + let expected_items = store.list_items(items.clone()).await.unwrap(); + fs::remove_file(parent_path).unwrap(); + assert_eq!( + serde_json::to_value(store.list_turns(turns).await.unwrap()).unwrap(), + serde_json::to_value(expected_turns).unwrap() + ); + assert_eq!( + serde_json::to_value(store.list_items(items).await.unwrap()).unwrap(), + serde_json::to_value(expected_items).unwrap() + ); + // At exactly the local row count the reader must still discover whether an ancestor + // contributes another row, rather than incorrectly returning a terminal cursor. + assert!( + store + .list_turns(turn_params( + child_id, + /*cursor*/ None, + /*page_size*/ 3, + SortDirection::Desc, + StoredTurnItemsView::NotLoaded + )) + .await + .is_err() + ); +} + +#[tokio::test] +async fn descending_pages_stop_in_the_parent_even_when_resuming_there() { + let (home, store, child) = store_with_mode(ThreadHistoryMode::Paginated).await; + let root = ThreadId::default(); + let parent = ThreadId::default(); + let root_path = rollout_path(home.path(), root); + let parent_path = rollout_path(home.path(), parent); + write_rollout_with_end( + &root_path, root, /*history_base*/ None, /*next_ordinal*/ 4, + ); + write_rollout_with_end( + &parent_path, + parent, + Some(history_position( + &root_path, root, /*end_ordinal_exclusive*/ 3, + )), + /*next_ordinal*/ 12, + ); + write_rollout_with_end( + &rollout_path(home.path(), child), + child, + Some(history_position( + &parent_path, + parent, + /*end_ordinal_exclusive*/ 13, + )), + /*next_ordinal*/ 4, + ); + let db = history_db(&store).await; + for (id, turn_id, ordinal) in [ + (root, "child", 1), + (parent, "oldest", 4), + (parent, "older", 5), + (parent, "child", 7), + (parent, "middle", 9), + (parent, "newest", 11), + (child, "child", 14), + ] { + insert_turn( + db, + id, + turn_id, + ordinal, + if id == child { + "interrupted" + } else { + "completed" + }, + /*error_json*/ None, + (id == root).then_some("item-1"), + /*final_agent_item_id*/ None, + ) + .await; + insert_item(db, id, turn_id, &format!("item-{ordinal}"), ordinal + 1).await; + } + + let mut turns = turn_params( + child, + /*cursor*/ None, + /*page_size*/ 2, + SortDirection::Desc, + StoredTurnItemsView::NotLoaded, + ); + // The leaf and parent fill the page, but only the grandparent has the original summary. + let summary = store + .list_turns(ListTurnsParams { + items_view: StoredTurnItemsView::Summary, + page_size: 1, + ..turns.clone() + }) + .await + .unwrap(); + assert_eq!( + summary.turns[0].items, + [expected_item("child", "item-1", /*rollout_ordinal*/ 2)] + ); + + fs::remove_file(root_path).unwrap(); + let first = store.list_turns(turns.clone()).await.unwrap(); + assert_eq!(turn_ids(&first), ["child", "newest"]); + turns.cursor = Some(first.next_cursor.unwrap()); + let second = store.list_turns(turns).await.unwrap(); + // The cursor skipped the leaf, but its copy of "child" must still shadow the parent's. + assert_eq!(turn_ids(&second), ["middle", "older"]); + assert!(second.next_cursor.is_some()); + + let mut items = item_params( + child, + /*turn_id*/ None, + /*cursor*/ None, + /*page_size*/ 2, + SortDirection::Desc, + ); + let first = store.list_items(items.clone()).await.unwrap(); + assert_eq!(item_ids(&first), ["item-14", "item-11"]); + items.position = Some(ListItemsPosition::Cursor(first.next_cursor.unwrap())); + let second = store.list_items(items).await.unwrap(); + assert_eq!(item_ids(&second), ["item-9", "item-7"]); + assert!(second.next_cursor.is_some()); +} + #[tokio::test] async fn list_turns_pages_projected_rows_and_applies_item_views() { let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await; @@ -1062,6 +1240,17 @@ async fn list_items_update_ordinals_use_selected_rollout_id() { "turn-1", "item-1", /*rollout_ordinal*/ 1 )] ); + let descending_page = store + .list_items(item_params( + thread_id, + /*turn_id*/ None, + /*cursor*/ None, + /*page_size*/ 1, + SortDirection::Desc, + )) + .await + .expect("creation-ordered selected rollout page"); + assert_eq!(descending_page.items, page.items); } #[tokio::test] diff --git a/codex-rs/thread-store/src/local/thread_history/segment_paging.rs b/codex-rs/thread-store/src/local/thread_history/segment_paging.rs index 86a13ce1c9..a84ab09afc 100644 --- a/codex-rs/thread-store/src/local/thread_history/segment_paging.rs +++ b/codex-rs/thread-store/src/local/thread_history/segment_paging.rs @@ -3,15 +3,15 @@ use sqlx::FromRow; use sqlx::QueryBuilder; use sqlx::Sqlite; -use super::super::rollout_lineage::RolloutLineage; +use super::super::rollout_lineage::LineageWalk; use super::super::rollout_lineage::RolloutLineageSegment; +use super::page_segments::PageSegments; use super::read::CursorScope; use super::read::HistoryCursor; use super::read::RolloutHistoryPosition; use super::read::StoredSummaryColumns; use super::read::StoredThreadItemRow; use super::read::StoredTurnRow; -use super::read::invalid_cursor; use super::read::parse_cursor; use super::read::serialize_cursor; use super::read::stored_thread_item_row; @@ -85,7 +85,7 @@ pub(super) fn validate_page_size(page_size: usize) -> ThreadStoreResult<()> { pub(super) async fn page_turn_rows( pool: &sqlx::SqlitePool, requested_thread_id: ThreadId, - lineage: &RolloutLineage, + lineage: &mut LineageWalk<'_>, cursor: Option<&str>, page_size: usize, direction: SortDirection, @@ -94,13 +94,13 @@ pub(super) async fn page_turn_rows( let cursor = parse_cursor(cursor, requested_thread_id, CursorScope::Turns)?; let ordinal_column = HistoryOrdinalColumn::Rollout; let mut rows = Vec::new(); - for (segment_index, segment, segment_cursor) in - segments_from_cursor(lineage, direction, cursor.as_ref())? - { - let remaining = remaining_limit(page_size, rows.len())?; - if remaining == 0 { + let mut segments = PageSegments::from_cursor(lineage, direction, cursor).await?; + while rows.len() <= page_size { + let Some(range) = segments.next().await? else { break; - } + }; + let segment = range.segment; + let remaining = remaining_limit(page_size, rows.len())?; let mut query = QueryBuilder::::new(""); if matches!(items_view, StoredTurnItemsView::Summary) { query.push("WITH page_turns AS ("); @@ -123,7 +123,7 @@ WHERE thread_id = ); query.push_bind(segment.rollout_id().to_string()); push_segment_range(&mut query, segment, ordinal_column)?; - for newer_segment in &lineage.segments()[segment_index + 1..] { + for newer_segment in range.newer { query .push(" AND NOT EXISTS (SELECT 1 FROM thread_turns AS newer_turn WHERE newer_turn.thread_id = ") .push_bind(newer_segment.rollout_id().to_string()) @@ -136,7 +136,7 @@ WHERE thread_id = } query.push(")"); } - push_cursor_clause(&mut query, direction, segment_cursor, ordinal_column)?; + push_cursor_clause(&mut query, direction, range.cursor.as_ref(), ordinal_column)?; push_order_and_limit(&mut query, direction, remaining, ordinal_column); if matches!(items_view, StoredTurnItemsView::Summary) { query.push( @@ -234,7 +234,7 @@ fn push_summary_item_join( pub(super) async fn page_item_rows( pool: &sqlx::SqlitePool, - lineage: &RolloutLineage, + lineage: &mut LineageWalk<'_>, params: &ListItemsParams, ) -> ThreadStoreResult> { let (cursor, anchor_item_id) = match params.position.as_ref() { @@ -255,6 +255,10 @@ pub(super) async fn page_item_rows( (None, Some(ordinal)) => Some(UpdateOrdinalBoundary::Before(ordinal)), (None, None) => None, }; + // Keep update replay eager, including when its rows are sorted by creation time. + if update_boundary.is_some() { + lineage.resolve_all().await?; + } if anchor_item_id.is_some() && (params.sort_key != ItemSortKey::CreatedAtOrdinal || update_boundary.is_some()) { @@ -283,7 +287,7 @@ pub(super) async fn page_item_rows( return Err(invalid_item_anchor()); } let mut anchor = None; - for segment in lineage.segments() { + for segment in lineage.resolve_all().await?.iter().rev() { let mut query = QueryBuilder::::new( "SELECT rollout_ordinal FROM thread_items WHERE thread_id = ", ); @@ -317,13 +321,13 @@ pub(super) async fn page_item_rows( }; let ordinal_column = HistoryOrdinalColumn::Rollout; let mut rows = Vec::new(); - for (_, segment, segment_cursor) in - segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())? - { - let remaining = remaining_limit(params.page_size, rows.len())?; - if remaining == 0 { + let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?; + while rows.len() <= params.page_size { + let Some(range) = segments.next().await? else { break; - } + }; + let segment = range.segment; + let remaining = remaining_limit(params.page_size, rows.len())?; let mut query = QueryBuilder::::new( r#" SELECT turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json @@ -358,7 +362,7 @@ FROM thread_items"#, push_cursor_clause( &mut query, params.sort_direction, - segment_cursor, + range.cursor.as_ref(), ordinal_column, )?; push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column); @@ -383,7 +387,7 @@ FROM thread_items"#, async fn page_updated_item_rows( pool: &sqlx::SqlitePool, - lineage: &RolloutLineage, + lineage: &mut LineageWalk<'_>, params: &ListItemsParams, cursor: Option<&str>, update_boundary: UpdateOrdinalBoundary, @@ -395,13 +399,13 @@ async fn page_updated_item_rows( )?; let ordinal_column = HistoryOrdinalColumn::UpdatedAt; let mut rows = Vec::new(); - for (_, segment, segment_cursor) in - segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())? - { - let remaining = remaining_limit(params.page_size, rows.len())?; - if remaining == 0 { + let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?; + while rows.len() <= params.page_size { + let Some(range) = segments.next().await? else { break; - } + }; + let segment = range.segment; + let remaining = remaining_limit(params.page_size, rows.len())?; let mut query = QueryBuilder::::new( r#" SELECT turn_id, item_id, updated_at_ordinal AS rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json @@ -418,7 +422,7 @@ WHERE thread_id = push_cursor_clause( &mut query, params.sort_direction, - segment_cursor, + range.cursor.as_ref(), ordinal_column, )?; push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column); @@ -441,39 +445,6 @@ WHERE thread_id = ) } -fn segments_from_cursor<'a>( - lineage: &'a RolloutLineage, - direction: SortDirection, - cursor: Option<&'a HistoryCursor>, -) -> ThreadStoreResult)>> { - let segments = lineage.segments(); - let cursor_index = cursor - .map(|cursor| { - lineage - .segment_index_for_ordinal(cursor.rollout_ordinal) - .ok_or_else(|| invalid_cursor("position outside thread lineage")) - }) - .transpose()?; - let indexes: Vec = match direction { - SortDirection::Asc => (cursor_index.unwrap_or(0)..segments.len()).collect(), - SortDirection::Desc => { - let end = cursor_index.unwrap_or_else(|| segments.len().saturating_sub(1)); - (0..=end).rev().collect() - } - }; - Ok(indexes - .into_iter() - .map(|index| { - let segment_cursor = if Some(index) == cursor_index { - cursor - } else { - None - }; - (index, &segments[index], segment_cursor) - }) - .collect()) -} - fn push_segment_range( query: &mut QueryBuilder, segment: &RolloutLineageSegment, diff --git a/codex-rs/thread-store/src/local/thread_history/turn_lookup.rs b/codex-rs/thread-store/src/local/thread_history/turn_lookup.rs index ba37aa0721..22b904706c 100644 --- a/codex-rs/thread-store/src/local/thread_history/turn_lookup.rs +++ b/codex-rs/thread-store/src/local/thread_history/turn_lookup.rs @@ -85,7 +85,7 @@ pub(in crate::local) async fn find_visible_turn( find_turn(pool, lineage.segments().iter().rev(), turn_id).await } -async fn find_turn<'a>( +pub(super) async fn find_turn<'a>( pool: &sqlx::SqlitePool, segments: impl Iterator, turn_id: &str,