Resolve fork ancestry lazily for descending history pages (#61840)

## Why

Reading the newest turns or items in a fork previously resolved the entire rollout lineage, even when the page only needed recent segments. This caused unnecessary ancestor reads and could fail when an unused ancestor was missing.

## What changed

Introduce an incremental `LineageWalk` and use it to resolve descending history pages only as far as needed for the cursor, requested rows, and next-page lookahead. Defer sibling rollout discovery until a parent is needed, while preserving newer turns' overrides when resuming in an ancestor.

Keep full lineage resolution for ascending reads, update replay, item anchors, and inherited summary lookup.

## Testing

Add regression coverage for pages with missing unused ancestors, continuation within a parent, turn overrides, inherited summaries, and exact page-boundary lookahead. Extend coverage for descending item pages using the selected rollout.

GitOrigin-RevId: dad941f4f3c54a46121d4fa22b69e13f1830a385
This commit is contained in:
Charlie Marsh
2026-11-16 16:37:03 +00:00
committed by copyberry
parent 586203ad13
commit bc4a2544b8
9 changed files with 548 additions and 236 deletions
@@ -1,6 +1,4 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
@@ -12,8 +10,11 @@ use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::RolloutItem;
mod ancestor_reads;
mod walk;
pub(super) use ancestor_reads::AncestorReads;
use ancestor_reads::FailedAncestor;
pub(super) use walk::LineageWalk;
use super::LocalThreadStore;
use super::thread_rollout_resolver;
@@ -30,10 +31,7 @@ pub(super) struct RolloutLineageSegment {
hides_parent: bool,
}
/// Ordered rollout ranges contributing to one forked history.
///
/// This is the only local abstraction that follows SessionMeta.history_base pointers. Readers
/// consume its bounded rollout segments without resolving or mutating fork pointers themselves.
/// The resolved ranges of a forked history, ordered from oldest to newest.
#[derive(Clone, Debug)]
pub(super) struct RolloutLineage {
pub(super) segments: Vec<RolloutLineageSegment>,
@@ -48,12 +46,14 @@ impl LocalThreadStore {
requested_thread_id: ThreadId,
initial_path: Option<PathBuf>,
) -> ThreadStoreResult<RolloutLineage> {
self.resolve_rollout_lineage_with_mode(
LineageWalk::new(
self,
requested_thread_id,
LineageResolutionMode::Existing,
initial_path,
&HashMap::new(),
/*excluded*/ None,
)
.finish()
.await
}
@@ -61,155 +61,25 @@ impl LocalThreadStore {
&self,
requested_thread_id: ThreadId,
) -> ThreadStoreResult<RolloutLineage> {
self.resolve_rollout_lineage_with_mode(
requested_thread_id,
LineageResolutionMode::HistoryPage,
/*initial_path*/ None,
&HashMap::new(),
)
.await
LineageWalk::for_history_page(self, requested_thread_id)
.finish()
.await
}
pub(super) async fn resolve_rollout_lineage_for_reference(
&self,
requested_thread_id: ThreadId,
) -> ThreadStoreResult<RolloutLineage> {
self.resolve_rollout_lineage_with_mode(
LineageWalk::new(
self,
requested_thread_id,
LineageResolutionMode::PlainForReference,
/*initial_path*/ None,
&HashMap::new(),
/*excluded*/ None,
)
.finish()
.await
}
async fn resolve_rollout_lineage_with_mode(
&self,
requested_thread_id: ThreadId,
mode: LineageResolutionMode,
mut initial_path: Option<PathBuf>,
excluded: &HashMap<ThreadId, FailedAncestor>,
) -> ThreadStoreResult<RolloutLineage> {
let mut segments = Vec::new();
let mut seen = HashSet::new();
let mut end: Option<HistoryPosition> = None;
let mut prefetched_paths = HashMap::new();
let mut ancestor_paths = HashMap::new();
loop {
let coordination_id = end.map_or(requested_thread_id, |end| end.thread_id);
let _writer_guard = match mode {
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None,
LineageResolutionMode::PlainForReference => {
Some(self.live_writer_locks.lock(coordination_id).await)
}
};
let (rollout_id, (rollout_path, bounds)) = if let Some(end) = end {
let ancestor = resolve_ancestor_rollout(
self,
requested_thread_id,
end,
mode,
prefetched_paths.remove(&end.thread_id),
excluded.get(&end.thread_id),
)
.await?;
(end.thread_id, ancestor)
} else {
let (rollout_id, path) = match initial_path.take() {
Some(path) => {
let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id(
&path,
requested_thread_id,
ThreadHistoryMode::Paginated,
)?;
(id, path)
}
None => {
let resolved = thread_rollout_resolver::resolve_current_including_archived(
self,
requested_thread_id,
)
.await?
.ok_or_else(|| {
malformed_lineage(requested_thread_id, "missing source rollout")
})?;
(resolved.rollout_id, resolved.path)
}
};
let (path, mut bounds) = mode
.read_rollout(self, path, requested_thread_id, Some(requested_thread_id))
.await?;
if matches!(mode, LineageResolutionMode::HistoryPage) {
bounds
.read_history_page_bounds(&path, requested_thread_id)
.await?;
}
(rollout_id, (path, bounds))
};
if !seen.insert(rollout_id) {
return Err(malformed_lineage(requested_thread_id, "cycle detected"));
}
let rollout_path = match mode {
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => {
rollout_path
}
LineageResolutionMode::PlainForReference
if end.is_none() && bounds.parent.is_none() =>
{
// A newly shared standalone source must remain readable by older binaries.
codex_rollout::materialize_rollout_for_reference(rollout_path.as_path())
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!(
"failed to materialize referenced rollout {}: {err}",
rollout_path.display()
),
})?
}
// Already-shared compressed history requires a compatible reader regardless of
// new forks. Read it without publishing decoded copies into ancestors' folders;
// their owners may concurrently archive or unarchive those immutable files.
LineageResolutionMode::PlainForReference => rollout_path,
};
if let Some(end) = end
&& !matches!(mode, LineageResolutionMode::HistoryPage)
{
ancestor_paths.insert(rollout_path.clone(), (end, bounds.owner));
}
segments.push(RolloutLineageSegment {
rollout_id,
rollout_path,
start_ordinal: bounds.start_ordinal,
end,
hides_parent: bounds.hides_parent,
});
let Some(base) = bounds.parent else {
break;
};
if matches!(mode, LineageResolutionMode::HistoryPage)
&& let [child] = segments.as_slice()
{
prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths(
&self.config.codex_home,
&child.rollout_path,
base.thread_id,
)
.await
.unwrap_or_default();
}
end = Some(base);
}
segments.reverse();
Ok(RolloutLineage {
segments,
reads: AncestorReads::new(ancestor_paths),
requested_thread_id,
mode,
})
}
}
/// A mid-turn fork starts with its header, settings, optional model guidance, and parent abort.
@@ -533,12 +403,9 @@ impl RolloutLineage {
}
pub(super) fn segment_index_for_ordinal(&self, ordinal: u64) -> Option<usize> {
self.segments.iter().position(|segment| {
ordinal >= segment.start_ordinal()
&& segment
.end_ordinal()
.is_none_or(|end_ordinal| ordinal < end_ordinal)
})
self.segments
.iter()
.position(|segment| segment.contains(ordinal))
}
pub(super) async fn truncate_at(
@@ -585,6 +452,10 @@ impl RolloutLineage {
}
impl RolloutLineageSegment {
pub(super) fn contains(&self, ordinal: u64) -> bool {
ordinal >= self.start_ordinal && self.end_ordinal().is_none_or(|end| ordinal < end)
}
pub(super) fn rollout_id(&self) -> ThreadId {
self.rollout_id
}
@@ -14,6 +14,7 @@ use std::sync::OnceLock;
use codex_protocol::ThreadId;
use codex_protocol::protocol::HistoryPosition;
use super::LineageWalk;
use super::LocalThreadStore;
use super::RolloutLineage;
use crate::ThreadStoreError;
@@ -133,15 +134,16 @@ impl RolloutLineage {
error: error.clone(),
},
);
lineage = store
.resolve_rollout_lineage_with_mode(
requested_thread_id,
mode,
leaf.clone(),
&excluded,
)
.await
.map_err(|_| error)?;
lineage = LineageWalk::new(
store,
requested_thread_id,
mode,
leaf.clone(),
Some(&excluded),
)
.finish()
.await
.map_err(|_| error)?;
}
}
}
@@ -0,0 +1,215 @@
//! Walks immutable rollout ranges from the selected leaf toward its ancestors on demand.
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
use codex_protocol::ThreadId;
use codex_protocol::protocol::HistoryPosition;
use codex_protocol::protocol::ThreadHistoryMode;
use super::AncestorReads;
use super::FailedAncestor;
use super::LineageResolutionMode;
use super::LocalThreadStore;
use super::RolloutLineage;
use super::RolloutLineageSegment;
use super::malformed_lineage;
use super::resolve_ancestor_rollout;
use super::thread_rollout_resolver;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
/// The next file to open. A parent bookmark bounds the rows inherited from that file.
enum NextRollout {
Selected(Option<PathBuf>),
Parent(HistoryPosition),
}
pub(in crate::local) struct LineageWalk<'a> {
store: &'a LocalThreadStore,
requested_thread_id: ThreadId,
mode: LineageResolutionMode,
next: Option<NextRollout>,
seen: HashSet<ThreadId>,
prefetched_paths: HashMap<ThreadId, PathBuf>,
excluded: Option<&'a HashMap<ThreadId, FailedAncestor>>,
ancestor_paths: HashMap<PathBuf, (HistoryPosition, ThreadId)>,
// Newest first, so discovering one older segment is a push. Only finish reverses this.
segments: Vec<RolloutLineageSegment>,
}
impl<'a> LineageWalk<'a> {
pub(super) fn new(
store: &'a LocalThreadStore,
requested_thread_id: ThreadId,
mode: LineageResolutionMode,
initial_path: Option<PathBuf>,
excluded: Option<&'a HashMap<ThreadId, FailedAncestor>>,
) -> Self {
Self {
store,
requested_thread_id,
mode,
next: Some(NextRollout::Selected(initial_path)),
seen: HashSet::new(),
prefetched_paths: HashMap::new(),
excluded,
ancestor_paths: HashMap::new(),
segments: Vec::new(),
}
}
pub(in crate::local) fn for_history_page(
store: &'a LocalThreadStore,
thread_id: ThreadId,
) -> Self {
Self::new(
store,
thread_id,
LineageResolutionMode::HistoryPage,
/*initial_path*/ None,
/*excluded*/ None,
)
}
/// Opens all remaining ancestors, oldest first in the resulting lineage.
pub(in crate::local) async fn finish(mut self) -> ThreadStoreResult<RolloutLineage> {
self.resolve_all().await?;
self.segments.reverse();
Ok(RolloutLineage {
segments: self.segments,
reads: AncestorReads::new(self.ancestor_paths),
requested_thread_id: self.requested_thread_id,
mode: self.mode,
})
}
pub(in crate::local) async fn resolve_all(
&mut self,
) -> ThreadStoreResult<&[RolloutLineageSegment]> {
while self.next_older().await? {}
Ok(&self.segments)
}
/// Returns the segment at the given depth from the leaf and all newer segments.
pub(in crate::local) async fn at_depth(
&mut self,
depth: usize,
) -> ThreadStoreResult<Option<(&RolloutLineageSegment, &[RolloutLineageSegment])>> {
while self.segments.len() <= depth {
if !self.next_older().await? {
return Ok(None);
}
}
Ok(self.segments[..=depth].split_last())
}
async fn next_older(&mut self) -> ThreadStoreResult<bool> {
let Some(next) = self.next.take() else {
return Ok(false);
};
let requested_thread_id = self.requested_thread_id;
let store = self.store;
let _writer_guard = match self.mode {
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None,
LineageResolutionMode::PlainForReference => {
let id = match &next {
NextRollout::Selected(_) => requested_thread_id,
NextRollout::Parent(end) => end.thread_id,
};
Some(store.live_writer_locks.lock(id).await)
}
};
let (rollout_id, rollout_path, bounds, end) = match next {
NextRollout::Selected(initial_path) => {
let (rollout_id, path) = match initial_path {
Some(path) => {
let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id(
&path,
requested_thread_id,
ThreadHistoryMode::Paginated,
)?;
(id, path)
}
None => {
let resolved = thread_rollout_resolver::resolve_current_including_archived(
store,
requested_thread_id,
)
.await?
.ok_or_else(|| {
malformed_lineage(requested_thread_id, "missing source rollout")
})?;
(resolved.rollout_id, resolved.path)
}
};
let (mut path, mut bounds) = self
.mode
.read_rollout(store, path, requested_thread_id, Some(requested_thread_id))
.await?;
match self.mode {
LineageResolutionMode::HistoryPage => {
bounds
.read_history_page_bounds(&path, requested_thread_id)
.await?
}
LineageResolutionMode::PlainForReference if bounds.parent.is_none() => {
// A newly shared standalone source must remain readable by older binaries.
path = codex_rollout::materialize_rollout_for_reference(&path)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!(
"failed to materialize referenced rollout {}: {err}",
path.display()
),
})?;
}
LineageResolutionMode::Existing | LineageResolutionMode::PlainForReference => {}
}
(rollout_id, path, bounds, None)
}
NextRollout::Parent(end) => {
// Defer the directory scan too: pages contained in the leaf never need it.
if matches!(self.mode, LineageResolutionMode::HistoryPage)
&& let [child] = self.segments.as_slice()
{
self.prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths(
&store.config.codex_home,
&child.rollout_path,
end.thread_id,
)
.await
.unwrap_or_default();
}
let (path, bounds) = resolve_ancestor_rollout(
store,
requested_thread_id,
end,
self.mode,
self.prefetched_paths.remove(&end.thread_id),
self.excluded
.and_then(|excluded| excluded.get(&end.thread_id)),
)
.await?;
if !matches!(self.mode, LineageResolutionMode::HistoryPage) {
self.ancestor_paths
.insert(path.clone(), (end, bounds.owner));
}
(end.thread_id, path, bounds, Some(end))
}
};
if !self.seen.insert(rollout_id) {
return Err(malformed_lineage(requested_thread_id, "cycle detected"));
}
self.next = bounds.parent.map(NextRollout::Parent);
self.segments.push(RolloutLineageSegment {
rollout_id,
rollout_path,
start_ordinal: bounds.start_ordinal,
end,
hides_parent: bounds.hides_parent,
});
Ok(true)
}
}
@@ -10,6 +10,7 @@ use super::LocalThreadStore;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
mod page_segments;
mod read;
mod realtime;
mod realtime_stream;
@@ -0,0 +1,67 @@
//! Seeks a history cursor and supplies the pager with each segment and its newer overrides.
use super::super::rollout_lineage::LineageWalk;
use super::super::rollout_lineage::RolloutLineageSegment;
use super::read::HistoryCursor;
use super::read::invalid_cursor;
use crate::SortDirection;
use crate::ThreadStoreResult;
pub(super) struct PageSegments<'a, 'store> {
lineage: &'a mut LineageWalk<'store>,
step: isize,
next_depth: Option<usize>,
cursor: Option<HistoryCursor>,
}
pub(super) struct PageSegment<'a> {
pub segment: &'a RolloutLineageSegment,
pub newer: &'a [RolloutLineageSegment],
pub cursor: Option<HistoryCursor>,
}
impl<'a, 'store> PageSegments<'a, 'store> {
pub(super) async fn from_cursor(
lineage: &'a mut LineageWalk<'store>,
direction: SortDirection,
cursor: Option<HistoryCursor>,
) -> ThreadStoreResult<Self> {
// Ascending reads must discover the oldest segment up front. Descending reads start
// at the leaf and seek only as far as the cursor, retaining all newer shadowing turns.
let (start, step) = match direction {
SortDirection::Asc => (lineage.resolve_all().await?.len().checked_sub(1), -1),
SortDirection::Desc => (Some(0), 1),
};
let mut next_depth = start;
if let Some(cursor) = &cursor {
loop {
let missing = || invalid_cursor("position outside thread lineage");
let depth = next_depth.ok_or_else(missing)?;
let (segment, _) = lineage.at_depth(depth).await?.ok_or_else(missing)?;
if segment.contains(cursor.rollout_ordinal) {
break;
}
next_depth = depth.checked_add_signed(step);
}
}
Ok(Self {
lineage,
step,
next_depth,
cursor,
})
}
pub(super) async fn next(&mut self) -> ThreadStoreResult<Option<PageSegment<'_>>> {
let Some(depth) = self.next_depth else {
return Ok(None);
};
let segment = self.lineage.at_depth(depth).await?;
self.next_depth = depth.checked_add_signed(self.step);
Ok(segment.map(|(segment, newer)| PageSegment {
segment,
newer,
cursor: self.cursor.take(),
}))
}
}
@@ -5,12 +5,12 @@ use serde::Serialize;
use sqlx::Row;
use super::super::LocalThreadStore;
use super::super::rollout_lineage::RolloutLineage;
use super::super::rollout_lineage::LineageWalk;
use super::segment_paging::page_item_rows;
use super::segment_paging::page_turn_rows;
use super::segment_paging::validate_page_size;
use super::sqlite_integer;
use super::turn_lookup::find_source_turn;
use super::turn_lookup::find_turn;
use crate::ItemPage;
use crate::ListItemsParams;
use crate::ListTurnsParams;
@@ -111,14 +111,12 @@ pub(in crate::local) async fn list_turns(
)
.await?;
validate_page_size(params.page_size)?;
let lineage = store
.resolve_rollout_lineage_for_history_page(params.thread_id)
.await?;
let pool = store.thread_history_db().await?;
let mut lineage = LineageWalk::for_history_page(store, params.thread_id);
let page = page_turn_rows(
pool,
params.thread_id,
&lineage,
&mut lineage,
params.cursor.as_deref(),
params.page_size,
params.sort_direction,
@@ -136,7 +134,7 @@ pub(in crate::local) async fn list_turns(
{
// Synthetic fork-boundary rows are interrupted without local summary IDs.
// Load their summary from the earliest visible source turn.
load_inherited_summary_items(pool, &lineage, &turn).await?
load_inherited_summary_items(pool, &mut lineage, &turn).await?
}
StoredTurnItemsView::Summary => turn.summary_items,
};
@@ -188,11 +186,9 @@ pub(in crate::local) async fn list_items(
params.sort_direction = crate::SortDirection::Asc;
}
validate_page_size(params.page_size)?;
let lineage = store
.resolve_rollout_lineage_for_history_page(params.thread_id)
.await?;
let pool = store.thread_history_db().await?;
let page = page_item_rows(pool, &lineage, &params).await?;
let mut lineage = LineageWalk::for_history_page(store, params.thread_id);
let page = page_item_rows(pool, &mut lineage, &params).await?;
Ok(ItemPage {
items: page.rows.into_iter().map(|row| row.item).collect(),
@@ -233,12 +229,12 @@ pub(super) async fn validate_thread_for_paginated_reads(
async fn load_inherited_summary_items(
pool: &sqlx::SqlitePool,
lineage: &RolloutLineage,
lineage: &mut LineageWalk<'_>,
turn: &StoredTurnRow,
) -> ThreadStoreResult<Vec<StoredThreadItem>> {
let source = find_source_turn(pool, lineage, turn.turn_id.as_str()).await?;
let lineage = lineage.resolve_all().await?;
let source = find_turn(pool, lineage.iter().rev(), turn.turn_id.as_str()).await?;
let Some(segment) = lineage
.segments()
.iter()
.find(|segment| segment.rollout_id() == source.rollout_id)
else {
@@ -31,6 +31,184 @@ use crate::StoredTurnError;
use crate::StoredTurnStatus;
use crate::local::test_support::test_config;
#[tokio::test]
async fn newest_indexed_pages_do_not_read_unused_ancestors() {
let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
let parent_id = ThreadId::default();
let parent_path = rollout_path(home.path(), parent_id);
write_rollout_with_end(
&parent_path,
parent_id,
/*history_base*/ None,
/*next_ordinal*/ 5,
);
write_rollout_with_end(
&rollout_path(home.path(), child_id),
child_id,
Some(history_position(
&parent_path,
parent_id,
/*end_ordinal_exclusive*/ 4,
)),
/*next_ordinal*/ 8,
);
let db = history_db(&store).await;
for (turn_id, item_id, ordinal) in [("one", "a", 5), ("two", "b", 7), ("three", "c", 9)] {
insert_turn(
db,
child_id,
turn_id,
ordinal,
"completed",
/*error_json*/ None,
Some(item_id),
/*final_agent_item_id*/ None,
)
.await;
insert_item(db, child_id, turn_id, item_id, ordinal + 1).await;
}
let turns = turn_params(
child_id,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Desc,
StoredTurnItemsView::Summary,
);
let items = item_params(
child_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Desc,
);
let expected_turns = store.list_turns(turns.clone()).await.unwrap();
let expected_items = store.list_items(items.clone()).await.unwrap();
fs::remove_file(parent_path).unwrap();
assert_eq!(
serde_json::to_value(store.list_turns(turns).await.unwrap()).unwrap(),
serde_json::to_value(expected_turns).unwrap()
);
assert_eq!(
serde_json::to_value(store.list_items(items).await.unwrap()).unwrap(),
serde_json::to_value(expected_items).unwrap()
);
// At exactly the local row count the reader must still discover whether an ancestor
// contributes another row, rather than incorrectly returning a terminal cursor.
assert!(
store
.list_turns(turn_params(
child_id,
/*cursor*/ None,
/*page_size*/ 3,
SortDirection::Desc,
StoredTurnItemsView::NotLoaded
))
.await
.is_err()
);
}
#[tokio::test]
async fn descending_pages_stop_in_the_parent_even_when_resuming_there() {
let (home, store, child) = store_with_mode(ThreadHistoryMode::Paginated).await;
let root = ThreadId::default();
let parent = ThreadId::default();
let root_path = rollout_path(home.path(), root);
let parent_path = rollout_path(home.path(), parent);
write_rollout_with_end(
&root_path, root, /*history_base*/ None, /*next_ordinal*/ 4,
);
write_rollout_with_end(
&parent_path,
parent,
Some(history_position(
&root_path, root, /*end_ordinal_exclusive*/ 3,
)),
/*next_ordinal*/ 12,
);
write_rollout_with_end(
&rollout_path(home.path(), child),
child,
Some(history_position(
&parent_path,
parent,
/*end_ordinal_exclusive*/ 13,
)),
/*next_ordinal*/ 4,
);
let db = history_db(&store).await;
for (id, turn_id, ordinal) in [
(root, "child", 1),
(parent, "oldest", 4),
(parent, "older", 5),
(parent, "child", 7),
(parent, "middle", 9),
(parent, "newest", 11),
(child, "child", 14),
] {
insert_turn(
db,
id,
turn_id,
ordinal,
if id == child {
"interrupted"
} else {
"completed"
},
/*error_json*/ None,
(id == root).then_some("item-1"),
/*final_agent_item_id*/ None,
)
.await;
insert_item(db, id, turn_id, &format!("item-{ordinal}"), ordinal + 1).await;
}
let mut turns = turn_params(
child,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Desc,
StoredTurnItemsView::NotLoaded,
);
// The leaf and parent fill the page, but only the grandparent has the original summary.
let summary = store
.list_turns(ListTurnsParams {
items_view: StoredTurnItemsView::Summary,
page_size: 1,
..turns.clone()
})
.await
.unwrap();
assert_eq!(
summary.turns[0].items,
[expected_item("child", "item-1", /*rollout_ordinal*/ 2)]
);
fs::remove_file(root_path).unwrap();
let first = store.list_turns(turns.clone()).await.unwrap();
assert_eq!(turn_ids(&first), ["child", "newest"]);
turns.cursor = Some(first.next_cursor.unwrap());
let second = store.list_turns(turns).await.unwrap();
// The cursor skipped the leaf, but its copy of "child" must still shadow the parent's.
assert_eq!(turn_ids(&second), ["middle", "older"]);
assert!(second.next_cursor.is_some());
let mut items = item_params(
child,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Desc,
);
let first = store.list_items(items.clone()).await.unwrap();
assert_eq!(item_ids(&first), ["item-14", "item-11"]);
items.position = Some(ListItemsPosition::Cursor(first.next_cursor.unwrap()));
let second = store.list_items(items).await.unwrap();
assert_eq!(item_ids(&second), ["item-9", "item-7"]);
assert!(second.next_cursor.is_some());
}
#[tokio::test]
async fn list_turns_pages_projected_rows_and_applies_item_views() {
let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
@@ -1062,6 +1240,17 @@ async fn list_items_update_ordinals_use_selected_rollout_id() {
"turn-1", "item-1", /*rollout_ordinal*/ 1
)]
);
let descending_page = store
.list_items(item_params(
thread_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 1,
SortDirection::Desc,
))
.await
.expect("creation-ordered selected rollout page");
assert_eq!(descending_page.items, page.items);
}
#[tokio::test]
@@ -3,15 +3,15 @@ use sqlx::FromRow;
use sqlx::QueryBuilder;
use sqlx::Sqlite;
use super::super::rollout_lineage::RolloutLineage;
use super::super::rollout_lineage::LineageWalk;
use super::super::rollout_lineage::RolloutLineageSegment;
use super::page_segments::PageSegments;
use super::read::CursorScope;
use super::read::HistoryCursor;
use super::read::RolloutHistoryPosition;
use super::read::StoredSummaryColumns;
use super::read::StoredThreadItemRow;
use super::read::StoredTurnRow;
use super::read::invalid_cursor;
use super::read::parse_cursor;
use super::read::serialize_cursor;
use super::read::stored_thread_item_row;
@@ -85,7 +85,7 @@ pub(super) fn validate_page_size(page_size: usize) -> ThreadStoreResult<()> {
pub(super) async fn page_turn_rows(
pool: &sqlx::SqlitePool,
requested_thread_id: ThreadId,
lineage: &RolloutLineage,
lineage: &mut LineageWalk<'_>,
cursor: Option<&str>,
page_size: usize,
direction: SortDirection,
@@ -94,13 +94,13 @@ pub(super) async fn page_turn_rows(
let cursor = parse_cursor(cursor, requested_thread_id, CursorScope::Turns)?;
let ordinal_column = HistoryOrdinalColumn::Rollout;
let mut rows = Vec::new();
for (segment_index, segment, segment_cursor) in
segments_from_cursor(lineage, direction, cursor.as_ref())?
{
let remaining = remaining_limit(page_size, rows.len())?;
if remaining == 0 {
let mut segments = PageSegments::from_cursor(lineage, direction, cursor).await?;
while rows.len() <= page_size {
let Some(range) = segments.next().await? else {
break;
}
};
let segment = range.segment;
let remaining = remaining_limit(page_size, rows.len())?;
let mut query = QueryBuilder::<Sqlite>::new("");
if matches!(items_view, StoredTurnItemsView::Summary) {
query.push("WITH page_turns AS (");
@@ -123,7 +123,7 @@ WHERE thread_id =
);
query.push_bind(segment.rollout_id().to_string());
push_segment_range(&mut query, segment, ordinal_column)?;
for newer_segment in &lineage.segments()[segment_index + 1..] {
for newer_segment in range.newer {
query
.push(" AND NOT EXISTS (SELECT 1 FROM thread_turns AS newer_turn WHERE newer_turn.thread_id = ")
.push_bind(newer_segment.rollout_id().to_string())
@@ -136,7 +136,7 @@ WHERE thread_id =
}
query.push(")");
}
push_cursor_clause(&mut query, direction, segment_cursor, ordinal_column)?;
push_cursor_clause(&mut query, direction, range.cursor.as_ref(), ordinal_column)?;
push_order_and_limit(&mut query, direction, remaining, ordinal_column);
if matches!(items_view, StoredTurnItemsView::Summary) {
query.push(
@@ -234,7 +234,7 @@ fn push_summary_item_join(
pub(super) async fn page_item_rows(
pool: &sqlx::SqlitePool,
lineage: &RolloutLineage,
lineage: &mut LineageWalk<'_>,
params: &ListItemsParams,
) -> ThreadStoreResult<SegmentPage<StoredThreadItemRow>> {
let (cursor, anchor_item_id) = match params.position.as_ref() {
@@ -255,6 +255,10 @@ pub(super) async fn page_item_rows(
(None, Some(ordinal)) => Some(UpdateOrdinalBoundary::Before(ordinal)),
(None, None) => None,
};
// Keep update replay eager, including when its rows are sorted by creation time.
if update_boundary.is_some() {
lineage.resolve_all().await?;
}
if anchor_item_id.is_some()
&& (params.sort_key != ItemSortKey::CreatedAtOrdinal || update_boundary.is_some())
{
@@ -283,7 +287,7 @@ pub(super) async fn page_item_rows(
return Err(invalid_item_anchor());
}
let mut anchor = None;
for segment in lineage.segments() {
for segment in lineage.resolve_all().await?.iter().rev() {
let mut query = QueryBuilder::<Sqlite>::new(
"SELECT rollout_ordinal FROM thread_items WHERE thread_id = ",
);
@@ -317,13 +321,13 @@ pub(super) async fn page_item_rows(
};
let ordinal_column = HistoryOrdinalColumn::Rollout;
let mut rows = Vec::new();
for (_, segment, segment_cursor) in
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
{
let remaining = remaining_limit(params.page_size, rows.len())?;
if remaining == 0 {
let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?;
while rows.len() <= params.page_size {
let Some(range) = segments.next().await? else {
break;
}
};
let segment = range.segment;
let remaining = remaining_limit(params.page_size, rows.len())?;
let mut query = QueryBuilder::<Sqlite>::new(
r#"
SELECT turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json
@@ -358,7 +362,7 @@ FROM thread_items"#,
push_cursor_clause(
&mut query,
params.sort_direction,
segment_cursor,
range.cursor.as_ref(),
ordinal_column,
)?;
push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column);
@@ -383,7 +387,7 @@ FROM thread_items"#,
async fn page_updated_item_rows(
pool: &sqlx::SqlitePool,
lineage: &RolloutLineage,
lineage: &mut LineageWalk<'_>,
params: &ListItemsParams,
cursor: Option<&str>,
update_boundary: UpdateOrdinalBoundary,
@@ -395,13 +399,13 @@ async fn page_updated_item_rows(
)?;
let ordinal_column = HistoryOrdinalColumn::UpdatedAt;
let mut rows = Vec::new();
for (_, segment, segment_cursor) in
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
{
let remaining = remaining_limit(params.page_size, rows.len())?;
if remaining == 0 {
let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?;
while rows.len() <= params.page_size {
let Some(range) = segments.next().await? else {
break;
}
};
let segment = range.segment;
let remaining = remaining_limit(params.page_size, rows.len())?;
let mut query = QueryBuilder::<Sqlite>::new(
r#"
SELECT turn_id, item_id, updated_at_ordinal AS rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json
@@ -418,7 +422,7 @@ WHERE thread_id =
push_cursor_clause(
&mut query,
params.sort_direction,
segment_cursor,
range.cursor.as_ref(),
ordinal_column,
)?;
push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column);
@@ -441,39 +445,6 @@ WHERE thread_id =
)
}
fn segments_from_cursor<'a>(
lineage: &'a RolloutLineage,
direction: SortDirection,
cursor: Option<&'a HistoryCursor>,
) -> ThreadStoreResult<Vec<(usize, &'a RolloutLineageSegment, Option<&'a HistoryCursor>)>> {
let segments = lineage.segments();
let cursor_index = cursor
.map(|cursor| {
lineage
.segment_index_for_ordinal(cursor.rollout_ordinal)
.ok_or_else(|| invalid_cursor("position outside thread lineage"))
})
.transpose()?;
let indexes: Vec<usize> = match direction {
SortDirection::Asc => (cursor_index.unwrap_or(0)..segments.len()).collect(),
SortDirection::Desc => {
let end = cursor_index.unwrap_or_else(|| segments.len().saturating_sub(1));
(0..=end).rev().collect()
}
};
Ok(indexes
.into_iter()
.map(|index| {
let segment_cursor = if Some(index) == cursor_index {
cursor
} else {
None
};
(index, &segments[index], segment_cursor)
})
.collect())
}
fn push_segment_range(
query: &mut QueryBuilder<Sqlite>,
segment: &RolloutLineageSegment,
@@ -85,7 +85,7 @@ pub(in crate::local) async fn find_visible_turn(
find_turn(pool, lineage.segments().iter().rev(), turn_id).await
}
async fn find_turn<'a>(
pub(super) async fn find_turn<'a>(
pool: &sqlx::SqlitePool,
segments: impl Iterator<Item = &'a RolloutLineageSegment>,
turn_id: &str,