mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Resolve fork ancestry lazily for descending history pages (#61840)
## Why Reading the newest turns or items in a fork previously resolved the entire rollout lineage, even when the page only needed recent segments. This caused unnecessary ancestor reads and could fail when an unused ancestor was missing. ## What changed Introduce an incremental `LineageWalk` and use it to resolve descending history pages only as far as needed for the cursor, requested rows, and next-page lookahead. Defer sibling rollout discovery until a parent is needed, while preserving newer turns' overrides when resuming in an ancestor. Keep full lineage resolution for ascending reads, update replay, item anchors, and inherited summary lookup. ## Testing Add regression coverage for pages with missing unused ancestors, continuation within a parent, turn overrides, inherited summaries, and exact page-boundary lookahead. Extend coverage for descending item pages using the selected rollout. GitOrigin-RevId: dad941f4f3c54a46121d4fa22b69e13f1830a385
This commit is contained in:
@@ -1,6 +1,4 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -12,8 +10,11 @@ use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_rollout::RolloutItem;
|
||||
|
||||
mod ancestor_reads;
|
||||
mod walk;
|
||||
|
||||
pub(super) use ancestor_reads::AncestorReads;
|
||||
use ancestor_reads::FailedAncestor;
|
||||
pub(super) use walk::LineageWalk;
|
||||
|
||||
use super::LocalThreadStore;
|
||||
use super::thread_rollout_resolver;
|
||||
@@ -30,10 +31,7 @@ pub(super) struct RolloutLineageSegment {
|
||||
hides_parent: bool,
|
||||
}
|
||||
|
||||
/// Ordered rollout ranges contributing to one forked history.
|
||||
///
|
||||
/// This is the only local abstraction that follows SessionMeta.history_base pointers. Readers
|
||||
/// consume its bounded rollout segments without resolving or mutating fork pointers themselves.
|
||||
/// The resolved ranges of a forked history, ordered from oldest to newest.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct RolloutLineage {
|
||||
pub(super) segments: Vec<RolloutLineageSegment>,
|
||||
@@ -48,12 +46,14 @@ impl LocalThreadStore {
|
||||
requested_thread_id: ThreadId,
|
||||
initial_path: Option<PathBuf>,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_rollout_lineage_with_mode(
|
||||
LineageWalk::new(
|
||||
self,
|
||||
requested_thread_id,
|
||||
LineageResolutionMode::Existing,
|
||||
initial_path,
|
||||
&HashMap::new(),
|
||||
/*excluded*/ None,
|
||||
)
|
||||
.finish()
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -61,155 +61,25 @@ impl LocalThreadStore {
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_rollout_lineage_with_mode(
|
||||
requested_thread_id,
|
||||
LineageResolutionMode::HistoryPage,
|
||||
/*initial_path*/ None,
|
||||
&HashMap::new(),
|
||||
)
|
||||
.await
|
||||
LineageWalk::for_history_page(self, requested_thread_id)
|
||||
.finish()
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_rollout_lineage_for_reference(
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_rollout_lineage_with_mode(
|
||||
LineageWalk::new(
|
||||
self,
|
||||
requested_thread_id,
|
||||
LineageResolutionMode::PlainForReference,
|
||||
/*initial_path*/ None,
|
||||
&HashMap::new(),
|
||||
/*excluded*/ None,
|
||||
)
|
||||
.finish()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_rollout_lineage_with_mode(
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
mode: LineageResolutionMode,
|
||||
mut initial_path: Option<PathBuf>,
|
||||
excluded: &HashMap<ThreadId, FailedAncestor>,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
let mut segments = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut end: Option<HistoryPosition> = None;
|
||||
let mut prefetched_paths = HashMap::new();
|
||||
let mut ancestor_paths = HashMap::new();
|
||||
|
||||
loop {
|
||||
let coordination_id = end.map_or(requested_thread_id, |end| end.thread_id);
|
||||
let _writer_guard = match mode {
|
||||
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None,
|
||||
LineageResolutionMode::PlainForReference => {
|
||||
Some(self.live_writer_locks.lock(coordination_id).await)
|
||||
}
|
||||
};
|
||||
let (rollout_id, (rollout_path, bounds)) = if let Some(end) = end {
|
||||
let ancestor = resolve_ancestor_rollout(
|
||||
self,
|
||||
requested_thread_id,
|
||||
end,
|
||||
mode,
|
||||
prefetched_paths.remove(&end.thread_id),
|
||||
excluded.get(&end.thread_id),
|
||||
)
|
||||
.await?;
|
||||
(end.thread_id, ancestor)
|
||||
} else {
|
||||
let (rollout_id, path) = match initial_path.take() {
|
||||
Some(path) => {
|
||||
let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id(
|
||||
&path,
|
||||
requested_thread_id,
|
||||
ThreadHistoryMode::Paginated,
|
||||
)?;
|
||||
(id, path)
|
||||
}
|
||||
None => {
|
||||
let resolved = thread_rollout_resolver::resolve_current_including_archived(
|
||||
self,
|
||||
requested_thread_id,
|
||||
)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
malformed_lineage(requested_thread_id, "missing source rollout")
|
||||
})?;
|
||||
(resolved.rollout_id, resolved.path)
|
||||
}
|
||||
};
|
||||
let (path, mut bounds) = mode
|
||||
.read_rollout(self, path, requested_thread_id, Some(requested_thread_id))
|
||||
.await?;
|
||||
if matches!(mode, LineageResolutionMode::HistoryPage) {
|
||||
bounds
|
||||
.read_history_page_bounds(&path, requested_thread_id)
|
||||
.await?;
|
||||
}
|
||||
(rollout_id, (path, bounds))
|
||||
};
|
||||
if !seen.insert(rollout_id) {
|
||||
return Err(malformed_lineage(requested_thread_id, "cycle detected"));
|
||||
}
|
||||
let rollout_path = match mode {
|
||||
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => {
|
||||
rollout_path
|
||||
}
|
||||
LineageResolutionMode::PlainForReference
|
||||
if end.is_none() && bounds.parent.is_none() =>
|
||||
{
|
||||
// A newly shared standalone source must remain readable by older binaries.
|
||||
codex_rollout::materialize_rollout_for_reference(rollout_path.as_path())
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"failed to materialize referenced rollout {}: {err}",
|
||||
rollout_path.display()
|
||||
),
|
||||
})?
|
||||
}
|
||||
// Already-shared compressed history requires a compatible reader regardless of
|
||||
// new forks. Read it without publishing decoded copies into ancestors' folders;
|
||||
// their owners may concurrently archive or unarchive those immutable files.
|
||||
LineageResolutionMode::PlainForReference => rollout_path,
|
||||
};
|
||||
if let Some(end) = end
|
||||
&& !matches!(mode, LineageResolutionMode::HistoryPage)
|
||||
{
|
||||
ancestor_paths.insert(rollout_path.clone(), (end, bounds.owner));
|
||||
}
|
||||
segments.push(RolloutLineageSegment {
|
||||
rollout_id,
|
||||
rollout_path,
|
||||
start_ordinal: bounds.start_ordinal,
|
||||
end,
|
||||
hides_parent: bounds.hides_parent,
|
||||
});
|
||||
|
||||
let Some(base) = bounds.parent else {
|
||||
break;
|
||||
};
|
||||
if matches!(mode, LineageResolutionMode::HistoryPage)
|
||||
&& let [child] = segments.as_slice()
|
||||
{
|
||||
prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths(
|
||||
&self.config.codex_home,
|
||||
&child.rollout_path,
|
||||
base.thread_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
}
|
||||
end = Some(base);
|
||||
}
|
||||
|
||||
segments.reverse();
|
||||
Ok(RolloutLineage {
|
||||
segments,
|
||||
reads: AncestorReads::new(ancestor_paths),
|
||||
requested_thread_id,
|
||||
mode,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A mid-turn fork starts with its header, settings, optional model guidance, and parent abort.
|
||||
@@ -533,12 +403,9 @@ impl RolloutLineage {
|
||||
}
|
||||
|
||||
pub(super) fn segment_index_for_ordinal(&self, ordinal: u64) -> Option<usize> {
|
||||
self.segments.iter().position(|segment| {
|
||||
ordinal >= segment.start_ordinal()
|
||||
&& segment
|
||||
.end_ordinal()
|
||||
.is_none_or(|end_ordinal| ordinal < end_ordinal)
|
||||
})
|
||||
self.segments
|
||||
.iter()
|
||||
.position(|segment| segment.contains(ordinal))
|
||||
}
|
||||
|
||||
pub(super) async fn truncate_at(
|
||||
@@ -585,6 +452,10 @@ impl RolloutLineage {
|
||||
}
|
||||
|
||||
impl RolloutLineageSegment {
|
||||
pub(super) fn contains(&self, ordinal: u64) -> bool {
|
||||
ordinal >= self.start_ordinal && self.end_ordinal().is_none_or(|end| ordinal < end)
|
||||
}
|
||||
|
||||
pub(super) fn rollout_id(&self) -> ThreadId {
|
||||
self.rollout_id
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use std::sync::OnceLock;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
|
||||
use super::LineageWalk;
|
||||
use super::LocalThreadStore;
|
||||
use super::RolloutLineage;
|
||||
use crate::ThreadStoreError;
|
||||
@@ -133,15 +134,16 @@ impl RolloutLineage {
|
||||
error: error.clone(),
|
||||
},
|
||||
);
|
||||
lineage = store
|
||||
.resolve_rollout_lineage_with_mode(
|
||||
requested_thread_id,
|
||||
mode,
|
||||
leaf.clone(),
|
||||
&excluded,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| error)?;
|
||||
lineage = LineageWalk::new(
|
||||
store,
|
||||
requested_thread_id,
|
||||
mode,
|
||||
leaf.clone(),
|
||||
Some(&excluded),
|
||||
)
|
||||
.finish()
|
||||
.await
|
||||
.map_err(|_| error)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
//! Walks immutable rollout ranges from the selected leaf toward its ancestors on demand.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
|
||||
use super::AncestorReads;
|
||||
use super::FailedAncestor;
|
||||
use super::LineageResolutionMode;
|
||||
use super::LocalThreadStore;
|
||||
use super::RolloutLineage;
|
||||
use super::RolloutLineageSegment;
|
||||
use super::malformed_lineage;
|
||||
use super::resolve_ancestor_rollout;
|
||||
use super::thread_rollout_resolver;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
/// The next file to open. A parent bookmark bounds the rows inherited from that file.
|
||||
enum NextRollout {
|
||||
Selected(Option<PathBuf>),
|
||||
Parent(HistoryPosition),
|
||||
}
|
||||
|
||||
pub(in crate::local) struct LineageWalk<'a> {
|
||||
store: &'a LocalThreadStore,
|
||||
requested_thread_id: ThreadId,
|
||||
mode: LineageResolutionMode,
|
||||
next: Option<NextRollout>,
|
||||
seen: HashSet<ThreadId>,
|
||||
prefetched_paths: HashMap<ThreadId, PathBuf>,
|
||||
excluded: Option<&'a HashMap<ThreadId, FailedAncestor>>,
|
||||
ancestor_paths: HashMap<PathBuf, (HistoryPosition, ThreadId)>,
|
||||
// Newest first, so discovering one older segment is a push. Only finish reverses this.
|
||||
segments: Vec<RolloutLineageSegment>,
|
||||
}
|
||||
|
||||
impl<'a> LineageWalk<'a> {
|
||||
pub(super) fn new(
|
||||
store: &'a LocalThreadStore,
|
||||
requested_thread_id: ThreadId,
|
||||
mode: LineageResolutionMode,
|
||||
initial_path: Option<PathBuf>,
|
||||
excluded: Option<&'a HashMap<ThreadId, FailedAncestor>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
requested_thread_id,
|
||||
mode,
|
||||
next: Some(NextRollout::Selected(initial_path)),
|
||||
seen: HashSet::new(),
|
||||
prefetched_paths: HashMap::new(),
|
||||
excluded,
|
||||
ancestor_paths: HashMap::new(),
|
||||
segments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::local) fn for_history_page(
|
||||
store: &'a LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
) -> Self {
|
||||
Self::new(
|
||||
store,
|
||||
thread_id,
|
||||
LineageResolutionMode::HistoryPage,
|
||||
/*initial_path*/ None,
|
||||
/*excluded*/ None,
|
||||
)
|
||||
}
|
||||
|
||||
/// Opens all remaining ancestors, oldest first in the resulting lineage.
|
||||
pub(in crate::local) async fn finish(mut self) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_all().await?;
|
||||
self.segments.reverse();
|
||||
Ok(RolloutLineage {
|
||||
segments: self.segments,
|
||||
reads: AncestorReads::new(self.ancestor_paths),
|
||||
requested_thread_id: self.requested_thread_id,
|
||||
mode: self.mode,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::local) async fn resolve_all(
|
||||
&mut self,
|
||||
) -> ThreadStoreResult<&[RolloutLineageSegment]> {
|
||||
while self.next_older().await? {}
|
||||
Ok(&self.segments)
|
||||
}
|
||||
|
||||
/// Returns the segment at the given depth from the leaf and all newer segments.
|
||||
pub(in crate::local) async fn at_depth(
|
||||
&mut self,
|
||||
depth: usize,
|
||||
) -> ThreadStoreResult<Option<(&RolloutLineageSegment, &[RolloutLineageSegment])>> {
|
||||
while self.segments.len() <= depth {
|
||||
if !self.next_older().await? {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(self.segments[..=depth].split_last())
|
||||
}
|
||||
|
||||
async fn next_older(&mut self) -> ThreadStoreResult<bool> {
|
||||
let Some(next) = self.next.take() else {
|
||||
return Ok(false);
|
||||
};
|
||||
let requested_thread_id = self.requested_thread_id;
|
||||
let store = self.store;
|
||||
let _writer_guard = match self.mode {
|
||||
LineageResolutionMode::Existing | LineageResolutionMode::HistoryPage => None,
|
||||
LineageResolutionMode::PlainForReference => {
|
||||
let id = match &next {
|
||||
NextRollout::Selected(_) => requested_thread_id,
|
||||
NextRollout::Parent(end) => end.thread_id,
|
||||
};
|
||||
Some(store.live_writer_locks.lock(id).await)
|
||||
}
|
||||
};
|
||||
let (rollout_id, rollout_path, bounds, end) = match next {
|
||||
NextRollout::Selected(initial_path) => {
|
||||
let (rollout_id, path) = match initial_path {
|
||||
Some(path) => {
|
||||
let id = thread_rollout_resolver::rollout_id_from_path_or_legacy_thread_id(
|
||||
&path,
|
||||
requested_thread_id,
|
||||
ThreadHistoryMode::Paginated,
|
||||
)?;
|
||||
(id, path)
|
||||
}
|
||||
None => {
|
||||
let resolved = thread_rollout_resolver::resolve_current_including_archived(
|
||||
store,
|
||||
requested_thread_id,
|
||||
)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
malformed_lineage(requested_thread_id, "missing source rollout")
|
||||
})?;
|
||||
(resolved.rollout_id, resolved.path)
|
||||
}
|
||||
};
|
||||
let (mut path, mut bounds) = self
|
||||
.mode
|
||||
.read_rollout(store, path, requested_thread_id, Some(requested_thread_id))
|
||||
.await?;
|
||||
match self.mode {
|
||||
LineageResolutionMode::HistoryPage => {
|
||||
bounds
|
||||
.read_history_page_bounds(&path, requested_thread_id)
|
||||
.await?
|
||||
}
|
||||
LineageResolutionMode::PlainForReference if bounds.parent.is_none() => {
|
||||
// A newly shared standalone source must remain readable by older binaries.
|
||||
path = codex_rollout::materialize_rollout_for_reference(&path)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"failed to materialize referenced rollout {}: {err}",
|
||||
path.display()
|
||||
),
|
||||
})?;
|
||||
}
|
||||
LineageResolutionMode::Existing | LineageResolutionMode::PlainForReference => {}
|
||||
}
|
||||
(rollout_id, path, bounds, None)
|
||||
}
|
||||
NextRollout::Parent(end) => {
|
||||
// Defer the directory scan too: pages contained in the leaf never need it.
|
||||
if matches!(self.mode, LineageResolutionMode::HistoryPage)
|
||||
&& let [child] = self.segments.as_slice()
|
||||
{
|
||||
self.prefetched_paths = codex_rollout::prefetch_sibling_rollout_paths(
|
||||
&store.config.codex_home,
|
||||
&child.rollout_path,
|
||||
end.thread_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
}
|
||||
let (path, bounds) = resolve_ancestor_rollout(
|
||||
store,
|
||||
requested_thread_id,
|
||||
end,
|
||||
self.mode,
|
||||
self.prefetched_paths.remove(&end.thread_id),
|
||||
self.excluded
|
||||
.and_then(|excluded| excluded.get(&end.thread_id)),
|
||||
)
|
||||
.await?;
|
||||
if !matches!(self.mode, LineageResolutionMode::HistoryPage) {
|
||||
self.ancestor_paths
|
||||
.insert(path.clone(), (end, bounds.owner));
|
||||
}
|
||||
(end.thread_id, path, bounds, Some(end))
|
||||
}
|
||||
};
|
||||
if !self.seen.insert(rollout_id) {
|
||||
return Err(malformed_lineage(requested_thread_id, "cycle detected"));
|
||||
}
|
||||
self.next = bounds.parent.map(NextRollout::Parent);
|
||||
self.segments.push(RolloutLineageSegment {
|
||||
rollout_id,
|
||||
rollout_path,
|
||||
start_ordinal: bounds.start_ordinal,
|
||||
end,
|
||||
hides_parent: bounds.hides_parent,
|
||||
});
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ use super::LocalThreadStore;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
mod page_segments;
|
||||
mod read;
|
||||
mod realtime;
|
||||
mod realtime_stream;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Seeks a history cursor and supplies the pager with each segment and its newer overrides.
|
||||
|
||||
use super::super::rollout_lineage::LineageWalk;
|
||||
use super::super::rollout_lineage::RolloutLineageSegment;
|
||||
use super::read::HistoryCursor;
|
||||
use super::read::invalid_cursor;
|
||||
use crate::SortDirection;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
pub(super) struct PageSegments<'a, 'store> {
|
||||
lineage: &'a mut LineageWalk<'store>,
|
||||
step: isize,
|
||||
next_depth: Option<usize>,
|
||||
cursor: Option<HistoryCursor>,
|
||||
}
|
||||
|
||||
pub(super) struct PageSegment<'a> {
|
||||
pub segment: &'a RolloutLineageSegment,
|
||||
pub newer: &'a [RolloutLineageSegment],
|
||||
pub cursor: Option<HistoryCursor>,
|
||||
}
|
||||
|
||||
impl<'a, 'store> PageSegments<'a, 'store> {
|
||||
pub(super) async fn from_cursor(
|
||||
lineage: &'a mut LineageWalk<'store>,
|
||||
direction: SortDirection,
|
||||
cursor: Option<HistoryCursor>,
|
||||
) -> ThreadStoreResult<Self> {
|
||||
// Ascending reads must discover the oldest segment up front. Descending reads start
|
||||
// at the leaf and seek only as far as the cursor, retaining all newer shadowing turns.
|
||||
let (start, step) = match direction {
|
||||
SortDirection::Asc => (lineage.resolve_all().await?.len().checked_sub(1), -1),
|
||||
SortDirection::Desc => (Some(0), 1),
|
||||
};
|
||||
let mut next_depth = start;
|
||||
if let Some(cursor) = &cursor {
|
||||
loop {
|
||||
let missing = || invalid_cursor("position outside thread lineage");
|
||||
let depth = next_depth.ok_or_else(missing)?;
|
||||
let (segment, _) = lineage.at_depth(depth).await?.ok_or_else(missing)?;
|
||||
if segment.contains(cursor.rollout_ordinal) {
|
||||
break;
|
||||
}
|
||||
next_depth = depth.checked_add_signed(step);
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
lineage,
|
||||
step,
|
||||
next_depth,
|
||||
cursor,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn next(&mut self) -> ThreadStoreResult<Option<PageSegment<'_>>> {
|
||||
let Some(depth) = self.next_depth else {
|
||||
return Ok(None);
|
||||
};
|
||||
let segment = self.lineage.at_depth(depth).await?;
|
||||
self.next_depth = depth.checked_add_signed(self.step);
|
||||
Ok(segment.map(|(segment, newer)| PageSegment {
|
||||
segment,
|
||||
newer,
|
||||
cursor: self.cursor.take(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,12 @@ use serde::Serialize;
|
||||
use sqlx::Row;
|
||||
|
||||
use super::super::LocalThreadStore;
|
||||
use super::super::rollout_lineage::RolloutLineage;
|
||||
use super::super::rollout_lineage::LineageWalk;
|
||||
use super::segment_paging::page_item_rows;
|
||||
use super::segment_paging::page_turn_rows;
|
||||
use super::segment_paging::validate_page_size;
|
||||
use super::sqlite_integer;
|
||||
use super::turn_lookup::find_source_turn;
|
||||
use super::turn_lookup::find_turn;
|
||||
use crate::ItemPage;
|
||||
use crate::ListItemsParams;
|
||||
use crate::ListTurnsParams;
|
||||
@@ -111,14 +111,12 @@ pub(in crate::local) async fn list_turns(
|
||||
)
|
||||
.await?;
|
||||
validate_page_size(params.page_size)?;
|
||||
let lineage = store
|
||||
.resolve_rollout_lineage_for_history_page(params.thread_id)
|
||||
.await?;
|
||||
let pool = store.thread_history_db().await?;
|
||||
let mut lineage = LineageWalk::for_history_page(store, params.thread_id);
|
||||
let page = page_turn_rows(
|
||||
pool,
|
||||
params.thread_id,
|
||||
&lineage,
|
||||
&mut lineage,
|
||||
params.cursor.as_deref(),
|
||||
params.page_size,
|
||||
params.sort_direction,
|
||||
@@ -136,7 +134,7 @@ pub(in crate::local) async fn list_turns(
|
||||
{
|
||||
// Synthetic fork-boundary rows are interrupted without local summary IDs.
|
||||
// Load their summary from the earliest visible source turn.
|
||||
load_inherited_summary_items(pool, &lineage, &turn).await?
|
||||
load_inherited_summary_items(pool, &mut lineage, &turn).await?
|
||||
}
|
||||
StoredTurnItemsView::Summary => turn.summary_items,
|
||||
};
|
||||
@@ -188,11 +186,9 @@ pub(in crate::local) async fn list_items(
|
||||
params.sort_direction = crate::SortDirection::Asc;
|
||||
}
|
||||
validate_page_size(params.page_size)?;
|
||||
let lineage = store
|
||||
.resolve_rollout_lineage_for_history_page(params.thread_id)
|
||||
.await?;
|
||||
let pool = store.thread_history_db().await?;
|
||||
let page = page_item_rows(pool, &lineage, ¶ms).await?;
|
||||
let mut lineage = LineageWalk::for_history_page(store, params.thread_id);
|
||||
let page = page_item_rows(pool, &mut lineage, ¶ms).await?;
|
||||
|
||||
Ok(ItemPage {
|
||||
items: page.rows.into_iter().map(|row| row.item).collect(),
|
||||
@@ -233,12 +229,12 @@ pub(super) async fn validate_thread_for_paginated_reads(
|
||||
|
||||
async fn load_inherited_summary_items(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
lineage: &mut LineageWalk<'_>,
|
||||
turn: &StoredTurnRow,
|
||||
) -> ThreadStoreResult<Vec<StoredThreadItem>> {
|
||||
let source = find_source_turn(pool, lineage, turn.turn_id.as_str()).await?;
|
||||
let lineage = lineage.resolve_all().await?;
|
||||
let source = find_turn(pool, lineage.iter().rev(), turn.turn_id.as_str()).await?;
|
||||
let Some(segment) = lineage
|
||||
.segments()
|
||||
.iter()
|
||||
.find(|segment| segment.rollout_id() == source.rollout_id)
|
||||
else {
|
||||
|
||||
@@ -31,6 +31,184 @@ use crate::StoredTurnError;
|
||||
use crate::StoredTurnStatus;
|
||||
use crate::local::test_support::test_config;
|
||||
|
||||
#[tokio::test]
|
||||
async fn newest_indexed_pages_do_not_read_unused_ancestors() {
|
||||
let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
let parent_id = ThreadId::default();
|
||||
let parent_path = rollout_path(home.path(), parent_id);
|
||||
write_rollout_with_end(
|
||||
&parent_path,
|
||||
parent_id,
|
||||
/*history_base*/ None,
|
||||
/*next_ordinal*/ 5,
|
||||
);
|
||||
write_rollout_with_end(
|
||||
&rollout_path(home.path(), child_id),
|
||||
child_id,
|
||||
Some(history_position(
|
||||
&parent_path,
|
||||
parent_id,
|
||||
/*end_ordinal_exclusive*/ 4,
|
||||
)),
|
||||
/*next_ordinal*/ 8,
|
||||
);
|
||||
let db = history_db(&store).await;
|
||||
for (turn_id, item_id, ordinal) in [("one", "a", 5), ("two", "b", 7), ("three", "c", 9)] {
|
||||
insert_turn(
|
||||
db,
|
||||
child_id,
|
||||
turn_id,
|
||||
ordinal,
|
||||
"completed",
|
||||
/*error_json*/ None,
|
||||
Some(item_id),
|
||||
/*final_agent_item_id*/ None,
|
||||
)
|
||||
.await;
|
||||
insert_item(db, child_id, turn_id, item_id, ordinal + 1).await;
|
||||
}
|
||||
let turns = turn_params(
|
||||
child_id,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Desc,
|
||||
StoredTurnItemsView::Summary,
|
||||
);
|
||||
let items = item_params(
|
||||
child_id,
|
||||
/*turn_id*/ None,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Desc,
|
||||
);
|
||||
let expected_turns = store.list_turns(turns.clone()).await.unwrap();
|
||||
let expected_items = store.list_items(items.clone()).await.unwrap();
|
||||
fs::remove_file(parent_path).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(store.list_turns(turns).await.unwrap()).unwrap(),
|
||||
serde_json::to_value(expected_turns).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(store.list_items(items).await.unwrap()).unwrap(),
|
||||
serde_json::to_value(expected_items).unwrap()
|
||||
);
|
||||
// At exactly the local row count the reader must still discover whether an ancestor
|
||||
// contributes another row, rather than incorrectly returning a terminal cursor.
|
||||
assert!(
|
||||
store
|
||||
.list_turns(turn_params(
|
||||
child_id,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 3,
|
||||
SortDirection::Desc,
|
||||
StoredTurnItemsView::NotLoaded
|
||||
))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn descending_pages_stop_in_the_parent_even_when_resuming_there() {
|
||||
let (home, store, child) = store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
let root = ThreadId::default();
|
||||
let parent = ThreadId::default();
|
||||
let root_path = rollout_path(home.path(), root);
|
||||
let parent_path = rollout_path(home.path(), parent);
|
||||
write_rollout_with_end(
|
||||
&root_path, root, /*history_base*/ None, /*next_ordinal*/ 4,
|
||||
);
|
||||
write_rollout_with_end(
|
||||
&parent_path,
|
||||
parent,
|
||||
Some(history_position(
|
||||
&root_path, root, /*end_ordinal_exclusive*/ 3,
|
||||
)),
|
||||
/*next_ordinal*/ 12,
|
||||
);
|
||||
write_rollout_with_end(
|
||||
&rollout_path(home.path(), child),
|
||||
child,
|
||||
Some(history_position(
|
||||
&parent_path,
|
||||
parent,
|
||||
/*end_ordinal_exclusive*/ 13,
|
||||
)),
|
||||
/*next_ordinal*/ 4,
|
||||
);
|
||||
let db = history_db(&store).await;
|
||||
for (id, turn_id, ordinal) in [
|
||||
(root, "child", 1),
|
||||
(parent, "oldest", 4),
|
||||
(parent, "older", 5),
|
||||
(parent, "child", 7),
|
||||
(parent, "middle", 9),
|
||||
(parent, "newest", 11),
|
||||
(child, "child", 14),
|
||||
] {
|
||||
insert_turn(
|
||||
db,
|
||||
id,
|
||||
turn_id,
|
||||
ordinal,
|
||||
if id == child {
|
||||
"interrupted"
|
||||
} else {
|
||||
"completed"
|
||||
},
|
||||
/*error_json*/ None,
|
||||
(id == root).then_some("item-1"),
|
||||
/*final_agent_item_id*/ None,
|
||||
)
|
||||
.await;
|
||||
insert_item(db, id, turn_id, &format!("item-{ordinal}"), ordinal + 1).await;
|
||||
}
|
||||
|
||||
let mut turns = turn_params(
|
||||
child,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Desc,
|
||||
StoredTurnItemsView::NotLoaded,
|
||||
);
|
||||
// The leaf and parent fill the page, but only the grandparent has the original summary.
|
||||
let summary = store
|
||||
.list_turns(ListTurnsParams {
|
||||
items_view: StoredTurnItemsView::Summary,
|
||||
page_size: 1,
|
||||
..turns.clone()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
summary.turns[0].items,
|
||||
[expected_item("child", "item-1", /*rollout_ordinal*/ 2)]
|
||||
);
|
||||
|
||||
fs::remove_file(root_path).unwrap();
|
||||
let first = store.list_turns(turns.clone()).await.unwrap();
|
||||
assert_eq!(turn_ids(&first), ["child", "newest"]);
|
||||
turns.cursor = Some(first.next_cursor.unwrap());
|
||||
let second = store.list_turns(turns).await.unwrap();
|
||||
// The cursor skipped the leaf, but its copy of "child" must still shadow the parent's.
|
||||
assert_eq!(turn_ids(&second), ["middle", "older"]);
|
||||
assert!(second.next_cursor.is_some());
|
||||
|
||||
let mut items = item_params(
|
||||
child,
|
||||
/*turn_id*/ None,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Desc,
|
||||
);
|
||||
let first = store.list_items(items.clone()).await.unwrap();
|
||||
assert_eq!(item_ids(&first), ["item-14", "item-11"]);
|
||||
items.position = Some(ListItemsPosition::Cursor(first.next_cursor.unwrap()));
|
||||
let second = store.list_items(items).await.unwrap();
|
||||
assert_eq!(item_ids(&second), ["item-9", "item-7"]);
|
||||
assert!(second.next_cursor.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_turns_pages_projected_rows_and_applies_item_views() {
|
||||
let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
@@ -1062,6 +1240,17 @@ async fn list_items_update_ordinals_use_selected_rollout_id() {
|
||||
"turn-1", "item-1", /*rollout_ordinal*/ 1
|
||||
)]
|
||||
);
|
||||
let descending_page = store
|
||||
.list_items(item_params(
|
||||
thread_id,
|
||||
/*turn_id*/ None,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 1,
|
||||
SortDirection::Desc,
|
||||
))
|
||||
.await
|
||||
.expect("creation-ordered selected rollout page");
|
||||
assert_eq!(descending_page.items, page.items);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -3,15 +3,15 @@ use sqlx::FromRow;
|
||||
use sqlx::QueryBuilder;
|
||||
use sqlx::Sqlite;
|
||||
|
||||
use super::super::rollout_lineage::RolloutLineage;
|
||||
use super::super::rollout_lineage::LineageWalk;
|
||||
use super::super::rollout_lineage::RolloutLineageSegment;
|
||||
use super::page_segments::PageSegments;
|
||||
use super::read::CursorScope;
|
||||
use super::read::HistoryCursor;
|
||||
use super::read::RolloutHistoryPosition;
|
||||
use super::read::StoredSummaryColumns;
|
||||
use super::read::StoredThreadItemRow;
|
||||
use super::read::StoredTurnRow;
|
||||
use super::read::invalid_cursor;
|
||||
use super::read::parse_cursor;
|
||||
use super::read::serialize_cursor;
|
||||
use super::read::stored_thread_item_row;
|
||||
@@ -85,7 +85,7 @@ pub(super) fn validate_page_size(page_size: usize) -> ThreadStoreResult<()> {
|
||||
pub(super) async fn page_turn_rows(
|
||||
pool: &sqlx::SqlitePool,
|
||||
requested_thread_id: ThreadId,
|
||||
lineage: &RolloutLineage,
|
||||
lineage: &mut LineageWalk<'_>,
|
||||
cursor: Option<&str>,
|
||||
page_size: usize,
|
||||
direction: SortDirection,
|
||||
@@ -94,13 +94,13 @@ pub(super) async fn page_turn_rows(
|
||||
let cursor = parse_cursor(cursor, requested_thread_id, CursorScope::Turns)?;
|
||||
let ordinal_column = HistoryOrdinalColumn::Rollout;
|
||||
let mut rows = Vec::new();
|
||||
for (segment_index, segment, segment_cursor) in
|
||||
segments_from_cursor(lineage, direction, cursor.as_ref())?
|
||||
{
|
||||
let remaining = remaining_limit(page_size, rows.len())?;
|
||||
if remaining == 0 {
|
||||
let mut segments = PageSegments::from_cursor(lineage, direction, cursor).await?;
|
||||
while rows.len() <= page_size {
|
||||
let Some(range) = segments.next().await? else {
|
||||
break;
|
||||
}
|
||||
};
|
||||
let segment = range.segment;
|
||||
let remaining = remaining_limit(page_size, rows.len())?;
|
||||
let mut query = QueryBuilder::<Sqlite>::new("");
|
||||
if matches!(items_view, StoredTurnItemsView::Summary) {
|
||||
query.push("WITH page_turns AS (");
|
||||
@@ -123,7 +123,7 @@ WHERE thread_id =
|
||||
);
|
||||
query.push_bind(segment.rollout_id().to_string());
|
||||
push_segment_range(&mut query, segment, ordinal_column)?;
|
||||
for newer_segment in &lineage.segments()[segment_index + 1..] {
|
||||
for newer_segment in range.newer {
|
||||
query
|
||||
.push(" AND NOT EXISTS (SELECT 1 FROM thread_turns AS newer_turn WHERE newer_turn.thread_id = ")
|
||||
.push_bind(newer_segment.rollout_id().to_string())
|
||||
@@ -136,7 +136,7 @@ WHERE thread_id =
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
push_cursor_clause(&mut query, direction, segment_cursor, ordinal_column)?;
|
||||
push_cursor_clause(&mut query, direction, range.cursor.as_ref(), ordinal_column)?;
|
||||
push_order_and_limit(&mut query, direction, remaining, ordinal_column);
|
||||
if matches!(items_view, StoredTurnItemsView::Summary) {
|
||||
query.push(
|
||||
@@ -234,7 +234,7 @@ fn push_summary_item_join(
|
||||
|
||||
pub(super) async fn page_item_rows(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
lineage: &mut LineageWalk<'_>,
|
||||
params: &ListItemsParams,
|
||||
) -> ThreadStoreResult<SegmentPage<StoredThreadItemRow>> {
|
||||
let (cursor, anchor_item_id) = match params.position.as_ref() {
|
||||
@@ -255,6 +255,10 @@ pub(super) async fn page_item_rows(
|
||||
(None, Some(ordinal)) => Some(UpdateOrdinalBoundary::Before(ordinal)),
|
||||
(None, None) => None,
|
||||
};
|
||||
// Keep update replay eager, including when its rows are sorted by creation time.
|
||||
if update_boundary.is_some() {
|
||||
lineage.resolve_all().await?;
|
||||
}
|
||||
if anchor_item_id.is_some()
|
||||
&& (params.sort_key != ItemSortKey::CreatedAtOrdinal || update_boundary.is_some())
|
||||
{
|
||||
@@ -283,7 +287,7 @@ pub(super) async fn page_item_rows(
|
||||
return Err(invalid_item_anchor());
|
||||
}
|
||||
let mut anchor = None;
|
||||
for segment in lineage.segments() {
|
||||
for segment in lineage.resolve_all().await?.iter().rev() {
|
||||
let mut query = QueryBuilder::<Sqlite>::new(
|
||||
"SELECT rollout_ordinal FROM thread_items WHERE thread_id = ",
|
||||
);
|
||||
@@ -317,13 +321,13 @@ pub(super) async fn page_item_rows(
|
||||
};
|
||||
let ordinal_column = HistoryOrdinalColumn::Rollout;
|
||||
let mut rows = Vec::new();
|
||||
for (_, segment, segment_cursor) in
|
||||
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
|
||||
{
|
||||
let remaining = remaining_limit(params.page_size, rows.len())?;
|
||||
if remaining == 0 {
|
||||
let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?;
|
||||
while rows.len() <= params.page_size {
|
||||
let Some(range) = segments.next().await? else {
|
||||
break;
|
||||
}
|
||||
};
|
||||
let segment = range.segment;
|
||||
let remaining = remaining_limit(params.page_size, rows.len())?;
|
||||
let mut query = QueryBuilder::<Sqlite>::new(
|
||||
r#"
|
||||
SELECT turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json
|
||||
@@ -358,7 +362,7 @@ FROM thread_items"#,
|
||||
push_cursor_clause(
|
||||
&mut query,
|
||||
params.sort_direction,
|
||||
segment_cursor,
|
||||
range.cursor.as_ref(),
|
||||
ordinal_column,
|
||||
)?;
|
||||
push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column);
|
||||
@@ -383,7 +387,7 @@ FROM thread_items"#,
|
||||
|
||||
async fn page_updated_item_rows(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
lineage: &mut LineageWalk<'_>,
|
||||
params: &ListItemsParams,
|
||||
cursor: Option<&str>,
|
||||
update_boundary: UpdateOrdinalBoundary,
|
||||
@@ -395,13 +399,13 @@ async fn page_updated_item_rows(
|
||||
)?;
|
||||
let ordinal_column = HistoryOrdinalColumn::UpdatedAt;
|
||||
let mut rows = Vec::new();
|
||||
for (_, segment, segment_cursor) in
|
||||
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
|
||||
{
|
||||
let remaining = remaining_limit(params.page_size, rows.len())?;
|
||||
if remaining == 0 {
|
||||
let mut segments = PageSegments::from_cursor(lineage, params.sort_direction, cursor).await?;
|
||||
while rows.len() <= params.page_size {
|
||||
let Some(range) = segments.next().await? else {
|
||||
break;
|
||||
}
|
||||
};
|
||||
let segment = range.segment;
|
||||
let remaining = remaining_limit(params.page_size, rows.len())?;
|
||||
let mut query = QueryBuilder::<Sqlite>::new(
|
||||
r#"
|
||||
SELECT turn_id, item_id, updated_at_ordinal AS rollout_ordinal, updated_at_ordinal, created_at_ms, started_at_ms, completed_at_ms, item_json
|
||||
@@ -418,7 +422,7 @@ WHERE thread_id =
|
||||
push_cursor_clause(
|
||||
&mut query,
|
||||
params.sort_direction,
|
||||
segment_cursor,
|
||||
range.cursor.as_ref(),
|
||||
ordinal_column,
|
||||
)?;
|
||||
push_order_and_limit(&mut query, params.sort_direction, remaining, ordinal_column);
|
||||
@@ -441,39 +445,6 @@ WHERE thread_id =
|
||||
)
|
||||
}
|
||||
|
||||
fn segments_from_cursor<'a>(
|
||||
lineage: &'a RolloutLineage,
|
||||
direction: SortDirection,
|
||||
cursor: Option<&'a HistoryCursor>,
|
||||
) -> ThreadStoreResult<Vec<(usize, &'a RolloutLineageSegment, Option<&'a HistoryCursor>)>> {
|
||||
let segments = lineage.segments();
|
||||
let cursor_index = cursor
|
||||
.map(|cursor| {
|
||||
lineage
|
||||
.segment_index_for_ordinal(cursor.rollout_ordinal)
|
||||
.ok_or_else(|| invalid_cursor("position outside thread lineage"))
|
||||
})
|
||||
.transpose()?;
|
||||
let indexes: Vec<usize> = match direction {
|
||||
SortDirection::Asc => (cursor_index.unwrap_or(0)..segments.len()).collect(),
|
||||
SortDirection::Desc => {
|
||||
let end = cursor_index.unwrap_or_else(|| segments.len().saturating_sub(1));
|
||||
(0..=end).rev().collect()
|
||||
}
|
||||
};
|
||||
Ok(indexes
|
||||
.into_iter()
|
||||
.map(|index| {
|
||||
let segment_cursor = if Some(index) == cursor_index {
|
||||
cursor
|
||||
} else {
|
||||
None
|
||||
};
|
||||
(index, &segments[index], segment_cursor)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn push_segment_range(
|
||||
query: &mut QueryBuilder<Sqlite>,
|
||||
segment: &RolloutLineageSegment,
|
||||
|
||||
@@ -85,7 +85,7 @@ pub(in crate::local) async fn find_visible_turn(
|
||||
find_turn(pool, lineage.segments().iter().rev(), turn_id).await
|
||||
}
|
||||
|
||||
async fn find_turn<'a>(
|
||||
pub(super) async fn find_turn<'a>(
|
||||
pool: &sqlx::SqlitePool,
|
||||
segments: impl Iterator<Item = &'a RolloutLineageSegment>,
|
||||
turn_id: &str,
|
||||
|
||||
Reference in New Issue
Block a user