Support incremental replay of updated thread items (#35013)

## Why

Persisted item snapshots can be updated after their initial projection while retaining their creation order. Callers need a way to read those updates incrementally without replaying every item.

## What changed

- Track the rollout ordinal of each item's latest snapshot separately from its creation ordinal.
- Let thread-store callers filter items after an exclusive update watermark and sort pages by either creation or update ordinal.
- Keep existing app-server reads creation-ordered, reject incremental reads across forked lineages, and preserve compatibility with older writers.

## Testing

- Cover update-watermark filtering, pagination, sort directions, per-turn reads, cursor validation, forked lineages, migration compatibility, and refreshed item snapshots.

GitOrigin-RevId: d57de849abec27f788e9c384dbff073a0aca1053
This commit is contained in:
richardopenai
2026-07-23 21:17:45 +00:00
committed by copyberry
parent 5b402270f9
commit b834702b27
13 changed files with 436 additions and 37 deletions
@@ -467,6 +467,7 @@ use codex_state::log_db::LogDbLayer;
use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams;
use codex_thread_store::DeleteThreadsParams as StoreDeleteThreadsParams;
use codex_thread_store::GitInfoPatch as StoreGitInfoPatch;
use codex_thread_store::ItemSortKey as StoreItemSortKey;
use codex_thread_store::ListItemsParams as StoreListItemsParams;
use codex_thread_store::ListThreadsParams as StoreListThreadsParams;
use codex_thread_store::ListTurnsParams as StoreListTurnsParams;
@@ -2715,6 +2715,8 @@ impl ThreadRequestProcessor {
cursor: cursor.clone(),
page_size: THREAD_ITEMS_MAX_LIMIT,
sort_direction: StoreSortDirection::Asc,
sort_key: StoreItemSortKey::CreatedAtOrdinal,
after_updated_at_ordinal: None,
})
.await
.map_err(paginated_history_list_error)?;
@@ -2829,6 +2831,8 @@ impl ThreadRequestProcessor {
cursor: None,
page_size: 1,
sort_direction: StoreSortDirection::Desc,
sort_key: StoreItemSortKey::CreatedAtOrdinal,
after_updated_at_ordinal: None,
})
.await
.map_err(paginated_history_list_error)?;
@@ -2864,6 +2868,8 @@ impl ThreadRequestProcessor {
SortDirection::Asc => StoreSortDirection::Asc,
SortDirection::Desc => StoreSortDirection::Desc,
},
sort_key: StoreItemSortKey::CreatedAtOrdinal,
after_updated_at_ordinal: None,
})
.await
.map_err(|err| match err {
@@ -1518,6 +1518,15 @@ async fn paginated_history_lists_use_projected_turns_and_items() -> Result<()> {
memory_citation: None,
}),
),
paginated_completed_item(
thread_id,
"turn-1",
CoreTurnItem::UserMessage(UserMessageItem {
id: "steer-1".to_string(),
client_id: Some("updated-steer".to_string()),
content: Vec::new(),
}),
),
paginated_turn_completed("turn-1"),
paginated_turn_started("turn-2"),
paginated_completed_item(
@@ -1550,7 +1559,7 @@ async fn paginated_history_lists_use_projected_turns_and_items() -> Result<()> {
},
ThreadItem::UserMessage {
id: "steer-1".to_string(),
client_id: None,
client_id: Some("updated-steer".to_string()),
content: Vec::new(),
},
ThreadItem::AgentMessage {
+69
View File
@@ -6,6 +6,7 @@ use sqlx::migrate::Migrator;
use std::borrow::Cow;
use super::STATE_MIGRATOR;
use super::THREAD_HISTORY_MIGRATOR;
use super::repair_legacy_recency_migration_version;
fn migrator_through(version: i64) -> Migrator {
@@ -100,6 +101,74 @@ INSERT INTO threads (
pool.close().await;
}
#[tokio::test]
async fn thread_item_update_ordinals_allow_older_writers() {
let sqlite_home = crate::runtime::test_support::unique_temp_dir();
tokio::fs::create_dir_all(&sqlite_home)
.await
.expect("sqlite home should be created");
let _cleanup = scopeguard::guard(sqlite_home.clone(), |sqlite_home| {
let _ = std::fs::remove_dir_all(sqlite_home);
});
let sqlite = crate::SqliteConfig::new_for_testing(sqlite_home.as_path().abs());
let pre_update_ordinal_migrator = Migrator {
migrations: Cow::Owned(
THREAD_HISTORY_MIGRATOR
.migrations
.iter()
.filter(|migration| migration.version < 4)
.cloned()
.collect(),
),
ignore_missing: THREAD_HISTORY_MIGRATOR.ignore_missing,
locking: THREAD_HISTORY_MIGRATOR.locking,
table_name: THREAD_HISTORY_MIGRATOR.table_name.clone(),
create_schemas: THREAD_HISTORY_MIGRATOR.create_schemas.clone(),
no_tx: THREAD_HISTORY_MIGRATOR.no_tx,
};
let pool = sqlite
.open_thread_history_db(
&pre_update_ordinal_migrator,
/*telemetry_override*/ None,
)
.await
.expect("pre-update-ordinal migrations should apply");
sqlx::query(
r#"
INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json) VALUES
('thread-1', 'turn-1', 'existing-item-1', 11, 1_100, 'userMessage', '{}'),
('thread-1', 'turn-1', 'existing-item-2', 12, 1_200, 'userMessage', '{}')
"#,
)
.execute(&pool)
.await
.expect("pre-migration items should be inserted");
THREAD_HISTORY_MIGRATOR
.run(&pool)
.await
.expect("update-ordinal migration should apply");
sqlx::query(
r#"
INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json) VALUES
('thread-1', 'turn-1', 'old-writer-item-1', 13, 1_300, 'userMessage', '{}'),
('thread-1', 'turn-1', 'old-writer-item-2', 14, 1_400, 'userMessage', '{}')
"#,
)
.execute(&pool)
.await
.expect("older writers should be able to append multiple items after migration");
let ordinals = sqlx::query_as::<_, (i64, i64)>(
"SELECT rollout_ordinal, updated_at_ordinal FROM thread_items WHERE thread_id = ? ORDER BY rollout_ordinal",
)
.bind("thread-1")
.fetch_all(&pool)
.await
.expect("old-writer items should load");
assert_eq!(ordinals, vec![(11, 11), (12, 12), (13, 0), (14, 0)]);
pool.close().await;
}
#[tokio::test]
async fn agent_job_tables_are_dropped_when_upgrading() {
let sqlite_home = crate::runtime::test_support::unique_temp_dir();
@@ -0,0 +1,15 @@
ALTER TABLE thread_items ADD COLUMN updated_at_ordinal INTEGER NOT NULL DEFAULT 0;
-- As of this migration, existing projected items originate from exactly one ItemCompleted rollout
-- event, so their creation ordinal is also their update ordinal.
UPDATE thread_items
SET updated_at_ordinal = rollout_ordinal;
-- Older writers can still append items with the zero default after this migration. Incremental
-- replay excludes those items until a newer writer projects an update for them.
-- Keep this index non-unique so mixed-version writers can continue to persist history.
CREATE INDEX idx_thread_items_updated_page
ON thread_items(thread_id, updated_at_ordinal);
CREATE INDEX idx_thread_items_by_turn_updated_page
ON thread_items(thread_id, turn_id, updated_at_ordinal);
+3
View File
@@ -50,6 +50,7 @@ fn stores() -> &'static Mutex<HashMap<String, Arc<InMemoryThreadStore>>> {
#[cfg(test)]
mod tests {
use super::*;
use crate::ItemSortKey;
use crate::ListItemsParams;
use crate::ListTurnsParams;
use crate::SortDirection;
@@ -90,6 +91,8 @@ mod tests {
cursor: None,
page_size: 10,
sort_direction: SortDirection::Asc,
sort_key: ItemSortKey::CreatedAtOrdinal,
after_updated_at_ordinal: None,
})
.await
.expect_err("default list_items should be unsupported");
+1
View File
@@ -31,6 +31,7 @@ pub use types::DeleteThreadsParams;
pub use types::ExtraConfig;
pub use types::GitInfoPatch;
pub use types::ItemPage;
pub use types::ItemSortKey;
pub use types::ListItemsParams;
pub use types::ListThreadsParams;
pub use types::ListTurnsParams;
@@ -312,8 +312,8 @@ WHERE thread_id = ?
let item_id = item.item.id().to_string();
let item_json = serde_json::to_string(&item.item).map_err(thread_history_error)?;
// The same item can appear again with a newer snapshot. Replace its JSON, but keep the
// ordinal and creation timestamp from the first record so item ordering and age stay
// stable.
// creation ordinal and timestamp from the first record so item identity and age stay
// stable. Track the latest snapshot separately for incremental replay.
sqlx::query(
r#"
INSERT INTO thread_items (
@@ -321,11 +321,13 @@ INSERT INTO thread_items (
turn_id,
item_id,
rollout_ordinal,
updated_at_ordinal,
created_at_ms,
item_type,
item_json
) VALUES (?, ?, ?, ?, ?, json_extract(?, '$.type'), ?)
) VALUES (?, ?, ?, ?, ?, ?, json_extract(?, '$.type'), ?)
ON CONFLICT(thread_id, turn_id, item_id) DO UPDATE SET
updated_at_ordinal = excluded.updated_at_ordinal,
item_type = excluded.item_type,
item_json = excluded.item_json
"#,
@@ -334,6 +336,7 @@ ON CONFLICT(thread_id, turn_id, item_id) DO UPDATE SET
.bind(item.turn_id.as_str())
.bind(item_id.as_str())
.bind(rollout_ordinal)
.bind(rollout_ordinal)
.bind(created_at_ms)
.bind(item_json.as_str())
.bind(item_json)
@@ -28,7 +28,8 @@ mod tests;
#[serde(tag = "kind", rename_all = "camelCase")]
pub(super) enum CursorScope {
Turns,
Items,
ItemsByCreatedAtOrdinal,
ItemsByUpdatedAtOrdinal,
}
#[derive(Deserialize, Serialize)]
@@ -126,16 +127,7 @@ pub(in crate::local) async fn list_items(
validate_page_size(params.page_size)?;
let lineage = store.resolve_rollout_lineage(params.thread_id).await?;
let pool = store.thread_history_db().await?;
let page = page_item_rows(
pool,
params.thread_id,
&lineage,
params.turn_id.as_deref(),
params.cursor.as_deref(),
params.page_size,
params.sort_direction,
)
.await?;
let page = page_item_rows(pool, &lineage, &params).await?;
Ok(ItemPage {
items: page.rows.into_iter().map(|row| row.item).collect(),
@@ -180,7 +172,7 @@ async fn load_summary_items(
) -> ThreadStoreResult<Vec<StoredThreadItem>> {
let rows = sqlx::query(
r#"
SELECT turn_id, item_id, created_at_ms, item_json
SELECT turn_id, item_id, updated_at_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id = ?
AND turn_id = ?
@@ -289,9 +281,15 @@ pub(super) fn stored_thread_item_row_for_thread(
}
fn stored_thread_item(row: sqlx::sqlite::SqliteRow) -> ThreadStoreResult<StoredThreadItem> {
let updated_at_ordinal = row.try_get::<i64, _>("updated_at_ordinal")?;
let updated_at_ordinal =
u64::try_from(updated_at_ordinal).map_err(|_| ThreadStoreError::Internal {
message: format!("invalid stored item updated-at ordinal: {updated_at_ordinal}"),
})?;
Ok(StoredThreadItem {
turn_id: row.try_get("turn_id")?,
item_id: row.try_get("item_id")?,
updated_at_ordinal,
created_at_ms: row.try_get("created_at_ms")?,
item_json: row.try_get::<String, _>("item_json")?.into_bytes(),
})
@@ -15,6 +15,7 @@ use pretty_assertions::assert_eq;
use tempfile::TempDir;
use super::*;
use crate::ItemSortKey;
use crate::SortDirection;
use crate::StoredTurnError;
use crate::StoredTurnStatus;
@@ -222,6 +223,159 @@ async fn list_items_pages_whole_thread_and_per_turn_rows() {
assert_eq!(item_ids(&next_turn_page), vec!["item-3"]);
}
#[tokio::test]
async fn list_items_filters_exclusive_update_ordinals_across_pages_and_turns() {
let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
let db = history_db(&store).await;
for (turn_id, item_id, ordinal) in [
("turn-1", "item-1", 1),
("turn-2", "item-2", 2),
("turn-2", "item-3", 3),
] {
insert_item(db, thread_id, turn_id, item_id, ordinal).await;
}
sqlx::query(
"UPDATE thread_items SET updated_at_ordinal = 4 WHERE thread_id = ? AND item_id = 'item-1'",
)
.bind(thread_id.to_string())
.execute(db)
.await
.expect("advance first item update ordinal");
let item_1 = StoredThreadItem {
updated_at_ordinal: 4,
..expected_item("turn-1", "item-1", /*rollout_ordinal*/ 1)
};
let item_2 = expected_item("turn-2", "item-2", /*rollout_ordinal*/ 2);
let item_3 = expected_item("turn-2", "item-3", /*rollout_ordinal*/ 3);
let creation_page = store
.list_items(item_params(
thread_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Asc,
))
.await
.expect("creation-ordered item page");
assert_eq!(creation_page.items, vec![item_1.clone(), item_2.clone()]);
for (sort_direction, expected) in [
(SortDirection::Asc, vec![item_1.clone(), item_3.clone()]),
(SortDirection::Desc, vec![item_3.clone(), item_1.clone()]),
] {
let page = store
.list_items(ListItemsParams {
after_updated_at_ordinal: Some(2),
..item_params(
thread_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
sort_direction,
)
})
.await
.expect("creation-ordered filtered item page");
assert_eq!(page.items, expected);
}
let first_page = store
.list_items(updated_item_params(
thread_id, /*after_updated_at_ordinal*/ 0,
))
.await
.expect("first filtered item page");
assert_eq!(first_page.items, vec![item_2.clone(), item_3.clone()]);
for params in [
ListItemsParams {
cursor: creation_page.next_cursor,
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ 0)
},
item_params(
thread_id,
/*turn_id*/ None,
first_page.next_cursor.clone(),
/*page_size*/ 2,
SortDirection::Asc,
),
] {
let error = store
.list_items(params)
.await
.expect_err("creation and update cursors should not be interchangeable");
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
}
let second_page = store
.list_items(ListItemsParams {
cursor: first_page.next_cursor,
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ 0)
})
.await
.expect("second filtered item page");
assert_eq!(second_page.items, vec![item_1.clone()]);
assert!(second_page.next_cursor.is_none());
let exclusive_page = store
.list_items(ListItemsParams {
turn_id: Some("turn-2".to_string()),
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ 2)
})
.await
.expect("exclusive filtered turn page");
assert_eq!(exclusive_page.items, vec![item_3.clone()]);
let descending_page = store
.list_items(ListItemsParams {
sort_direction: SortDirection::Desc,
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ 0)
})
.await
.expect("descending update-ordered item page");
assert_eq!(descending_page.items, vec![item_1, item_3]);
let descending_next_page = store
.list_items(ListItemsParams {
cursor: descending_page.next_cursor,
sort_direction: SortDirection::Desc,
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ 0)
})
.await
.expect("next descending update-ordered item page");
assert_eq!(descending_next_page.items, vec![item_2]);
let error = store
.list_items(ListItemsParams {
sort_key: ItemSortKey::UpdatedAtOrdinal,
..item_params(
thread_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Asc,
)
})
.await
.expect_err("update-ordinal sorting should require a watermark");
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
}
#[tokio::test]
async fn list_items_rejects_update_ordinals_outside_sqlite_integer_range() {
let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
for sort_key in [ItemSortKey::CreatedAtOrdinal, ItemSortKey::UpdatedAtOrdinal] {
let error = store
.list_items(ListItemsParams {
sort_key,
..updated_item_params(thread_id, /*after_updated_at_ordinal*/ u64::MAX)
})
.await
.expect_err("out-of-range SQLite update ordinal should fail");
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
}
}
#[tokio::test]
async fn list_history_keeps_legacy_threads_unsupported() {
let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Legacy).await;
@@ -403,6 +557,17 @@ async fn lineage_reads_page_across_parent_and_child_segments() {
vec!["root-user", "root-agent"]
);
for sort_key in [ItemSortKey::CreatedAtOrdinal, ItemSortKey::UpdatedAtOrdinal] {
let error = store
.list_items(ListItemsParams {
sort_key,
..updated_item_params(child_id, /*after_updated_at_ordinal*/ 0)
})
.await
.expect_err("incremental replay should reject forked lineages");
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
}
let (_other_home, other_store, other_thread_id) =
store_with_mode(ThreadHistoryMode::Paginated).await;
let error = other_store
@@ -657,12 +822,13 @@ async fn insert_item(
rollout_ordinal: i64,
) {
sqlx::query(
"INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_json) VALUES (?, ?, ?, ?, ?, ?)",
"INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, item_json) VALUES (?, ?, ?, ?, ?, ?, ?)",
)
.bind(thread_id.to_string())
.bind(turn_id)
.bind(item_id)
.bind(rollout_ordinal)
.bind(rollout_ordinal)
.bind(rollout_ordinal * 1_000)
.bind(format!(r#"{{"type":"userMessage","id":"{item_id}","content":[]}}"#))
.execute(db)
@@ -701,6 +867,22 @@ fn item_params(
cursor,
page_size,
sort_direction,
sort_key: ItemSortKey::CreatedAtOrdinal,
after_updated_at_ordinal: None,
}
}
fn updated_item_params(thread_id: ThreadId, after_updated_at_ordinal: u64) -> ListItemsParams {
ListItemsParams {
sort_key: ItemSortKey::UpdatedAtOrdinal,
after_updated_at_ordinal: Some(after_updated_at_ordinal),
..item_params(
thread_id,
/*turn_id*/ None,
/*cursor*/ None,
/*page_size*/ 2,
SortDirection::Asc,
)
}
}
@@ -708,6 +890,7 @@ fn expected_item(turn_id: &str, item_id: &str, rollout_ordinal: u64) -> StoredTh
StoredThreadItem {
turn_id: turn_id.to_string(),
item_id: item_id.to_string(),
updated_at_ordinal: rollout_ordinal,
created_at_ms: i64::try_from(rollout_ordinal).expect("fixture ordinal fits i64") * 1_000,
item_json: format!(r#"{{"type":"userMessage","id":"{item_id}","content":[]}}"#)
.into_bytes(),
@@ -15,6 +15,8 @@ use super::read::serialize_cursor;
use super::read::stored_thread_item_row_for_thread;
use super::read::stored_turn_row;
use super::thread_history_error;
use crate::ItemSortKey;
use crate::ListItemsParams;
use crate::SortDirection;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
@@ -87,34 +89,56 @@ WHERE thread_id =
pub(super) async fn page_item_rows(
pool: &sqlx::SqlitePool,
requested_thread_id: ThreadId,
lineage: &RolloutLineage,
turn_id: Option<&str>,
cursor: Option<&str>,
page_size: usize,
direction: SortDirection,
params: &ListItemsParams,
) -> ThreadStoreResult<SegmentPage<StoredThreadItemRow>> {
let cursor = parse_cursor(cursor, requested_thread_id, CursorScope::Items)?;
// Update ordinals are local to a physical rollout. Forked lineages need a structured
// watermark before incremental replay can safely span their segments.
if params.after_updated_at_ordinal.is_some() && lineage.segments().len() > 1 {
return Err(ThreadStoreError::InvalidRequest {
message: "incremental item replay is not supported for forked threads".to_string(),
});
}
if matches!(params.sort_key, ItemSortKey::UpdatedAtOrdinal) {
let Some(after_updated_at_ordinal) = params.after_updated_at_ordinal else {
return Err(ThreadStoreError::InvalidRequest {
message: "update-ordinal item sorting requires an update watermark".to_string(),
});
};
return page_updated_item_rows(pool, params, after_updated_at_ordinal).await;
}
let cursor = parse_cursor(
params.cursor.as_deref(),
params.thread_id,
CursorScope::ItemsByCreatedAtOrdinal,
)?;
let mut rows = Vec::new();
for (segment, segment_cursor) in segments_from_cursor(lineage, direction, cursor.as_ref())? {
let remaining = remaining_limit(page_size, rows.len())?;
for (segment, segment_cursor) in
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
{
let remaining = remaining_limit(params.page_size, rows.len())?;
if remaining == 0 {
break;
}
let mut query = QueryBuilder::<Sqlite>::new(
r#"
SELECT turn_id, item_id, rollout_ordinal, created_at_ms, item_json
SELECT turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id =
"#,
);
query.push_bind(segment.thread_id().to_string());
push_segment_range(&mut query, segment)?;
if let Some(turn_id) = turn_id {
if let Some(after_updated_at_ordinal) = params.after_updated_at_ordinal {
query
.push(" AND updated_at_ordinal > ")
.push_bind(sqlite_integer(after_updated_at_ordinal)?);
}
if let Some(turn_id) = params.turn_id.as_deref() {
query.push(" AND turn_id = ").push_bind(turn_id);
}
push_cursor_clause(&mut query, direction, segment_cursor)?;
push_order_and_limit(&mut query, direction, remaining);
push_cursor_clause(&mut query, params.sort_direction, segment_cursor)?;
push_order_and_limit(&mut query, params.sort_direction, remaining);
rows.extend(
query
.build()
@@ -126,7 +150,80 @@ WHERE thread_id =
.collect::<ThreadStoreResult<Vec<_>>>()?,
);
}
finish_page(requested_thread_id, CursorScope::Items, rows, page_size)
finish_page(
params.thread_id,
CursorScope::ItemsByCreatedAtOrdinal,
rows,
params.page_size,
)
}
async fn page_updated_item_rows(
pool: &sqlx::SqlitePool,
params: &ListItemsParams,
after_updated_at_ordinal: u64,
) -> ThreadStoreResult<SegmentPage<StoredThreadItemRow>> {
let cursor = parse_cursor(
params.cursor.as_deref(),
params.thread_id,
CursorScope::ItemsByUpdatedAtOrdinal,
)?;
if cursor
.as_ref()
.is_some_and(|cursor| cursor.physical_thread_id != params.thread_id)
{
return Err(invalid_cursor("unknown physical segment"));
}
let mut query = QueryBuilder::<Sqlite>::new(
r#"
SELECT turn_id, item_id, updated_at_ordinal AS rollout_ordinal, updated_at_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id =
"#,
);
query
.push_bind(params.thread_id.to_string())
.push(" AND updated_at_ordinal > ")
.push_bind(sqlite_integer(after_updated_at_ordinal)?);
if let Some(turn_id) = params.turn_id.as_deref() {
query.push(" AND turn_id = ").push_bind(turn_id);
}
if let Some(cursor) = cursor {
let comparator = match (params.sort_direction, cursor.include_anchor) {
(SortDirection::Asc, true) => ">=",
(SortDirection::Asc, false) => ">",
(SortDirection::Desc, true) => "<=",
(SortDirection::Desc, false) => "<",
};
query
.push(" AND updated_at_ordinal ")
.push(comparator)
.push(" ")
.push_bind(sqlite_integer(cursor.rollout_ordinal)?);
}
let order = match params.sort_direction {
SortDirection::Asc => "ASC",
SortDirection::Desc => "DESC",
};
query
.push(" ORDER BY updated_at_ordinal ")
.push(order)
.push(" LIMIT ")
.push_bind(remaining_limit(params.page_size, /*row_count*/ 0)?);
let rows = query
.build()
.fetch_all(pool)
.await
.map_err(thread_history_error)?
.into_iter()
.map(|row| stored_thread_item_row_for_thread(params.thread_id, row))
.collect::<ThreadStoreResult<Vec<_>>>()?;
finish_page(
params.thread_id,
CursorScope::ItemsByUpdatedAtOrdinal,
rows,
params.page_size,
)
}
fn segments_from_cursor<'a>(
@@ -491,7 +491,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() {
}
#[tokio::test]
async fn replayed_item_snapshot_updates_content_without_reordering() {
async fn replayed_item_snapshot_preserves_creation_ordinal_and_advances_update_ordinal() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
@@ -545,9 +545,9 @@ async fn replayed_item_snapshot_updates_content_without_reordering() {
.await
.expect("append replayed item snapshot");
let item = sqlx::query_as::<_, (i64, i64, String)>(
let item = sqlx::query_as::<_, (i64, i64, i64, String)>(
r#"
SELECT rollout_ordinal, created_at_ms, item_json
SELECT rollout_ordinal, updated_at_ordinal, created_at_ms, item_json
FROM thread_items
WHERE thread_id = ? AND turn_id = ? AND item_id = ?
"#,
@@ -558,10 +558,9 @@ WHERE thread_id = ? AND turn_id = ? AND item_id = ?
.fetch_one(&pool)
.await
.expect("read projected item");
assert_eq!(item.0, 2);
assert_eq!(item.1, first_created_at_ms);
assert_eq!((item.0, item.1, item.2), (2, 3, first_created_at_ms));
assert_eq!(
serde_json::from_str::<ThreadItem>(item.2.as_str()).expect("parse projected item"),
serde_json::from_str::<ThreadItem>(item.3.as_str()).expect("parse projected item"),
ThreadItem::UserMessage {
id: "user-1".to_string(),
client_id: Some("updated".to_string()),
+16 -1
View File
@@ -399,8 +399,21 @@ pub struct ListItemsParams {
pub cursor: Option<String>,
/// Maximum number of items to return.
pub page_size: usize,
/// Sort direction requested by the caller.
/// Direction to sort items by the selected ordinal.
pub sort_direction: SortDirection,
/// Ordinal to sort items by. Update-ordinal sorting requires an update watermark.
pub sort_key: ItemSortKey,
/// Filters out items with an update ordinal less than or equal to the provided value.
pub after_updated_at_ordinal: Option<u64>,
}
/// The ordinal to use when listing persisted items.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ItemSortKey {
/// Sort by the ordinal where the item was first projected.
CreatedAtOrdinal,
/// Sort by the ordinal where the item was last updated.
UpdatedAtOrdinal,
}
/// A projected app-server `ThreadItem` snapshot within a turn.
@@ -410,6 +423,8 @@ pub struct StoredThreadItem {
pub turn_id: String,
/// Stable item identifier within the turn.
pub item_id: String,
/// Rollout ordinal of the latest persisted update to this item.
pub updated_at_ordinal: u64,
/// Unix timestamp (milliseconds) when this logical item was first projected.
pub created_at_ms: i64,
/// Serialized app-server ThreadItem snapshot.