From b8cbdbfeff72869e5d715480572aad24f739aa65 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 19:13:30 +0000 Subject: [PATCH] Preserve SQLite row metadata in thread listings (#61866) ## Why Direct SQLite listings already contain thread titles and section ordering metadata. Converting those rows to rollout summaries and fetching metadata again discards the original per-row values. ## What changed Build thread listings directly from the collected SQLite rows, retaining their titles, section positions, and section entry times. Preserve row order and each occurrence's metadata when a thread appears more than once during pagination refill. Continue resolving legacy names from the session index with the existing title precedence. Keep rollout listings and search name enrichment best effort, and avoid redundant metadata queries for direct SQLite listings. ## Testing Add coverage for repeated legacy rows retaining indexed names and distinct timestamps and section positions. Extend listing tests to check SQLite titles and section metadata in recent-thread results. GitOrigin-RevId: 838036250d38e6ca1b403741de3e6dd3eafdd04b --- codex-rs/rollout/src/lib.rs | 1 + codex-rs/rollout/src/recorder.rs | 3 +- codex-rs/thread-store/src/local/helpers.rs | 77 +++++---- .../thread-store/src/local/list_threads.rs | 155 ++++++++++++++---- .../src/local/list_threads_tests.rs | 72 ++++++++ .../thread-store/src/local/search_threads.rs | 9 +- 6 files changed, 246 insertions(+), 71 deletions(-) create mode 100644 codex-rs/thread-store/src/local/list_threads_tests.rs diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index dad54bb1e6..33402ee38e 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -157,6 +157,7 @@ pub use policy::should_persist_response_item_for_memories; pub use recorder::RolloutRecorder; pub use recorder::RolloutRecorderParams; pub use recorder::append_rollout_item_to_path; +pub use recorder::thread_item_from_state_metadata; pub use reverse_jsonl_scanner::ReverseJsonlScanner; pub use reverse_jsonl_scanner::ScanOutcome; pub use rollout_reference_index::RolloutReferenceIndex; diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 44fffde8e5..b4b8bda7f9 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -2158,7 +2158,8 @@ impl From for ThreadsPage { } } -fn thread_item_from_state_metadata( +/// Projects a SQLite row into the rollout summary used by thread listings. +pub fn thread_item_from_state_metadata( item: codex_state::ThreadMetadata, parent_thread_id: Option, ) -> ThreadItem { diff --git a/codex-rs/thread-store/src/local/helpers.rs b/codex-rs/thread-store/src/local/helpers.rs index 1b5d0d74de..c5b2a05e6e 100644 --- a/codex-rs/thread-store/src/local/helpers.rs +++ b/codex-rs/thread-store/src/local/helpers.rs @@ -228,59 +228,68 @@ pub(super) async fn resolve_thread_section_metadata( .unwrap_or_default() } -/// DB-only listings require complete names; ordinary listings retain best-effort enrichment. +/// Enriches rollout listings and search results with names on a best-effort basis. pub(super) async fn resolve_thread_names( store: &LocalThreadStore, thread_history_modes: &HashMap, - use_state_db_only: bool, -) -> ThreadStoreResult> { +) -> HashMap { let legacy_thread_ids = thread_history_modes .iter() .filter_map(|(&thread_id, &history_mode)| { (history_mode == ThreadHistoryMode::Legacy).then_some(thread_id) }) .collect::>(); - let mut names = - match find_thread_names_by_ids(store.config.codex_home.as_path(), &legacy_thread_ids).await - { - Ok(names) => names, - Err(err) if use_state_db_only => { - return Err(ThreadStoreError::Internal { - message: format!("failed to read indexed thread names: {err}"), - }); - } - Err(_) => HashMap::new(), - }; + let mut names = find_thread_names_by_ids(store.config.codex_home.as_path(), &legacy_thread_ids) + .await + .unwrap_or_default(); if let Some(state_db_ctx) = store.state_db().await { let thread_ids = thread_history_modes.keys().copied().collect::>(); - let metadata_by_id = match state_db_ctx.get_threads(&thread_ids).await { - Ok(metadata) => metadata, - Err(err) if use_state_db_only => { - return Err(ThreadStoreError::Internal { - message: format!("failed to read thread names from state DB: {err}"), - }); - } - Err(_) => return Ok(names), + let Ok(metadata_by_id) = state_db_ctx.get_threads(&thread_ids).await else { + return names; }; for (&thread_id, &history_mode) in thread_history_modes { let Some(metadata) = metadata_by_id.get(&thread_id) else { continue; }; - let name = match history_mode { - ThreadHistoryMode::Legacy => distinct_thread_metadata_title(metadata), - ThreadHistoryMode::Paginated => sqlite_thread_name(metadata), - }; - if let Some(name) = name { - if history_mode == ThreadHistoryMode::Legacy && has_guardian_default_title(metadata) - { - names.entry(thread_id).or_insert(name); - } else { - names.insert(thread_id, name); - } + if let Some(name) = listed_thread_name(history_mode, metadata, names.remove(&thread_id)) + { + names.insert(thread_id, name); } } } - Ok(names) + names +} + +pub(super) async fn indexed_thread_names( + store: &LocalThreadStore, + legacy_thread_ids: &HashSet, + use_state_db_only: bool, +) -> ThreadStoreResult> { + match find_thread_names_by_ids(store.config.codex_home.as_path(), legacy_thread_ids).await { + Ok(names) => Ok(names), + Err(err) if use_state_db_only => Err(ThreadStoreError::Internal { + message: format!("failed to read indexed thread names: {err}"), + }), + Err(_) => Ok(HashMap::new()), + } +} + +pub(super) fn listed_thread_name( + history_mode: ThreadHistoryMode, + metadata: &ThreadMetadata, + indexed_name: Option, +) -> Option { + match history_mode { + ThreadHistoryMode::Legacy => { + let title = distinct_thread_metadata_title(metadata); + if has_guardian_default_title(metadata) { + indexed_name.or(title) + } else { + title.or(indexed_name) + } + } + ThreadHistoryMode::Paginated => sqlite_thread_name(metadata), + } } /// Identifies the derived Guardian label, which must yield to legacy indexed names. diff --git a/codex-rs/thread-store/src/local/list_threads.rs b/codex-rs/thread-store/src/local/list_threads.rs index e645558ca1..ccf7f3df8b 100644 --- a/codex-rs/thread-store/src/local/list_threads.rs +++ b/codex-rs/thread-store/src/local/list_threads.rs @@ -2,12 +2,16 @@ use std::collections::HashMap; use chrono::DateTime; use chrono::Utc; +use codex_protocol::ThreadId; +use codex_protocol::protocol::ThreadHistoryMode; use codex_rollout::RolloutConfig; use codex_rollout::RolloutRecorder; use codex_rollout::parse_cursor; use codex_state::ThreadFilterOptions; use super::LocalThreadStore; +use super::helpers::indexed_thread_names; +use super::helpers::listed_thread_name; use super::helpers::resolve_thread_names; use super::helpers::resolve_thread_section_metadata; use super::helpers::set_thread_name; @@ -21,6 +25,24 @@ use crate::ThreadSortKey; use crate::ThreadStoreError; use crate::ThreadStoreResult; +/// A local thread listing that preserves full rows from the direct SQLite path. +pub(super) enum LocalThreadsPage { + /// Rows from a direct SQLite listing, with titles and section metadata intact. + State(codex_state::ThreadsPage), + /// Summaries from the rollout recorder, built from rollout files or SQLite rows. + /// The thread store resolves titles and section ordering metadata separately. + Rollouts(codex_rollout::ThreadsPage), +} + +impl From for codex_rollout::ThreadsPage { + fn from(page: LocalThreadsPage) -> Self { + match page { + LocalThreadsPage::State(page) => page.into(), + LocalThreadsPage::Rollouts(page) => page, + } + } +} + pub(super) async fn list_threads( store: &LocalThreadStore, params: ListThreadsParams, @@ -66,11 +88,26 @@ pub(super) async fn list_threads( ) .await?; - let next_cursor = page - .next_cursor - .as_ref() - .and_then(|cursor| serde_json::to_value(cursor).ok()) - .and_then(|value| value.as_str().map(str::to_owned)); + let page = match page { + LocalThreadsPage::State(page) => { + let legacy_thread_ids = page + .items + .iter() + .filter(|row| row.history_mode == ThreadHistoryMode::Legacy) + .map(|row| row.id) + .collect(); + let indexed_names = + indexed_thread_names(store, &legacy_thread_ids, params.use_state_db_only).await?; + return Ok(thread_page_from_state( + page, + &indexed_names, + params.archived, + &store.config.default_model_provider_id, + )); + } + LocalThreadsPage::Rollouts(page) => page, + }; + let next_cursor = encode_cursor(page.next_cursor.as_ref()); let mut items = page .items .into_iter() @@ -87,8 +124,7 @@ pub(super) async fn list_threads( .iter() .map(|thread| (thread.thread_id, thread.history_mode)) .collect::>(); - let names = - resolve_thread_names(store, &thread_history_modes, params.use_state_db_only).await?; + let names = resolve_thread_names(store, &thread_history_modes).await; for thread in &mut items { if let Some(name) = names.get(&thread.thread_id).cloned() { set_thread_name(thread, name); @@ -112,13 +148,51 @@ pub(super) async fn list_threads( } } - if let Some(project_id) = params.project_id.as_ref() { - items.retain(|thread| &thread.project_id == project_id); - } - Ok(ThreadPage { items, next_cursor }) } +/// Projects the collected SQLite rows in order, including repeats across a refill. +fn thread_page_from_state( + page: codex_state::ThreadsPage, + indexed_names: &HashMap, + archived: bool, + default_model_provider_id: &str, +) -> ThreadPage { + let items = page + .items + .into_iter() + .filter_map(|row| { + let name = + listed_thread_name(row.history_mode, &row, indexed_names.get(&row.id).cloned()); + let position = row.section_position; + let entered_at = row.section_entered_at; + let parent = page.parent_thread_ids.get(&row.id).copied(); + let item = codex_rollout::thread_item_from_state_metadata(row, parent); + let mut thread = + stored_thread_from_rollout_item(item, archived, default_model_provider_id)?; + if let Some(name) = name { + set_thread_name(&mut thread, name); + } + if thread.section.is_some() { + thread.section_position = position; + thread.section_entered_at = entered_at; + } + Some(thread) + }) + .collect(); + let cursor = page.next_anchor.map(codex_rollout::Cursor::from); + ThreadPage { + items, + next_cursor: encode_cursor(cursor.as_ref()), + } +} + +fn encode_cursor(cursor: Option<&codex_rollout::Cursor>) -> Option { + cursor + .and_then(|cursor| serde_json::to_value(cursor).ok()) + .and_then(|value| value.as_str().map(str::to_owned)) +} + async fn list_section_threads( store: &LocalThreadStore, params: ListThreadsParams, @@ -241,7 +315,7 @@ pub(super) async fn list_rollout_threads( cursor: Option<&codex_rollout::Cursor>, sort_key: codex_rollout::ThreadSortKey, sort_direction: codex_rollout::SortDirection, -) -> ThreadStoreResult { +) -> ThreadStoreResult { if params.use_state_db_only || params.included_thread_ids.is_some() || params @@ -271,7 +345,7 @@ pub(super) async fn list_rollout_threads( .ok_or_else(|| ThreadStoreError::Internal { message: "state DB unavailable for filtered thread listing".to_string(), })?; - return Ok(page.into()); + return Ok(LocalThreadsPage::State(page)); } let include_attached_threads = @@ -309,9 +383,10 @@ pub(super) async fn list_rollout_threads( ) .await }; - page.map_err(|err| ThreadStoreError::Internal { - message: format!("failed to list threads: {err}"), - }) + page.map(LocalThreadsPage::Rollouts) + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to list threads: {err}"), + }) } fn state_relation_filter(filter: ThreadRelationFilter) -> codex_state::ThreadRelationFilter { @@ -329,6 +404,10 @@ fn state_relation_filter(filter: ThreadRelationFilter) -> codex_state::ThreadRel } } +#[cfg(test)] +#[path = "list_threads_tests.rs"] +mod state_page_tests; + #[cfg(test)] mod tests { use chrono::Utc; @@ -449,15 +528,16 @@ mod tests { .await .expect("thread listing"); - let ids = page - .items - .iter() - .map(|item| item.thread_id) - .collect::>(); - assert_eq!(ids, vec![thread_id]); + let [thread] = page.items.as_slice() else { + panic!("expected one thread matching the SQLite title"); + }; assert_eq!( - page.items[0].first_user_message.as_deref(), - Some("plain preview") + ( + thread.thread_id, + thread.name.as_deref(), + thread.first_user_message.as_deref(), + ), + (thread_id, Some("needle title"), Some("plain preview")) ); } @@ -658,15 +738,11 @@ mod tests { assert_eq!(page.items[0].cli_version, "test_version"); assert_eq!(page.items[0].source, SessionSource::Cli); - // Name query errors must abort DB-only lookup while ordinary listing keeps its fallback. - let history_modes = HashMap::from([(thread_id, ThreadHistoryMode::Legacy)]); + // Ordinary listing keeps its indexed-name fallback when the state DB is unavailable. codex_rollout::append_thread_name(home.path(), thread_id, "indexed name") .await .expect("write indexed name"); runtime.close().await; - resolve_thread_names(&store, &history_modes, /*use_state_db_only*/ true) - .await - .expect_err("name query errors must be reported"); let fallback = store .list_threads(params) .await @@ -751,6 +827,27 @@ mod tests { ); assert_eq!(page.next_cursor, Some(format!("2000000|{}", thread_ids[1]))); + let recent_page = store + .list_threads(ListThreadsParams { + page_size: 3, + sort_key: ThreadSortKey::UpdatedAt, + sort_direction: SortDirection::Desc, + section: None, + ..params.clone() + }) + .await + .expect("list recent threads with section metadata"); + let section_member = recent_page + .items + .iter() + .find(|thread| thread.thread_id == thread_ids[1]) + .expect("listed section member"); + let entered_at = page.items[1] + .section_entered_at + .expect("section entry time"); + assert_eq!(section_member.section_position, Some(2_000_000)); + assert_eq!(section_member.section_entered_at, Some(entered_at)); + let next_page = store .list_threads(ListThreadsParams { cursor: page.next_cursor, diff --git a/codex-rs/thread-store/src/local/list_threads_tests.rs b/codex-rs/thread-store/src/local/list_threads_tests.rs new file mode 100644 index 0000000000..01dc4b06d0 --- /dev/null +++ b/codex-rs/thread-store/src/local/list_threads_tests.rs @@ -0,0 +1,72 @@ +#![cfg(test)] + +//! Tests SQLite thread listing. Each row for a repeated legacy thread retains +//! its own metadata and the name from the session index. + +use std::collections::HashMap; + +use chrono::DateTime; +use chrono::TimeDelta; +use codex_protocol::ThreadId; +use codex_protocol::protocol::SessionSource; +use codex_state::ThreadMetadataBuilder; +use codex_state::ThreadSection; +use codex_state::ThreadsPage; +use pretty_assertions::assert_eq; + +use super::thread_page_from_state; + +#[test] +fn duplicate_legacy_rows_keep_the_indexed_name() { + let id = ThreadId::new(); + let first_update = DateTime::UNIX_EPOCH; + let second_update = first_update + TimeDelta::seconds(2); + let mut first = + ThreadMetadataBuilder::new(id, "rollout.jsonl".into(), first_update, SessionSource::Cli) + .build("test-provider"); + first.section = Some(ThreadSection { + id: "section".to_owned(), + name: "Section".to_owned(), + appearance: None, + }); + first.section_position = Some(1_000_000); + // A thread updated across the cursor can appear again when a missing rollout forces a refill. + let mut repeated = first.clone(); + repeated.updated_at = second_update; + repeated.section_position = Some(2_000_000); + let page = ThreadsPage { + items: vec![first, repeated], + parent_thread_ids: HashMap::new(), + next_anchor: None, + num_scanned_rows: 3, + }; + let name = "Indexed name"; + let indexed_names = HashMap::from([(id, name.to_owned())]); + + let page = thread_page_from_state( + page, + &indexed_names, + /*archived*/ false, + "test-provider", + ); + + let rows: Vec<_> = page + .items + .iter() + .map(|thread| { + ( + thread.thread_id, + thread.updated_at, + thread.name.as_deref(), + thread.section_position, + ) + }) + .collect(); + assert_eq!( + rows, + vec![ + (id, first_update, Some(name), Some(1_000_000)), + (id, second_update, Some(name), Some(2_000_000)), + ] + ); +} diff --git a/codex-rs/thread-store/src/local/search_threads.rs b/codex-rs/thread-store/src/local/search_threads.rs index a579d42b95..b58cc6723c 100644 --- a/codex-rs/thread-store/src/local/search_threads.rs +++ b/codex-rs/thread-store/src/local/search_threads.rs @@ -128,6 +128,7 @@ pub(super) async fn search_threads( sort_direction, ) .await?; + let page = codex_rollout::ThreadsPage::from(page); let page_thread_ids: Vec<_> = page .items .iter() @@ -274,13 +275,7 @@ async fn set_thread_search_result_names( .iter() .map(|item| (item.thread.thread_id, item.thread.history_mode)) .collect::>(); - let names = resolve_thread_names( - store, - &thread_history_modes, - /*use_state_db_only*/ false, - ) - .await - .unwrap_or_default(); + let names = resolve_thread_names(store, &thread_history_modes).await; for item in items { if let Some(name) = names.get(&item.thread.thread_id).cloned() { set_thread_name(&mut item.thread, name);