Files
codex/codex-rs/thread-store/src/local/search_threads_tests.rs
T
Charlie Marsh 378364e784 Reuse regexes across rollout snippet searches (#62076)
## Why

Thread search compiled the same snippet-matching regexes for each rollout file. Reusing them avoids repeated compilation and preserves regex scratch caches across file scans.

## What changed

Introduce `RolloutSnippetMatcher` to lazily compile and share patterns for one search term. Reuse the matcher across thread search catalog pages and within compressed rollout search workers, preserving per-file I/O error precedence.

## Testing

Add regression coverage for lazy, once-per-matcher compilation, matching decoded text across files, and independent concurrent searches spanning catalog refills. Adapt compressed rollout tests to the matcher API.

GitOrigin-RevId: 1a7ade8c8c4ee533a452658808886a1d83e90741
2026-11-17 16:27:00 +00:00

409 lines
14 KiB
Rust

use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::ThreadItem;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use uuid::Uuid;
use super::super::LocalThreadStore;
use super::super::test_support::test_config;
use super::super::test_support::write_session_file_with;
use super::super::test_support::write_session_file_with_fork;
use super::ThreadSearchItem;
use super::cursor_from_thread_search_item;
use crate::ListThreadsParams;
use crate::SearchThreadsParams;
use crate::SortDirection;
use crate::ThreadRelationFilter;
use crate::ThreadSortKey;
use crate::ThreadStore;
#[test]
fn time_sort_cursors_include_thread_id_tie_breaker() {
let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000123")
.expect("thread ID should parse");
let item = ThreadSearchItem {
item: ThreadItem {
thread_id: Some(thread_id),
created_at: Some("2026-01-27T12:34:56Z".to_string()),
updated_at: Some("2026-01-27T12:34:56Z".to_string()),
recency_at: Some("2026-01-27T12:34:56Z".to_string()),
..Default::default()
},
snippet: String::new(),
};
for sort_key in [
ThreadSortKey::CreatedAt,
ThreadSortKey::UpdatedAt,
ThreadSortKey::RecencyAt,
] {
let cursor = cursor_from_thread_search_item(&item, sort_key).expect("cursor should build");
assert_eq!(
serde_json::to_string(&cursor).expect("cursor should serialize"),
format!("\"2026-01-27T12:34:56Z|{thread_id}\"")
);
}
}
#[tokio::test]
async fn search_matches_selected_rollout_across_path_spellings_and_compression() {
for compressed in [false, true] {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let state_db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("initialize state database");
let uuid = Uuid::new_v4();
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("thread ID should parse");
let selected_path = write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
uuid,
"needle in selected rollout",
Some("test-provider"),
ThreadHistoryMode::Legacy,
)
.expect("write selected rollout");
// Reverting a thread leaves its previous rollout on disk with the same thread ID.
write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T11-00-00",
uuid,
"obsolete needle in previous rollout",
Some("test-provider"),
ThreadHistoryMode::Legacy,
)
.expect("write unselected rollout");
let home_paths = [home.path().to_path_buf()];
#[cfg(windows)]
let home_paths = {
let verbatim = std::fs::canonicalize(&home_paths[0]).expect("canonicalize home");
let ordinary =
codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path(&verbatim)
.expect("normalize home")
.into_path_buf();
[ordinary, verbatim]
};
let relative_path = selected_path
.strip_prefix(home.path())
.expect("relative rollout path");
let selected_paths = home_paths.each_ref().map(|home| home.join(relative_path));
#[cfg(windows)]
assert_ne!(selected_paths[0], selected_paths[1]);
if compressed {
let contents = std::fs::read(&selected_path).expect("read selected rollout");
let contents = zstd::stream::encode_all(contents.as_slice(), /*level*/ 3)
.expect("compress selected rollout");
std::fs::write(selected_path.with_extension("jsonl.zst"), contents)
.expect("write compressed rollout");
std::fs::remove_file(&selected_path).expect("remove plain rollout");
}
let selected_paths = selected_paths
.into_iter()
.flat_map(|path| {
let compressed_path = compressed.then(|| path.with_extension("jsonl.zst"));
std::iter::once(path).chain(compressed_path)
})
.collect::<Vec<_>>();
for codex_home in home_paths {
let store = LocalThreadStore::new(
super::super::LocalThreadStoreConfig {
codex_home,
..config.clone()
},
Some(state_db.clone()),
);
for selected_path in &selected_paths {
let mut metadata = codex_state::ThreadMetadataBuilder::new(
thread_id,
selected_path.clone(),
Utc::now(),
SessionSource::Cli,
)
.build("test-provider");
metadata.first_user_message = Some("needle in selected rollout".to_string());
state_db
.upsert_thread(&metadata)
.await
.expect("select rollout in database");
for (search_term, expected) in [
("needle", vec![(thread_id, "needle in selected rollout")]),
("obsolete", Vec::new()),
] {
let page = store
.search_threads(SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
archived: false,
search_term: search_term.to_string(),
})
.await
.expect("search selected rollout");
assert_eq!(
(
page.items
.iter()
.map(|item| (item.thread.thread_id, item.snippet.as_str()))
.collect::<Vec<_>>(),
page.next_cursor,
),
(expected, None),
"compressed={compressed}, selected_path={selected_path:?}",
);
}
}
}
}
}
#[tokio::test]
async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn std::error::Error>>
{
let home = TempDir::new()?;
let config = test_config(home.path());
let db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await?;
let store = LocalThreadStore::new(config, Some(db.clone()));
// At this page size the catalog returns 256 rows. Put a hit on either side.
let created_at = Utc::now();
for id in 1..=258 {
let message = match id {
258 => "first needle",
257 => "other query",
1 => "last NEEDLE",
_ => "ordinary conversation",
};
let path = write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
Uuid::from_u128(id),
message,
Some("test-provider"),
ThreadHistoryMode::Legacy,
)?;
let mut row = codex_state::ThreadMetadataBuilder::new(
ThreadId::from_u128(id),
path,
created_at,
SessionSource::Cli,
)
.build("test-provider");
row.first_user_message = Some(message.to_owned());
db.upsert_thread(&row).await?;
}
let search = async |query: &str| {
let page = store
.search_threads(SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 2,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
archived: false,
search_term: query.to_owned(),
})
.await?;
Ok::<_, crate::ThreadStoreError>((
page.items
.into_iter()
.map(|item| (item.thread.thread_id, item.snippet))
.collect::<Vec<_>>(),
page.next_cursor,
))
};
let (many, one, none) = tokio::try_join!(search("needle"), search("other"), search("absent"))?;
assert_eq!(
(many, one, none),
(
(
vec![
(
ThreadId::from_u128(/*value*/ 258),
"first needle".to_owned()
),
(ThreadId::from_u128(/*value*/ 1), "last NEEDLE".to_owned()),
],
None,
),
(
vec![(ThreadId::from_u128(/*value*/ 257), "other query".to_owned())],
None
),
(vec![], None),
)
);
Ok(())
}
async fn write_named_rollout(home: &Path, id: Uuid, attached_to: Option<Uuid>) -> PathBuf {
let path = write_session_file_with_fork(
home,
home.join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
id,
"searchable message",
Some("test-provider"),
attached_to,
attached_to.is_some(),
ThreadHistoryMode::Legacy,
)
.expect("write rollout");
codex_rollout::append_thread_name(home, ThreadId::from_u128(id.as_u128()), "matching title")
.await
.expect("save thread name");
path
}
fn inclusive_title_search() -> ListThreadsParams {
ListThreadsParams {
included_thread_ids: None,
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
model_providers: None,
cwd_filters: None,
section: None,
project_id: None,
archived: false,
search_term: Some("matching title".to_string()),
relation_filter: Some(ThreadRelationFilter::IncludeAttached),
use_state_db_only: false,
}
}
#[tokio::test]
async fn title_and_content_search_return_attached_parent_without_state_db() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let parent = Uuid::new_v4();
let child = Uuid::new_v4();
write_named_rollout(home.path(), child, Some(parent)).await;
let expected = vec![(
ThreadId::from_u128(child.as_u128()),
Some(ThreadId::from_u128(parent.as_u128())),
)];
let list = store
.list_threads(inclusive_title_search())
.await
.expect("search titles on disk");
assert_eq!(
list.items
.into_iter()
.map(|t| (t.thread_id, t.attached_to_thread_id))
.collect::<Vec<_>>(),
expected
);
let search = store
.search_threads(SearchThreadsParams {
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
archived: false,
search_term: "searchable message".to_string(),
include_attached_threads: true,
account: None,
})
.await
.expect("search messages on disk");
assert_eq!(
search
.items
.into_iter()
.map(|hit| (hit.thread.thread_id, hit.thread.attached_to_thread_id))
.collect::<Vec<_>>(),
expected
);
}
#[tokio::test]
async fn inclusive_title_search_finds_unindexed_matches_when_filesystem_scanning_is_allowed() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let state_db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("initialize state database");
let parent = Uuid::new_v4();
let child = Uuid::new_v4();
let standalone = Uuid::new_v4();
let parent_path = write_named_rollout(home.path(), parent, /*attached_to*/ None).await;
write_named_rollout(home.path(), standalone, /*attached_to*/ None).await;
write_named_rollout(home.path(), child, Some(parent)).await;
let parent_id = ThreadId::from_u128(parent.as_u128());
let child_id = ThreadId::from_u128(child.as_u128());
let mut row = codex_state::ThreadMetadataBuilder::new(
parent_id,
parent_path,
Utc::now(),
SessionSource::Cli,
)
.build("test-provider");
row.first_user_message = Some("searchable message".to_string());
row.title = "matching title".to_string();
state_db
.upsert_thread(&row)
.await
.expect("index only the parent");
let store = LocalThreadStore::new(config, Some(state_db));
let all = HashMap::from([
(parent_id, None),
(child_id, Some(parent_id)),
(ThreadId::from_u128(standalone.as_u128()), None),
]);
for (use_state_db_only, expected) in [
(true, HashMap::from([(parent_id, None)])),
(false, all.clone()),
(true, all),
] {
let page = store
.list_threads(ListThreadsParams {
use_state_db_only,
..inclusive_title_search()
})
.await
.expect("search titles");
assert_eq!(
page.items
.into_iter()
.map(|t| (t.thread_id, t.attached_to_thread_id))
.collect::<HashMap<_, _>>(),
expected
);
}
}