diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index 83909df0e3..2c1db321e6 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -27,8 +27,8 @@ use crate::RolloutItem; use crate::RolloutLine; use crate::RolloutRecorder; use crate::RolloutRecorderParams; +use crate::RolloutSnippetMatcher; use crate::append_rollout_item_to_path; -use crate::first_rollout_content_match_snippet; use crate::read_session_meta_line; use crate::search_rollout_matches; @@ -344,7 +344,7 @@ async fn compressed_search_preserves_first_visible_snippet_and_no_match() -> any ("absent", None), ] { assert_eq!( - first_rollout_content_match_snippet(&path, query).await?, + RolloutSnippetMatcher::new(query).first_match(&path).await?, expected.map(str::to_string), "compressed={compressed}, query={query}" ); @@ -365,11 +365,14 @@ async fn compressed_search_only_reports_stream_errors_before_a_match() -> anyhow fs::write(&path, contents)?; compress_now(&path)?; assert_eq!( - first_rollout_content_match_snippet(&path, "needle").await?, + RolloutSnippetMatcher::new("needle") + .first_match(&path) + .await?, Some("needle before unreadable tail".to_owned()) ); assert_eq!( - first_rollout_content_match_snippet(&path, "absent") + RolloutSnippetMatcher::new("absent") + .first_match(&path) .await .unwrap_err() .kind(), diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 33402ee38e..acd15073e5 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -161,7 +161,7 @@ pub use recorder::thread_item_from_state_metadata; pub use reverse_jsonl_scanner::ReverseJsonlScanner; pub use reverse_jsonl_scanner::ScanOutcome; pub use rollout_reference_index::RolloutReferenceIndex; -pub use search::first_rollout_content_match_snippet; +pub use search::RolloutSnippetMatcher; pub use search::search_rollout_matches; pub use search::search_rollout_paths; pub use session_index::append_thread_name; diff --git a/codex-rs/rollout/src/search.rs b/codex-rs/rollout/src/search.rs index 7d0af20a10..842c707855 100644 --- a/codex-rs/rollout/src/search.rs +++ b/codex-rs/rollout/src/search.rs @@ -4,6 +4,7 @@ use std::io; use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -125,7 +126,7 @@ async fn scan_rollout_matches( let mut matches = HashMap::new(); let mut dirs = vec![root.to_path_buf()]; let json_search_term = case_insensitive_literal_regex(json_search_term)?; - let mut patterns = None; + let mut matcher = RolloutSnippetMatcher::new(search_term); while let Some(dir) = dirs.pop() { let mut entries = match tokio::fs::read_dir(dir).await { @@ -147,17 +148,17 @@ async fn scan_rollout_matches( continue; }; if rollout_file.is_compressed() { - if let Some(snippet) = - first_content_match_snippet(rollout_file.path(), search_term, &mut patterns) - .await - .unwrap_or_else(|err| { - tracing::warn!( - path = %rollout_file.path().display(), - %err, - "Failed to search compressed rollout" - ); - None - }) + if let Some(snippet) = matcher + .first_match(rollout_file.path()) + .await + .unwrap_or_else(|err| { + tracing::warn!( + path = %rollout_file.path().display(), + %err, + "Failed to search compressed rollout" + ); + None + }) { matches.insert( compression::plain_rollout_path(rollout_file.path()), @@ -185,38 +186,44 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result Ok(false) } -pub async fn first_rollout_content_match_snippet( - path: &Path, - search_term: &str, -) -> io::Result> { - first_content_match_snippet(path, search_term, &mut None).await +/// Finds snippets across rollouts for one search term, compiling only when a file is read. +pub struct RolloutSnippetMatcher<'a> { + search_term: &'a str, + patterns: Option>, } -async fn first_content_match_snippet( - path: &Path, - search_term: &str, - patterns: &mut Option<(Regex, Regex)>, -) -> io::Result> { - let lines = compression::open_rollout_line_reader(path).await?; - // Keep compilation errors inside the caller's per-file error handler. - let (json_search_term, search_term) = match patterns { - Some(patterns) => patterns, - None => patterns.insert(( - case_insensitive_literal_regex(json_escaped_search_term(search_term)?)?, - case_insensitive_literal_regex(search_term)?, - )), - }; - let json_search_term = json_search_term.clone(); - let search_term = search_term.clone(); - lines - .find_map(move |line| { - if json_search_term.is_match(line) { - content_match_snippet(line, &search_term) - } else { - None - } - }) - .await +impl<'a> RolloutSnippetMatcher<'a> { + pub fn new(search_term: &'a str) -> Self { + Self { + search_term, + patterns: None, + } + } + + /// Returns the first matching conversation excerpt. The file is opened before compiling + /// the patterns so per-file I/O errors take precedence even on the first call. + pub async fn first_match(&mut self, path: &Path) -> io::Result> { + let lines = compression::open_rollout_line_reader(path).await?; + let patterns = match &self.patterns { + Some(patterns) => patterns, + None => self.patterns.insert(Arc::new(( + case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?, + case_insensitive_literal_regex(self.search_term)?, + ))), + }; + // Share the regexes across file scans to reuse their scratch caches. + let patterns = Arc::clone(patterns); + lines + .find_map(move |line| { + let (json_search_term, search_term) = patterns.as_ref(); + if json_search_term.is_match(line) { + content_match_snippet(line, search_term) + } else { + None + } + }) + .await + } } async fn scan_compressed_rollout_matches( @@ -256,20 +263,16 @@ async fn scan_compressed_rollout_matches( const COMPRESSED_SEARCH_WORKERS: usize = 4; let scan = async |paths: &[PathBuf]| { let mut matches = HashMap::new(); - let mut patterns = None; + let mut matcher = RolloutSnippetMatcher::new(search_term); for path in paths { - if let Some(snippet) = - first_content_match_snippet(path.as_path(), search_term, &mut patterns) - .await - .unwrap_or_else(|err| { - tracing::warn!( - path = %path.as_path().display(), - %err, - "Failed to search compressed rollout" - ); - None - }) - { + if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| { + tracing::warn!( + path = %path.as_path().display(), + %err, + "Failed to search compressed rollout" + ); + None + }) { matches.insert( compression::plain_rollout_path(path.as_path()), Some(snippet), @@ -291,6 +294,8 @@ fn json_escaped_search_term(search_term: &str) -> io::Result { } fn case_insensitive_literal_regex(search_term: impl AsRef) -> io::Result { + #[cfg(test)] + tests::REGEX_COMPILATIONS.set(tests::REGEX_COMPILATIONS.get() + 1); RegexBuilder::new(regex::escape(search_term.as_ref()).as_str()) .case_insensitive(true) .build() diff --git a/codex-rs/rollout/src/search_tests.rs b/codex-rs/rollout/src/search_tests.rs index 5ad05090ff..5ae57bda0c 100644 --- a/codex-rs/rollout/src/search_tests.rs +++ b/codex-rs/rollout/src/search_tests.rs @@ -1,9 +1,45 @@ -//! Regression coverage for rollout search error handling. +//! Regression coverage for rollout snippet matching and per-file error handling. use super::*; use pretty_assertions::assert_eq; use tempfile::TempDir; +thread_local! { + pub(super) static REGEX_COMPILATIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[tokio::test] +async fn plain_snippets_compile_once_and_match_decoded_text_across_files() -> io::Result<()> { + let root = TempDir::new()?; + REGEX_COMPILATIONS.set(/*value*/ 0); + let mut matcher = RolloutSnippetMatcher::new("é\"\n["); + let mut other = RolloutSnippetMatcher::new("unrelated"); + assert_eq!(REGEX_COMPILATIONS.get(), 0); + + for (index, message) in ["unrelated", "first É\"\n[hit]", "next é\"\n[hit]"] + .into_iter() + .enumerate() + { + let path = root.path().join(format!("{index}.jsonl")); + let line = serde_json::json!({ + "timestamp": "2025-01-01T00:00:00Z", + "type": "event_msg", + "payload": { "type": "agent_message", "message": message } + }); + std::fs::write(&path, line.to_string())?; + let expected = (index != 0).then(|| message.replace('\n', " ")); + assert_eq!(matcher.first_match(&path).await?, expected); + assert_eq!(REGEX_COMPILATIONS.get(), 2); + } + + assert_eq!( + other.first_match(&root.path().join("0.jsonl")).await?, + Some("unrelated".to_string()) + ); + assert_eq!(REGEX_COMPILATIONS.get(), 4); + Ok(()) +} + #[tokio::test] async fn compressed_search_preserves_error_handling() -> io::Result<()> { let root = TempDir::new()?; @@ -28,7 +64,8 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> { RolloutSearchMatches::new() ); assert_eq!( - first_rollout_content_match_snippet(&root.path().join("missing.jsonl"), &query) + RolloutSnippetMatcher::new(&query) + .first_match(&root.path().join("missing.jsonl")) .await .unwrap_err() .kind(), diff --git a/codex-rs/thread-store/src/local/search_threads.rs b/codex-rs/thread-store/src/local/search_threads.rs index b58cc6723c..e422a06029 100644 --- a/codex-rs/thread-store/src/local/search_threads.rs +++ b/codex-rs/thread-store/src/local/search_threads.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use codex_install_context::InstallContext; use codex_rollout::RolloutConfig; -use codex_rollout::first_rollout_content_match_snippet; +use codex_rollout::RolloutSnippetMatcher; use codex_rollout::parse_cursor; use codex_rollout::search_rollout_matches; use codex_utils_absolute_path::normalize_windows_device_path; @@ -117,6 +117,7 @@ pub(super) async fn search_threads( .map(|(path, snippet)| (rollout_search_path(&path), snippet)) .collect::>(); + let mut snippet_matcher = RolloutSnippetMatcher::new(search_term); loop { let page = list_rollout_threads( state_db.clone(), @@ -158,7 +159,8 @@ pub(super) async fn search_threads( } } if snippet.is_none() { - snippet = first_rollout_content_match_snippet(item.path.as_path(), search_term) + snippet = snippet_matcher + .first_match(&item.path) .await .map_err(|err| ThreadStoreError::Internal { message: format!("failed to read rollout search match: {err}"), diff --git a/codex-rs/thread-store/src/local/search_threads_tests.rs b/codex-rs/thread-store/src/local/search_threads_tests.rs index 636e288a83..50b1aeee3e 100644 --- a/codex-rs/thread-store/src/local/search_threads_tests.rs +++ b/codex-rs/thread-store/src/local/search_threads_tests.rs @@ -177,6 +177,91 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression() } } +#[tokio::test] +async fn searches_are_independent_across_catalog_refills() -> Result<(), Box> +{ + let home = TempDir::new()?; + let config = test_config(home.path()); + let db = codex_state::StateRuntime::init( + config.sqlite.clone(), + config.default_model_provider_id.clone(), + ) + .await?; + let store = LocalThreadStore::new(config, Some(db.clone())); + // At this page size the catalog returns 256 rows. Put a hit on either side. + let created_at = Utc::now(); + for id in 1..=258 { + let message = match id { + 258 => "first needle", + 257 => "other query", + 1 => "last NEEDLE", + _ => "ordinary conversation", + }; + let path = write_session_file_with( + home.path(), + home.path().join("sessions/2025/01/03"), + "2025-01-03T12-00-00", + Uuid::from_u128(id), + message, + Some("test-provider"), + ThreadHistoryMode::Legacy, + )?; + let mut row = codex_state::ThreadMetadataBuilder::new( + ThreadId::from_u128(id), + path, + created_at, + SessionSource::Cli, + ) + .build("test-provider"); + row.first_user_message = Some(message.to_owned()); + db.upsert_thread(&row).await?; + } + let search = async |query: &str| { + let page = store + .search_threads(SearchThreadsParams { + include_attached_threads: false, + account: None, + page_size: 2, + cursor: None, + sort_key: ThreadSortKey::CreatedAt, + sort_direction: SortDirection::Desc, + allowed_sources: vec![SessionSource::Cli], + archived: false, + search_term: query.to_owned(), + }) + .await?; + Ok::<_, crate::ThreadStoreError>(( + page.items + .into_iter() + .map(|item| (item.thread.thread_id, item.snippet)) + .collect::>(), + page.next_cursor, + )) + }; + let (many, one, none) = tokio::try_join!(search("needle"), search("other"), search("absent"))?; + assert_eq!( + (many, one, none), + ( + ( + vec![ + ( + ThreadId::from_u128(/*value*/ 258), + "first needle".to_owned() + ), + (ThreadId::from_u128(/*value*/ 1), "last NEEDLE".to_owned()), + ], + None, + ), + ( + vec![(ThreadId::from_u128(/*value*/ 257), "other query".to_owned())], + None + ), + (vec![], None), + ) + ); + Ok(()) +} + async fn write_named_rollout(home: &Path, id: Uuid, attached_to: Option) -> PathBuf { let path = write_session_file_with_fork( home,