Reuse regexes across rollout snippet searches (#62076)

## Why

Thread search compiled the same snippet-matching regexes for each rollout file. Reusing them avoids repeated compilation and preserves regex scratch caches across file scans.

## What changed

Introduce `RolloutSnippetMatcher` to lazily compile and share patterns for one search term. Reuse the matcher across thread search catalog pages and within compressed rollout search workers, preserving per-file I/O error precedence.

## Testing

Add regression coverage for lazy, once-per-matcher compilation, matching decoded text across files, and independent concurrent searches spanning catalog refills. Adapt compressed rollout tests to the matcher API.

GitOrigin-RevId: 1a7ade8c8c4ee533a452658808886a1d83e90741
This commit is contained in:
Charlie Marsh
2026-11-17 16:27:00 +00:00
committed by copyberry
parent ce5a565ba0
commit 378364e784
6 changed files with 196 additions and 64 deletions
+7 -4
View File
@@ -27,8 +27,8 @@ use crate::RolloutItem;
use crate::RolloutLine;
use crate::RolloutRecorder;
use crate::RolloutRecorderParams;
use crate::RolloutSnippetMatcher;
use crate::append_rollout_item_to_path;
use crate::first_rollout_content_match_snippet;
use crate::read_session_meta_line;
use crate::search_rollout_matches;
@@ -344,7 +344,7 @@ async fn compressed_search_preserves_first_visible_snippet_and_no_match() -> any
("absent", None),
] {
assert_eq!(
first_rollout_content_match_snippet(&path, query).await?,
RolloutSnippetMatcher::new(query).first_match(&path).await?,
expected.map(str::to_string),
"compressed={compressed}, query={query}"
);
@@ -365,11 +365,14 @@ async fn compressed_search_only_reports_stream_errors_before_a_match() -> anyhow
fs::write(&path, contents)?;
compress_now(&path)?;
assert_eq!(
first_rollout_content_match_snippet(&path, "needle").await?,
RolloutSnippetMatcher::new("needle")
.first_match(&path)
.await?,
Some("needle before unreadable tail".to_owned())
);
assert_eq!(
first_rollout_content_match_snippet(&path, "absent")
RolloutSnippetMatcher::new("absent")
.first_match(&path)
.await
.unwrap_err()
.kind(),
+1 -1
View File
@@ -161,7 +161,7 @@ pub use recorder::thread_item_from_state_metadata;
pub use reverse_jsonl_scanner::ReverseJsonlScanner;
pub use reverse_jsonl_scanner::ScanOutcome;
pub use rollout_reference_index::RolloutReferenceIndex;
pub use search::first_rollout_content_match_snippet;
pub use search::RolloutSnippetMatcher;
pub use search::search_rollout_matches;
pub use search::search_rollout_paths;
pub use session_index::append_thread_name;
+60 -55
View File
@@ -4,6 +4,7 @@ use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Arc;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
@@ -125,7 +126,7 @@ async fn scan_rollout_matches(
let mut matches = HashMap::new();
let mut dirs = vec![root.to_path_buf()];
let json_search_term = case_insensitive_literal_regex(json_search_term)?;
let mut patterns = None;
let mut matcher = RolloutSnippetMatcher::new(search_term);
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
@@ -147,17 +148,17 @@ async fn scan_rollout_matches(
continue;
};
if rollout_file.is_compressed() {
if let Some(snippet) =
first_content_match_snippet(rollout_file.path(), search_term, &mut patterns)
.await
.unwrap_or_else(|err| {
tracing::warn!(
path = %rollout_file.path().display(),
%err,
"Failed to search compressed rollout"
);
None
})
if let Some(snippet) = matcher
.first_match(rollout_file.path())
.await
.unwrap_or_else(|err| {
tracing::warn!(
path = %rollout_file.path().display(),
%err,
"Failed to search compressed rollout"
);
None
})
{
matches.insert(
compression::plain_rollout_path(rollout_file.path()),
@@ -185,38 +186,44 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result<bool>
Ok(false)
}
pub async fn first_rollout_content_match_snippet(
path: &Path,
search_term: &str,
) -> io::Result<Option<String>> {
first_content_match_snippet(path, search_term, &mut None).await
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
pub struct RolloutSnippetMatcher<'a> {
search_term: &'a str,
patterns: Option<Arc<(Regex, Regex)>>,
}
async fn first_content_match_snippet(
path: &Path,
search_term: &str,
patterns: &mut Option<(Regex, Regex)>,
) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
// Keep compilation errors inside the caller's per-file error handler.
let (json_search_term, search_term) = match patterns {
Some(patterns) => patterns,
None => patterns.insert((
case_insensitive_literal_regex(json_escaped_search_term(search_term)?)?,
case_insensitive_literal_regex(search_term)?,
)),
};
let json_search_term = json_search_term.clone();
let search_term = search_term.clone();
lines
.find_map(move |line| {
if json_search_term.is_match(line) {
content_match_snippet(line, &search_term)
} else {
None
}
})
.await
impl<'a> RolloutSnippetMatcher<'a> {
pub fn new(search_term: &'a str) -> Self {
Self {
search_term,
patterns: None,
}
}
/// Returns the first matching conversation excerpt. The file is opened before compiling
/// the patterns so per-file I/O errors take precedence even on the first call.
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
let patterns = match &self.patterns {
Some(patterns) => patterns,
None => self.patterns.insert(Arc::new((
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
case_insensitive_literal_regex(self.search_term)?,
))),
};
// Share the regexes across file scans to reuse their scratch caches.
let patterns = Arc::clone(patterns);
lines
.find_map(move |line| {
let (json_search_term, search_term) = patterns.as_ref();
if json_search_term.is_match(line) {
content_match_snippet(line, search_term)
} else {
None
}
})
.await
}
}
async fn scan_compressed_rollout_matches(
@@ -256,20 +263,16 @@ async fn scan_compressed_rollout_matches(
const COMPRESSED_SEARCH_WORKERS: usize = 4;
let scan = async |paths: &[PathBuf]| {
let mut matches = HashMap::new();
let mut patterns = None;
let mut matcher = RolloutSnippetMatcher::new(search_term);
for path in paths {
if let Some(snippet) =
first_content_match_snippet(path.as_path(), search_term, &mut patterns)
.await
.unwrap_or_else(|err| {
tracing::warn!(
path = %path.as_path().display(),
%err,
"Failed to search compressed rollout"
);
None
})
{
if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| {
tracing::warn!(
path = %path.as_path().display(),
%err,
"Failed to search compressed rollout"
);
None
}) {
matches.insert(
compression::plain_rollout_path(path.as_path()),
Some(snippet),
@@ -291,6 +294,8 @@ fn json_escaped_search_term(search_term: &str) -> io::Result<String> {
}
fn case_insensitive_literal_regex(search_term: impl AsRef<str>) -> io::Result<Regex> {
#[cfg(test)]
tests::REGEX_COMPILATIONS.set(tests::REGEX_COMPILATIONS.get() + 1);
RegexBuilder::new(regex::escape(search_term.as_ref()).as_str())
.case_insensitive(true)
.build()
+39 -2
View File
@@ -1,9 +1,45 @@
//! Regression coverage for rollout search error handling.
//! Regression coverage for rollout snippet matching and per-file error handling.
use super::*;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
thread_local! {
pub(super) static REGEX_COMPILATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[tokio::test]
async fn plain_snippets_compile_once_and_match_decoded_text_across_files() -> io::Result<()> {
let root = TempDir::new()?;
REGEX_COMPILATIONS.set(/*value*/ 0);
let mut matcher = RolloutSnippetMatcher::new("é\"\n[");
let mut other = RolloutSnippetMatcher::new("unrelated");
assert_eq!(REGEX_COMPILATIONS.get(), 0);
for (index, message) in ["unrelated", "first É\"\n[hit]", "next é\"\n[hit]"]
.into_iter()
.enumerate()
{
let path = root.path().join(format!("{index}.jsonl"));
let line = serde_json::json!({
"timestamp": "2025-01-01T00:00:00Z",
"type": "event_msg",
"payload": { "type": "agent_message", "message": message }
});
std::fs::write(&path, line.to_string())?;
let expected = (index != 0).then(|| message.replace('\n', " "));
assert_eq!(matcher.first_match(&path).await?, expected);
assert_eq!(REGEX_COMPILATIONS.get(), 2);
}
assert_eq!(
other.first_match(&root.path().join("0.jsonl")).await?,
Some("unrelated".to_string())
);
assert_eq!(REGEX_COMPILATIONS.get(), 4);
Ok(())
}
#[tokio::test]
async fn compressed_search_preserves_error_handling() -> io::Result<()> {
let root = TempDir::new()?;
@@ -28,7 +64,8 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
RolloutSearchMatches::new()
);
assert_eq!(
first_rollout_content_match_snippet(&root.path().join("missing.jsonl"), &query)
RolloutSnippetMatcher::new(&query)
.first_match(&root.path().join("missing.jsonl"))
.await
.unwrap_err()
.kind(),
@@ -4,7 +4,7 @@ use std::path::PathBuf;
use codex_install_context::InstallContext;
use codex_rollout::RolloutConfig;
use codex_rollout::first_rollout_content_match_snippet;
use codex_rollout::RolloutSnippetMatcher;
use codex_rollout::parse_cursor;
use codex_rollout::search_rollout_matches;
use codex_utils_absolute_path::normalize_windows_device_path;
@@ -117,6 +117,7 @@ pub(super) async fn search_threads(
.map(|(path, snippet)| (rollout_search_path(&path), snippet))
.collect::<HashMap<_, _>>();
let mut snippet_matcher = RolloutSnippetMatcher::new(search_term);
loop {
let page = list_rollout_threads(
state_db.clone(),
@@ -158,7 +159,8 @@ pub(super) async fn search_threads(
}
}
if snippet.is_none() {
snippet = first_rollout_content_match_snippet(item.path.as_path(), search_term)
snippet = snippet_matcher
.first_match(&item.path)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to read rollout search match: {err}"),
@@ -177,6 +177,91 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression()
}
}
#[tokio::test]
async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn std::error::Error>>
{
let home = TempDir::new()?;
let config = test_config(home.path());
let db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await?;
let store = LocalThreadStore::new(config, Some(db.clone()));
// At this page size the catalog returns 256 rows. Put a hit on either side.
let created_at = Utc::now();
for id in 1..=258 {
let message = match id {
258 => "first needle",
257 => "other query",
1 => "last NEEDLE",
_ => "ordinary conversation",
};
let path = write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
Uuid::from_u128(id),
message,
Some("test-provider"),
ThreadHistoryMode::Legacy,
)?;
let mut row = codex_state::ThreadMetadataBuilder::new(
ThreadId::from_u128(id),
path,
created_at,
SessionSource::Cli,
)
.build("test-provider");
row.first_user_message = Some(message.to_owned());
db.upsert_thread(&row).await?;
}
let search = async |query: &str| {
let page = store
.search_threads(SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 2,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
archived: false,
search_term: query.to_owned(),
})
.await?;
Ok::<_, crate::ThreadStoreError>((
page.items
.into_iter()
.map(|item| (item.thread.thread_id, item.snippet))
.collect::<Vec<_>>(),
page.next_cursor,
))
};
let (many, one, none) = tokio::try_join!(search("needle"), search("other"), search("absent"))?;
assert_eq!(
(many, one, none),
(
(
vec![
(
ThreadId::from_u128(/*value*/ 258),
"first needle".to_owned()
),
(ThreadId::from_u128(/*value*/ 1), "last NEEDLE".to_owned()),
],
None,
),
(
vec![(ThreadId::from_u128(/*value*/ 257), "other query".to_owned())],
None
),
(vec![], None),
)
);
Ok(())
}
async fn write_named_rollout(home: &Path, id: Uuid, attached_to: Option<Uuid>) -> PathBuf {
let path = write_session_file_with_fork(
home,