mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Reuse regexes across rollout snippet searches (#62076)
## Why Thread search compiled the same snippet-matching regexes for each rollout file. Reusing them avoids repeated compilation and preserves regex scratch caches across file scans. ## What changed Introduce `RolloutSnippetMatcher` to lazily compile and share patterns for one search term. Reuse the matcher across thread search catalog pages and within compressed rollout search workers, preserving per-file I/O error precedence. ## Testing Add regression coverage for lazy, once-per-matcher compilation, matching decoded text across files, and independent concurrent searches spanning catalog refills. Adapt compressed rollout tests to the matcher API. GitOrigin-RevId: 1a7ade8c8c4ee533a452658808886a1d83e90741
This commit is contained in:
@@ -27,8 +27,8 @@ use crate::RolloutItem;
|
||||
use crate::RolloutLine;
|
||||
use crate::RolloutRecorder;
|
||||
use crate::RolloutRecorderParams;
|
||||
use crate::RolloutSnippetMatcher;
|
||||
use crate::append_rollout_item_to_path;
|
||||
use crate::first_rollout_content_match_snippet;
|
||||
use crate::read_session_meta_line;
|
||||
use crate::search_rollout_matches;
|
||||
|
||||
@@ -344,7 +344,7 @@ async fn compressed_search_preserves_first_visible_snippet_and_no_match() -> any
|
||||
("absent", None),
|
||||
] {
|
||||
assert_eq!(
|
||||
first_rollout_content_match_snippet(&path, query).await?,
|
||||
RolloutSnippetMatcher::new(query).first_match(&path).await?,
|
||||
expected.map(str::to_string),
|
||||
"compressed={compressed}, query={query}"
|
||||
);
|
||||
@@ -365,11 +365,14 @@ async fn compressed_search_only_reports_stream_errors_before_a_match() -> anyhow
|
||||
fs::write(&path, contents)?;
|
||||
compress_now(&path)?;
|
||||
assert_eq!(
|
||||
first_rollout_content_match_snippet(&path, "needle").await?,
|
||||
RolloutSnippetMatcher::new("needle")
|
||||
.first_match(&path)
|
||||
.await?,
|
||||
Some("needle before unreadable tail".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
first_rollout_content_match_snippet(&path, "absent")
|
||||
RolloutSnippetMatcher::new("absent")
|
||||
.first_match(&path)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
|
||||
@@ -161,7 +161,7 @@ pub use recorder::thread_item_from_state_metadata;
|
||||
pub use reverse_jsonl_scanner::ReverseJsonlScanner;
|
||||
pub use reverse_jsonl_scanner::ScanOutcome;
|
||||
pub use rollout_reference_index::RolloutReferenceIndex;
|
||||
pub use search::first_rollout_content_match_snippet;
|
||||
pub use search::RolloutSnippetMatcher;
|
||||
pub use search::search_rollout_matches;
|
||||
pub use search::search_rollout_paths;
|
||||
pub use session_index::append_thread_name;
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::io;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -125,7 +126,7 @@ async fn scan_rollout_matches(
|
||||
let mut matches = HashMap::new();
|
||||
let mut dirs = vec![root.to_path_buf()];
|
||||
let json_search_term = case_insensitive_literal_regex(json_search_term)?;
|
||||
let mut patterns = None;
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
|
||||
while let Some(dir) = dirs.pop() {
|
||||
let mut entries = match tokio::fs::read_dir(dir).await {
|
||||
@@ -147,17 +148,17 @@ async fn scan_rollout_matches(
|
||||
continue;
|
||||
};
|
||||
if rollout_file.is_compressed() {
|
||||
if let Some(snippet) =
|
||||
first_content_match_snippet(rollout_file.path(), search_term, &mut patterns)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %rollout_file.path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
})
|
||||
if let Some(snippet) = matcher
|
||||
.first_match(rollout_file.path())
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %rollout_file.path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
})
|
||||
{
|
||||
matches.insert(
|
||||
compression::plain_rollout_path(rollout_file.path()),
|
||||
@@ -185,38 +186,44 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result<bool>
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub async fn first_rollout_content_match_snippet(
|
||||
path: &Path,
|
||||
search_term: &str,
|
||||
) -> io::Result<Option<String>> {
|
||||
first_content_match_snippet(path, search_term, &mut None).await
|
||||
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
|
||||
pub struct RolloutSnippetMatcher<'a> {
|
||||
search_term: &'a str,
|
||||
patterns: Option<Arc<(Regex, Regex)>>,
|
||||
}
|
||||
|
||||
async fn first_content_match_snippet(
|
||||
path: &Path,
|
||||
search_term: &str,
|
||||
patterns: &mut Option<(Regex, Regex)>,
|
||||
) -> io::Result<Option<String>> {
|
||||
let lines = compression::open_rollout_line_reader(path).await?;
|
||||
// Keep compilation errors inside the caller's per-file error handler.
|
||||
let (json_search_term, search_term) = match patterns {
|
||||
Some(patterns) => patterns,
|
||||
None => patterns.insert((
|
||||
case_insensitive_literal_regex(json_escaped_search_term(search_term)?)?,
|
||||
case_insensitive_literal_regex(search_term)?,
|
||||
)),
|
||||
};
|
||||
let json_search_term = json_search_term.clone();
|
||||
let search_term = search_term.clone();
|
||||
lines
|
||||
.find_map(move |line| {
|
||||
if json_search_term.is_match(line) {
|
||||
content_match_snippet(line, &search_term)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.await
|
||||
impl<'a> RolloutSnippetMatcher<'a> {
|
||||
pub fn new(search_term: &'a str) -> Self {
|
||||
Self {
|
||||
search_term,
|
||||
patterns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first matching conversation excerpt. The file is opened before compiling
|
||||
/// the patterns so per-file I/O errors take precedence even on the first call.
|
||||
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
|
||||
let lines = compression::open_rollout_line_reader(path).await?;
|
||||
let patterns = match &self.patterns {
|
||||
Some(patterns) => patterns,
|
||||
None => self.patterns.insert(Arc::new((
|
||||
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
|
||||
case_insensitive_literal_regex(self.search_term)?,
|
||||
))),
|
||||
};
|
||||
// Share the regexes across file scans to reuse their scratch caches.
|
||||
let patterns = Arc::clone(patterns);
|
||||
lines
|
||||
.find_map(move |line| {
|
||||
let (json_search_term, search_term) = patterns.as_ref();
|
||||
if json_search_term.is_match(line) {
|
||||
content_match_snippet(line, search_term)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_compressed_rollout_matches(
|
||||
@@ -256,20 +263,16 @@ async fn scan_compressed_rollout_matches(
|
||||
const COMPRESSED_SEARCH_WORKERS: usize = 4;
|
||||
let scan = async |paths: &[PathBuf]| {
|
||||
let mut matches = HashMap::new();
|
||||
let mut patterns = None;
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
for path in paths {
|
||||
if let Some(snippet) =
|
||||
first_content_match_snippet(path.as_path(), search_term, &mut patterns)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %path.as_path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
})
|
||||
{
|
||||
if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %path.as_path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
}) {
|
||||
matches.insert(
|
||||
compression::plain_rollout_path(path.as_path()),
|
||||
Some(snippet),
|
||||
@@ -291,6 +294,8 @@ fn json_escaped_search_term(search_term: &str) -> io::Result<String> {
|
||||
}
|
||||
|
||||
fn case_insensitive_literal_regex(search_term: impl AsRef<str>) -> io::Result<Regex> {
|
||||
#[cfg(test)]
|
||||
tests::REGEX_COMPILATIONS.set(tests::REGEX_COMPILATIONS.get() + 1);
|
||||
RegexBuilder::new(regex::escape(search_term.as_ref()).as_str())
|
||||
.case_insensitive(true)
|
||||
.build()
|
||||
|
||||
@@ -1,9 +1,45 @@
|
||||
//! Regression coverage for rollout search error handling.
|
||||
//! Regression coverage for rollout snippet matching and per-file error handling.
|
||||
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tempfile::TempDir;
|
||||
|
||||
thread_local! {
|
||||
pub(super) static REGEX_COMPILATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plain_snippets_compile_once_and_match_decoded_text_across_files() -> io::Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
REGEX_COMPILATIONS.set(/*value*/ 0);
|
||||
let mut matcher = RolloutSnippetMatcher::new("é\"\n[");
|
||||
let mut other = RolloutSnippetMatcher::new("unrelated");
|
||||
assert_eq!(REGEX_COMPILATIONS.get(), 0);
|
||||
|
||||
for (index, message) in ["unrelated", "first É\"\n[hit]", "next é\"\n[hit]"]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let path = root.path().join(format!("{index}.jsonl"));
|
||||
let line = serde_json::json!({
|
||||
"timestamp": "2025-01-01T00:00:00Z",
|
||||
"type": "event_msg",
|
||||
"payload": { "type": "agent_message", "message": message }
|
||||
});
|
||||
std::fs::write(&path, line.to_string())?;
|
||||
let expected = (index != 0).then(|| message.replace('\n', " "));
|
||||
assert_eq!(matcher.first_match(&path).await?, expected);
|
||||
assert_eq!(REGEX_COMPILATIONS.get(), 2);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
other.first_match(&root.path().join("0.jsonl")).await?,
|
||||
Some("unrelated".to_string())
|
||||
);
|
||||
assert_eq!(REGEX_COMPILATIONS.get(), 4);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compressed_search_preserves_error_handling() -> io::Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
@@ -28,7 +64,8 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
|
||||
RolloutSearchMatches::new()
|
||||
);
|
||||
assert_eq!(
|
||||
first_rollout_content_match_snippet(&root.path().join("missing.jsonl"), &query)
|
||||
RolloutSnippetMatcher::new(&query)
|
||||
.first_match(&root.path().join("missing.jsonl"))
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
|
||||
Reference in New Issue
Block a user