mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Stream partial thread search results and support cancellation (#62132)
## Why Clients using `thread/search` have to wait for the search to finish before they can show any matches. A search can also outlive the query that triggered it, such as when a user changes their task mention query. ## What changed - Add experimental `thread/search/stream` with best-effort `thread/search/partial` notifications. Each notification carries the original `requestId` and a batch of verified matches. Clients replace provisional matches with the final page, which determines results, ordering, and cursors. - Support connection-scoped cancellation through `thread/search/stream/cancel` and `thread/search/cancel` for their respective search methods. Disconnecting also stops active searches and reaps their search subprocesses. - Cancel superseded or timed-out TUI task mention searches without delaying available title matches. ## Testing Add coverage for incremental results and final-page parity, slow clients, connection-scoped cancellation and subprocess cleanup, and TUI task search cancellation. GitOrigin-RevId: b58abf7a1df424f191c8ede806d1d5266d299548
This commit is contained in:
committed by
copyberry
parent
c90234de95
commit
3bf4e138fe
@@ -302,10 +302,7 @@ async fn search_rollout_matches_uses_logical_path_for_compressed_rollout() -> an
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
matches.get(rollout_path.as_path()),
|
||||
Some(&Some("targeted search term".to_string()))
|
||||
);
|
||||
assert_eq!(matches, std::collections::HashSet::from([rollout_path]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -161,8 +161,11 @@ pub use recorder::thread_item_from_state_metadata;
|
||||
pub use reverse_jsonl_scanner::ReverseJsonlScanner;
|
||||
pub use reverse_jsonl_scanner::ScanOutcome;
|
||||
pub use rollout_reference_index::RolloutReferenceIndex;
|
||||
pub use search::RolloutSearchCandidate;
|
||||
pub use search::RolloutSearchMatches;
|
||||
pub use search::RolloutSnippetMatcher;
|
||||
pub use search::search_rollout_matches;
|
||||
pub use search::search_rollout_matches_with_progress;
|
||||
pub use search::search_rollout_paths;
|
||||
pub use session_index::append_thread_name;
|
||||
pub use session_index::find_thread_meta_by_name_str;
|
||||
|
||||
+180
-166
@@ -1,19 +1,23 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::strip_user_message_prefix;
|
||||
use futures::future::join_all;
|
||||
use futures::FutureExt;
|
||||
use futures::future::BoxFuture;
|
||||
use futures::future::pending;
|
||||
use regex::Regex;
|
||||
use regex::RegexBuilder;
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::ARCHIVED_SESSIONS_SUBDIR;
|
||||
use super::SESSIONS_SUBDIR;
|
||||
@@ -21,11 +25,22 @@ use super::compression;
|
||||
use crate::ResponseItemEnvelope;
|
||||
use crate::RolloutItem;
|
||||
|
||||
mod compressed;
|
||||
mod snippet_matcher;
|
||||
use compressed::scan_compressed_rollout_matches;
|
||||
pub use snippet_matcher::RolloutSnippetMatcher;
|
||||
|
||||
const MATCH_CONTEXT_BEFORE_CHARS: usize = 48;
|
||||
const MATCH_CONTEXT_AFTER_CHARS: usize = 96;
|
||||
|
||||
/// Search matches keyed by the canonical `.jsonl` path for each rollout.
|
||||
pub type RolloutSearchMatches = HashMap<PathBuf, Option<String>>;
|
||||
/// Logical `.jsonl` paths discovered by search, regardless of physical representation.
|
||||
pub type RolloutSearchMatches = HashSet<PathBuf>;
|
||||
|
||||
/// A potential match discovered by the rollout scanner.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct RolloutSearchCandidate {
|
||||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
pub async fn search_rollout_paths(
|
||||
rg_command: &Path,
|
||||
@@ -33,12 +48,7 @@ pub async fn search_rollout_paths(
|
||||
archived: bool,
|
||||
search_term: &str,
|
||||
) -> io::Result<HashSet<PathBuf>> {
|
||||
Ok(
|
||||
search_rollout_matches(rg_command, codex_home, archived, search_term)
|
||||
.await?
|
||||
.into_keys()
|
||||
.collect(),
|
||||
)
|
||||
search_rollout_matches(rg_command, codex_home, archived, search_term).await
|
||||
}
|
||||
|
||||
pub async fn search_rollout_matches(
|
||||
@@ -46,6 +56,26 @@ pub async fn search_rollout_matches(
|
||||
codex_home: &Path,
|
||||
archived: bool,
|
||||
search_term: &str,
|
||||
) -> io::Result<RolloutSearchMatches> {
|
||||
search_rollout_matches_with_progress(
|
||||
rg_command,
|
||||
codex_home,
|
||||
archived,
|
||||
search_term,
|
||||
/*progress*/ None,
|
||||
Box::pin(pending()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Search plain rollouts, then compressed rollouts, reporting candidates without blocking.
|
||||
pub async fn search_rollout_matches_with_progress(
|
||||
rg_command: &Path,
|
||||
codex_home: &Path,
|
||||
archived: bool,
|
||||
search_term: &str,
|
||||
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
|
||||
cancelled: BoxFuture<'_, ()>,
|
||||
) -> io::Result<RolloutSearchMatches> {
|
||||
let root = codex_home.join(if archived {
|
||||
ARCHIVED_SESSIONS_SUBDIR
|
||||
@@ -53,28 +83,63 @@ pub async fn search_rollout_matches(
|
||||
SESSIONS_SUBDIR
|
||||
});
|
||||
let json_search_term = json_escaped_search_term(search_term)?;
|
||||
let Some(plain_matches) =
|
||||
ripgrep_rollout_paths(rg_command, root.as_path(), json_search_term.as_str()).await?
|
||||
else {
|
||||
return scan_rollout_matches(root.as_path(), json_search_term.as_str(), search_term).await;
|
||||
let cancelled = cancelled.shared();
|
||||
let mut cancellation: BoxFuture<'_, ()> = Box::pin(cancelled.clone());
|
||||
let mut matches = match ripgrep_rollout_paths(
|
||||
rg_command,
|
||||
&root,
|
||||
&json_search_term,
|
||||
progress.clone(),
|
||||
&mut cancellation,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Some(paths) => paths,
|
||||
None => tokio::select! {
|
||||
biased;
|
||||
() = cancelled.clone() => return Err(search_cancelled()),
|
||||
result = scan_plain_rollout_matches(&root, &json_search_term, progress.clone()) => result?,
|
||||
},
|
||||
};
|
||||
let mut matches: RolloutSearchMatches =
|
||||
plain_matches.into_iter().map(|path| (path, None)).collect();
|
||||
matches.extend(scan_compressed_rollout_matches(root.as_path(), search_term).await?);
|
||||
// Compression publishes the archive before removing its plain sibling. Discover archives
|
||||
// after the plain scan so a transition cannot fall between the two inventories.
|
||||
let compressed = tokio::select! {
|
||||
biased;
|
||||
() = cancelled => return Err(search_cancelled()),
|
||||
result = scan_compressed_rollout_matches(&root, search_term, progress) => result?,
|
||||
};
|
||||
matches.extend(compressed);
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
fn search_cancelled() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::Interrupted, "rollout search cancelled")
|
||||
}
|
||||
|
||||
async fn ripgrep_rollout_paths(
|
||||
rg_command: &Path,
|
||||
root: &Path,
|
||||
search_term: &str,
|
||||
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
|
||||
cancelled: &mut BoxFuture<'_, ()>,
|
||||
) -> io::Result<Option<HashSet<PathBuf>>> {
|
||||
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
|
||||
let exists = tokio::select! {
|
||||
biased;
|
||||
() = &mut *cancelled => return Err(search_cancelled()),
|
||||
exists = tokio::fs::try_exists(root) => exists.unwrap_or(false),
|
||||
};
|
||||
if !exists {
|
||||
return Ok(Some(HashSet::new()));
|
||||
}
|
||||
|
||||
let output = match Command::from(codex_utils_process::background_command(rg_command))
|
||||
.arg("-l")
|
||||
let mut command = Command::from(codex_utils_process::background_command(rg_command));
|
||||
// A NUL-only file listing does not flush through ripgrep's line buffer. A one-match
|
||||
// count adds a newline after each NUL-delimited path and still stops at the first hit.
|
||||
command.args(["--count", "--max-count", "1", "--null"]);
|
||||
if progress.is_some() {
|
||||
command.arg("--line-buffered");
|
||||
}
|
||||
let mut child = match command
|
||||
.arg("--fixed-strings")
|
||||
.arg("--ignore-case")
|
||||
.arg("--no-ignore")
|
||||
@@ -84,49 +149,115 @@ async fn ripgrep_rollout_paths(
|
||||
.arg(search_term)
|
||||
.arg(root)
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.await
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
{
|
||||
Ok(output) => output,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(child) => child,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !output.status.success() {
|
||||
if output.status.code() == Some(1) && output.stderr.is_empty() {
|
||||
let mut stdout = BufReader::new(
|
||||
child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| io::Error::other("missing ripgrep stdout"))?,
|
||||
);
|
||||
let mut stderr = child
|
||||
.stderr
|
||||
.take()
|
||||
.ok_or_else(|| io::Error::other("missing ripgrep stderr"))?;
|
||||
let mut sink = tokio::io::sink();
|
||||
let read_paths = async {
|
||||
let mut paths = HashSet::new();
|
||||
let mut record = Vec::new();
|
||||
let mut count = [0; 2];
|
||||
while stdout.read_until(0, &mut record).await? != 0 {
|
||||
if record.pop() != Some(0) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unterminated ripgrep path",
|
||||
));
|
||||
}
|
||||
stdout.read_exact(&mut count).await?;
|
||||
if count != *b"1\n" {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"invalid ripgrep count",
|
||||
));
|
||||
}
|
||||
let path = path_from_rg_bytes(&record);
|
||||
let path = if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if paths.insert(path.clone()) {
|
||||
report_candidate(&mut progress, RolloutSearchCandidate { path });
|
||||
}
|
||||
record.clear();
|
||||
}
|
||||
Ok::<_, io::Error>(paths)
|
||||
};
|
||||
let (paths, stderr_bytes, status) = tokio::select! {
|
||||
biased;
|
||||
() = &mut *cancelled => {
|
||||
// Cancellation completes only after ripgrep is reaped.
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
return Err(search_cancelled());
|
||||
}
|
||||
result = async {
|
||||
tokio::try_join!(read_paths, tokio::io::copy(&mut stderr, &mut sink), child.wait())
|
||||
} => result?,
|
||||
};
|
||||
if !status.success() {
|
||||
if status.code() == Some(1) && stderr_bytes == 0 {
|
||||
return Ok(Some(HashSet::new()));
|
||||
}
|
||||
|
||||
return Err(io::Error::other(format!(
|
||||
"ripgrep rollout search failed under {}",
|
||||
root.display()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut matches = HashSet::new();
|
||||
for line in String::from_utf8_lossy(output.stdout.as_slice()).lines() {
|
||||
let path = PathBuf::from(line);
|
||||
let path = if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
matches.insert(path);
|
||||
}
|
||||
|
||||
Ok(Some(matches))
|
||||
Ok(Some(paths))
|
||||
}
|
||||
|
||||
async fn scan_rollout_matches(
|
||||
#[cfg(unix)]
|
||||
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
Path::new(std::ffi::OsStr::from_bytes(bytes)).to_path_buf()
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
|
||||
String::from_utf8_lossy(bytes).into_owned().into()
|
||||
}
|
||||
|
||||
fn report_candidate(
|
||||
progress: &mut Option<mpsc::Sender<RolloutSearchCandidate>>,
|
||||
candidate: RolloutSearchCandidate,
|
||||
) {
|
||||
if let Some(sender) = progress
|
||||
&& matches!(
|
||||
sender.try_send(candidate),
|
||||
Err(mpsc::error::TrySendError::Closed(_))
|
||||
)
|
||||
{
|
||||
*progress = None;
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_plain_rollout_matches(
|
||||
root: &Path,
|
||||
json_search_term: &str,
|
||||
search_term: &str,
|
||||
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
|
||||
) -> io::Result<RolloutSearchMatches> {
|
||||
let mut matches = HashMap::new();
|
||||
let mut matches = HashSet::new();
|
||||
let mut dirs = vec![root.to_path_buf()];
|
||||
let json_search_term = case_insensitive_literal_regex(json_search_term)?;
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
|
||||
while let Some(dir) = dirs.pop() {
|
||||
let mut entries = match tokio::fs::read_dir(dir).await {
|
||||
@@ -148,27 +279,12 @@ async fn scan_rollout_matches(
|
||||
continue;
|
||||
};
|
||||
if rollout_file.is_compressed() {
|
||||
if let Some(snippet) = matcher
|
||||
.first_match(rollout_file.path())
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %rollout_file.path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
})
|
||||
{
|
||||
matches.insert(
|
||||
compression::plain_rollout_path(rollout_file.path()),
|
||||
Some(snippet),
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if rollout_contains(rollout_file.path(), &json_search_term).await? {
|
||||
matches.insert(rollout_file.into_path(), None);
|
||||
let path = rollout_file.into_path();
|
||||
report_candidate(&mut progress, RolloutSearchCandidate { path: path.clone() });
|
||||
matches.insert(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -186,108 +302,6 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result<bool>
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
|
||||
pub struct RolloutSnippetMatcher<'a> {
|
||||
search_term: &'a str,
|
||||
patterns: Option<Arc<(Regex, Regex)>>,
|
||||
}
|
||||
|
||||
impl<'a> RolloutSnippetMatcher<'a> {
|
||||
pub fn new(search_term: &'a str) -> Self {
|
||||
Self {
|
||||
search_term,
|
||||
patterns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first matching conversation excerpt. The file is opened before compiling
|
||||
/// the patterns so per-file I/O errors take precedence even on the first call.
|
||||
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
|
||||
let lines = compression::open_rollout_line_reader(path).await?;
|
||||
let patterns = match &self.patterns {
|
||||
Some(patterns) => patterns,
|
||||
None => self.patterns.insert(Arc::new((
|
||||
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
|
||||
case_insensitive_literal_regex(self.search_term)?,
|
||||
))),
|
||||
};
|
||||
// Share the regexes across file scans to reuse their scratch caches.
|
||||
let patterns = Arc::clone(patterns);
|
||||
lines
|
||||
.find_map(move |line| {
|
||||
let (json_search_term, search_term) = patterns.as_ref();
|
||||
if json_search_term.is_match(line) {
|
||||
content_match_snippet(line, search_term)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_compressed_rollout_matches(
|
||||
root: &Path,
|
||||
search_term: &str,
|
||||
) -> io::Result<RolloutSearchMatches> {
|
||||
let mut matches = HashMap::new();
|
||||
let mut paths = Vec::new();
|
||||
let mut dirs = vec![root.to_path_buf()];
|
||||
|
||||
while let Some(dir) = dirs.pop() {
|
||||
let mut entries = match tokio::fs::read_dir(dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type().await?;
|
||||
if file_type.is_dir() {
|
||||
dirs.push(path);
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
|
||||
continue;
|
||||
};
|
||||
if !rollout_file.is_compressed() {
|
||||
continue;
|
||||
}
|
||||
paths.push(rollout_file.into_path());
|
||||
}
|
||||
}
|
||||
|
||||
const COMPRESSED_SEARCH_WORKERS: usize = 4;
|
||||
let scan = async |paths: &[PathBuf]| {
|
||||
let mut matches = HashMap::new();
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
for path in paths {
|
||||
if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| {
|
||||
tracing::warn!(
|
||||
path = %path.as_path().display(),
|
||||
%err,
|
||||
"Failed to search compressed rollout"
|
||||
);
|
||||
None
|
||||
}) {
|
||||
matches.insert(
|
||||
compression::plain_rollout_path(path.as_path()),
|
||||
Some(snippet),
|
||||
);
|
||||
}
|
||||
}
|
||||
matches
|
||||
};
|
||||
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
|
||||
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
|
||||
matches.extend(batch);
|
||||
}
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
fn json_escaped_search_term(search_term: &str) -> io::Result<String> {
|
||||
let serialized = serde_json::to_string(search_term).map_err(io::Error::other)?;
|
||||
Ok(serialized[1..serialized.len() - 1].to_string())
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
//! Compressed rollout discovery and best-effort candidate delivery.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use futures::future::join_all;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::RolloutSearchCandidate;
|
||||
use super::RolloutSearchMatches;
|
||||
use super::RolloutSnippetMatcher;
|
||||
use super::compression;
|
||||
use super::report_candidate;
|
||||
|
||||
pub(super) async fn scan_compressed_rollout_matches(
|
||||
root: &Path,
|
||||
search_term: &str,
|
||||
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
|
||||
) -> io::Result<RolloutSearchMatches> {
|
||||
let mut matches = HashSet::new();
|
||||
let mut paths = Vec::new();
|
||||
let mut dirs = vec![root.to_path_buf()];
|
||||
|
||||
while let Some(dir) = dirs.pop() {
|
||||
let mut entries = match tokio::fs::read_dir(dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type().await?;
|
||||
if file_type.is_dir() {
|
||||
dirs.push(path);
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
|
||||
continue;
|
||||
};
|
||||
if !rollout_file.is_compressed() {
|
||||
continue;
|
||||
}
|
||||
paths.push(rollout_file.into_path());
|
||||
}
|
||||
}
|
||||
|
||||
if paths.is_empty() {
|
||||
return Ok(matches);
|
||||
}
|
||||
const COMPRESSED_SEARCH_WORKERS: usize = 4;
|
||||
let scan = async |paths: &[PathBuf]| {
|
||||
let mut matches = HashSet::new();
|
||||
let mut progress = progress.clone();
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
for path in paths {
|
||||
match matcher.first_match(path).await {
|
||||
Ok(Some(_)) => {
|
||||
let logical_path = compression::plain_rollout_path(path);
|
||||
report_candidate(
|
||||
&mut progress,
|
||||
RolloutSearchCandidate {
|
||||
path: logical_path.clone(),
|
||||
},
|
||||
);
|
||||
matches.insert(logical_path);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => {
|
||||
tracing::warn!(path = %path.display(), %err, "Failed to search compressed rollout")
|
||||
}
|
||||
}
|
||||
}
|
||||
matches
|
||||
};
|
||||
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
|
||||
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
|
||||
matches.extend(batch);
|
||||
}
|
||||
Ok(matches)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Lazy, request-scoped matching of visible conversation snippets in rollout files.
|
||||
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use super::case_insensitive_literal_regex;
|
||||
use super::compression;
|
||||
use super::content_match_snippet;
|
||||
use super::json_escaped_search_term;
|
||||
|
||||
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
|
||||
pub struct RolloutSnippetMatcher<'a> {
|
||||
search_term: &'a str,
|
||||
patterns: Option<Arc<(Regex, Regex)>>,
|
||||
}
|
||||
|
||||
impl<'a> RolloutSnippetMatcher<'a> {
|
||||
pub fn new(search_term: &'a str) -> Self {
|
||||
Self {
|
||||
search_term,
|
||||
patterns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compiles the expressions now when a caller needs to validate a query before discovery.
|
||||
pub fn compile(&mut self) -> io::Result<()> {
|
||||
self.compiled_patterns().map(|_| ())
|
||||
}
|
||||
|
||||
fn compiled_patterns(&mut self) -> io::Result<&Arc<(Regex, Regex)>> {
|
||||
Ok(match &mut self.patterns {
|
||||
Some(patterns) => patterns,
|
||||
slot @ None => slot.insert(Arc::new((
|
||||
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
|
||||
case_insensitive_literal_regex(self.search_term)?,
|
||||
))),
|
||||
})
|
||||
}
|
||||
|
||||
/// Opens the file before compiling so per-file I/O errors retain their priority.
|
||||
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
|
||||
let lines = compression::open_rollout_line_reader(path).await?;
|
||||
// Share the regexes across file scans to reuse their scratch caches.
|
||||
let patterns = Arc::clone(self.compiled_patterns()?);
|
||||
lines
|
||||
.find_map(move |line| {
|
||||
let (json_search_term, search_term) = patterns.as_ref();
|
||||
if json_search_term.is_match(line) {
|
||||
content_match_snippet(line, search_term)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
//! Regression coverage for rollout snippet matching and per-file error handling.
|
||||
//! Regression coverage for rollout snippet matching, progress, cancellation, and errors.
|
||||
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -48,19 +50,20 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
|
||||
let mut expected = RolloutSearchMatches::new();
|
||||
for index in 0..7 {
|
||||
let path = root.path().join(format!("rollout-{index:04}.jsonl"));
|
||||
std::fs::write(path.with_extension("jsonl.zst"), &compressed)?;
|
||||
expected.insert(path, Some("needle".to_string()));
|
||||
let physical_path = path.with_extension("jsonl.zst");
|
||||
std::fs::write(&physical_path, &compressed)?;
|
||||
expected.insert(path);
|
||||
}
|
||||
std::fs::write(root.path().join("rollout-broken.jsonl.zst"), b"not zstd")?;
|
||||
assert_eq!(
|
||||
scan_compressed_rollout_matches(root.path(), "needle").await?,
|
||||
scan_compressed_rollout_matches(root.path(), "needle", /*progress*/ None).await?,
|
||||
expected
|
||||
);
|
||||
let query = "a".repeat(500_000);
|
||||
|
||||
assert!(case_insensitive_literal_regex(&query).is_err());
|
||||
assert_eq!(
|
||||
scan_compressed_rollout_matches(root.path(), &query).await?,
|
||||
scan_compressed_rollout_matches(root.path(), &query, /*progress*/ None).await?,
|
||||
RolloutSearchMatches::new()
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -73,3 +76,230 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fallback_reports_plain_and_compressed_candidates() -> io::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let root = home.path().join(SESSIONS_SUBDIR);
|
||||
std::fs::create_dir(&root)?;
|
||||
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
|
||||
let plain = root.join("rollout-plain.jsonl");
|
||||
let compressed = root.join("rollout-compressed.jsonl");
|
||||
std::fs::write(&plain, line)?;
|
||||
std::fs::write(
|
||||
compressed.with_extension("jsonl.zst"),
|
||||
zstd::stream::encode_all(&line[..], 3)?,
|
||||
)?;
|
||||
let (sender, mut receiver) = mpsc::channel(4);
|
||||
let matches = search_rollout_matches_with_progress(
|
||||
&home.path().join("missing-rg"),
|
||||
home.path(),
|
||||
/*archived*/ false,
|
||||
"needle",
|
||||
Some(sender),
|
||||
Box::pin(pending()),
|
||||
)
|
||||
.await?;
|
||||
let mut progress = HashSet::new();
|
||||
while let Some(candidate) = receiver.recv().await {
|
||||
progress.insert(candidate.path);
|
||||
}
|
||||
assert_eq!(
|
||||
(matches, progress),
|
||||
(
|
||||
HashSet::from([plain.clone(), compressed.clone()]),
|
||||
HashSet::from([plain, compressed])
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod streaming {
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::time::Duration;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::timeout;
|
||||
|
||||
fn gated_rg(home: &Path) -> io::Result<PathBuf> {
|
||||
let root = home.join(SESSIONS_SUBDIR);
|
||||
std::fs::create_dir(&root)?;
|
||||
let rg = home.join("gated-rg");
|
||||
codex_utils_cargo_bin::write_executable(
|
||||
&rg,
|
||||
r#"#!/bin/sh
|
||||
for root do :; done
|
||||
printf '%s\n' "$$" > "$root/rg.pid"
|
||||
printf '%s\0001\n' "$root/first
|
||||
file.jsonl"
|
||||
while ! test -f "$root/release"; do sleep 0.02; done
|
||||
printf '%s\0001\n' "$root/second.jsonl"
|
||||
"#,
|
||||
)?;
|
||||
Ok(rg)
|
||||
}
|
||||
|
||||
/// A path discovered by ripgrep reaches the consumer while the child is still running.
|
||||
#[tokio::test]
|
||||
async fn reports_paths_before_ripgrep_exits() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let rg = gated_rg(home.path())?;
|
||||
let root = home.path().join(SESSIONS_SUBDIR);
|
||||
let codex_home = home.path().to_path_buf();
|
||||
let (sender, mut paths) = mpsc::channel(2);
|
||||
let search = tokio::spawn(async move {
|
||||
search_rollout_matches_with_progress(
|
||||
&rg,
|
||||
&codex_home,
|
||||
/*archived*/ false,
|
||||
"needle",
|
||||
Some(sender),
|
||||
Box::pin(pending()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(5), paths.recv()).await?,
|
||||
Some(RolloutSearchCandidate {
|
||||
path: root.join("first\nfile.jsonl"),
|
||||
})
|
||||
);
|
||||
assert!(!search.is_finished());
|
||||
tokio::fs::write(root.join("release"), []).await?;
|
||||
let result = timeout(Duration::from_secs(5), search).await???;
|
||||
assert_eq!(
|
||||
result,
|
||||
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")])
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full progress queues never prevent ripgrep or the final search from completing.
|
||||
#[tokio::test]
|
||||
async fn full_progress_queue_does_not_stall_search() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let rg = gated_rg(home.path())?;
|
||||
let root = home.path().join(SESSIONS_SUBDIR);
|
||||
tokio::fs::write(root.join("release"), []).await?;
|
||||
let (sender, mut paths) = mpsc::channel(1);
|
||||
let matches = timeout(
|
||||
Duration::from_secs(5),
|
||||
search_rollout_matches_with_progress(
|
||||
&rg,
|
||||
home.path(),
|
||||
/*archived*/ false,
|
||||
"needle",
|
||||
Some(sender),
|
||||
Box::pin(pending()),
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(
|
||||
(matches, paths.recv().await, paths.recv().await,),
|
||||
(
|
||||
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")]),
|
||||
Some(RolloutSearchCandidate {
|
||||
path: root.join("first\nfile.jsonl"),
|
||||
}),
|
||||
None,
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cancelling a held search reaps ripgrep and closes its progress stream.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn cancellation_reaps_ripgrep() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let rg = gated_rg(home.path())?;
|
||||
let root = home.path().join(SESSIONS_SUBDIR);
|
||||
let codex_home = home.path().to_path_buf();
|
||||
let (sender, mut paths) = mpsc::channel(2);
|
||||
let (cancel, cancelled) = oneshot::channel::<()>();
|
||||
let search = tokio::spawn(async move {
|
||||
search_rollout_matches_with_progress(
|
||||
&rg,
|
||||
&codex_home,
|
||||
/*archived*/ false,
|
||||
"needle",
|
||||
Some(sender),
|
||||
Box::pin(async {
|
||||
let _ = cancelled.await;
|
||||
}),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
let candidate = timeout(Duration::from_secs(5), paths.recv()).await?;
|
||||
if candidate.is_none() {
|
||||
let outcome = timeout(Duration::from_secs(5), search).await?;
|
||||
anyhow::bail!("search ended before reporting a candidate: {outcome:?}");
|
||||
}
|
||||
assert_eq!(
|
||||
candidate,
|
||||
Some(RolloutSearchCandidate {
|
||||
path: root.join("first\nfile.jsonl"),
|
||||
})
|
||||
);
|
||||
let pid = tokio::fs::read_to_string(root.join("rg.pid")).await?;
|
||||
let child = Path::new("/proc").join(pid.trim());
|
||||
assert!(child.exists());
|
||||
cancel.send(()).expect("search is still running");
|
||||
let error = timeout(Duration::from_secs(5), search).await??.unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
assert_eq!(paths.recv().await, None);
|
||||
assert!(!child.exists());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn search_finds_archive_published_while_ripgrep_is_running() -> io::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let root = home.path().join(SESSIONS_SUBDIR);
|
||||
std::fs::create_dir(&root)?;
|
||||
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
|
||||
let target = root.join("rollout-target.jsonl");
|
||||
std::fs::write(&target, line)?;
|
||||
let rg = home.path().join("rg");
|
||||
codex_utils_cargo_bin::write_executable(
|
||||
&rg,
|
||||
"#!/bin/sh\nfor root do :; done\nprintf '%s\\0001\\n' \"$root/ready\"\nread release < \"$root/release\"\nexit 1\n",
|
||||
)?;
|
||||
let fifo = root.join("release");
|
||||
assert!(
|
||||
std::process::Command::new("mkfifo")
|
||||
.arg(&fifo)
|
||||
.status()?
|
||||
.success()
|
||||
);
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
let search = search_rollout_matches_with_progress(
|
||||
&rg,
|
||||
home.path(),
|
||||
/*archived*/ false,
|
||||
"needle",
|
||||
Some(tx),
|
||||
Box::pin(pending()),
|
||||
);
|
||||
tokio::pin!(search);
|
||||
tokio::select! {
|
||||
result = &mut search => panic!("search completed before archive publication: {result:?}"),
|
||||
candidate = rx.recv() => assert_eq!(candidate.expect("rg ready").path, root.join("ready")),
|
||||
}
|
||||
std::fs::write(
|
||||
target.with_extension("jsonl.zst"),
|
||||
zstd::stream::encode_all(&line[..], 3)?,
|
||||
)?;
|
||||
std::fs::remove_file(&target)?;
|
||||
let release = tokio::task::spawn_blocking(move || std::fs::write(fifo, b"release\n"));
|
||||
let matches = search.await?;
|
||||
release.await.map_err(io::Error::other)??;
|
||||
assert!(matches.contains(&target));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user