Stream partial thread search results and support cancellation (#62132)

## Why

Clients using `thread/search` have to wait for the search to finish before
they can show any matches. A search can also outlive the query that triggered
it, such as when a user changes their task mention query.

## What changed

- Add experimental `thread/search/stream` with best-effort
  `thread/search/partial` notifications. Each notification carries the original
  `requestId` and a batch of verified matches. Clients replace provisional
  matches with the final page, which determines results, ordering, and cursors.
- Support connection-scoped cancellation through `thread/search/stream/cancel`
  and `thread/search/cancel` for their respective search methods. Disconnecting
  also stops active searches and reaps their search subprocesses.
- Cancel superseded or timed-out TUI task mention searches without delaying
  available title matches.

## Testing

Add coverage for incremental results and final-page parity, slow clients,
connection-scoped cancellation and subprocess cleanup, and TUI task search
cancellation.

GitOrigin-RevId: b58abf7a1df424f191c8ede806d1d5266d299548
This commit is contained in:
matthewdavis-oai
2026-11-17 23:46:48 +00:00
committed by copyberry
parent c90234de95
commit 3bf4e138fe
53 changed files with 5965 additions and 420 deletions
+1
View File
@@ -57,4 +57,5 @@ libc = { workspace = true }
[dev-dependencies]
codex-utils-absolute-path = { workspace = true }
codex-utils-cargo-bin = { workspace = true }
pretty_assertions = { workspace = true }
+1 -4
View File
@@ -302,10 +302,7 @@ async fn search_rollout_matches_uses_logical_path_for_compressed_rollout() -> an
)
.await?;
assert_eq!(
matches.get(rollout_path.as_path()),
Some(&Some("targeted search term".to_string()))
);
assert_eq!(matches, std::collections::HashSet::from([rollout_path]));
Ok(())
}
+3
View File
@@ -161,8 +161,11 @@ pub use recorder::thread_item_from_state_metadata;
pub use reverse_jsonl_scanner::ReverseJsonlScanner;
pub use reverse_jsonl_scanner::ScanOutcome;
pub use rollout_reference_index::RolloutReferenceIndex;
pub use search::RolloutSearchCandidate;
pub use search::RolloutSearchMatches;
pub use search::RolloutSnippetMatcher;
pub use search::search_rollout_matches;
pub use search::search_rollout_matches_with_progress;
pub use search::search_rollout_paths;
pub use session_index::append_thread_name;
pub use session_index::find_thread_meta_by_name_str;
+180 -166
View File
@@ -1,19 +1,23 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Arc;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::strip_user_message_prefix;
use futures::future::join_all;
use futures::FutureExt;
use futures::future::BoxFuture;
use futures::future::pending;
use regex::Regex;
use regex::RegexBuilder;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncReadExt;
use tokio::io::BufReader;
use tokio::process::Command;
use tokio::sync::mpsc;
use super::ARCHIVED_SESSIONS_SUBDIR;
use super::SESSIONS_SUBDIR;
@@ -21,11 +25,22 @@ use super::compression;
use crate::ResponseItemEnvelope;
use crate::RolloutItem;
mod compressed;
mod snippet_matcher;
use compressed::scan_compressed_rollout_matches;
pub use snippet_matcher::RolloutSnippetMatcher;
const MATCH_CONTEXT_BEFORE_CHARS: usize = 48;
const MATCH_CONTEXT_AFTER_CHARS: usize = 96;
/// Search matches keyed by the canonical `.jsonl` path for each rollout.
pub type RolloutSearchMatches = HashMap<PathBuf, Option<String>>;
/// Logical `.jsonl` paths discovered by search, regardless of physical representation.
pub type RolloutSearchMatches = HashSet<PathBuf>;
/// A potential match discovered by the rollout scanner.
#[derive(Debug, PartialEq, Eq)]
pub struct RolloutSearchCandidate {
pub path: PathBuf,
}
pub async fn search_rollout_paths(
rg_command: &Path,
@@ -33,12 +48,7 @@ pub async fn search_rollout_paths(
archived: bool,
search_term: &str,
) -> io::Result<HashSet<PathBuf>> {
Ok(
search_rollout_matches(rg_command, codex_home, archived, search_term)
.await?
.into_keys()
.collect(),
)
search_rollout_matches(rg_command, codex_home, archived, search_term).await
}
pub async fn search_rollout_matches(
@@ -46,6 +56,26 @@ pub async fn search_rollout_matches(
codex_home: &Path,
archived: bool,
search_term: &str,
) -> io::Result<RolloutSearchMatches> {
search_rollout_matches_with_progress(
rg_command,
codex_home,
archived,
search_term,
/*progress*/ None,
Box::pin(pending()),
)
.await
}
/// Search plain rollouts, then compressed rollouts, reporting candidates without blocking.
pub async fn search_rollout_matches_with_progress(
rg_command: &Path,
codex_home: &Path,
archived: bool,
search_term: &str,
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
cancelled: BoxFuture<'_, ()>,
) -> io::Result<RolloutSearchMatches> {
let root = codex_home.join(if archived {
ARCHIVED_SESSIONS_SUBDIR
@@ -53,28 +83,63 @@ pub async fn search_rollout_matches(
SESSIONS_SUBDIR
});
let json_search_term = json_escaped_search_term(search_term)?;
let Some(plain_matches) =
ripgrep_rollout_paths(rg_command, root.as_path(), json_search_term.as_str()).await?
else {
return scan_rollout_matches(root.as_path(), json_search_term.as_str(), search_term).await;
let cancelled = cancelled.shared();
let mut cancellation: BoxFuture<'_, ()> = Box::pin(cancelled.clone());
let mut matches = match ripgrep_rollout_paths(
rg_command,
&root,
&json_search_term,
progress.clone(),
&mut cancellation,
)
.await?
{
Some(paths) => paths,
None => tokio::select! {
biased;
() = cancelled.clone() => return Err(search_cancelled()),
result = scan_plain_rollout_matches(&root, &json_search_term, progress.clone()) => result?,
},
};
let mut matches: RolloutSearchMatches =
plain_matches.into_iter().map(|path| (path, None)).collect();
matches.extend(scan_compressed_rollout_matches(root.as_path(), search_term).await?);
// Compression publishes the archive before removing its plain sibling. Discover archives
// after the plain scan so a transition cannot fall between the two inventories.
let compressed = tokio::select! {
biased;
() = cancelled => return Err(search_cancelled()),
result = scan_compressed_rollout_matches(&root, search_term, progress) => result?,
};
matches.extend(compressed);
Ok(matches)
}
fn search_cancelled() -> io::Error {
io::Error::new(io::ErrorKind::Interrupted, "rollout search cancelled")
}
async fn ripgrep_rollout_paths(
rg_command: &Path,
root: &Path,
search_term: &str,
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
cancelled: &mut BoxFuture<'_, ()>,
) -> io::Result<Option<HashSet<PathBuf>>> {
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
let exists = tokio::select! {
biased;
() = &mut *cancelled => return Err(search_cancelled()),
exists = tokio::fs::try_exists(root) => exists.unwrap_or(false),
};
if !exists {
return Ok(Some(HashSet::new()));
}
let output = match Command::from(codex_utils_process::background_command(rg_command))
.arg("-l")
let mut command = Command::from(codex_utils_process::background_command(rg_command));
// A NUL-only file listing does not flush through ripgrep's line buffer. A one-match
// count adds a newline after each NUL-delimited path and still stops at the first hit.
command.args(["--count", "--max-count", "1", "--null"]);
if progress.is_some() {
command.arg("--line-buffered");
}
let mut child = match command
.arg("--fixed-strings")
.arg("--ignore-case")
.arg("--no-ignore")
@@ -84,49 +149,115 @@ async fn ripgrep_rollout_paths(
.arg(search_term)
.arg(root)
.stdin(Stdio::null())
.output()
.await
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
{
Ok(output) => output,
Err(err) if err.kind() == io::ErrorKind::NotFound => {
return Ok(None);
}
Ok(child) => child,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err),
};
if !output.status.success() {
if output.status.code() == Some(1) && output.stderr.is_empty() {
let mut stdout = BufReader::new(
child
.stdout
.take()
.ok_or_else(|| io::Error::other("missing ripgrep stdout"))?,
);
let mut stderr = child
.stderr
.take()
.ok_or_else(|| io::Error::other("missing ripgrep stderr"))?;
let mut sink = tokio::io::sink();
let read_paths = async {
let mut paths = HashSet::new();
let mut record = Vec::new();
let mut count = [0; 2];
while stdout.read_until(0, &mut record).await? != 0 {
if record.pop() != Some(0) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unterminated ripgrep path",
));
}
stdout.read_exact(&mut count).await?;
if count != *b"1\n" {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid ripgrep count",
));
}
let path = path_from_rg_bytes(&record);
let path = if path.is_absolute() {
path
} else {
root.join(path)
};
if paths.insert(path.clone()) {
report_candidate(&mut progress, RolloutSearchCandidate { path });
}
record.clear();
}
Ok::<_, io::Error>(paths)
};
let (paths, stderr_bytes, status) = tokio::select! {
biased;
() = &mut *cancelled => {
// Cancellation completes only after ripgrep is reaped.
let _ = child.start_kill();
let _ = child.wait().await;
return Err(search_cancelled());
}
result = async {
tokio::try_join!(read_paths, tokio::io::copy(&mut stderr, &mut sink), child.wait())
} => result?,
};
if !status.success() {
if status.code() == Some(1) && stderr_bytes == 0 {
return Ok(Some(HashSet::new()));
}
return Err(io::Error::other(format!(
"ripgrep rollout search failed under {}",
root.display()
)));
}
let mut matches = HashSet::new();
for line in String::from_utf8_lossy(output.stdout.as_slice()).lines() {
let path = PathBuf::from(line);
let path = if path.is_absolute() {
path
} else {
root.join(path)
};
matches.insert(path);
}
Ok(Some(matches))
Ok(Some(paths))
}
async fn scan_rollout_matches(
#[cfg(unix)]
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
use std::os::unix::ffi::OsStrExt;
Path::new(std::ffi::OsStr::from_bytes(bytes)).to_path_buf()
}
#[cfg(not(unix))]
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
String::from_utf8_lossy(bytes).into_owned().into()
}
fn report_candidate(
progress: &mut Option<mpsc::Sender<RolloutSearchCandidate>>,
candidate: RolloutSearchCandidate,
) {
if let Some(sender) = progress
&& matches!(
sender.try_send(candidate),
Err(mpsc::error::TrySendError::Closed(_))
)
{
*progress = None;
}
}
async fn scan_plain_rollout_matches(
root: &Path,
json_search_term: &str,
search_term: &str,
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashMap::new();
let mut matches = HashSet::new();
let mut dirs = vec![root.to_path_buf()];
let json_search_term = case_insensitive_literal_regex(json_search_term)?;
let mut matcher = RolloutSnippetMatcher::new(search_term);
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
@@ -148,27 +279,12 @@ async fn scan_rollout_matches(
continue;
};
if rollout_file.is_compressed() {
if let Some(snippet) = matcher
.first_match(rollout_file.path())
.await
.unwrap_or_else(|err| {
tracing::warn!(
path = %rollout_file.path().display(),
%err,
"Failed to search compressed rollout"
);
None
})
{
matches.insert(
compression::plain_rollout_path(rollout_file.path()),
Some(snippet),
);
}
continue;
}
if rollout_contains(rollout_file.path(), &json_search_term).await? {
matches.insert(rollout_file.into_path(), None);
let path = rollout_file.into_path();
report_candidate(&mut progress, RolloutSearchCandidate { path: path.clone() });
matches.insert(path);
}
}
}
@@ -186,108 +302,6 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result<bool>
Ok(false)
}
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
pub struct RolloutSnippetMatcher<'a> {
search_term: &'a str,
patterns: Option<Arc<(Regex, Regex)>>,
}
impl<'a> RolloutSnippetMatcher<'a> {
pub fn new(search_term: &'a str) -> Self {
Self {
search_term,
patterns: None,
}
}
/// Returns the first matching conversation excerpt. The file is opened before compiling
/// the patterns so per-file I/O errors take precedence even on the first call.
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
let patterns = match &self.patterns {
Some(patterns) => patterns,
None => self.patterns.insert(Arc::new((
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
case_insensitive_literal_regex(self.search_term)?,
))),
};
// Share the regexes across file scans to reuse their scratch caches.
let patterns = Arc::clone(patterns);
lines
.find_map(move |line| {
let (json_search_term, search_term) = patterns.as_ref();
if json_search_term.is_match(line) {
content_match_snippet(line, search_term)
} else {
None
}
})
.await
}
}
async fn scan_compressed_rollout_matches(
root: &Path,
search_term: &str,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashMap::new();
let mut paths = Vec::new();
let mut dirs = vec![root.to_path_buf()];
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let file_type = entry.file_type().await?;
if file_type.is_dir() {
dirs.push(path);
continue;
}
if !file_type.is_file() {
continue;
}
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
continue;
};
if !rollout_file.is_compressed() {
continue;
}
paths.push(rollout_file.into_path());
}
}
const COMPRESSED_SEARCH_WORKERS: usize = 4;
let scan = async |paths: &[PathBuf]| {
let mut matches = HashMap::new();
let mut matcher = RolloutSnippetMatcher::new(search_term);
for path in paths {
if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| {
tracing::warn!(
path = %path.as_path().display(),
%err,
"Failed to search compressed rollout"
);
None
}) {
matches.insert(
compression::plain_rollout_path(path.as_path()),
Some(snippet),
);
}
}
matches
};
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
matches.extend(batch);
}
Ok(matches)
}
fn json_escaped_search_term(search_term: &str) -> io::Result<String> {
let serialized = serde_json::to_string(search_term).map_err(io::Error::other)?;
Ok(serialized[1..serialized.len() - 1].to_string())
+85
View File
@@ -0,0 +1,85 @@
//! Compressed rollout discovery and best-effort candidate delivery.
use std::collections::HashSet;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use futures::future::join_all;
use tokio::sync::mpsc;
use super::RolloutSearchCandidate;
use super::RolloutSearchMatches;
use super::RolloutSnippetMatcher;
use super::compression;
use super::report_candidate;
pub(super) async fn scan_compressed_rollout_matches(
root: &Path,
search_term: &str,
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashSet::new();
let mut paths = Vec::new();
let mut dirs = vec![root.to_path_buf()];
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let file_type = entry.file_type().await?;
if file_type.is_dir() {
dirs.push(path);
continue;
}
if !file_type.is_file() {
continue;
}
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
continue;
};
if !rollout_file.is_compressed() {
continue;
}
paths.push(rollout_file.into_path());
}
}
if paths.is_empty() {
return Ok(matches);
}
const COMPRESSED_SEARCH_WORKERS: usize = 4;
let scan = async |paths: &[PathBuf]| {
let mut matches = HashSet::new();
let mut progress = progress.clone();
let mut matcher = RolloutSnippetMatcher::new(search_term);
for path in paths {
match matcher.first_match(path).await {
Ok(Some(_)) => {
let logical_path = compression::plain_rollout_path(path);
report_candidate(
&mut progress,
RolloutSearchCandidate {
path: logical_path.clone(),
},
);
matches.insert(logical_path);
}
Ok(None) => {}
Err(err) => {
tracing::warn!(path = %path.display(), %err, "Failed to search compressed rollout")
}
}
}
matches
};
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
matches.extend(batch);
}
Ok(matches)
}
@@ -0,0 +1,59 @@
//! Lazy, request-scoped matching of visible conversation snippets in rollout files.
use std::io;
use std::path::Path;
use std::sync::Arc;
use regex::Regex;
use super::case_insensitive_literal_regex;
use super::compression;
use super::content_match_snippet;
use super::json_escaped_search_term;
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
pub struct RolloutSnippetMatcher<'a> {
search_term: &'a str,
patterns: Option<Arc<(Regex, Regex)>>,
}
impl<'a> RolloutSnippetMatcher<'a> {
pub fn new(search_term: &'a str) -> Self {
Self {
search_term,
patterns: None,
}
}
/// Compiles the expressions now when a caller needs to validate a query before discovery.
pub fn compile(&mut self) -> io::Result<()> {
self.compiled_patterns().map(|_| ())
}
fn compiled_patterns(&mut self) -> io::Result<&Arc<(Regex, Regex)>> {
Ok(match &mut self.patterns {
Some(patterns) => patterns,
slot @ None => slot.insert(Arc::new((
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
case_insensitive_literal_regex(self.search_term)?,
))),
})
}
/// Opens the file before compiling so per-file I/O errors retain their priority.
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
// Share the regexes across file scans to reuse their scratch caches.
let patterns = Arc::clone(self.compiled_patterns()?);
lines
.find_map(move |line| {
let (json_search_term, search_term) = patterns.as_ref();
if json_search_term.is_match(line) {
content_match_snippet(line, search_term)
} else {
None
}
})
.await
}
}
+235 -5
View File
@@ -1,4 +1,6 @@
//! Regression coverage for rollout snippet matching and per-file error handling.
//! Regression coverage for rollout snippet matching, progress, cancellation, and errors.
#![cfg(test)]
use super::*;
use pretty_assertions::assert_eq;
@@ -48,19 +50,20 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
let mut expected = RolloutSearchMatches::new();
for index in 0..7 {
let path = root.path().join(format!("rollout-{index:04}.jsonl"));
std::fs::write(path.with_extension("jsonl.zst"), &compressed)?;
expected.insert(path, Some("needle".to_string()));
let physical_path = path.with_extension("jsonl.zst");
std::fs::write(&physical_path, &compressed)?;
expected.insert(path);
}
std::fs::write(root.path().join("rollout-broken.jsonl.zst"), b"not zstd")?;
assert_eq!(
scan_compressed_rollout_matches(root.path(), "needle").await?,
scan_compressed_rollout_matches(root.path(), "needle", /*progress*/ None).await?,
expected
);
let query = "a".repeat(500_000);
assert!(case_insensitive_literal_regex(&query).is_err());
assert_eq!(
scan_compressed_rollout_matches(root.path(), &query).await?,
scan_compressed_rollout_matches(root.path(), &query, /*progress*/ None).await?,
RolloutSearchMatches::new()
);
assert_eq!(
@@ -73,3 +76,230 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
);
Ok(())
}
#[tokio::test]
async fn fallback_reports_plain_and_compressed_candidates() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let plain = root.join("rollout-plain.jsonl");
let compressed = root.join("rollout-compressed.jsonl");
std::fs::write(&plain, line)?;
std::fs::write(
compressed.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
let (sender, mut receiver) = mpsc::channel(4);
let matches = search_rollout_matches_with_progress(
&home.path().join("missing-rg"),
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await?;
let mut progress = HashSet::new();
while let Some(candidate) = receiver.recv().await {
progress.insert(candidate.path);
}
assert_eq!(
(matches, progress),
(
HashSet::from([plain.clone(), compressed.clone()]),
HashSet::from([plain, compressed])
)
);
Ok(())
}
#[cfg(unix)]
mod streaming {
use super::*;
use pretty_assertions::assert_eq;
use std::time::Duration;
#[cfg(target_os = "linux")]
use tokio::sync::oneshot;
use tokio::time::timeout;
fn gated_rg(home: &Path) -> io::Result<PathBuf> {
let root = home.join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let rg = home.join("gated-rg");
codex_utils_cargo_bin::write_executable(
&rg,
r#"#!/bin/sh
for root do :; done
printf '%s\n' "$$" > "$root/rg.pid"
printf '%s\0001\n' "$root/first
file.jsonl"
while ! test -f "$root/release"; do sleep 0.02; done
printf '%s\0001\n' "$root/second.jsonl"
"#,
)?;
Ok(rg)
}
/// A path discovered by ripgrep reaches the consumer while the child is still running.
#[tokio::test]
async fn reports_paths_before_ripgrep_exits() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await
});
assert_eq!(
timeout(Duration::from_secs(5), paths.recv()).await?,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
assert!(!search.is_finished());
tokio::fs::write(root.join("release"), []).await?;
let result = timeout(Duration::from_secs(5), search).await???;
assert_eq!(
result,
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")])
);
Ok(())
}
/// Full progress queues never prevent ripgrep or the final search from completing.
#[tokio::test]
async fn full_progress_queue_does_not_stall_search() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
tokio::fs::write(root.join("release"), []).await?;
let (sender, mut paths) = mpsc::channel(1);
let matches = timeout(
Duration::from_secs(5),
search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
),
)
.await??;
assert_eq!(
(matches, paths.recv().await, paths.recv().await,),
(
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")]),
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
}),
None,
)
);
Ok(())
}
/// Cancelling a held search reaps ripgrep and closes its progress stream.
#[cfg(target_os = "linux")]
#[tokio::test]
async fn cancellation_reaps_ripgrep() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let (cancel, cancelled) = oneshot::channel::<()>();
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(async {
let _ = cancelled.await;
}),
)
.await
});
let candidate = timeout(Duration::from_secs(5), paths.recv()).await?;
if candidate.is_none() {
let outcome = timeout(Duration::from_secs(5), search).await?;
anyhow::bail!("search ended before reporting a candidate: {outcome:?}");
}
assert_eq!(
candidate,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
let pid = tokio::fs::read_to_string(root.join("rg.pid")).await?;
let child = Path::new("/proc").join(pid.trim());
assert!(child.exists());
cancel.send(()).expect("search is still running");
let error = timeout(Duration::from_secs(5), search).await??.unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
assert_eq!(paths.recv().await, None);
assert!(!child.exists());
Ok(())
}
}
#[cfg(unix)]
#[tokio::test]
async fn search_finds_archive_published_while_ripgrep_is_running() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let target = root.join("rollout-target.jsonl");
std::fs::write(&target, line)?;
let rg = home.path().join("rg");
codex_utils_cargo_bin::write_executable(
&rg,
"#!/bin/sh\nfor root do :; done\nprintf '%s\\0001\\n' \"$root/ready\"\nread release < \"$root/release\"\nexit 1\n",
)?;
let fifo = root.join("release");
assert!(
std::process::Command::new("mkfifo")
.arg(&fifo)
.status()?
.success()
);
let (tx, mut rx) = mpsc::channel(4);
let search = search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(tx),
Box::pin(pending()),
);
tokio::pin!(search);
tokio::select! {
result = &mut search => panic!("search completed before archive publication: {result:?}"),
candidate = rx.recv() => assert_eq!(candidate.expect("rg ready").path, root.join("ready")),
}
std::fs::write(
target.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
std::fs::remove_file(&target)?;
let release = tokio::task::spawn_blocking(move || std::fs::write(fifo, b"release\n"));
let matches = search.await?;
release.await.map_err(io::Error::other)??;
assert!(matches.contains(&target));
Ok(())
}