Files
codex/codex-rs/rollout/src/search_tests.rs
T
matthewdavis-oai 3bf4e138fe Stream partial thread search results and support cancellation (#62132)
## Why

Clients using `thread/search` have to wait for the search to finish before
they can show any matches. A search can also outlive the query that triggered
it, such as when a user changes their task mention query.

## What changed

- Add experimental `thread/search/stream` with best-effort
  `thread/search/partial` notifications. Each notification carries the original
  `requestId` and a batch of verified matches. Clients replace provisional
  matches with the final page, which determines results, ordering, and cursors.
- Support connection-scoped cancellation through `thread/search/stream/cancel`
  and `thread/search/cancel` for their respective search methods. Disconnecting
  also stops active searches and reaps their search subprocesses.
- Cancel superseded or timed-out TUI task mention searches without delaying
  available title matches.

## Testing

Add coverage for incremental results and final-page parity, slow clients,
connection-scoped cancellation and subprocess cleanup, and TUI task search
cancellation.

GitOrigin-RevId: b58abf7a1df424f191c8ede806d1d5266d299548
2026-11-17 23:46:48 +00:00

306 lines
10 KiB
Rust

//! Regression coverage for rollout snippet matching, progress, cancellation, and errors.
#![cfg(test)]
use super::*;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
thread_local! {
pub(super) static REGEX_COMPILATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[tokio::test]
async fn plain_snippets_compile_once_and_match_decoded_text_across_files() -> io::Result<()> {
let root = TempDir::new()?;
REGEX_COMPILATIONS.set(/*value*/ 0);
let mut matcher = RolloutSnippetMatcher::new("é\"\n[");
let mut other = RolloutSnippetMatcher::new("unrelated");
assert_eq!(REGEX_COMPILATIONS.get(), 0);
for (index, message) in ["unrelated", "first É\"\n[hit]", "next é\"\n[hit]"]
.into_iter()
.enumerate()
{
let path = root.path().join(format!("{index}.jsonl"));
let line = serde_json::json!({
"timestamp": "2025-01-01T00:00:00Z",
"type": "event_msg",
"payload": { "type": "agent_message", "message": message }
});
std::fs::write(&path, line.to_string())?;
let expected = (index != 0).then(|| message.replace('\n', " "));
assert_eq!(matcher.first_match(&path).await?, expected);
assert_eq!(REGEX_COMPILATIONS.get(), 2);
}
assert_eq!(
other.first_match(&root.path().join("0.jsonl")).await?,
Some("unrelated".to_string())
);
assert_eq!(REGEX_COMPILATIONS.get(), 4);
Ok(())
}
#[tokio::test]
async fn compressed_search_preserves_error_handling() -> io::Result<()> {
let root = TempDir::new()?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let compressed = zstd::stream::encode_all(&line[..], 3)?;
let mut expected = RolloutSearchMatches::new();
for index in 0..7 {
let path = root.path().join(format!("rollout-{index:04}.jsonl"));
let physical_path = path.with_extension("jsonl.zst");
std::fs::write(&physical_path, &compressed)?;
expected.insert(path);
}
std::fs::write(root.path().join("rollout-broken.jsonl.zst"), b"not zstd")?;
assert_eq!(
scan_compressed_rollout_matches(root.path(), "needle", /*progress*/ None).await?,
expected
);
let query = "a".repeat(500_000);
assert!(case_insensitive_literal_regex(&query).is_err());
assert_eq!(
scan_compressed_rollout_matches(root.path(), &query, /*progress*/ None).await?,
RolloutSearchMatches::new()
);
assert_eq!(
RolloutSnippetMatcher::new(&query)
.first_match(&root.path().join("missing.jsonl"))
.await
.unwrap_err()
.kind(),
io::ErrorKind::NotFound
);
Ok(())
}
#[tokio::test]
async fn fallback_reports_plain_and_compressed_candidates() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let plain = root.join("rollout-plain.jsonl");
let compressed = root.join("rollout-compressed.jsonl");
std::fs::write(&plain, line)?;
std::fs::write(
compressed.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
let (sender, mut receiver) = mpsc::channel(4);
let matches = search_rollout_matches_with_progress(
&home.path().join("missing-rg"),
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await?;
let mut progress = HashSet::new();
while let Some(candidate) = receiver.recv().await {
progress.insert(candidate.path);
}
assert_eq!(
(matches, progress),
(
HashSet::from([plain.clone(), compressed.clone()]),
HashSet::from([plain, compressed])
)
);
Ok(())
}
#[cfg(unix)]
mod streaming {
use super::*;
use pretty_assertions::assert_eq;
use std::time::Duration;
#[cfg(target_os = "linux")]
use tokio::sync::oneshot;
use tokio::time::timeout;
fn gated_rg(home: &Path) -> io::Result<PathBuf> {
let root = home.join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let rg = home.join("gated-rg");
codex_utils_cargo_bin::write_executable(
&rg,
r#"#!/bin/sh
for root do :; done
printf '%s\n' "$$" > "$root/rg.pid"
printf '%s\0001\n' "$root/first
file.jsonl"
while ! test -f "$root/release"; do sleep 0.02; done
printf '%s\0001\n' "$root/second.jsonl"
"#,
)?;
Ok(rg)
}
/// A path discovered by ripgrep reaches the consumer while the child is still running.
#[tokio::test]
async fn reports_paths_before_ripgrep_exits() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await
});
assert_eq!(
timeout(Duration::from_secs(5), paths.recv()).await?,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
assert!(!search.is_finished());
tokio::fs::write(root.join("release"), []).await?;
let result = timeout(Duration::from_secs(5), search).await???;
assert_eq!(
result,
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")])
);
Ok(())
}
/// Full progress queues never prevent ripgrep or the final search from completing.
#[tokio::test]
async fn full_progress_queue_does_not_stall_search() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
tokio::fs::write(root.join("release"), []).await?;
let (sender, mut paths) = mpsc::channel(1);
let matches = timeout(
Duration::from_secs(5),
search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
),
)
.await??;
assert_eq!(
(matches, paths.recv().await, paths.recv().await,),
(
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")]),
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
}),
None,
)
);
Ok(())
}
/// Cancelling a held search reaps ripgrep and closes its progress stream.
#[cfg(target_os = "linux")]
#[tokio::test]
async fn cancellation_reaps_ripgrep() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let (cancel, cancelled) = oneshot::channel::<()>();
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(async {
let _ = cancelled.await;
}),
)
.await
});
let candidate = timeout(Duration::from_secs(5), paths.recv()).await?;
if candidate.is_none() {
let outcome = timeout(Duration::from_secs(5), search).await?;
anyhow::bail!("search ended before reporting a candidate: {outcome:?}");
}
assert_eq!(
candidate,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
let pid = tokio::fs::read_to_string(root.join("rg.pid")).await?;
let child = Path::new("/proc").join(pid.trim());
assert!(child.exists());
cancel.send(()).expect("search is still running");
let error = timeout(Duration::from_secs(5), search).await??.unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
assert_eq!(paths.recv().await, None);
assert!(!child.exists());
Ok(())
}
}
#[cfg(unix)]
#[tokio::test]
async fn search_finds_archive_published_while_ripgrep_is_running() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let target = root.join("rollout-target.jsonl");
std::fs::write(&target, line)?;
let rg = home.path().join("rg");
codex_utils_cargo_bin::write_executable(
&rg,
"#!/bin/sh\nfor root do :; done\nprintf '%s\\0001\\n' \"$root/ready\"\nread release < \"$root/release\"\nexit 1\n",
)?;
let fifo = root.join("release");
assert!(
std::process::Command::new("mkfifo")
.arg(&fifo)
.status()?
.success()
);
let (tx, mut rx) = mpsc::channel(4);
let search = search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(tx),
Box::pin(pending()),
);
tokio::pin!(search);
tokio::select! {
result = &mut search => panic!("search completed before archive publication: {result:?}"),
candidate = rx.recv() => assert_eq!(candidate.expect("rg ready").path, root.join("ready")),
}
std::fs::write(
target.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
std::fs::remove_file(&target)?;
let release = tokio::task::spawn_blocking(move || std::fs::write(fifo, b"release\n"));
let matches = search.await?;
release.await.map_err(io::Error::other)??;
assert!(matches.contains(&target));
Ok(())
}