mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Stream partial thread search results and support cancellation (#62132)
## Why Clients using `thread/search` have to wait for the search to finish before they can show any matches. A search can also outlive the query that triggered it, such as when a user changes their task mention query. ## What changed - Add experimental `thread/search/stream` with best-effort `thread/search/partial` notifications. Each notification carries the original `requestId` and a batch of verified matches. Clients replace provisional matches with the final page, which determines results, ordering, and cursors. - Support connection-scoped cancellation through `thread/search/stream/cancel` and `thread/search/cancel` for their respective search methods. Disconnecting also stops active searches and reaps their search subprocesses. - Cancel superseded or timed-out TUI task mention searches without delaying available title matches. ## Testing Add coverage for incremental results and final-page parity, slow clients, connection-scoped cancellation and subprocess cleanup, and TUI task search cancellation. GitOrigin-RevId: b58abf7a1df424f191c8ede806d1d5266d299548
This commit is contained in:
committed by
copyberry
parent
c90234de95
commit
3bf4e138fe
@@ -66,6 +66,7 @@ pub use realtime::ListRealtimeParams;
|
||||
pub use realtime::RealtimePage;
|
||||
pub use store::PendingDaybreakPreference;
|
||||
pub use store::PersistContext;
|
||||
pub use store::ThreadSearchObserver;
|
||||
pub use store::ThreadStore;
|
||||
pub use store::ThreadStoreFuture;
|
||||
pub use store::ThreadStoreResumePreparation;
|
||||
@@ -124,6 +125,7 @@ pub use types::ThreadPage;
|
||||
pub use types::ThreadPersistenceMetadata;
|
||||
pub use types::ThreadRelationFilter;
|
||||
pub use types::ThreadSearchPage;
|
||||
pub use types::ThreadSearchSourceKind;
|
||||
pub use types::ThreadSortKey;
|
||||
pub use types::TimelinePage;
|
||||
pub use types::TurnPage;
|
||||
|
||||
@@ -840,6 +840,16 @@ impl ThreadStore for LocalThreadStore {
|
||||
Box::pin(async move { search_threads::search_threads(self, params).await })
|
||||
}
|
||||
|
||||
fn search_threads_observed(
|
||||
&self,
|
||||
params: SearchThreadsParams,
|
||||
observer: crate::ThreadSearchObserver,
|
||||
) -> ThreadStoreFuture<'_, ThreadSearchPage> {
|
||||
Box::pin(search_threads::search_threads_observed(
|
||||
self, params, observer,
|
||||
))
|
||||
}
|
||||
|
||||
fn search_thread_occurrences(
|
||||
&self,
|
||||
params: SearchThreadOccurrencesParams,
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//! Optional progress from the same rollout discovery and content verification used by final search.
|
||||
|
||||
use super::LocalThreadStore;
|
||||
use super::SearchThreadsParams;
|
||||
use super::StoredThreadSearchResult;
|
||||
use super::ThreadSearchItem;
|
||||
use super::eligible_search_item;
|
||||
use super::materialize_search_results;
|
||||
use super::rollout_search_path;
|
||||
use super::verified_snippet;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_rollout::RolloutSearchCandidate;
|
||||
use codex_rollout::RolloutSnippetMatcher;
|
||||
use codex_rollout::state_db::list_threads_db;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const PROGRESS_RESULT_LIMIT: usize = 8;
|
||||
const CANDIDATE_LIMIT: usize = 32;
|
||||
const CANDIDATE_BATCH: usize = 16;
|
||||
|
||||
pub(super) struct SearchProgressContext<'a, 'term> {
|
||||
pub(super) store: &'a LocalThreadStore,
|
||||
pub(super) params: &'a SearchThreadsParams,
|
||||
pub(super) state_db: Option<&'a codex_state::StateRuntime>,
|
||||
pub(super) cursor: Option<&'a codex_rollout::Cursor>,
|
||||
pub(super) sort_key: codex_rollout::ThreadSortKey,
|
||||
pub(super) sort_direction: codex_rollout::SortDirection,
|
||||
pub(super) matcher: &'a mut RolloutSnippetMatcher<'term>,
|
||||
}
|
||||
|
||||
impl SearchProgressContext<'_, '_> {
|
||||
pub(super) async fn report_search_candidates(
|
||||
self,
|
||||
mut candidates: mpsc::Receiver<RolloutSearchCandidate>,
|
||||
progress: mpsc::Sender<Vec<StoredThreadSearchResult>>,
|
||||
) {
|
||||
let Some(db) = self.state_db else { return };
|
||||
let mut emitted = HashSet::<ThreadId>::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut scanned = 0;
|
||||
let mut batch = Vec::new();
|
||||
while scanned < CANDIDATE_LIMIT
|
||||
&& emitted.len() < self.params.page_size.min(PROGRESS_RESULT_LIMIT)
|
||||
{
|
||||
if candidates
|
||||
.recv_many(&mut batch, CANDIDATE_BATCH.min(CANDIDATE_LIMIT - scanned))
|
||||
.await
|
||||
== 0
|
||||
{
|
||||
break;
|
||||
}
|
||||
scanned += batch.len();
|
||||
let mut ids = Vec::with_capacity(batch.len());
|
||||
let hits = batch
|
||||
.drain(..)
|
||||
.filter_map(|candidate| {
|
||||
let path = rollout_search_path(&candidate.path);
|
||||
let id = codex_rollout::thread_id_from_rollout_path(&path)?;
|
||||
if !seen.insert(path.clone()) || emitted.contains(&id) {
|
||||
return None;
|
||||
}
|
||||
ids.push(id);
|
||||
Some((id, path))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if hits.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let relation = self
|
||||
.params
|
||||
.include_attached_threads
|
||||
.then_some(codex_state::ThreadRelationFilter::IncludeAttached);
|
||||
let Some(page) = list_threads_db(
|
||||
Some(db),
|
||||
db.sqlite(),
|
||||
ids.len(),
|
||||
self.cursor,
|
||||
self.sort_key,
|
||||
self.sort_direction,
|
||||
&self.params.allowed_sources,
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
Some(&ids),
|
||||
relation,
|
||||
self.params.archived,
|
||||
/*section*/ None,
|
||||
/*project_id*/ None,
|
||||
/*search_term*/ None,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let page: codex_rollout::ThreadsPage = page.into();
|
||||
let mut items = page
|
||||
.items
|
||||
.into_iter()
|
||||
.filter_map(|item| item.thread_id.map(|id| (id, item)))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut originals = None;
|
||||
for (id, path) in hits {
|
||||
let Some(item) = items.remove(&id) else {
|
||||
continue;
|
||||
};
|
||||
if rollout_search_path(&item.path) != path
|
||||
|| !eligible_search_item(self.store, self.params, &item, &ids, &mut originals)
|
||||
.await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Ok(Some(snippet)) = verified_snippet(self.matcher, &item.path).await else {
|
||||
continue;
|
||||
};
|
||||
let data = materialize_search_results(
|
||||
self.store,
|
||||
Some(db),
|
||||
self.params.archived,
|
||||
vec![ThreadSearchItem { item, snippet }],
|
||||
)
|
||||
.await;
|
||||
if data.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if progress.try_send(data).is_ok() {
|
||||
emitted.insert(id);
|
||||
}
|
||||
if emitted.len() >= self.params.page_size.min(PROGRESS_RESULT_LIMIT) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,11 @@ use codex_install_context::InstallContext;
|
||||
use codex_rollout::RolloutConfig;
|
||||
use codex_rollout::RolloutSnippetMatcher;
|
||||
use codex_rollout::parse_cursor;
|
||||
use codex_rollout::search_rollout_matches;
|
||||
use codex_rollout::search_rollout_matches_with_progress;
|
||||
use codex_utils_absolute_path::normalize_windows_device_path;
|
||||
use futures::FutureExt;
|
||||
use futures::future::pending;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::LocalThreadStore;
|
||||
use super::helpers::resolve_thread_names;
|
||||
@@ -19,6 +22,7 @@ use crate::ListThreadsParams;
|
||||
use crate::SearchThreadsParams;
|
||||
use crate::SortDirection;
|
||||
use crate::StoredThreadSearchResult;
|
||||
use crate::ThreadSearchObserver;
|
||||
use crate::ThreadSearchPage;
|
||||
use crate::ThreadSortKey;
|
||||
use crate::ThreadStoreError;
|
||||
@@ -28,6 +32,9 @@ use crate::ThreadStoreResult;
|
||||
#[path = "search_threads_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
#[path = "search_progress.rs"]
|
||||
mod progress;
|
||||
|
||||
struct ThreadSearchItem {
|
||||
item: codex_rollout::ThreadItem,
|
||||
snippet: String,
|
||||
@@ -37,6 +44,23 @@ pub(super) async fn search_threads(
|
||||
store: &LocalThreadStore,
|
||||
params: SearchThreadsParams,
|
||||
) -> ThreadStoreResult<ThreadSearchPage> {
|
||||
search_threads_observed(
|
||||
store,
|
||||
params,
|
||||
ThreadSearchObserver {
|
||||
progress: None,
|
||||
cancelled: Box::pin(pending()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn search_threads_observed(
|
||||
store: &LocalThreadStore,
|
||||
params: SearchThreadsParams,
|
||||
observer: ThreadSearchObserver,
|
||||
) -> ThreadStoreResult<ThreadSearchPage> {
|
||||
let cancelled = observer.cancelled.shared();
|
||||
let search_term = params.search_term.as_str();
|
||||
if search_term.is_empty() {
|
||||
return Err(ThreadStoreError::InvalidRequest {
|
||||
@@ -66,7 +90,11 @@ pub(super) async fn search_threads(
|
||||
SortDirection::Asc => codex_rollout::SortDirection::Asc,
|
||||
SortDirection::Desc => codex_rollout::SortDirection::Desc,
|
||||
};
|
||||
let state_db = store.state_db().await;
|
||||
let state_db = tokio::select! {
|
||||
biased;
|
||||
() = cancelled.clone() => return Err(search_cancelled()),
|
||||
db = store.state_db() => db,
|
||||
};
|
||||
let rollout_config = RolloutConfig {
|
||||
codex_home: store.config.codex_home.clone(),
|
||||
sqlite: store.config.sqlite.clone(),
|
||||
@@ -74,17 +102,99 @@ pub(super) async fn search_threads(
|
||||
model_provider_id: store.config.default_model_provider_id.clone(),
|
||||
generate_memories: false,
|
||||
};
|
||||
let mut matcher = RolloutSnippetMatcher::new(search_term);
|
||||
if observer.progress.is_some() {
|
||||
matcher
|
||||
.compile()
|
||||
.map_err(|err| ThreadStoreError::InvalidRequest {
|
||||
message: format!("invalid rollout search term: {err}"),
|
||||
})?;
|
||||
}
|
||||
let rg_command = InstallContext::current().rg_command();
|
||||
let matching_rollouts = search_rollout_matches(
|
||||
rg_command.as_path(),
|
||||
store.config.codex_home.as_path(),
|
||||
params.archived,
|
||||
search_term,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
let matching_rollouts = if let Some(progress) = observer.progress {
|
||||
let (paths, candidates) = mpsc::channel(16);
|
||||
let discovery = search_rollout_matches_with_progress(
|
||||
&rg_command,
|
||||
&store.config.codex_home,
|
||||
params.archived,
|
||||
search_term,
|
||||
Some(paths),
|
||||
Box::pin(cancelled.clone()),
|
||||
);
|
||||
tokio::pin!(discovery);
|
||||
{
|
||||
let context = progress::SearchProgressContext {
|
||||
store,
|
||||
params: ¶ms,
|
||||
state_db: state_db.as_deref(),
|
||||
cursor: cursor.as_ref(),
|
||||
sort_key,
|
||||
sort_direction,
|
||||
matcher: &mut matcher,
|
||||
};
|
||||
let reporter = context.report_search_candidates(candidates, progress);
|
||||
tokio::pin!(reporter);
|
||||
tokio::select! {
|
||||
biased;
|
||||
matches = &mut discovery => matches,
|
||||
() = &mut reporter => discovery.await,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
search_rollout_matches_with_progress(
|
||||
&rg_command,
|
||||
&store.config.codex_home,
|
||||
params.archived,
|
||||
search_term,
|
||||
/*progress*/ None,
|
||||
Box::pin(cancelled.clone()),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let matching_rollouts = matching_rollouts.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to search rollout contents: {err}"),
|
||||
})?;
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = cancelled => Err(search_cancelled()),
|
||||
result = finish_search(
|
||||
SearchContext { store, params: ¶ms, state_db, rollout_config: &rollout_config, cursor,
|
||||
sort_key, sort_direction, matcher: &mut matcher }, matching_rollouts,
|
||||
) => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn search_cancelled() -> ThreadStoreError {
|
||||
ThreadStoreError::Internal {
|
||||
message: "thread search cancelled".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
struct SearchContext<'a, 'term> {
|
||||
store: &'a LocalThreadStore,
|
||||
params: &'a SearchThreadsParams,
|
||||
state_db: Option<codex_rollout::StateDbHandle>,
|
||||
rollout_config: &'a RolloutConfig,
|
||||
cursor: Option<codex_rollout::Cursor>,
|
||||
sort_key: codex_rollout::ThreadSortKey,
|
||||
sort_direction: codex_rollout::SortDirection,
|
||||
matcher: &'a mut RolloutSnippetMatcher<'term>,
|
||||
}
|
||||
|
||||
async fn finish_search(
|
||||
context: SearchContext<'_, '_>,
|
||||
matching_rollouts: codex_rollout::RolloutSearchMatches,
|
||||
) -> ThreadStoreResult<ThreadSearchPage> {
|
||||
let SearchContext {
|
||||
store,
|
||||
params,
|
||||
state_db,
|
||||
rollout_config,
|
||||
cursor,
|
||||
sort_key,
|
||||
sort_direction,
|
||||
matcher,
|
||||
} = context;
|
||||
if matching_rollouts.is_empty() {
|
||||
return Ok(ThreadSearchPage {
|
||||
items: Vec::new(),
|
||||
@@ -114,14 +224,13 @@ pub(super) async fn search_threads(
|
||||
};
|
||||
let mut remaining_rollouts = matching_rollouts
|
||||
.into_iter()
|
||||
.map(|(path, snippet)| (rollout_search_path(&path), snippet))
|
||||
.collect::<HashMap<_, _>>();
|
||||
.map(|path| rollout_search_path(&path))
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
|
||||
let mut snippet_matcher = RolloutSnippetMatcher::new(search_term);
|
||||
loop {
|
||||
let page = list_rollout_threads(
|
||||
state_db.clone(),
|
||||
&rollout_config,
|
||||
rollout_config,
|
||||
store.config.default_model_provider_id.as_str(),
|
||||
&scan_params,
|
||||
page_cursor.as_ref(),
|
||||
@@ -138,34 +247,17 @@ pub(super) async fn search_threads(
|
||||
let mut original_paths = None;
|
||||
for item in page.items {
|
||||
let logical_path = rollout_search_path(item.path.as_path());
|
||||
let Some(mut snippet) = remaining_rollouts.remove(logical_path.as_path()) else {
|
||||
if !remaining_rollouts.remove(logical_path.as_path()) {
|
||||
continue;
|
||||
};
|
||||
if let Some(account) = ¶ms.account {
|
||||
let Some(thread_id) = item.thread_id else {
|
||||
continue;
|
||||
};
|
||||
if !account
|
||||
.includes(
|
||||
&store.config.codex_home,
|
||||
thread_id,
|
||||
Some(&item.path),
|
||||
&page_thread_ids,
|
||||
&mut original_paths,
|
||||
)
|
||||
.await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if snippet.is_none() {
|
||||
snippet = snippet_matcher
|
||||
.first_match(&item.path)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to read rollout search match: {err}"),
|
||||
})?;
|
||||
if !eligible_search_item(store, params, &item, &page_thread_ids, &mut original_paths)
|
||||
.await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Discovery can decode an archive, but its earlier match is not authoritative:
|
||||
// the archive or the catalog-selected rollout can change before this page is built.
|
||||
let snippet = verified_snippet(matcher, &item.path).await?;
|
||||
let Some(snippet) = snippet else { continue };
|
||||
matching_items.push(ThreadSearchItem { item, snippet });
|
||||
if matching_items.len() > params.page_size {
|
||||
@@ -198,12 +290,76 @@ pub(super) async fn search_threads(
|
||||
.and_then(|cursor| serde_json::to_value(cursor).ok())
|
||||
.and_then(|value| value.as_str().map(str::to_owned));
|
||||
|
||||
let items =
|
||||
materialize_search_results(store, state_db.as_deref(), params.archived, matching_items)
|
||||
.await;
|
||||
Ok(ThreadSearchPage { items, next_cursor })
|
||||
}
|
||||
|
||||
async fn verified_snippet(
|
||||
matcher: &mut RolloutSnippetMatcher<'_>,
|
||||
path: &Path,
|
||||
) -> ThreadStoreResult<Option<String>> {
|
||||
match matcher.first_match(path).await {
|
||||
Ok(snippet) => Ok(snippet),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(ThreadStoreError::Internal {
|
||||
message: format!("failed to read rollout search match: {err}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn eligible_search_item(
|
||||
store: &LocalThreadStore,
|
||||
params: &SearchThreadsParams,
|
||||
item: &codex_rollout::ThreadItem,
|
||||
ids: &[codex_protocol::ThreadId],
|
||||
originals: &mut Option<HashMap<codex_protocol::ThreadId, PathBuf>>,
|
||||
) -> bool {
|
||||
if !matches_source(params, item) {
|
||||
return false;
|
||||
}
|
||||
match (¶ms.account, item.thread_id) {
|
||||
(Some(account), Some(id)) => {
|
||||
account
|
||||
.includes(
|
||||
&store.config.codex_home,
|
||||
id,
|
||||
Some(&item.path),
|
||||
ids,
|
||||
originals,
|
||||
)
|
||||
.await
|
||||
}
|
||||
(Some(_), None) => false,
|
||||
(None, _) => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_source(params: &SearchThreadsParams, item: &codex_rollout::ThreadItem) -> bool {
|
||||
params.source_kinds.as_ref().is_none_or(|kinds| {
|
||||
kinds.iter().any(|kind| {
|
||||
kind.matches(
|
||||
item.source
|
||||
.as_ref()
|
||||
.unwrap_or(&codex_protocol::protocol::SessionSource::Unknown),
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async fn materialize_search_results(
|
||||
store: &LocalThreadStore,
|
||||
state_db: Option<&codex_state::StateRuntime>,
|
||||
archived: bool,
|
||||
matching_items: Vec<ThreadSearchItem>,
|
||||
) -> Vec<StoredThreadSearchResult> {
|
||||
let mut items = matching_items
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
stored_thread_from_rollout_item(
|
||||
item.item,
|
||||
params.archived,
|
||||
archived,
|
||||
store.config.default_model_provider_id.as_str(),
|
||||
)
|
||||
.map(|thread| StoredThreadSearchResult {
|
||||
@@ -219,7 +375,7 @@ pub(super) async fn search_threads(
|
||||
.map(|item| item.thread.thread_id)
|
||||
.collect::<Vec<_>>();
|
||||
let mut section_metadata =
|
||||
resolve_thread_section_metadata(state_db.as_ref(), §ioned_thread_ids).await;
|
||||
resolve_thread_section_metadata(state_db, §ioned_thread_ids).await;
|
||||
for item in &mut items {
|
||||
if let Some((section_position, section_entered_at)) =
|
||||
section_metadata.remove(&item.thread.thread_id)
|
||||
@@ -231,7 +387,7 @@ pub(super) async fn search_threads(
|
||||
}
|
||||
set_thread_search_result_names(store, &mut items).await;
|
||||
|
||||
Ok(ThreadSearchPage { items, next_cursor })
|
||||
items
|
||||
}
|
||||
|
||||
fn rollout_search_path(path: &Path) -> PathBuf {
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::path::PathBuf;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_rollout::ThreadItem;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -21,6 +22,7 @@ use crate::ListThreadsParams;
|
||||
use crate::SearchThreadsParams;
|
||||
use crate::SortDirection;
|
||||
use crate::ThreadRelationFilter;
|
||||
use crate::ThreadSearchSourceKind;
|
||||
use crate::ThreadSortKey;
|
||||
use crate::ThreadStore;
|
||||
|
||||
@@ -52,6 +54,65 @@ fn time_sort_cursors_include_thread_id_tie_breaker() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_search_rechecks_stale_compressed_discovery() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let store = LocalThreadStore::new(config.clone(), /*state_db*/ None);
|
||||
let id = Uuid::new_v4();
|
||||
let path = write_session_file_with(
|
||||
home.path(),
|
||||
home.path().join("sessions/2025/01/03"),
|
||||
"2025-01-03T12-00-00",
|
||||
id,
|
||||
"changed message",
|
||||
Some("test-provider"),
|
||||
ThreadHistoryMode::Legacy,
|
||||
)
|
||||
.expect("write selected rollout");
|
||||
let contents = std::fs::read(&path).expect("read rollout");
|
||||
let compressed =
|
||||
zstd::stream::encode_all(contents.as_slice(), /*level*/ 3).expect("compress rollout");
|
||||
std::fs::write(path.with_extension("jsonl.zst"), compressed).expect("write compressed rollout");
|
||||
std::fs::remove_file(&path).expect("remove plain rollout");
|
||||
let rollout_config = codex_rollout::RolloutConfig {
|
||||
codex_home: config.codex_home.clone(),
|
||||
sqlite: config.sqlite.clone(),
|
||||
cwd: config.codex_home,
|
||||
model_provider_id: config.default_model_provider_id,
|
||||
generate_memories: false,
|
||||
};
|
||||
let params = SearchThreadsParams {
|
||||
include_attached_threads: false,
|
||||
account: None,
|
||||
page_size: 10,
|
||||
cursor: None,
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Cli],
|
||||
source_kinds: None,
|
||||
archived: false,
|
||||
search_term: "needle".to_string(),
|
||||
};
|
||||
let mut matcher = codex_rollout::RolloutSnippetMatcher::new("needle");
|
||||
let page = super::finish_search(
|
||||
super::SearchContext {
|
||||
store: &store,
|
||||
params: ¶ms,
|
||||
state_db: None,
|
||||
rollout_config: &rollout_config,
|
||||
cursor: None,
|
||||
sort_key: codex_rollout::ThreadSortKey::CreatedAt,
|
||||
sort_direction: codex_rollout::SortDirection::Desc,
|
||||
matcher: &mut matcher,
|
||||
},
|
||||
std::collections::HashSet::from([path]),
|
||||
)
|
||||
.await
|
||||
.expect("finish search");
|
||||
assert_eq!((page.items.len(), page.next_cursor), (0, None));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_matches_selected_rollout_across_path_spellings_and_compression() {
|
||||
for compressed in [false, true] {
|
||||
@@ -155,6 +216,7 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression()
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Cli],
|
||||
source_kinds: None,
|
||||
archived: false,
|
||||
search_term: search_term.to_string(),
|
||||
})
|
||||
@@ -177,6 +239,89 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn source_kind_filter_applies_before_search_pagination() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let state_db = codex_state::StateRuntime::init(
|
||||
config.sqlite.clone(),
|
||||
config.default_model_provider_id.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("initialize state database");
|
||||
let mut review_ids = Vec::new();
|
||||
for (age, source) in [
|
||||
(0, SessionSource::SubAgent(SubAgentSource::Review)),
|
||||
(1, SessionSource::Cli),
|
||||
(2, SessionSource::SubAgent(SubAgentSource::Review)),
|
||||
] {
|
||||
let uuid = Uuid::new_v4();
|
||||
let id = ThreadId::from_u128(uuid.as_u128());
|
||||
let path = write_session_file_with(
|
||||
home.path(),
|
||||
home.path().join("sessions/2025/01/03"),
|
||||
"2025-01-03T12-00-00",
|
||||
uuid,
|
||||
"needle message",
|
||||
Some("test-provider"),
|
||||
ThreadHistoryMode::Legacy,
|
||||
)
|
||||
.expect("write rollout");
|
||||
let mut metadata = codex_state::ThreadMetadataBuilder::new(
|
||||
id,
|
||||
path,
|
||||
Utc::now() - chrono::TimeDelta::days(age),
|
||||
source.clone(),
|
||||
)
|
||||
.build("test-provider");
|
||||
metadata.first_user_message = Some("needle message".to_string());
|
||||
state_db
|
||||
.upsert_thread(&metadata)
|
||||
.await
|
||||
.expect("index rollout");
|
||||
if matches!(source, SessionSource::SubAgent(SubAgentSource::Review)) {
|
||||
review_ids.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
let store = LocalThreadStore::new(config, Some(state_db));
|
||||
let mut params = SearchThreadsParams {
|
||||
include_attached_threads: false,
|
||||
account: None,
|
||||
page_size: 1,
|
||||
cursor: None,
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: Vec::new(),
|
||||
source_kinds: Some(vec![ThreadSearchSourceKind::SubAgentReview]),
|
||||
archived: false,
|
||||
search_term: "needle".to_string(),
|
||||
};
|
||||
let first = store
|
||||
.search_threads(params.clone())
|
||||
.await
|
||||
.expect("first page");
|
||||
params.cursor = first.next_cursor.clone();
|
||||
let second = store.search_threads(params).await.expect("second page");
|
||||
assert_eq!(
|
||||
(
|
||||
first
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| item.thread.thread_id)
|
||||
.collect::<Vec<_>>(),
|
||||
first.next_cursor.is_some(),
|
||||
second
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| item.thread.thread_id)
|
||||
.collect::<Vec<_>>(),
|
||||
second.next_cursor,
|
||||
),
|
||||
(vec![review_ids[0]], true, vec![review_ids[1]], None),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn std::error::Error>>
|
||||
{
|
||||
@@ -226,6 +371,7 @@ async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Cli],
|
||||
source_kinds: None,
|
||||
archived: false,
|
||||
search_term: query.to_owned(),
|
||||
})
|
||||
@@ -331,6 +477,7 @@ async fn title_and_content_search_return_attached_parent_without_state_db() {
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Cli],
|
||||
source_kinds: None,
|
||||
archived: false,
|
||||
search_term: "searchable message".to_string(),
|
||||
include_attached_threads: true,
|
||||
@@ -406,3 +553,61 @@ async fn inclusive_title_search_finds_unindexed_matches_when_filesystem_scanning
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_search_never_reports_undiscovered_rows_from_a_shared_database() {
|
||||
let home = TempDir::new().unwrap();
|
||||
let other = TempDir::new().unwrap();
|
||||
let config = test_config(home.path());
|
||||
let db = codex_state::StateRuntime::init(
|
||||
config.sqlite.clone(),
|
||||
config.default_model_provider_id.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let uuid = Uuid::new_v4();
|
||||
let path = write_session_file_with(
|
||||
other.path(),
|
||||
other.path().join("sessions/2025/01/03"),
|
||||
"2025-01-03T12-00-00",
|
||||
uuid,
|
||||
"private needle",
|
||||
Some("test-provider"),
|
||||
ThreadHistoryMode::Legacy,
|
||||
)
|
||||
.unwrap();
|
||||
let metadata = codex_state::ThreadMetadataBuilder::new(
|
||||
ThreadId::from_u128(uuid.as_u128()),
|
||||
path,
|
||||
Utc::now(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.build("test-provider");
|
||||
db.upsert_thread(&metadata).await.unwrap();
|
||||
let store = LocalThreadStore::new(config, Some(db));
|
||||
let params = SearchThreadsParams {
|
||||
include_attached_threads: false,
|
||||
account: None,
|
||||
page_size: 10,
|
||||
cursor: None,
|
||||
sort_key: ThreadSortKey::CreatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Cli],
|
||||
source_kinds: None,
|
||||
archived: false,
|
||||
search_term: "needle".into(),
|
||||
};
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(4);
|
||||
let page = store
|
||||
.search_threads_observed(
|
||||
params,
|
||||
crate::ThreadSearchObserver {
|
||||
progress: Some(tx),
|
||||
cancelled: Box::pin(std::future::pending()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(page.items.is_empty());
|
||||
assert!(rx.recv().await.is_none());
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_rollout::RolloutItem;
|
||||
use futures::future::BoxFuture;
|
||||
use std::any::Any;
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::AddThreadAttachmentOutcome;
|
||||
use crate::AddThreadAttachmentParams;
|
||||
@@ -48,6 +50,7 @@ use crate::StoredProject;
|
||||
use crate::StoredProjectsPage;
|
||||
use crate::StoredThread;
|
||||
use crate::StoredThreadHistory;
|
||||
use crate::StoredThreadSearchResult;
|
||||
use crate::StoredThreadSection;
|
||||
use crate::StoredThreadSectionsPage;
|
||||
use crate::ThreadAttachmentOwnerPage;
|
||||
@@ -66,6 +69,14 @@ use crate::UpdatedProject;
|
||||
/// Future returned by [`ThreadStore`] operations.
|
||||
pub type ThreadStoreFuture<'a, T> = Pin<Box<dyn Future<Output = ThreadStoreResult<T>> + Send + 'a>>;
|
||||
|
||||
/// Request lifetime and optional progress sink for a thread search.
|
||||
pub struct ThreadSearchObserver {
|
||||
/// Verified matches discovered before the authoritative page is complete.
|
||||
pub progress: Option<mpsc::Sender<Vec<StoredThreadSearchResult>>>,
|
||||
/// Resolves when the requester no longer needs the result.
|
||||
pub cancelled: BoxFuture<'static, ()>,
|
||||
}
|
||||
|
||||
/// Data a [`ThreadStore`] has already loaded while preparing a thread for resume.
|
||||
///
|
||||
/// Core does not inspect this value. It gives the value back to the store when reopening the
|
||||
@@ -569,6 +580,23 @@ pub trait ThreadStore: Any + Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
/// Searches threads with cancellation and optional progress where supported.
|
||||
fn search_threads_observed(
|
||||
&self,
|
||||
params: SearchThreadsParams,
|
||||
observer: ThreadSearchObserver,
|
||||
) -> ThreadStoreFuture<'_, ThreadSearchPage> {
|
||||
Box::pin(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = observer.cancelled => Err(ThreadStoreError::Internal {
|
||||
message: "thread search cancelled".to_string(),
|
||||
}),
|
||||
result = self.search_threads(params) => result,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Searches visible message occurrences within one paginated thread.
|
||||
fn search_thread_occurrences(
|
||||
&self,
|
||||
|
||||
@@ -18,6 +18,7 @@ use codex_protocol::protocol::GitInfo;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_protocol::protocol::ThreadMemoryMode as MemoryMode;
|
||||
use codex_protocol::protocol::ThreadSource;
|
||||
@@ -394,6 +395,49 @@ pub struct ListThreadsParams {
|
||||
pub use_state_db_only: bool,
|
||||
}
|
||||
|
||||
/// Source categories accepted by thread content search.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum ThreadSearchSourceKind {
|
||||
Cli,
|
||||
VsCode,
|
||||
Exec,
|
||||
AppServer,
|
||||
SubAgent,
|
||||
SubAgentReview,
|
||||
SubAgentCompact,
|
||||
SubAgentThreadSpawn,
|
||||
SubAgentOther,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl ThreadSearchSourceKind {
|
||||
/// Determine whether a stored session source belongs to this category.
|
||||
pub fn matches(self, source: &SessionSource) -> bool {
|
||||
match self {
|
||||
Self::Cli => matches!(source, SessionSource::Cli),
|
||||
Self::VsCode => matches!(source, SessionSource::VSCode),
|
||||
Self::Exec => matches!(source, SessionSource::Exec),
|
||||
Self::AppServer => matches!(source, SessionSource::Mcp),
|
||||
Self::SubAgent => matches!(source, SessionSource::SubAgent(_)),
|
||||
Self::SubAgentReview => {
|
||||
matches!(source, SessionSource::SubAgent(SubAgentSource::Review))
|
||||
}
|
||||
Self::SubAgentCompact => {
|
||||
matches!(source, SessionSource::SubAgent(SubAgentSource::Compact))
|
||||
}
|
||||
Self::SubAgentThreadSpawn => matches!(
|
||||
source,
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. })
|
||||
),
|
||||
Self::SubAgentOther => {
|
||||
matches!(source, SessionSource::SubAgent(SubAgentSource::Other(_)))
|
||||
}
|
||||
Self::Unknown => matches!(source, SessionSource::Unknown),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for searching thread content.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SearchThreadsParams {
|
||||
@@ -412,6 +456,9 @@ pub struct SearchThreadsParams {
|
||||
pub sort_direction: SortDirection,
|
||||
/// Allowed session sources. Empty means implementation default.
|
||||
pub allowed_sources: Vec<SessionSource>,
|
||||
/// Exact source categories to apply before pagination. None accepts any allowed source.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source_kinds: Option<Vec<ThreadSearchSourceKind>>,
|
||||
/// Whether archived threads should be searched instead of active threads.
|
||||
pub archived: bool,
|
||||
/// Visible thread content to search for.
|
||||
|
||||
Reference in New Issue
Block a user