feat(app-server): add within-thread occurrence search

This commit is contained in:
Brent Traut
2026-07-08 10:43:14 -07:00
parent f1affbac5e
commit 786430670b
24 changed files with 1551 additions and 5 deletions
+2
View File
@@ -14,6 +14,7 @@ workspace = true
[dependencies]
chrono = { workspace = true, features = ["serde"] }
codex-app-server-protocol = { workspace = true }
codex-git-utils = { workspace = true }
codex-install-context = { workspace = true }
codex-otel = { workspace = true }
@@ -21,6 +22,7 @@ codex-protocol = { workspace = true }
codex-rollout = { workspace = true }
codex-state = { workspace = true }
codex-utils-path = { workspace = true }
pulldown-cmark = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
+4
View File
@@ -37,17 +37,21 @@ pub use types::LoadThreadHistoryParams;
pub use types::ReadThreadByRolloutPathParams;
pub use types::ReadThreadParams;
pub use types::ResumeThreadParams;
pub use types::SearchTextRange;
pub use types::SearchThreadOccurrencesParams;
pub use types::SearchThreadsParams;
pub use types::SortDirection;
pub use types::StoredThread;
pub use types::StoredThreadHistory;
pub use types::StoredThreadItem;
pub use types::StoredThreadOccurrence;
pub use types::StoredThreadSearchResult;
pub use types::StoredTurn;
pub use types::StoredTurnError;
pub use types::StoredTurnItemsView;
pub use types::StoredTurnStatus;
pub use types::ThreadMetadataPatch;
pub use types::ThreadOccurrenceSearchPage;
pub use types::ThreadPage;
pub use types::ThreadPersistenceMetadata;
pub use types::ThreadRelationFilter;
+14
View File
@@ -5,6 +5,7 @@ mod helpers;
mod list_threads;
mod live_writer;
mod read_thread;
mod search_thread_occurrences;
mod search_threads;
mod unarchive_thread;
mod update_thread_metadata;
@@ -31,9 +32,11 @@ use crate::LoadThreadHistoryParams;
use crate::ReadThreadByRolloutPathParams;
use crate::ReadThreadParams;
use crate::ResumeThreadParams;
use crate::SearchThreadOccurrencesParams;
use crate::SearchThreadsParams;
use crate::StoredThread;
use crate::StoredThreadHistory;
use crate::ThreadOccurrenceSearchPage;
use crate::ThreadPage;
use crate::ThreadSearchPage;
use crate::ThreadStore;
@@ -59,6 +62,7 @@ use crate::UpdateThreadMetadataParams;
pub struct LocalThreadStore {
pub(super) config: LocalThreadStoreConfig,
live_recorders: Arc<Mutex<HashMap<ThreadId, LiveRecorderEntry>>>,
occurrence_search_cache: Arc<Mutex<search_thread_occurrences::ThreadOccurrenceSearchCache>>,
state_db: Option<StateDbHandle>,
}
@@ -106,6 +110,7 @@ impl LocalThreadStore {
Self {
config,
live_recorders: Arc::new(Mutex::new(HashMap::new())),
occurrence_search_cache: Arc::new(Mutex::new(Default::default())),
state_db,
}
}
@@ -301,6 +306,15 @@ impl ThreadStore for LocalThreadStore {
Box::pin(async move { search_threads::search_threads(self, params).await })
}
fn search_thread_occurrences(
&self,
params: SearchThreadOccurrencesParams,
) -> ThreadStoreFuture<'_, ThreadOccurrenceSearchPage> {
Box::pin(
async move { search_thread_occurrences::search_thread_occurrences(self, params).await },
)
}
fn update_thread_metadata(
&self,
params: UpdateThreadMetadataParams,
@@ -206,7 +206,7 @@ async fn attach_history_if_requested(
Ok(())
}
async fn resolve_rollout_path(
pub(super) async fn resolve_rollout_path(
store: &LocalThreadStore,
thread_id: codex_protocol::ThreadId,
include_archived: bool,
@@ -0,0 +1,634 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
use chrono::DateTime;
use codex_app_server_protocol::ThreadHistoryBuilder;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::UserInput;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
use pulldown_cmark::Event;
use pulldown_cmark::Parser;
use pulldown_cmark::TagEnd;
use serde::Deserialize;
use serde::Serialize;
use super::LocalThreadStore;
use crate::SearchTextRange;
use crate::SearchThreadOccurrencesParams;
use crate::StoredThreadOccurrence;
use crate::ThreadOccurrenceSearchPage;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
const SNAPSHOT_TTL: Duration = Duration::from_secs(10 * 60);
const COMPLETED_SNAPSHOT_TTL: Duration = Duration::from_secs(30);
const MAX_ACTIVE_SNAPSHOTS: usize = 256;
const MAX_CACHED_PROJECTIONS: usize = 8;
const SNIPPET_CONTEXT_BEFORE_CHARS: usize = 48;
const SNIPPET_CONTEXT_AFTER_CHARS: usize = 96;
#[cfg(test)]
#[path = "search_thread_occurrences_tests.rs"]
mod tests;
#[derive(Default)]
pub(super) struct ThreadOccurrenceSearchCache {
next_snapshot_id: u64,
snapshots: HashMap<u64, ThreadOccurrenceSearchSnapshot>,
projections: HashMap<codex_protocol::ThreadId, CachedSearchProjection>,
}
#[derive(Clone)]
struct ThreadOccurrenceSearchSnapshot {
thread_id: codex_protocol::ThreadId,
search_term: String,
items: Vec<StoredThreadOccurrence>,
is_capped: bool,
expires_at: Instant,
completed: bool,
}
#[derive(Clone, PartialEq, Eq)]
struct RolloutFingerprint {
path: PathBuf,
len: u64,
modified_at: Option<SystemTime>,
}
#[derive(Clone)]
struct SearchProjection {
fingerprint: RolloutFingerprint,
items: Vec<SearchableThreadItem>,
}
struct CachedSearchProjection {
projection: Arc<SearchProjection>,
expires_at: Instant,
}
#[derive(Clone)]
struct SearchableThreadItem {
turn_id: String,
item_id: String,
text: String,
turn_started_at: i64,
}
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ThreadOccurrenceSearchCursor {
snapshot_id: u64,
offset: usize,
}
pub(super) async fn search_thread_occurrences(
store: &LocalThreadStore,
params: SearchThreadOccurrencesParams,
) -> ThreadStoreResult<ThreadOccurrenceSearchPage> {
if params.search_term.is_empty() {
return Err(ThreadStoreError::InvalidRequest {
message: "search_thread_occurrences requires search_term".to_string(),
});
}
if params.max_results == 0 {
return Err(ThreadStoreError::InvalidRequest {
message: "search_thread_occurrences requires max_results greater than zero".to_string(),
});
}
let page_size = params.page_size.clamp(1, params.max_results);
if let Some(cursor) = params.cursor.as_deref() {
return page_from_cached_snapshot(store, &params, cursor, page_size).await;
}
let snapshot = build_snapshot(store, &params).await?;
let is_capped = snapshot.is_capped;
let page_end = page_size.min(snapshot.items.len());
let items = snapshot.items[..page_end].to_vec();
let next_cursor = if page_end < snapshot.items.len() {
let snapshot_id = store
.occurrence_search_cache
.lock()
.await
.insert_snapshot(snapshot)?;
Some(serialize_cursor(ThreadOccurrenceSearchCursor {
snapshot_id,
offset: page_end,
})?)
} else {
None
};
Ok(ThreadOccurrenceSearchPage {
items,
next_cursor,
is_capped,
})
}
async fn page_from_cached_snapshot(
store: &LocalThreadStore,
params: &SearchThreadOccurrencesParams,
cursor: &str,
page_size: usize,
) -> ThreadStoreResult<ThreadOccurrenceSearchPage> {
let cursor = parse_cursor(cursor)?;
let mut cache = store.occurrence_search_cache.lock().await;
cache.remove_expired();
let snapshot = cache.snapshots.get(&cursor.snapshot_id).ok_or_else(|| {
ThreadStoreError::InvalidRequest {
message: "invalid cursor: search snapshot is no longer available".to_string(),
}
})?;
if snapshot.thread_id != params.thread_id || snapshot.search_term != params.search_term {
return Err(ThreadStoreError::InvalidRequest {
message: "invalid cursor: thread or search term does not match snapshot".to_string(),
});
}
if cursor.offset >= snapshot.items.len() {
return Err(ThreadStoreError::InvalidRequest {
message: "invalid cursor: search offset is out of range".to_string(),
});
}
let page_end = cursor
.offset
.saturating_add(page_size)
.min(snapshot.items.len());
let items = snapshot.items[cursor.offset..page_end].to_vec();
let has_next_page = page_end < snapshot.items.len();
let next_cursor = has_next_page
.then(|| {
serialize_cursor(ThreadOccurrenceSearchCursor {
snapshot_id: cursor.snapshot_id,
offset: page_end,
})
})
.transpose()?;
let page = ThreadOccurrenceSearchPage {
items,
next_cursor,
is_capped: snapshot.is_capped,
};
if !has_next_page && let Some(snapshot) = cache.snapshots.get_mut(&cursor.snapshot_id) {
snapshot.completed = true;
snapshot.expires_at = Instant::now() + COMPLETED_SNAPSHOT_TTL;
}
Ok(page)
}
async fn build_snapshot(
store: &LocalThreadStore,
params: &SearchThreadOccurrencesParams,
) -> ThreadStoreResult<ThreadOccurrenceSearchSnapshot> {
let projection = load_search_projection(store, params.thread_id).await?;
let matcher = LiteralMatcher::new(params.search_term.as_str());
let mut items = Vec::with_capacity(params.max_results.saturating_add(1));
for item in &projection.items {
let remaining = params
.max_results
.saturating_add(1)
.saturating_sub(items.len());
items.extend(occurrences_in_item(
params.thread_id,
item.turn_id.clone(),
item.item_id.clone(),
item.text.as_str(),
item.turn_started_at,
&matcher,
remaining,
));
if items.len() > params.max_results {
break;
}
}
let is_capped = items.len() > params.max_results;
items.truncate(params.max_results);
Ok(ThreadOccurrenceSearchSnapshot {
thread_id: params.thread_id,
search_term: params.search_term.clone(),
items,
is_capped,
expires_at: Instant::now() + SNAPSHOT_TTL,
completed: false,
})
}
async fn load_search_projection(
store: &LocalThreadStore,
thread_id: codex_protocol::ThreadId,
) -> ThreadStoreResult<Arc<SearchProjection>> {
let path =
super::read_thread::resolve_rollout_path(store, thread_id, /*include_archived*/ true)
.await?
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?;
let mut lines = match codex_rollout::open_rollout_line_reader(path.as_path()).await {
Ok(lines) => lines,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(ThreadStoreError::ThreadNotFound { thread_id });
}
Err(err) => {
return Err(ThreadStoreError::Internal {
message: format!("failed to open rollout for search: {err}"),
});
}
};
let physical_path = lines.source_path().to_path_buf();
let fingerprint = RolloutFingerprint {
path: physical_path.clone(),
len: lines.source_len(),
modified_at: lines.source_modified_at(),
};
{
let mut cache = store.occurrence_search_cache.lock().await;
cache.remove_expired();
if let Some(cached) = cache.projections.get_mut(&thread_id)
&& cached.projection.fingerprint == fingerprint
{
cached.expires_at = Instant::now() + SNAPSHOT_TTL;
return Ok(Arc::clone(&cached.projection));
}
}
let plain_path = codex_rollout::plain_rollout_path(physical_path.as_path());
let snapshot_len = if physical_path == plain_path {
Some(fingerprint.len)
} else {
// Compressed rollouts are immutable, so the decompressed byte count does not need a bound.
None
};
let mut builder = ThreadHistoryBuilder::new();
let mut turn_timestamps = HashMap::<String, i64>::new();
let mut bytes_read = 0_u64;
let mut parse_errors = 0_usize;
while let Some(line) = lines
.next_line()
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to read rollout during search: {err}"),
})?
{
if snapshot_len.is_some_and(|limit| bytes_read.saturating_add(line.len() as u64) > limit) {
break;
}
bytes_read = bytes_read.saturating_add(line.len() as u64 + 1);
let rollout_line = match serde_json::from_str::<RolloutLine>(line.trim()) {
Ok(rollout_line) => rollout_line,
Err(_) => {
parse_errors = parse_errors.saturating_add(1);
continue;
}
};
let changes = builder.handle_rollout_item_with_changes(&rollout_line.item);
let line_timestamp = if changes
.changed_turns
.iter()
.any(|turn| turn.started_at.is_none())
{
DateTime::parse_from_rfc3339(&rollout_line.timestamp)
.map(|timestamp| timestamp.timestamp())
.unwrap_or_default()
} else {
0
};
for removed_turn_id in changes.removed_turn_ids {
turn_timestamps.remove(&removed_turn_id);
}
for turn in changes.changed_turns {
if let Some(started_at) = turn.started_at {
turn_timestamps.insert(turn.turn_id, started_at);
} else {
turn_timestamps
.entry(turn.turn_id)
.or_insert(line_timestamp);
}
}
}
if parse_errors > 0 {
tracing::warn!(%thread_id, parse_errors, "skipped malformed rollout records during thread search");
}
let mut items = Vec::new();
for turn in builder.finish() {
let turn_started_at = turn
.started_at
.or_else(|| turn_timestamps.get(&turn.id).copied())
.unwrap_or_default();
let last_assistant_id = turn.items.iter().rev().find_map(|item| match item {
ThreadItem::AgentMessage { id, .. } => Some(id.clone()),
_ => None,
});
for item in turn.items {
if matches!(&item, ThreadItem::AgentMessage { id, .. } if Some(id) != last_assistant_id.as_ref())
{
continue;
}
let Some(text) = searchable_text(&item) else {
continue;
};
items.push(SearchableThreadItem {
turn_id: turn.id.clone(),
item_id: item.id().to_string(),
text: text.into_owned(),
turn_started_at,
});
}
}
let projection = Arc::new(SearchProjection { fingerprint, items });
store
.occurrence_search_cache
.lock()
.await
.insert_projection(
thread_id,
CachedSearchProjection {
projection: Arc::clone(&projection),
expires_at: Instant::now() + SNAPSHOT_TTL,
},
);
Ok(projection)
}
fn searchable_text(item: &ThreadItem) -> Option<Cow<'_, str>> {
match item {
ThreadItem::UserMessage { content, .. } => {
let mut text_parts = content
.iter()
.filter_map(|input| match input {
UserInput::Text { text, .. } => Some(strip_user_message_prefix(text)),
UserInput::Image { .. }
| UserInput::LocalImage { .. }
| UserInput::Skill { .. }
| UserInput::Mention { .. } => None,
})
.filter(|text| !text.is_empty())
.peekable();
let first = text_parts.next()?;
match text_parts.next() {
None => Some(Cow::Borrowed(first)),
Some(second) => {
let mut parts = vec![first, second];
parts.extend(text_parts);
Some(Cow::Owned(parts.join("\n")))
}
}
}
ThreadItem::AgentMessage { text, .. } => {
let text = markdown_to_search_text(text);
(!text.is_empty()).then_some(Cow::Owned(text))
}
_ => None,
}
}
fn strip_user_message_prefix(text: &str) -> &str {
match text.find(USER_MESSAGE_BEGIN) {
Some(index) => text[index + USER_MESSAGE_BEGIN.len()..].trim(),
None => text.trim(),
}
}
fn markdown_to_search_text(markdown: &str) -> String {
let mut text = String::new();
for event in Parser::new(markdown.trim()) {
match event {
Event::Text(value)
| Event::Code(value)
| Event::Html(value)
| Event::InlineHtml(value) => text.push_str(&value),
Event::SoftBreak | Event::Rule => text.push(' '),
Event::End(
TagEnd::Paragraph
| TagEnd::Heading(_)
| TagEnd::BlockQuote
| TagEnd::CodeBlock
| TagEnd::List(_)
| TagEnd::Item
| TagEnd::Table
| TagEnd::TableHead
| TagEnd::TableRow
| TagEnd::TableCell,
) => text.push(' '),
Event::Start(_)
| Event::End(
TagEnd::Emphasis
| TagEnd::Strong
| TagEnd::Strikethrough
| TagEnd::Link
| TagEnd::HtmlBlock
| TagEnd::FootnoteDefinition
| TagEnd::Image
| TagEnd::MetadataBlock(_),
)
| Event::HardBreak
| Event::FootnoteReference(_)
| Event::TaskListMarker(_) => {}
}
}
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
struct LiteralMatcher {
lowercase_needle: String,
}
impl LiteralMatcher {
fn new(needle: &str) -> Self {
Self {
lowercase_needle: needle.to_lowercase(),
}
}
fn find_ranges(&self, text: &str, limit: usize) -> Vec<std::ops::Range<usize>> {
let lowercase_text = text.to_lowercase();
let mut spans = Vec::with_capacity(text.chars().count());
let mut lowercase_start = 0;
for (original_start, character) in text.char_indices() {
let lowercase_end =
lowercase_start + character.to_lowercase().map(char::len_utf8).sum::<usize>();
spans.push((
lowercase_start..lowercase_end,
original_start..original_start + character.len_utf8(),
));
lowercase_start = lowercase_end;
}
lowercase_text
.match_indices(self.lowercase_needle.as_str())
.take(limit)
.filter_map(|(start, matched)| {
let end = start.saturating_add(matched.len());
let original_start = spans
.iter()
.find(|(lowercase, _)| lowercase.contains(&start))?
.1
.start;
let original_end = spans
.iter()
.find(|(lowercase, _)| lowercase.contains(&end.saturating_sub(1)))?
.1
.end;
Some(original_start..original_end)
})
.collect()
}
}
fn occurrences_in_item(
thread_id: codex_protocol::ThreadId,
turn_id: String,
item_id: String,
text: &str,
turn_started_at: i64,
matcher: &LiteralMatcher,
limit: usize,
) -> Vec<StoredThreadOccurrence> {
let mut previous_match_end = 0;
let mut utf16_offset = 0_u32;
matcher
.find_ranges(text, limit)
.into_iter()
.enumerate()
.map(|(occurrence_index, matched)| {
utf16_offset =
utf16_offset.saturating_add(utf16_len(&text[previous_match_end..matched.start]));
let match_start = utf16_offset;
let match_end = match_start.saturating_add(utf16_len(&text[matched.clone()]));
previous_match_end = matched.end;
utf16_offset = match_end;
let snippet_start =
char_start_before(text, matched.start, SNIPPET_CONTEXT_BEFORE_CHARS);
let snippet_end = char_end_after(text, matched.end, SNIPPET_CONTEXT_AFTER_CHARS);
let leading_ellipsis = snippet_start > 0;
let trailing_ellipsis = snippet_end < text.len();
let mut snippet = String::new();
if leading_ellipsis {
snippet.push_str("... ");
}
snippet.push_str(&text[snippet_start..snippet_end]);
if trailing_ellipsis {
snippet.push_str(" ...");
}
let snippet_match_start = if leading_ellipsis { 4 } else { 0 }
+ utf16_len(&text[snippet_start..matched.start]);
let match_len = utf16_len(&text[matched]);
let occurrence_index = u32::try_from(occurrence_index).unwrap_or(u32::MAX);
StoredThreadOccurrence {
thread_id,
occurrence_id: format!("{turn_id}:{item_id}:{match_start}"),
turn_id: turn_id.clone(),
item_id: item_id.clone(),
occurrence_index,
snippet,
match_range: SearchTextRange {
start: match_start,
end: match_end,
},
snippet_match_range: SearchTextRange {
start: snippet_match_start,
end: snippet_match_start.saturating_add(match_len),
},
turn_started_at,
}
})
.collect()
}
fn utf16_len(text: &str) -> u32 {
u32::try_from(text.encode_utf16().count()).unwrap_or(u32::MAX)
}
fn char_start_before(text: &str, byte_index: usize, chars_before: usize) -> usize {
text[..byte_index]
.char_indices()
.rev()
.nth(chars_before)
.map(|(index, _)| index)
.unwrap_or(0)
}
fn char_end_after(text: &str, byte_index: usize, chars_after: usize) -> usize {
text[byte_index..]
.char_indices()
.nth(chars_after)
.map(|(offset, _)| byte_index.saturating_add(offset))
.unwrap_or(text.len())
}
fn serialize_cursor(cursor: ThreadOccurrenceSearchCursor) -> ThreadStoreResult<String> {
serde_json::to_string(&cursor).map_err(|err| ThreadStoreError::Internal {
message: format!("failed to serialize search cursor: {err}"),
})
}
fn parse_cursor(cursor: &str) -> ThreadStoreResult<ThreadOccurrenceSearchCursor> {
serde_json::from_str(cursor).map_err(|_| ThreadStoreError::InvalidRequest {
message: format!("invalid cursor: {cursor}"),
})
}
impl ThreadOccurrenceSearchCache {
fn insert_snapshot(
&mut self,
snapshot: ThreadOccurrenceSearchSnapshot,
) -> ThreadStoreResult<u64> {
self.remove_expired();
if self.snapshots.len() >= MAX_ACTIVE_SNAPSHOTS
&& let Some(completed_snapshot_id) = self
.snapshots
.iter()
.filter(|(_, snapshot)| snapshot.completed)
.min_by_key(|(_, snapshot)| snapshot.expires_at)
.map(|(snapshot_id, _)| *snapshot_id)
{
self.snapshots.remove(&completed_snapshot_id);
}
if self.snapshots.len() >= MAX_ACTIVE_SNAPSHOTS {
return Err(ThreadStoreError::Internal {
message: "too many active thread search snapshots".to_string(),
});
}
self.next_snapshot_id = self.next_snapshot_id.wrapping_add(1);
let snapshot_id = self.next_snapshot_id;
self.snapshots.insert(snapshot_id, snapshot);
Ok(snapshot_id)
}
fn insert_projection(
&mut self,
thread_id: codex_protocol::ThreadId,
projection: CachedSearchProjection,
) {
self.remove_expired();
if self.projections.len() >= MAX_CACHED_PROJECTIONS
&& !self.projections.contains_key(&thread_id)
&& let Some(oldest_thread_id) = self
.projections
.iter()
.min_by_key(|(_, projection)| projection.expires_at)
.map(|(thread_id, _)| *thread_id)
{
self.projections.remove(&oldest_thread_id);
}
self.projections.insert(thread_id, projection);
}
fn remove_expired(&mut self) {
let now = Instant::now();
self.snapshots
.retain(|_, snapshot| snapshot.expires_at > now);
self.projections
.retain(|_, projection| projection.expires_at > now);
}
}
@@ -0,0 +1,219 @@
use std::sync::Arc;
use codex_protocol::ThreadId;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::protocol::AgentMessageEvent;
use codex_protocol::protocol::AgentReasoningEvent;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
use codex_protocol::protocol::UserMessageEvent;
use pretty_assertions::assert_eq;
use tempfile::TempDir;
use super::LocalThreadStore;
use crate::CreateThreadParams;
use crate::LiveThread;
use crate::SearchThreadOccurrencesParams;
use crate::ThreadPersistenceMetadata;
use crate::ThreadStore;
use crate::local::test_support::test_config;
#[tokio::test]
async fn searches_visible_occurrences_and_keeps_paginated_snapshot_stable() {
let home = TempDir::new().expect("temp dir");
let store = Arc::new(LocalThreadStore::new(test_config(home.path()), None));
let thread_id = ThreadId::default();
let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id))
.await
.expect("create thread");
live_thread
.append_items(&[
turn_started("turn-1", 1_700_000_000),
user_message(&format!(
"needle in hidden environment context\n{USER_MESSAGE_BEGIN}\nNeedle twice: NEEDLE"
)),
RolloutItem::EventMsg(EventMsg::AgentReasoning(AgentReasoningEvent {
text: "needle in hidden reasoning".to_string(),
})),
RolloutItem::ResponseItem(codex_protocol::models::ResponseItem::FunctionCallOutput {
id: None,
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_text("needle in tool output".to_string()),
internal_chat_message_metadata_passthrough: None,
}),
agent_message("needle in an earlier assistant message"),
agent_message("**Needle** from assistant; [needle](https://example.com)"),
turn_complete("turn-1"),
])
.await
.expect("append searchable history");
live_thread.flush().await.expect("flush history");
let first = store
.search_thread_occurrences(search_params(thread_id, None, 2))
.await
.expect("first search page");
assert_eq!(first.items.len(), 2);
assert_eq!(first.items[0].turn_id, "turn-1");
assert_eq!(first.items[0].item_id, "item-1");
assert_eq!(first.items[0].occurrence_index, 0);
assert_eq!(first.items[1].occurrence_index, 1);
assert_eq!(first.items[0].turn_started_at, 1_700_000_000);
assert_eq!(first.items[0].snippet_match_range.start, 0);
assert_eq!(first.items[0].snippet_match_range.end, 6);
let cursor = first.next_cursor.expect("second page cursor");
live_thread
.append_items(&[
turn_started("turn-2", 1_700_000_100),
agent_message("new needle after the snapshot"),
turn_complete("turn-2"),
])
.await
.expect("append after snapshot");
live_thread.flush().await.expect("flush appended history");
let second = store
.search_thread_occurrences(search_params(thread_id, Some(cursor), 2))
.await
.expect("second search page");
assert_eq!(second.items.len(), 2);
assert_eq!(second.items[0].turn_id, "turn-1");
assert_eq!(second.items[0].item_id, "item-4");
assert_eq!(second.items[1].item_id, "item-4");
assert_eq!(second.next_cursor, None);
let refreshed = store
.search_thread_occurrences(search_params(thread_id, None, 10))
.await
.expect("refreshed search");
assert_eq!(refreshed.items.len(), 5);
assert_eq!(refreshed.items.last().expect("new match").turn_id, "turn-2");
let capped = store
.search_thread_occurrences(SearchThreadOccurrencesParams {
max_results: 3,
page_size: 3,
..search_params(thread_id, None, 3)
})
.await
.expect("capped search");
assert_eq!(capped.items.len(), 3);
assert!(capped.is_capped);
}
#[test]
fn reports_utf16_ranges_for_javascript_clients() {
let matcher = super::LiteralMatcher::new("needle");
let matches = super::occurrences_in_item(
ThreadId::default(),
"turn-1".to_string(),
"item-1".to_string(),
"🙂NEEDLE needle",
1,
&matcher,
usize::MAX,
);
assert_eq!(
matches
.iter()
.map(|item| item.match_range.clone())
.collect::<Vec<_>>(),
vec![
crate::SearchTextRange { start: 2, end: 8 },
crate::SearchTextRange { start: 9, end: 15 },
]
);
let greek_matches = super::LiteralMatcher::new("ος").find_ranges("ΟΣ", usize::MAX);
assert_eq!(greek_matches, vec![0..4]);
let expanded_matches = super::LiteralMatcher::new("i").find_ranges("İ", usize::MAX);
assert_eq!(expanded_matches, vec![0..2]);
assert_eq!(
super::markdown_to_search_text("before <kbd>needle</kbd>"),
"before <kbd>needle</kbd>"
);
assert_eq!(super::markdown_to_search_text("foo \nbar"), "foobar");
}
fn search_params(
thread_id: ThreadId,
cursor: Option<String>,
page_size: usize,
) -> SearchThreadOccurrencesParams {
SearchThreadOccurrencesParams {
thread_id,
search_term: "needle".to_string(),
cursor,
page_size,
max_results: 250,
}
}
fn create_thread_params(thread_id: ThreadId) -> CreateThreadParams {
CreateThreadParams {
session_id: thread_id.into(),
thread_id,
extra_config: None,
forked_from_id: None,
parent_thread_id: None,
source: SessionSource::Exec,
thread_source: None,
originator: "test_originator".to_string(),
base_instructions: BaseInstructions::default(),
dynamic_tools: Vec::new(),
selected_capability_roots: Vec::new(),
multi_agent_version: None,
history_mode: ThreadHistoryMode::Legacy,
initial_window_id: uuid::Uuid::now_v7().to_string(),
metadata: ThreadPersistenceMetadata {
cwd: Some(std::env::current_dir().expect("cwd")),
model_provider: "test-provider".to_string(),
memory_mode: ThreadMemoryMode::Enabled,
},
}
}
fn turn_started(turn_id: &str, started_at: i64) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
turn_id: turn_id.to_string(),
trace_id: None,
started_at: Some(started_at),
model_context_window: None,
collaboration_mode_kind: Default::default(),
}))
}
fn turn_complete(turn_id: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
turn_id: turn_id.to_string(),
last_agent_message: None,
completed_at: None,
duration_ms: None,
time_to_first_token_ms: None,
}))
}
fn user_message(message: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: message.to_string(),
..Default::default()
}))
}
fn agent_message(message: &str) -> RolloutItem {
RolloutItem::EventMsg(EventMsg::AgentMessage(AgentMessageEvent {
message: message.to_string(),
phase: None,
memory_citation: None,
}))
}
+14
View File
@@ -16,9 +16,11 @@ use crate::LoadThreadHistoryParams;
use crate::ReadThreadByRolloutPathParams;
use crate::ReadThreadParams;
use crate::ResumeThreadParams;
use crate::SearchThreadOccurrencesParams;
use crate::SearchThreadsParams;
use crate::StoredThread;
use crate::StoredThreadHistory;
use crate::ThreadOccurrenceSearchPage;
use crate::ThreadPage;
use crate::ThreadSearchPage;
use crate::ThreadStoreError;
@@ -102,6 +104,18 @@ pub trait ThreadStore: Any + Send + Sync {
})
}
/// Searches visible user and assistant message occurrences within one thread.
fn search_thread_occurrences(
&self,
_params: SearchThreadOccurrencesParams,
) -> ThreadStoreFuture<'_, ThreadOccurrenceSearchPage> {
Box::pin(async {
Err(ThreadStoreError::Unsupported {
operation: "search_thread_occurrences",
})
})
}
/// Lists turns within a stored thread.
fn list_turns(&self, _params: ListTurnsParams) -> ThreadStoreFuture<'_, TurnPage> {
Box::pin(async {
+44
View File
@@ -284,6 +284,50 @@ pub struct ThreadSearchPage {
pub next_cursor: Option<String>,
}
/// Parameters for searching occurrences within one persisted thread.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SearchThreadOccurrencesParams {
/// Thread id to search.
pub thread_id: ThreadId,
/// Case-insensitive literal substring to find in visible message text.
pub search_term: String,
/// Opaque cursor returned by a previous search call for this query.
pub cursor: Option<String>,
/// Maximum number of occurrences to return in this page.
pub page_size: usize,
/// Maximum number of occurrences retained for this query generation.
pub max_results: usize,
}
/// UTF-16 code-unit range suitable for direct use by JavaScript clients.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SearchTextRange {
pub start: u32,
pub end: u32,
}
/// Lightweight stable anchor for one persisted message occurrence.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct StoredThreadOccurrence {
pub thread_id: ThreadId,
pub turn_id: String,
pub item_id: String,
pub occurrence_id: String,
pub occurrence_index: u32,
pub snippet: String,
pub match_range: SearchTextRange,
pub snippet_match_range: SearchTextRange,
pub turn_started_at: i64,
}
/// Point-in-time page of occurrence search results.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ThreadOccurrenceSearchPage {
pub items: Vec<StoredThreadOccurrence>,
pub next_cursor: Option<String>,
pub is_capped: bool,
}
/// Requested amount of item detail for stored turns.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub enum StoredTurnItemsView {