Stream partial thread search results and support cancellation (#62132)

## Why

Clients using `thread/search` have to wait for the search to finish before
they can show any matches. A search can also outlive the query that triggered
it, such as when a user changes their task mention query.

## What changed

- Add experimental `thread/search/stream` with best-effort
  `thread/search/partial` notifications. Each notification carries the original
  `requestId` and a batch of verified matches. Clients replace provisional
  matches with the final page, which determines results, ordering, and cursors.
- Support connection-scoped cancellation through `thread/search/stream/cancel`
  and `thread/search/cancel` for their respective search methods. Disconnecting
  also stops active searches and reaps their search subprocesses.
- Cancel superseded or timed-out TUI task mention searches without delaying
  available title matches.

## Testing

Add coverage for incremental results and final-page parity, slow clients,
connection-scoped cancellation and subprocess cleanup, and TUI task search
cancellation.

GitOrigin-RevId: b58abf7a1df424f191c8ede806d1d5266d299548
This commit is contained in:
matthewdavis-oai
2026-11-17 23:46:48 +00:00
committed by copyberry
parent c90234de95
commit 3bf4e138fe
53 changed files with 5965 additions and 420 deletions
+1
View File
@@ -4899,6 +4899,7 @@ dependencies = [
"codex-protocol",
"codex-state",
"codex-utils-absolute-path",
"codex-utils-cargo-bin",
"codex-utils-output-truncation",
"codex-utils-path",
"codex-utils-process",
+5 -3
View File
@@ -410,13 +410,15 @@ impl InProcessAppServerClient {
// this loop can keep draining runtime events
// while the request is blocked on client input.
tokio::spawn(async move {
// Device ceremonies belong to the waiting UI. Preserve
// its cancellation through this buffering task.
// Dropping the UI's request must drop the detached runtime
// future too: its drop guard cancels verification and search.
let cancellable = matches!(*request,
ClientRequest::UserVerificationStatus { .. }
| ClientRequest::UserVerificationEnroll { .. }
| ClientRequest::UserVerificationDelete { .. }
| ClientRequest::UserVerificationVerify { .. });
| ClientRequest::UserVerificationVerify { .. }
| ClientRequest::ThreadSearch { .. }
| ClientRequest::ThreadSearchStream { .. });
let mut response_tx = response_tx;
tokio::select! {
_ = response_tx.closed(), if cancellable => {}
@@ -6570,6 +6570,40 @@
],
"type": "object"
},
"ThreadSearchPartialNotification": {
"description": "A best-effort incremental batch for `thread/search/stream`. Match `requestId` to the request on this connection. Only a small subset of verified matches may be emitted; rows are unique and arrive in discovery order, but may be dropped under load or fall outside the final sorted page. Thread fields use stored metadata and the current watched status; live thread settings are enriched in the final response. Replace provisional rows with the authoritative final response. No partials follow it or an error.",
"properties": {
"data": {
"items": {
"$ref": "#/definitions/ThreadSearchResult"
},
"type": "array"
},
"requestId": {
"$ref": "#/definitions/RequestId"
}
},
"required": [
"data",
"requestId"
],
"type": "object"
},
"ThreadSearchResult": {
"properties": {
"snippet": {
"type": "string"
},
"thread": {
"$ref": "#/definitions/Thread"
}
},
"required": [
"snippet",
"thread"
],
"type": "object"
},
"ThreadSection": {
"description": "An independently persisted, user-visible thread section.",
"properties": {
@@ -8417,6 +8451,26 @@
"title": "Process/outputDeltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/search/partial"
],
"title": "Thread/search/partialNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadSearchPartialNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/search/partialNotification",
"type": "object"
},
{
"description": "Final exit notification for a `process/spawn` session.",
"properties": {
@@ -5541,6 +5541,26 @@
"title": "Process/outputDeltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/search/partial"
],
"title": "Thread/search/partialNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/v2/ThreadSearchPartialNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/search/partialNotification",
"type": "object"
},
{
"description": "Final exit notification for a `process/spawn` session.",
"properties": {
@@ -26456,6 +26476,27 @@
"title": "ThreadRevertedNotification",
"type": "object"
},
"ThreadSearchPartialNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "A best-effort incremental batch for `thread/search/stream`. Match `requestId` to the request on this connection. Only a small subset of verified matches may be emitted; rows are unique and arrive in discovery order, but may be dropped under load or fall outside the final sorted page. Thread fields use stored metadata and the current watched status; live thread settings are enriched in the final response. Replace provisional rows with the authoritative final response. No partials follow it or an error.",
"properties": {
"data": {
"items": {
"$ref": "#/definitions/v2/ThreadSearchResult"
},
"type": "array"
},
"requestId": {
"$ref": "#/definitions/v2/RequestId"
}
},
"required": [
"data",
"requestId"
],
"title": "ThreadSearchPartialNotification",
"type": "object"
},
"ThreadSearchResult": {
"properties": {
"snippet": {
@@ -18715,6 +18715,26 @@
"title": "Process/outputDeltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/search/partial"
],
"title": "Thread/search/partialNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadSearchPartialNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/search/partialNotification",
"type": "object"
},
{
"description": "Final exit notification for a `process/spawn` session.",
"properties": {
@@ -24152,6 +24172,27 @@
"title": "ThreadRevertedNotification",
"type": "object"
},
"ThreadSearchPartialNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "A best-effort incremental batch for `thread/search/stream`. Match `requestId` to the request on this connection. Only a small subset of verified matches may be emitted; rows are unique and arrive in discovery order, but may be dropped under load or fall outside the final sorted page. Thread fields use stored metadata and the current watched status; live thread settings are enriched in the final response. Replace provisional rows with the authoritative final response. No partials follow it or an error.",
"properties": {
"data": {
"items": {
"$ref": "#/definitions/ThreadSearchResult"
},
"type": "array"
},
"requestId": {
"$ref": "#/definitions/RequestId"
}
},
"required": [
"data",
"requestId"
],
"title": "ThreadSearchPartialNotification",
"type": "object"
},
"ThreadSearchResult": {
"properties": {
"snippet": {
File diff suppressed because it is too large Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,15 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { RequestId } from "../RequestId";
import type { ThreadSearchResult } from "./ThreadSearchResult";
/**
* A best-effort incremental batch for `thread/search/stream`. Match `requestId` to the request
* on this connection. Only a small subset of verified matches may be emitted; rows are unique
* and arrive in discovery order, but may be dropped under load or fall outside the final sorted
* page. Thread fields use stored metadata
* and the current watched status; live thread settings are enriched in the final response.
* Replace provisional rows with the authoritative final response. No partials follow it or an error.
*/
export type ThreadSearchPartialNotification = { requestId: RequestId, data: Array<ThreadSearchResult>, };
@@ -611,6 +611,7 @@ export type { ThreadResumeResponse } from "./ThreadResumeResponse";
export type { ThreadRevertParams } from "./ThreadRevertParams";
export type { ThreadRevertResponse } from "./ThreadRevertResponse";
export type { ThreadRevertedNotification } from "./ThreadRevertedNotification";
export type { ThreadSearchPartialNotification } from "./ThreadSearchPartialNotification";
export type { ThreadSearchResult } from "./ThreadSearchResult";
export type { ThreadSearchSortKey } from "./ThreadSearchSortKey";
export type { ThreadSection } from "./ThreadSection";
@@ -885,6 +885,29 @@ client_request_definitions! {
serialization: None,
response: v2::ThreadSearchResponse,
},
#[experimental("thread/search/stream")]
/// Search with best-effort incremental `thread/search/partial` notifications. The final
/// response is authoritative for results, ordering, and cursors.
ThreadSearchStream => "thread/search/stream" {
params: v2::ThreadSearchParams,
serialization: None,
manual_payload_conversion: manual,
response: v2::ThreadSearchResponse,
},
#[experimental("thread/search/cancel")]
ThreadSearchCancel => "thread/search/cancel" {
params: v2::ThreadSearchCancelParams,
serialization: None,
response: v2::ThreadSearchCancelResponse,
},
#[experimental("thread/search/stream/cancel")]
/// Cancel a `thread/search/stream` request on this connection.
ThreadSearchStreamCancel => "thread/search/stream/cancel" {
params: v2::ThreadSearchCancelParams,
serialization: None,
manual_payload_conversion: manual,
response: v2::ThreadSearchCancelResponse,
},
#[experimental("thread/searchOccurrences")]
ThreadSearchOccurrences => "thread/searchOccurrences" {
params: v2::ThreadSearchOccurrencesParams,
@@ -2109,6 +2132,8 @@ server_notification_definitions! {
/// Stream base64-encoded stdout/stderr chunks for a running `process/spawn` session.
#[experimental("process/outputDelta")]
ProcessOutputDelta => "process/outputDelta" (v2::ProcessOutputDeltaNotification),
#[experimental("thread/search/partial")]
ThreadSearchPartial => "thread/search/partial" (v2::ThreadSearchPartialNotification),
/// Final exit notification for a `process/spawn` session.
#[experimental("process/exited")]
ProcessExited => "process/exited" (v2::ProcessExitedNotification),
@@ -19,6 +19,7 @@ use super::TurnStatus;
use super::UserInput;
use super::shared::v2_enum_from_core;
use crate::JsonSchema;
use crate::RequestId;
use crate::TS;
use codex_experimental_api_macros::ExperimentalApi;
pub use codex_protocol::capabilities::CapabilityRootLocation;
@@ -1573,7 +1574,8 @@ pub struct ThreadSearchParams {
/// If false or null, only non-archived threads are returned.
#[ts(optional = nullable)]
pub archived: Option<bool>,
/// Required substring/full-text query for thread search.
/// Required substring/full-text query. `thread/search/stream` limits this to 65,536
/// UTF-8 bytes after trimming surrounding whitespace.
pub search_term: String,
}
@@ -1671,6 +1673,37 @@ pub struct ThreadSearchResponse {
pub backwards_cursor: Option<String>,
}
/// A best-effort incremental batch for `thread/search/stream`. Match `requestId` to the request
/// on this connection. Only a small subset of verified matches may be emitted; rows are unique
/// and arrive in discovery order, but may be dropped under load or fall outside the final sorted
/// page. Thread fields use stored metadata
/// and the current watched status; live thread settings are enriched in the final response.
/// Replace provisional rows with the authoritative final response. No partials follow it or an error.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadSearchPartialNotification {
pub request_id: RequestId,
pub data: Vec<ThreadSearchResult>,
}
/// Request cancellation on the same connection. `thread/search/cancel` targets only
/// `thread/search`; `thread/search/stream/cancel` targets only `thread/search/stream`.
/// Cancellation is idempotent: the cancel response acknowledges the request, and an interrupted
/// search completes with an error after its subprocess is stopped and reaped. If completion races
/// cancellation, the original search may still succeed. An already-completed search is unaffected.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadSearchCancelParams {
pub request_id: RequestId,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadSearchCancelResponse {}
/// Parameters for searching visible message occurrences within one paginated thread.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
+17
View File
@@ -37,6 +37,23 @@ across account switches. Resume and revert preserve that distinction; forks are
Scoped requests skip candidates whose rollout header cannot be read. API-key
logins see all local history. Direct access by thread ID remains available.
# Streaming thread search (experimental)
`thread/search` continues to return a single paginated response. Clients that initialize
with `experimentalApi: true` can instead call `thread/search/stream` with the same params
(a maximum of 65,536 UTF-8 bytes for the trimmed search term).
It emits best-effort `thread/search/partial` notifications containing the original
`requestId` and a small subset of verified results. Partials may be dropped, may fall
outside the final page, and use stored thread metadata; replace them with the final
response, which is authoritative for results, ordering, and cursors.
Send `thread/search/stream/cancel` with `{requestId}` on the same connection to cancel
the original streaming request. A client can use a UUID string as the search request ID. The cancel RPC has its own ID and returns `{}`; the
search completes independently with an error if cancellation wins, or may succeed if
completion races cancellation. Unknown or finished IDs are no-ops. `thread/search/cancel`
targets only the existing nonstreaming method. A JavaScript client can connect an
`AbortSignal` listener to the corresponding cancel RPC.
# Thread list ID filters
`thread/list` accepts `includedThreadIds` to list specific saved threads and
+6 -4
View File
@@ -9,6 +9,7 @@ use codex_core::config::Config;
use codex_feedback::CodexFeedback;
use codex_state::LogWriteFailureReporter;
use crate::outgoing_message::NotificationTarget;
use crate::outgoing_message::OutgoingMessageSender;
const LOG_WRITE_WARNING_WITH_FEEDBACK: &str = "Codex couldn't save diagnostic logs to its local database. Use /feedback with logs included before closing Codex, or run `codex doctor` for diagnostics.";
@@ -45,12 +46,13 @@ impl LogWriteWarningReporter {
if !self.has_failure.swap(true, Ordering::Relaxed)
&& let Some(outgoing) = self.outgoing.upgrade()
{
outgoing.try_send_server_notification(ServerNotification::Warning(
WarningNotification {
outgoing.try_send_server_notification(
NotificationTarget::Broadcast,
ServerNotification::Warning(WarningNotification {
thread_id: None,
message: self.message.to_string(),
},
));
}),
);
}
}
}
+88 -1
View File
@@ -47,6 +47,7 @@ use crate::request_processors::ThreadGoalRequestProcessor;
use crate::request_processors::ThreadQueueRequestProcessor;
use crate::request_processors::ThreadRequestProcessor;
use crate::request_processors::ThreadResumeTarget;
use crate::request_processors::ThreadSearchDelivery;
use crate::request_processors::TurnRequestProcessor;
use crate::request_processors::WindowsSandboxRequestProcessor;
use crate::request_processors::account_processor::handle_gcp_setup;
@@ -75,6 +76,8 @@ use codex_app_server_protocol::JSONRPCNotification;
use codex_app_server_protocol::JSONRPCRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::ThreadSampleCancelResponse;
use codex_app_server_protocol::ThreadSearchCancelResponse;
use codex_app_server_protocol::ThreadSearchParams;
use codex_app_server_protocol::UserVerificationCancelResponse;
use codex_app_server_protocol::experimental_required_message;
use codex_arg0::Arg0DispatchPaths;
@@ -1084,6 +1087,31 @@ impl MessageProcessor {
),
_ => None,
};
let search_cancel_target = match &codex_request {
ClientRequest::ThreadSearchCancel { params, .. } => {
self.outgoing
.search_cancellation_target(
&ConnectionRequestId {
connection_id,
request_id: params.request_id.clone(),
},
/*streaming*/ false,
)
.await
}
ClientRequest::ThreadSearchStreamCancel { params, .. } => {
self.outgoing
.search_cancellation_target(
&ConnectionRequestId {
connection_id,
request_id: params.request_id.clone(),
},
/*streaming*/ true,
)
.await
}
_ => None,
};
let serialization_scope = codex_request.serialization_scope();
let error_request_id = connection_request_id.clone();
let rpc_gate = Arc::clone(&session.rpc_gate);
@@ -1112,6 +1140,7 @@ impl MessageProcessor {
request_context,
session,
event_stream_ready,
search_cancel_target,
),
))
.await;
@@ -1135,6 +1164,32 @@ impl MessageProcessor {
Ok(())
}
async fn handle_thread_search(
&self,
params: ThreadSearchParams,
request_id: ConnectionRequestId,
request_context: &RequestContext,
session: &ConnectionSessionState,
delivery: ThreadSearchDelivery,
) -> Result<Option<ClientResponsePayload>, JSONRPCErrorError> {
let cancellation = &request_context.cancellation;
let search =
self.thread_processor
.thread_search(params, request_id, cancellation.clone(), delivery);
tokio::pin!(search);
let response = tokio::select! {
result = &mut search => result,
_ = session.disconnect.cancelled() => {
cancellation.cancel();
search.await
}
}?;
Ok(Some(match delivery {
ThreadSearchDelivery::FinalOnly => ClientResponsePayload::ThreadSearch(response),
ThreadSearchDelivery::Streaming => ClientResponsePayload::ThreadSearchStream(response),
}))
}
async fn handle_initialized_client_request(
self: Arc<Self>,
connection_request_id: ConnectionRequestId,
@@ -1142,6 +1197,7 @@ impl MessageProcessor {
request_context: RequestContext,
session: Arc<ConnectionSessionState>,
event_stream_ready: Option<McpEventStreamReady>,
search_cancel_target: Option<tokio_util::sync::CancellationToken>,
) -> Result<(), JSONRPCErrorError> {
let connection_id = connection_request_id.connection_id;
let app_server_client_name = session.app_server_client_name().map(str::to_string);
@@ -1607,7 +1663,38 @@ impl MessageProcessor {
self.project_processor.project_delete(params).await
}
ClientRequest::ThreadSearch { params, .. } => {
self.thread_processor.thread_search(params).await
self.handle_thread_search(
params,
request_id.clone(),
&request_context,
&session,
ThreadSearchDelivery::FinalOnly,
)
.await
}
ClientRequest::ThreadSearchStream { params, .. } => {
self.handle_thread_search(
params,
request_id.clone(),
&request_context,
&session,
ThreadSearchDelivery::Streaming,
)
.await
}
ClientRequest::ThreadSearchCancel { .. } => {
if let Some(target) = search_cancel_target {
target.cancel();
}
Ok(Some(ThreadSearchCancelResponse {}.into()))
}
ClientRequest::ThreadSearchStreamCancel { .. } => {
if let Some(target) = search_cancel_target {
target.cancel();
}
Ok(Some(ClientResponsePayload::ThreadSearchStreamCancel(
ThreadSearchCancelResponse {},
)))
}
ClientRequest::ThreadSearchOccurrences { params, .. } => {
self.thread_processor
@@ -139,6 +139,7 @@ pub(crate) fn without_notification_media(notification: ServerNotification) -> Se
| ServerNotification::WindowsWorldWritableWarning(_)
| ServerNotification::WindowsSandboxSetupCompleted(_)
| ServerNotification::AccountLoginCompleted(_)
| ServerNotification::ThreadSearchPartial(_)
| ServerNotification::GatewayOAuthChanged(_) => notification,
}
}
+135 -6
View File
@@ -78,6 +78,8 @@ enum RequestCancellationScope {
Unavailable,
UserVerification,
ThreadSample,
ThreadSearch,
ThreadSearchStream,
}
impl RequestContext {
@@ -100,6 +102,8 @@ impl RequestContext {
| "userVerification/delete"
| "userVerification/verify" => RequestCancellationScope::UserVerification,
"thread/sample" => RequestCancellationScope::ThreadSample,
"thread/search" => RequestCancellationScope::ThreadSearch,
"thread/search/stream" => RequestCancellationScope::ThreadSearchStream,
_ => RequestCancellationScope::Unavailable,
},
span,
@@ -133,6 +137,11 @@ pub(crate) enum OutgoingEnvelope {
},
}
pub(crate) enum NotificationTarget {
Broadcast,
Connection(ConnectionId),
}
/// Sends messages to the client and manages request callbacks.
pub(crate) struct OutgoingMessageSender {
verification_auth: OnceLock<Arc<codex_login::AuthManager>>,
@@ -319,6 +328,25 @@ impl OutgoingMessageSender {
}
}
/// Capture the exact target at ordered ingress; cancellation must not resolve a reused ID later.
pub(crate) async fn search_cancellation_target(
&self,
request_id: &ConnectionRequestId,
streaming: bool,
) -> Option<tokio_util::sync::CancellationToken> {
let scope = if streaming {
RequestCancellationScope::ThreadSearchStream
} else {
RequestCancellationScope::ThreadSearch
};
self.request_contexts
.lock()
.await
.get(request_id)
.filter(|context| context.cancellation_scope == scope)
.map(|context| context.cancellation.clone())
}
pub(crate) async fn enable_human_approval_connection(&self, connection_id: ConnectionId) {
self.human_approval_connections
.lock()
@@ -334,7 +362,19 @@ impl OutgoingMessageSender {
self.disconnect_user_verification_connection(connection_id)
.await;
let mut request_contexts = self.request_contexts.lock().await;
request_contexts.retain(|request_id, _| request_id.connection_id != connection_id);
request_contexts.retain(|request_id, context| {
if request_id.connection_id != connection_id {
return true;
}
if matches!(
context.cancellation_scope,
RequestCancellationScope::ThreadSearch
| RequestCancellationScope::ThreadSearchStream
) {
context.cancellation.cancel();
}
false
});
}
pub(crate) async fn request_trace_context(
@@ -858,12 +898,35 @@ impl OutgoingMessageSender {
.await;
}
pub(crate) fn try_send_server_notification(&self, notification: ServerNotification) {
if let Err(error) = self.sender.try_send(OutgoingEnvelope::Broadcast {
message: timestamped_server_notification(notification),
}) {
warn!("failed to queue server notification to clients: {error}");
/// Queue an optional notification without delaying the request or other connections.
pub(crate) fn try_send_server_notification(
&self,
target: NotificationTarget,
notification: ServerNotification,
) -> bool {
let permit = match self.sender.try_reserve() {
Ok(permit) => permit,
Err(error) => {
if matches!(target, NotificationTarget::Broadcast) {
warn!("failed to queue server notification to clients: {error}");
}
return false;
}
};
// Leave room for the authoritative response when progress fills the queue.
if matches!(target, NotificationTarget::Connection(_)) && self.sender.capacity() == 0 {
return false;
}
let message = timestamped_server_notification(notification);
permit.send(match target {
NotificationTarget::Broadcast => OutgoingEnvelope::Broadcast { message },
NotificationTarget::Connection(connection_id) => OutgoingEnvelope::ToConnection {
connection_id,
message,
write_complete_tx: None,
},
});
true
}
pub(crate) async fn send_server_notification_to_connections(
@@ -1410,6 +1473,72 @@ mod tests {
}
}
/// Search cancellation is scoped to the owning connection, including on disconnect.
#[tokio::test]
async fn thread_search_cancellation_is_connection_scoped() {
let (tx, _rx) = mpsc::channel::<OutgoingEnvelope>(4);
let outgoing =
OutgoingMessageSender::new(tx, codex_analytics::AnalyticsEventsClient::disabled());
let request = |connection_id| ConnectionRequestId {
connection_id: ConnectionId(connection_id),
request_id: RequestId::String("bf76b226-4c49-4394-a28c-3ff976abb756".to_string()),
};
let first = RequestContext::new(
request(1),
"thread/search",
Span::none(),
/*parent_trace*/ None,
);
let second = RequestContext::new(
request(2),
"thread/search/stream",
Span::none(),
/*parent_trace*/ None,
);
outgoing.register_request_context(first.clone()).await;
outgoing.register_request_context(second.clone()).await;
assert!(
outgoing
.search_cancellation_target(&request(2), /*streaming*/ false)
.await
.is_none()
);
assert!(
outgoing
.search_cancellation_target(&request(1), /*streaming*/ true)
.await
.is_none()
);
assert!(!first.cancellation.is_cancelled());
assert!(!second.cancellation.is_cancelled());
outgoing
.search_cancellation_target(&request(2), /*streaming*/ true)
.await
.expect("stream target")
.cancel();
assert!(second.cancellation.is_cancelled());
assert!(!first.cancellation.is_cancelled());
let captured = outgoing
.search_cancellation_target(&request(1), /*streaming*/ false)
.await
.expect("original target");
let replacement = RequestContext::new(
request(1),
"thread/search",
Span::none(),
/*parent_trace*/ None,
);
outgoing.register_request_context(replacement.clone()).await;
captured.cancel();
assert!(first.cancellation.is_cancelled());
assert!(!replacement.cancellation.is_cancelled());
outgoing.connection_closed(ConnectionId(1)).await;
assert!(replacement.cancellation.is_cancelled());
}
#[tokio::test]
async fn send_response_clears_registered_request_context() {
let (tx, _rx) = mpsc::channel::<OutgoingEnvelope>(4);
@@ -604,6 +604,7 @@ pub(crate) use search::SearchRequestProcessor;
pub(crate) use thread_goal_processor::ThreadGoalRequestProcessor;
pub(crate) use thread_processor::ThreadRequestProcessor;
pub(crate) use thread_processor::ThreadResumeTarget;
pub(crate) use thread_processor::ThreadSearchDelivery;
pub(crate) use thread_queue_processor::ThreadQueueRequestProcessor;
pub(crate) use turn_processor::TurnRequestProcessor;
pub(crate) use windows_sandbox_processor::WindowsSandboxRequestProcessor;
@@ -40,6 +40,7 @@ use codex_app_server_protocol::ThreadRevertResponse;
use codex_app_server_protocol::ThreadRevertedNotification;
use codex_app_server_protocol::ThreadSampleParams;
use codex_app_server_protocol::ThreadSampleResponse;
use codex_app_server_protocol::ThreadSearchPartialNotification;
use codex_app_server_protocol::ThreadSection;
use codex_app_server_protocol::ThreadSectionAppearance;
use codex_app_server_protocol::ThreadSectionChangedNotification;
@@ -57,7 +58,11 @@ use codex_protocol::protocol::ThreadHistoryMode;
use codex_thread_store::LocalHistoryAccount;
use codex_thread_store::PendingDaybreakPreference;
use codex_thread_store::PersistContext;
use codex_thread_store::StoredThreadSearchResult;
use codex_thread_store::ThreadSearchObserver;
use codex_thread_store::ThreadSearchSourceKind;
use std::ops::ControlFlow;
use tokio::sync::mpsc;
pub(super) const THREAD_LIST_DEFAULT_LIMIT: usize = 25;
pub(super) const THREAD_LIST_MAX_LIMIT: usize = 100;
@@ -65,6 +70,14 @@ const THREAD_LIST_MAX_FILTER_IDS: usize = 100;
const PAGINATED_FULL_HISTORY_DEPRECATION_SUMMARY: &str = "Full-history hydration is deprecated for paginated threads; use `excludeTurns: true`, then page with `thread/turns/list` and `thread/items/list`.";
const PAGINATED_THREAD_READ_DEPRECATION_SUMMARY: &str = "Full-history hydration is deprecated for paginated threads; omit `includeTurns` or set it to `false`, then page with `thread/turns/list` and `thread/items/list`.";
const MAX_STREAM_SEARCH_TERM_BYTES: usize = 64 * 1024;
#[derive(Clone, Copy)]
pub(crate) enum ThreadSearchDelivery {
FinalOnly,
Streaming,
}
async fn stage_pending_thread_metadata(
thread_manager: &ThreadManager,
thread_store: &dyn ThreadStore,
@@ -967,10 +980,12 @@ impl ThreadRequestProcessor {
pub(crate) async fn thread_search(
&self,
params: ThreadSearchParams,
) -> Result<Option<ClientResponsePayload>, JSONRPCErrorError> {
self.thread_search_response_inner(params)
request_id: ConnectionRequestId,
cancellation: CancellationToken,
delivery: ThreadSearchDelivery,
) -> Result<ThreadSearchResponse, JSONRPCErrorError> {
self.thread_search_response_inner(params, request_id, cancellation, delivery)
.await
.map(|response| Some(response.into()))
}
pub(crate) async fn thread_search_occurrences(
@@ -2940,6 +2955,9 @@ impl ThreadRequestProcessor {
async fn thread_search_response_inner(
&self,
params: ThreadSearchParams,
request_id: ConnectionRequestId,
cancellation: CancellationToken,
delivery: ThreadSearchDelivery,
) -> Result<ThreadSearchResponse, JSONRPCErrorError> {
let ThreadSearchParams {
include_attached_threads,
@@ -2956,6 +2974,13 @@ impl ThreadRequestProcessor {
let search_term = (!search_term.is_empty())
.then_some(search_term)
.ok_or_else(|| invalid_request("thread/search requires a non-empty searchTerm"))?;
if matches!(delivery, ThreadSearchDelivery::Streaming)
&& search_term.len() > MAX_STREAM_SEARCH_TERM_BYTES
{
return Err(invalid_params(format!(
"thread/search/stream searchTerm exceeds {MAX_STREAM_SEARCH_TERM_BYTES} UTF-8 bytes"
)));
}
let requested_page_size = limit
.map(|value| value as usize)
.unwrap_or(THREAD_LIST_DEFAULT_LIMIT)
@@ -2967,66 +2992,117 @@ impl ThreadRequestProcessor {
};
let store_sort_direction = sort_direction.unwrap_or(SortDirection::Desc);
let (allowed_sources, source_kind_filter) = compute_source_filters(source_kinds);
let mut cursor_obj = cursor;
let mut last_cursor = cursor_obj.clone();
let mut remaining = requested_page_size;
let account = self.local_history_account(current_account_only);
let mut search_results = Vec::with_capacity(requested_page_size);
let mut next_cursor = None;
while remaining > 0 {
let page = self
.thread_store
.search_threads(StoreSearchThreadsParams {
include_attached_threads,
account: account.clone(),
page_size: remaining.min(THREAD_LIST_MAX_LIMIT),
cursor: cursor_obj.clone(),
sort_key: store_sort_key,
sort_direction: match store_sort_direction {
SortDirection::Asc => StoreSortDirection::Asc,
SortDirection::Desc => StoreSortDirection::Desc,
},
allowed_sources: allowed_sources.clone(),
archived: archived.unwrap_or(false),
search_term: search_term.clone(),
let source_kinds = source_kind_filter.map(|kinds| {
kinds
.into_iter()
.map(|kind| match kind {
ThreadSourceKind::Cli => ThreadSearchSourceKind::Cli,
ThreadSourceKind::VsCode => ThreadSearchSourceKind::VsCode,
ThreadSourceKind::Exec => ThreadSearchSourceKind::Exec,
ThreadSourceKind::AppServer => ThreadSearchSourceKind::AppServer,
ThreadSourceKind::SubAgent => ThreadSearchSourceKind::SubAgent,
ThreadSourceKind::SubAgentReview => ThreadSearchSourceKind::SubAgentReview,
ThreadSourceKind::SubAgentCompact => ThreadSearchSourceKind::SubAgentCompact,
ThreadSourceKind::SubAgentThreadSpawn => {
ThreadSearchSourceKind::SubAgentThreadSpawn
}
ThreadSourceKind::SubAgentOther => ThreadSearchSourceKind::SubAgentOther,
ThreadSourceKind::Unknown => ThreadSearchSourceKind::Unknown,
})
.await
.map_err(thread_store_list_error)?;
for result in page.items {
let source = with_thread_spawn_agent_metadata(
result.thread.source.clone(),
result.thread.agent_nickname.clone(),
result.thread.agent_role.clone(),
);
if source_kind_filter
.as_ref()
.is_none_or(|filter| source_kind_matches(&source, filter))
{
search_results.push(result);
if search_results.len() >= requested_page_size {
break;
.collect()
});
let account = self.local_history_account(current_account_only);
let (progress, mut batches) = if matches!(delivery, ThreadSearchDelivery::Streaming) {
let (sender, receiver) = mpsc::channel(2);
(Some(sender), Some(receiver))
} else {
(None, None)
};
let search = self.thread_store.search_threads_observed(
StoreSearchThreadsParams {
include_attached_threads,
account,
page_size: requested_page_size,
cursor,
sort_key: store_sort_key,
sort_direction: match store_sort_direction {
SortDirection::Asc => StoreSortDirection::Asc,
SortDirection::Desc => StoreSortDirection::Desc,
},
allowed_sources,
source_kinds,
archived: archived.unwrap_or(false),
search_term,
},
ThreadSearchObserver {
progress,
cancelled: Box::pin(cancellation.clone().cancelled_owned()),
},
);
tokio::pin!(search);
let mut emitted = HashSet::new();
let page = loop {
tokio::select! {
biased;
page = &mut search => break page,
batch = async {
match &mut batches {
Some(receiver) => receiver.recv().await,
None => std::future::pending().await,
}
} => {
let Some(batch) = batch else {
batches = None;
continue;
};
let mut batch_ids = HashSet::new();
let mut accepted = Vec::new();
for result in batch {
let id = result.thread.thread_id;
if !emitted.contains(&id) && batch_ids.insert(id) {
accepted.push(result);
if accepted.len() == requested_page_size.saturating_sub(emitted.len()) {
break;
}
}
}
let batch = accepted;
if batch.is_empty() {
continue;
}
let data = tokio::select! {
biased;
page = &mut search => break page,
data = self.thread_search_provisional_rows(batch) => data,
};
if self.outgoing.try_send_server_notification(
crate::outgoing_message::NotificationTarget::Connection(request_id.connection_id),
ServerNotification::ThreadSearchPartial(ThreadSearchPartialNotification {
request_id: request_id.request_id.clone(),
data,
}),
) {
emitted.extend(batch_ids);
}
if emitted.len() >= requested_page_size {
batches = None;
}
}
}
remaining = requested_page_size.saturating_sub(search_results.len());
next_cursor = page.next_cursor;
if remaining == 0 {
break;
}
let Some(cursor_val) = next_cursor.clone() else {
break;
};
if last_cursor.as_ref() == Some(&cursor_val) {
next_cursor = None;
break;
}
last_cursor = Some(cursor_val.clone());
cursor_obj = Some(cursor_val);
};
if cancellation.is_cancelled() {
return Err(invalid_request("thread search cancelled"));
}
let page = page.map_err(|err| match err {
ThreadStoreError::InvalidRequest { message }
if matches!(delivery, ThreadSearchDelivery::Streaming) =>
{
invalid_params(message)
}
err => thread_store_list_error(err),
})?;
let search_results = page.items;
let next_cursor = page.next_cursor;
let backwards_cursor = search_results.first().and_then(|result| {
thread_backwards_cursor_for_sort_key(
@@ -3035,12 +3111,27 @@ impl ThreadRequestProcessor {
store_sort_direction,
)
});
let fallback_provider = self.config.model_provider_id.clone();
let data = tokio::select! {
biased;
() = cancellation.cancelled() => return Err(invalid_request("thread search cancelled")),
data = self.thread_search_rows(search_results) => data,
};
Ok(ThreadSearchResponse {
data,
next_cursor,
backwards_cursor,
})
}
fn thread_search_stored_rows(
&self,
search_results: Vec<StoredThreadSearchResult>,
) -> Vec<ThreadSearchResult> {
let mut data = Vec::with_capacity(search_results.len());
for result in search_results {
let (thread, _) = thread_from_stored_thread(
result.thread,
fallback_provider.as_str(),
&self.config.model_provider_id,
&self.config.cwd,
);
data.push(ThreadSearchResult {
@@ -3049,6 +3140,31 @@ impl ThreadRequestProcessor {
});
}
data
}
async fn thread_search_provisional_rows(
&self,
search_results: Vec<StoredThreadSearchResult>,
) -> Vec<ThreadSearchResult> {
let mut data = self.thread_search_stored_rows(search_results);
let statuses = self
.thread_watch_manager
.loaded_statuses_for_threads(data.iter().map(|result| result.thread.id.clone()))
.await;
for result in &mut data {
if let Some(status) = statuses.get(&result.thread.id) {
result.thread.status = status.clone();
}
}
data
}
async fn thread_search_rows(
&self,
search_results: Vec<StoredThreadSearchResult>,
) -> Vec<ThreadSearchResult> {
let mut data = self.thread_search_stored_rows(search_results);
enrich_loaded_threads(
&self.thread_manager,
&self.thread_watch_manager,
@@ -3057,11 +3173,7 @@ impl ThreadRequestProcessor {
)
.await;
Ok(ThreadSearchResponse {
data,
next_cursor,
backwards_cursor,
})
data
}
async fn thread_loaded_list_response_inner(
+16 -1
View File
@@ -1,6 +1,7 @@
use crate::message_processor::ConnectionSessionState;
use crate::outgoing_message::OutgoingEnvelope;
use codex_app_server_protocol::ExperimentalApi;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::ServerRequest;
use std::collections::HashMap;
use std::collections::HashSet;
@@ -154,11 +155,25 @@ async fn send_message_to_connection(
}
let writer = connection_state.writer.clone();
let optional_progress = matches!(
&message,
OutgoingMessage::AppServerNotification(envelope)
if matches!(&envelope.notification, ServerNotification::ThreadSearchPartial(_))
);
let queued_message = QueuedOutgoingMessage {
message,
write_complete_tx,
};
if connection_state.can_disconnect() {
if optional_progress {
// Keep room for the required response that follows optional progress.
if writer.capacity() <= 1 {
return false;
}
if let Err(mpsc::error::TrySendError::Closed(_)) = writer.try_send(queued_message) {
return disconnect_connection(connections, connection_id);
}
false
} else if connection_state.can_disconnect() {
match writer.try_send(queued_message) {
Ok(()) => false,
Err(mpsc::error::TrySendError::Full(_)) => {
@@ -5,6 +5,7 @@ use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::ServerNotificationEnvelope;
use codex_app_server_protocol::ThreadRealtimeStartedNotification;
use codex_app_server_protocol::ThreadSearchPartialNotification;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_utils_absolute_path::AbsolutePathBuf;
use pretty_assertions::assert_eq;
@@ -31,6 +32,50 @@ fn app_server_notification(notification: ServerNotification) -> OutgoingMessage
})
}
#[tokio::test]
async fn optional_search_progress_does_not_disconnect_a_slow_connection() {
let connection_id = ConnectionId(7);
let (writer_tx, mut writer_rx) = mpsc::channel(1);
let disconnect = CancellationToken::new();
let mut connections = HashMap::from([(
connection_id,
OutboundConnectionState::new(
writer_tx.clone(),
Arc::new(AtomicBool::new(true)),
Arc::new(AtomicBool::new(true)),
Arc::new(RwLock::new(HashSet::new())),
Some(disconnect.clone()),
),
)]);
route_outgoing_envelope(
&mut connections,
OutgoingEnvelope::ToConnection {
connection_id,
message: app_server_notification(ServerNotification::ThreadSearchPartial(
ThreadSearchPartialNotification {
request_id: RequestId::Integer(4),
data: Vec::new(),
},
)),
write_complete_tx: None,
},
)
.await;
route_outgoing_envelope(
&mut connections,
OutgoingEnvelope::ToConnection {
connection_id,
message: app_server_notification(thread_realtime_started_notification()),
write_complete_tx: None,
},
)
.await;
assert!(!disconnect.is_cancelled());
assert!(connections.contains_key(&connection_id));
assert!(writer_rx.try_recv().is_ok());
assert!(writer_rx.try_recv().is_err());
}
#[tokio::test]
async fn to_connection_notification_respects_opt_out_filters() {
let connection_id = ConnectionId(7);
@@ -567,6 +567,15 @@ pub(super) async fn spawn_websocket_server_with_args(
codex_home: &Path,
listen_url: &str,
extra_args: &[String],
) -> Result<(Child, SocketAddr)> {
spawn_websocket_server_with_env(codex_home, listen_url, extra_args, &[]).await
}
pub(super) async fn spawn_websocket_server_with_env(
codex_home: &Path,
listen_url: &str,
extra_args: &[String],
env_overrides: &[(&str, &std::ffi::OsStr)],
) -> Result<(Child, SocketAddr)> {
let program = codex_utils_cargo_bin::cargo_bin("codex-app-server")
.context("should find app-server binary")?;
@@ -581,6 +590,9 @@ pub(super) async fn spawn_websocket_server_with_args(
.env("CODEX_HOME", codex_home)
.env("RUST_LOG", "warn")
.env("OTEL_METRIC_EXPORT_INTERVAL", "200");
for (key, value) in env_overrides {
cmd.env(key, value);
}
let mut process = cmd
.kill_on_drop(true)
@@ -172,6 +172,9 @@ mod thread_read_state;
mod thread_resume;
mod thread_revert;
mod thread_sample;
#[cfg(target_os = "linux")]
#[path = "thread_search_streaming_tests.rs"]
mod thread_search_streaming;
mod thread_sections;
mod thread_settings_update;
mod thread_shell_command;
@@ -1290,6 +1290,22 @@ async fn thread_search_filters_by_source_kind() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn thread_search_preserves_large_query_compatibility_on_empty_history() -> Result<()> {
let codex_home = TempDir::new()?;
create_minimal_config(codex_home.path())?;
let mut server = init_mcp(codex_home.path()).await?;
for term in ["s".repeat(65_536), "x".repeat(65_537)] {
let request_id = server
.send_raw_request("thread/search", Some(json!({"searchTerm": term})))
.await?;
let response: ThreadSearchResponse =
timeout(DEFAULT_READ_TIMEOUT, server.read_response(request_id)).await??;
assert!(response.data.is_empty());
}
Ok(())
}
#[tokio::test]
async fn thread_list_db_only_errors_distinguish_unavailable_database_from_empty_history()
-> Result<()> {
@@ -0,0 +1,418 @@
#![cfg(test)]
//! Exercises optional search progress and cancellation over the public JSON-RPC transport.
use super::connection_handling_websocket as ws;
use anyhow::Context;
use anyhow::Result;
use app_test_support::create_fake_rollout_with_source;
use app_test_support::rollout_path;
use chrono::DateTime;
use chrono::Utc;
use codex_app_server_protocol::JSONRPCMessage;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ThreadSearchPartialNotification;
use codex_app_server_protocol::ThreadSearchResponse;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_utils_absolute_path::test_support::PathExt;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::HashSet;
use std::ffi::OsString;
use std::path::Path;
use std::path::PathBuf;
use tempfile::TempDir;
use tokio::time::Duration;
use tokio::time::timeout;
struct SearchFixture {
home: TempDir,
path: OsString,
exec_ids: Vec<String>,
}
impl SearchFixture {
async fn new() -> Result<Self> {
let home = TempDir::new()?;
ws::create_config_toml(home.path(), "http://127.0.0.1:9", "never")?;
let sqlite = codex_state::SqliteConfig::new_for_testing(home.path().abs());
let db = codex_state::StateRuntime::init(sqlite, "mock_provider".to_string()).await?;
let mut exec_ids = Vec::new();
let mut paths = Vec::new();
for (hour, source) in [
(10, SessionSource::Exec),
(11, SessionSource::Cli),
(12, SessionSource::Exec),
(13, SessionSource::Exec),
] {
let filename_ts = format!("2025-02-01T{hour:02}-00-00");
let timestamp = format!("2025-02-01T{hour:02}:00:00Z");
let id = create_fake_rollout_with_source(
home.path(),
&filename_ts,
&timestamp,
"visible needle message",
Some("mock_provider"),
/*git_info*/ None,
source.clone(),
)?;
let path = rollout_path(home.path(), &filename_ts, &id);
let created_at = DateTime::parse_from_rfc3339(&timestamp)?.with_timezone(&Utc);
let mut metadata = codex_state::ThreadMetadataBuilder::new(
ThreadId::from_string(&id)?,
path.clone(),
created_at,
source.clone(),
)
.with_model_provider(Some("mock_provider".to_string()))
.with_cwd(home.path().to_path_buf())
.with_cli_version("0.0.0".to_string())
.build("mock_provider");
metadata.first_user_message = Some("visible needle message".to_string());
db.upsert_thread(&metadata).await?;
if matches!(source, SessionSource::Exec) {
exec_ids.push(id);
}
paths.push(path);
}
db.mark_backfill_complete(/*last_watermark*/ None).await?;
let root = home.path().join("sessions");
let candidates = paths
.iter()
.map(|path| format!("{}\0{}\n", path.display(), 1))
.collect::<String>();
std::fs::write(root.join("rg.candidates"), candidates)?;
let bin = home.path().join("bin");
std::fs::create_dir(&bin)?;
let rg = bin.join("rg");
codex_utils_cargo_bin::write_executable(
&rg,
r#"#!/bin/sh
for root do :; done
printf '%s\n' "$$" > "$root/rg.pid"
cat "$root/rg.candidates"
while ! test -f "$root/rg.release"; do sleep 0.02; done
"#,
)?;
let inherited_path = std::env::var_os("PATH").unwrap_or_default();
let path = std::env::join_paths(
std::iter::once(bin).chain(std::env::split_paths(&inherited_path)),
)?;
Ok(Self {
home,
path,
exec_ids,
})
}
fn root(&self) -> PathBuf {
self.home.path().join("sessions")
}
async fn spawn(&self) -> Result<(tokio::process::Child, std::net::SocketAddr)> {
ws::spawn_websocket_server_with_env(
self.home.path(),
"ws://127.0.0.1:0",
&[],
&[("PATH", self.path.as_os_str())],
)
.await
}
}
async fn initialize(stream: &mut ws::WsClient, id: i64) -> Result<()> {
ws::send_request(
stream,
"initialize",
id,
Some(json!({
"clientInfo": {"name": "search-test", "version": "0.1.0"},
"capabilities": {"experimentalApi": true}
})),
)
.await?;
ws::read_response_for_id(stream, id).await?;
Ok(())
}
fn params() -> serde_json::Value {
json!({
"searchTerm": "needle",
"sourceKinds": ["exec"],
"sortKey": "created_at",
"sortDirection": "desc",
"limit": 2
})
}
async fn pid(root: &Path) -> Result<String> {
timeout(Duration::from_secs(5), async {
loop {
if let Ok(pid) = tokio::fs::read_to_string(root.join("rg.pid")).await
&& let Ok(pid) = pid.trim().parse::<u32>()
&& pid > 0
{
return Ok::<_, anyhow::Error>(pid.to_string());
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.context("gated ripgrep did not start")?
}
async fn assert_reaped(pid: &str) -> Result<()> {
let proc_path = Path::new("/proc").join(pid);
timeout(Duration::from_secs(5), async {
while proc_path.exists() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.context("ripgrep was not promptly reaped")?;
Ok(())
}
#[tokio::test]
async fn streaming_search_is_incremental_filtered_and_matches_final_response() -> Result<()> {
let fixture = SearchFixture::new().await?;
let (mut server, addr) = fixture.spawn().await?;
let mut denied = ws::connect_websocket(addr).await?;
ws::send_initialize_request(&mut denied, /*id*/ 1, "search-without-experimental").await?;
ws::read_response_for_id(&mut denied, /*id*/ 1).await?;
ws::send_request(
&mut denied,
"thread/search/stream",
/*id*/ 2,
Some(params()),
)
.await?;
let error = ws::read_error_for_id(&mut denied, /*id*/ 2).await?;
assert_eq!(error.error.code, -32600);
assert_eq!(
error.error.message,
"thread/search/stream requires experimentalApi capability"
);
ws::send_request(
&mut denied,
"thread/search/stream/cancel",
/*id*/ 3,
Some(json!({"requestId": 2})),
)
.await?;
let error = ws::read_error_for_id(&mut denied, /*id*/ 3).await?;
assert_eq!(error.error.code, -32600);
assert_eq!(
error.error.message,
"thread/search/stream/cancel requires experimentalApi capability"
);
let mut client = ws::connect_websocket(addr).await?;
initialize(&mut client, /*id*/ 1).await?;
let mut oversized = params();
oversized["searchTerm"] = json!("é".repeat(32_769));
ws::send_request(
&mut client,
"thread/search/stream",
/*id*/ 90,
Some(oversized),
)
.await?;
assert_eq!(
ws::read_error_for_id(&mut client, /*id*/ 90)
.await?
.error
.code,
-32602
);
let mut expensive = params();
expensive["searchTerm"] = json!("s".repeat(65_536));
ws::send_request(
&mut client,
"thread/search/stream",
/*id*/ 91,
Some(expensive),
)
.await?;
assert_eq!(
ws::read_error_for_id(&mut client, /*id*/ 91)
.await?
.error
.code,
-32602
);
let mut streaming_params = params();
streaming_params["sortDirection"] = json!("asc");
ws::send_request(
&mut client,
"thread/search/stream",
/*id*/ 2,
Some(streaming_params),
)
.await?;
let child_pid = pid(&fixture.root()).await?;
let notice = ws::read_notification_for_method(&mut client, "thread/search/partial").await?;
let partial: ThreadSearchPartialNotification =
serde_json::from_value(notice.params.context("missing partial params")?)?;
assert_eq!(partial.request_id, RequestId::Integer(2));
assert!(!partial.data.is_empty());
assert!(
partial
.data
.iter()
.all(|row| fixture.exec_ids.contains(&row.thread.id))
);
let mut streamed_ids = partial
.data
.into_iter()
.map(|row| row.thread.id)
.collect::<Vec<_>>();
assert!(Path::new("/proc").join(&child_pid).exists());
tokio::fs::write(fixture.root().join("rg.release"), []).await?;
let streamed: ThreadSearchResponse = loop {
match ws::read_jsonrpc_message(&mut client).await? {
JSONRPCMessage::Notification(notice) if notice.method == "thread/search/partial" => {
let partial: ThreadSearchPartialNotification =
serde_json::from_value(notice.params.context("missing partial params")?)?;
assert_eq!(partial.request_id, RequestId::Integer(2));
streamed_ids.extend(partial.data.into_iter().map(|row| row.thread.id));
}
JSONRPCMessage::Response(response) if response.id == RequestId::Integer(2) => {
break serde_json::from_value(response.result)?;
}
_ => {}
}
};
assert!(streamed_ids.len() <= 2);
assert_eq!(
streamed_ids.iter().collect::<HashSet<_>>().len(),
streamed_ids.len()
);
assert!(streamed_ids.iter().all(|id| fixture.exec_ids.contains(id)));
assert_eq!(
streamed
.data
.iter()
.map(|row| row.thread.id.clone())
.collect::<Vec<_>>(),
fixture.exec_ids.iter().take(2).cloned().collect::<Vec<_>>()
);
assert!(streamed.next_cursor.is_some());
let mut plain_params = params();
plain_params["sortDirection"] = json!("asc");
ws::send_request(
&mut client,
"thread/search",
/*id*/ 3,
Some(plain_params),
)
.await?;
let plain: ThreadSearchResponse = loop {
match ws::read_jsonrpc_message(&mut client).await? {
JSONRPCMessage::Notification(notice) if notice.method == "thread/search/partial" => {
anyhow::bail!("thread/search unexpectedly emitted partial results");
}
JSONRPCMessage::Response(response) if response.id == RequestId::Integer(3) => {
break serde_json::from_value(response.result)?;
}
_ => {}
}
};
assert_eq!(streamed, plain);
server.kill().await?;
Ok(())
}
#[tokio::test]
async fn search_cancel_and_disconnect_stop_only_the_owning_search() -> Result<()> {
let fixture = SearchFixture::new().await?;
let (mut server, addr) = fixture.spawn().await?;
let mut owner = ws::connect_websocket(addr).await?;
let mut other = ws::connect_websocket(addr).await?;
initialize(&mut owner, /*id*/ 1).await?;
initialize(&mut other, /*id*/ 1).await?;
for (method, wrong_cancel, cancel, search_id, wrong_id, cancel_id) in [
(
"thread/search/stream",
"thread/search/cancel",
"thread/search/stream/cancel",
7,
8,
10,
),
(
"thread/search",
"thread/search/stream/cancel",
"thread/search/cancel",
9,
11,
12,
),
] {
ws::send_request(&mut owner, method, search_id, Some(params())).await?;
let child = pid(&fixture.root()).await?;
if search_id == 7 {
ws::send_request(
&mut other,
cancel,
/*id*/ 8,
Some(json!({"requestId": search_id})),
)
.await?;
ws::read_response_for_id(&mut other, /*id*/ 8).await?;
assert!(Path::new("/proc").join(&child).exists());
}
ws::send_request(
&mut owner,
wrong_cancel,
wrong_id,
Some(json!({"requestId": search_id})),
)
.await?;
ws::read_response_for_id(&mut owner, wrong_id).await?;
assert!(Path::new("/proc").join(&child).exists());
ws::send_request(
&mut owner,
cancel,
cancel_id,
Some(json!({"requestId": search_id})),
)
.await?;
let (mut acknowledged, mut cancelled) = (false, false);
while !acknowledged || !cancelled {
match ws::read_jsonrpc_message(&mut owner).await? {
JSONRPCMessage::Response(response)
if response.id == RequestId::Integer(cancel_id) =>
{
acknowledged = true
}
JSONRPCMessage::Error(error) if error.id == RequestId::Integer(search_id) => {
assert!(error.error.message.contains("cancelled"));
cancelled = true;
}
_ => {}
}
}
assert_reaped(&child).await?;
tokio::fs::remove_file(fixture.root().join("rg.pid")).await?;
}
ws::send_request(
&mut owner,
"thread/search/stream",
/*id*/ 13,
Some(params()),
)
.await?;
let third_pid = pid(&fixture.root()).await?;
owner.close(None).await?;
assert_reaped(&third_pid).await?;
ws::send_request(
&mut other,
"thread/search/stream/cancel",
/*id*/ 10,
Some(json!({"requestId": 13})),
)
.await?;
ws::read_response_for_id(&mut other, /*id*/ 10).await?;
server.kill().await?;
Ok(())
}
+1
View File
@@ -57,4 +57,5 @@ libc = { workspace = true }
[dev-dependencies]
codex-utils-absolute-path = { workspace = true }
codex-utils-cargo-bin = { workspace = true }
pretty_assertions = { workspace = true }
+1 -4
View File
@@ -302,10 +302,7 @@ async fn search_rollout_matches_uses_logical_path_for_compressed_rollout() -> an
)
.await?;
assert_eq!(
matches.get(rollout_path.as_path()),
Some(&Some("targeted search term".to_string()))
);
assert_eq!(matches, std::collections::HashSet::from([rollout_path]));
Ok(())
}
+3
View File
@@ -161,8 +161,11 @@ pub use recorder::thread_item_from_state_metadata;
pub use reverse_jsonl_scanner::ReverseJsonlScanner;
pub use reverse_jsonl_scanner::ScanOutcome;
pub use rollout_reference_index::RolloutReferenceIndex;
pub use search::RolloutSearchCandidate;
pub use search::RolloutSearchMatches;
pub use search::RolloutSnippetMatcher;
pub use search::search_rollout_matches;
pub use search::search_rollout_matches_with_progress;
pub use search::search_rollout_paths;
pub use session_index::append_thread_name;
pub use session_index::find_thread_meta_by_name_str;
+180 -166
View File
@@ -1,19 +1,23 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Arc;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::strip_user_message_prefix;
use futures::future::join_all;
use futures::FutureExt;
use futures::future::BoxFuture;
use futures::future::pending;
use regex::Regex;
use regex::RegexBuilder;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncReadExt;
use tokio::io::BufReader;
use tokio::process::Command;
use tokio::sync::mpsc;
use super::ARCHIVED_SESSIONS_SUBDIR;
use super::SESSIONS_SUBDIR;
@@ -21,11 +25,22 @@ use super::compression;
use crate::ResponseItemEnvelope;
use crate::RolloutItem;
mod compressed;
mod snippet_matcher;
use compressed::scan_compressed_rollout_matches;
pub use snippet_matcher::RolloutSnippetMatcher;
const MATCH_CONTEXT_BEFORE_CHARS: usize = 48;
const MATCH_CONTEXT_AFTER_CHARS: usize = 96;
/// Search matches keyed by the canonical `.jsonl` path for each rollout.
pub type RolloutSearchMatches = HashMap<PathBuf, Option<String>>;
/// Logical `.jsonl` paths discovered by search, regardless of physical representation.
pub type RolloutSearchMatches = HashSet<PathBuf>;
/// A potential match discovered by the rollout scanner.
#[derive(Debug, PartialEq, Eq)]
pub struct RolloutSearchCandidate {
pub path: PathBuf,
}
pub async fn search_rollout_paths(
rg_command: &Path,
@@ -33,12 +48,7 @@ pub async fn search_rollout_paths(
archived: bool,
search_term: &str,
) -> io::Result<HashSet<PathBuf>> {
Ok(
search_rollout_matches(rg_command, codex_home, archived, search_term)
.await?
.into_keys()
.collect(),
)
search_rollout_matches(rg_command, codex_home, archived, search_term).await
}
pub async fn search_rollout_matches(
@@ -46,6 +56,26 @@ pub async fn search_rollout_matches(
codex_home: &Path,
archived: bool,
search_term: &str,
) -> io::Result<RolloutSearchMatches> {
search_rollout_matches_with_progress(
rg_command,
codex_home,
archived,
search_term,
/*progress*/ None,
Box::pin(pending()),
)
.await
}
/// Search plain rollouts, then compressed rollouts, reporting candidates without blocking.
pub async fn search_rollout_matches_with_progress(
rg_command: &Path,
codex_home: &Path,
archived: bool,
search_term: &str,
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
cancelled: BoxFuture<'_, ()>,
) -> io::Result<RolloutSearchMatches> {
let root = codex_home.join(if archived {
ARCHIVED_SESSIONS_SUBDIR
@@ -53,28 +83,63 @@ pub async fn search_rollout_matches(
SESSIONS_SUBDIR
});
let json_search_term = json_escaped_search_term(search_term)?;
let Some(plain_matches) =
ripgrep_rollout_paths(rg_command, root.as_path(), json_search_term.as_str()).await?
else {
return scan_rollout_matches(root.as_path(), json_search_term.as_str(), search_term).await;
let cancelled = cancelled.shared();
let mut cancellation: BoxFuture<'_, ()> = Box::pin(cancelled.clone());
let mut matches = match ripgrep_rollout_paths(
rg_command,
&root,
&json_search_term,
progress.clone(),
&mut cancellation,
)
.await?
{
Some(paths) => paths,
None => tokio::select! {
biased;
() = cancelled.clone() => return Err(search_cancelled()),
result = scan_plain_rollout_matches(&root, &json_search_term, progress.clone()) => result?,
},
};
let mut matches: RolloutSearchMatches =
plain_matches.into_iter().map(|path| (path, None)).collect();
matches.extend(scan_compressed_rollout_matches(root.as_path(), search_term).await?);
// Compression publishes the archive before removing its plain sibling. Discover archives
// after the plain scan so a transition cannot fall between the two inventories.
let compressed = tokio::select! {
biased;
() = cancelled => return Err(search_cancelled()),
result = scan_compressed_rollout_matches(&root, search_term, progress) => result?,
};
matches.extend(compressed);
Ok(matches)
}
fn search_cancelled() -> io::Error {
io::Error::new(io::ErrorKind::Interrupted, "rollout search cancelled")
}
async fn ripgrep_rollout_paths(
rg_command: &Path,
root: &Path,
search_term: &str,
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
cancelled: &mut BoxFuture<'_, ()>,
) -> io::Result<Option<HashSet<PathBuf>>> {
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
let exists = tokio::select! {
biased;
() = &mut *cancelled => return Err(search_cancelled()),
exists = tokio::fs::try_exists(root) => exists.unwrap_or(false),
};
if !exists {
return Ok(Some(HashSet::new()));
}
let output = match Command::from(codex_utils_process::background_command(rg_command))
.arg("-l")
let mut command = Command::from(codex_utils_process::background_command(rg_command));
// A NUL-only file listing does not flush through ripgrep's line buffer. A one-match
// count adds a newline after each NUL-delimited path and still stops at the first hit.
command.args(["--count", "--max-count", "1", "--null"]);
if progress.is_some() {
command.arg("--line-buffered");
}
let mut child = match command
.arg("--fixed-strings")
.arg("--ignore-case")
.arg("--no-ignore")
@@ -84,49 +149,115 @@ async fn ripgrep_rollout_paths(
.arg(search_term)
.arg(root)
.stdin(Stdio::null())
.output()
.await
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
{
Ok(output) => output,
Err(err) if err.kind() == io::ErrorKind::NotFound => {
return Ok(None);
}
Ok(child) => child,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err),
};
if !output.status.success() {
if output.status.code() == Some(1) && output.stderr.is_empty() {
let mut stdout = BufReader::new(
child
.stdout
.take()
.ok_or_else(|| io::Error::other("missing ripgrep stdout"))?,
);
let mut stderr = child
.stderr
.take()
.ok_or_else(|| io::Error::other("missing ripgrep stderr"))?;
let mut sink = tokio::io::sink();
let read_paths = async {
let mut paths = HashSet::new();
let mut record = Vec::new();
let mut count = [0; 2];
while stdout.read_until(0, &mut record).await? != 0 {
if record.pop() != Some(0) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unterminated ripgrep path",
));
}
stdout.read_exact(&mut count).await?;
if count != *b"1\n" {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid ripgrep count",
));
}
let path = path_from_rg_bytes(&record);
let path = if path.is_absolute() {
path
} else {
root.join(path)
};
if paths.insert(path.clone()) {
report_candidate(&mut progress, RolloutSearchCandidate { path });
}
record.clear();
}
Ok::<_, io::Error>(paths)
};
let (paths, stderr_bytes, status) = tokio::select! {
biased;
() = &mut *cancelled => {
// Cancellation completes only after ripgrep is reaped.
let _ = child.start_kill();
let _ = child.wait().await;
return Err(search_cancelled());
}
result = async {
tokio::try_join!(read_paths, tokio::io::copy(&mut stderr, &mut sink), child.wait())
} => result?,
};
if !status.success() {
if status.code() == Some(1) && stderr_bytes == 0 {
return Ok(Some(HashSet::new()));
}
return Err(io::Error::other(format!(
"ripgrep rollout search failed under {}",
root.display()
)));
}
let mut matches = HashSet::new();
for line in String::from_utf8_lossy(output.stdout.as_slice()).lines() {
let path = PathBuf::from(line);
let path = if path.is_absolute() {
path
} else {
root.join(path)
};
matches.insert(path);
}
Ok(Some(matches))
Ok(Some(paths))
}
async fn scan_rollout_matches(
#[cfg(unix)]
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
use std::os::unix::ffi::OsStrExt;
Path::new(std::ffi::OsStr::from_bytes(bytes)).to_path_buf()
}
#[cfg(not(unix))]
fn path_from_rg_bytes(bytes: &[u8]) -> PathBuf {
String::from_utf8_lossy(bytes).into_owned().into()
}
fn report_candidate(
progress: &mut Option<mpsc::Sender<RolloutSearchCandidate>>,
candidate: RolloutSearchCandidate,
) {
if let Some(sender) = progress
&& matches!(
sender.try_send(candidate),
Err(mpsc::error::TrySendError::Closed(_))
)
{
*progress = None;
}
}
async fn scan_plain_rollout_matches(
root: &Path,
json_search_term: &str,
search_term: &str,
mut progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashMap::new();
let mut matches = HashSet::new();
let mut dirs = vec![root.to_path_buf()];
let json_search_term = case_insensitive_literal_regex(json_search_term)?;
let mut matcher = RolloutSnippetMatcher::new(search_term);
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
@@ -148,27 +279,12 @@ async fn scan_rollout_matches(
continue;
};
if rollout_file.is_compressed() {
if let Some(snippet) = matcher
.first_match(rollout_file.path())
.await
.unwrap_or_else(|err| {
tracing::warn!(
path = %rollout_file.path().display(),
%err,
"Failed to search compressed rollout"
);
None
})
{
matches.insert(
compression::plain_rollout_path(rollout_file.path()),
Some(snippet),
);
}
continue;
}
if rollout_contains(rollout_file.path(), &json_search_term).await? {
matches.insert(rollout_file.into_path(), None);
let path = rollout_file.into_path();
report_candidate(&mut progress, RolloutSearchCandidate { path: path.clone() });
matches.insert(path);
}
}
}
@@ -186,108 +302,6 @@ async fn rollout_contains(path: &Path, search_term: &Regex) -> io::Result<bool>
Ok(false)
}
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
pub struct RolloutSnippetMatcher<'a> {
search_term: &'a str,
patterns: Option<Arc<(Regex, Regex)>>,
}
impl<'a> RolloutSnippetMatcher<'a> {
pub fn new(search_term: &'a str) -> Self {
Self {
search_term,
patterns: None,
}
}
/// Returns the first matching conversation excerpt. The file is opened before compiling
/// the patterns so per-file I/O errors take precedence even on the first call.
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
let patterns = match &self.patterns {
Some(patterns) => patterns,
None => self.patterns.insert(Arc::new((
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
case_insensitive_literal_regex(self.search_term)?,
))),
};
// Share the regexes across file scans to reuse their scratch caches.
let patterns = Arc::clone(patterns);
lines
.find_map(move |line| {
let (json_search_term, search_term) = patterns.as_ref();
if json_search_term.is_match(line) {
content_match_snippet(line, search_term)
} else {
None
}
})
.await
}
}
async fn scan_compressed_rollout_matches(
root: &Path,
search_term: &str,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashMap::new();
let mut paths = Vec::new();
let mut dirs = vec![root.to_path_buf()];
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let file_type = entry.file_type().await?;
if file_type.is_dir() {
dirs.push(path);
continue;
}
if !file_type.is_file() {
continue;
}
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
continue;
};
if !rollout_file.is_compressed() {
continue;
}
paths.push(rollout_file.into_path());
}
}
const COMPRESSED_SEARCH_WORKERS: usize = 4;
let scan = async |paths: &[PathBuf]| {
let mut matches = HashMap::new();
let mut matcher = RolloutSnippetMatcher::new(search_term);
for path in paths {
if let Some(snippet) = matcher.first_match(path).await.unwrap_or_else(|err| {
tracing::warn!(
path = %path.as_path().display(),
%err,
"Failed to search compressed rollout"
);
None
}) {
matches.insert(
compression::plain_rollout_path(path.as_path()),
Some(snippet),
);
}
}
matches
};
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
matches.extend(batch);
}
Ok(matches)
}
fn json_escaped_search_term(search_term: &str) -> io::Result<String> {
let serialized = serde_json::to_string(search_term).map_err(io::Error::other)?;
Ok(serialized[1..serialized.len() - 1].to_string())
+85
View File
@@ -0,0 +1,85 @@
//! Compressed rollout discovery and best-effort candidate delivery.
use std::collections::HashSet;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use futures::future::join_all;
use tokio::sync::mpsc;
use super::RolloutSearchCandidate;
use super::RolloutSearchMatches;
use super::RolloutSnippetMatcher;
use super::compression;
use super::report_candidate;
pub(super) async fn scan_compressed_rollout_matches(
root: &Path,
search_term: &str,
progress: Option<mpsc::Sender<RolloutSearchCandidate>>,
) -> io::Result<RolloutSearchMatches> {
let mut matches = HashSet::new();
let mut paths = Vec::new();
let mut dirs = vec![root.to_path_buf()];
while let Some(dir) = dirs.pop() {
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
};
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let file_type = entry.file_type().await?;
if file_type.is_dir() {
dirs.push(path);
continue;
}
if !file_type.is_file() {
continue;
}
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
continue;
};
if !rollout_file.is_compressed() {
continue;
}
paths.push(rollout_file.into_path());
}
}
if paths.is_empty() {
return Ok(matches);
}
const COMPRESSED_SEARCH_WORKERS: usize = 4;
let scan = async |paths: &[PathBuf]| {
let mut matches = HashSet::new();
let mut progress = progress.clone();
let mut matcher = RolloutSnippetMatcher::new(search_term);
for path in paths {
match matcher.first_match(path).await {
Ok(Some(_)) => {
let logical_path = compression::plain_rollout_path(path);
report_candidate(
&mut progress,
RolloutSearchCandidate {
path: logical_path.clone(),
},
);
matches.insert(logical_path);
}
Ok(None) => {}
Err(err) => {
tracing::warn!(path = %path.display(), %err, "Failed to search compressed rollout")
}
}
}
matches
};
let chunk_size = paths.len().div_ceil(COMPRESSED_SEARCH_WORKERS).max(1);
for batch in join_all(paths.chunks(chunk_size).map(scan)).await {
matches.extend(batch);
}
Ok(matches)
}
@@ -0,0 +1,59 @@
//! Lazy, request-scoped matching of visible conversation snippets in rollout files.
use std::io;
use std::path::Path;
use std::sync::Arc;
use regex::Regex;
use super::case_insensitive_literal_regex;
use super::compression;
use super::content_match_snippet;
use super::json_escaped_search_term;
/// Finds snippets across rollouts for one search term, compiling only when a file is read.
pub struct RolloutSnippetMatcher<'a> {
search_term: &'a str,
patterns: Option<Arc<(Regex, Regex)>>,
}
impl<'a> RolloutSnippetMatcher<'a> {
pub fn new(search_term: &'a str) -> Self {
Self {
search_term,
patterns: None,
}
}
/// Compiles the expressions now when a caller needs to validate a query before discovery.
pub fn compile(&mut self) -> io::Result<()> {
self.compiled_patterns().map(|_| ())
}
fn compiled_patterns(&mut self) -> io::Result<&Arc<(Regex, Regex)>> {
Ok(match &mut self.patterns {
Some(patterns) => patterns,
slot @ None => slot.insert(Arc::new((
case_insensitive_literal_regex(json_escaped_search_term(self.search_term)?)?,
case_insensitive_literal_regex(self.search_term)?,
))),
})
}
/// Opens the file before compiling so per-file I/O errors retain their priority.
pub async fn first_match(&mut self, path: &Path) -> io::Result<Option<String>> {
let lines = compression::open_rollout_line_reader(path).await?;
// Share the regexes across file scans to reuse their scratch caches.
let patterns = Arc::clone(self.compiled_patterns()?);
lines
.find_map(move |line| {
let (json_search_term, search_term) = patterns.as_ref();
if json_search_term.is_match(line) {
content_match_snippet(line, search_term)
} else {
None
}
})
.await
}
}
+235 -5
View File
@@ -1,4 +1,6 @@
//! Regression coverage for rollout snippet matching and per-file error handling.
//! Regression coverage for rollout snippet matching, progress, cancellation, and errors.
#![cfg(test)]
use super::*;
use pretty_assertions::assert_eq;
@@ -48,19 +50,20 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
let mut expected = RolloutSearchMatches::new();
for index in 0..7 {
let path = root.path().join(format!("rollout-{index:04}.jsonl"));
std::fs::write(path.with_extension("jsonl.zst"), &compressed)?;
expected.insert(path, Some("needle".to_string()));
let physical_path = path.with_extension("jsonl.zst");
std::fs::write(&physical_path, &compressed)?;
expected.insert(path);
}
std::fs::write(root.path().join("rollout-broken.jsonl.zst"), b"not zstd")?;
assert_eq!(
scan_compressed_rollout_matches(root.path(), "needle").await?,
scan_compressed_rollout_matches(root.path(), "needle", /*progress*/ None).await?,
expected
);
let query = "a".repeat(500_000);
assert!(case_insensitive_literal_regex(&query).is_err());
assert_eq!(
scan_compressed_rollout_matches(root.path(), &query).await?,
scan_compressed_rollout_matches(root.path(), &query, /*progress*/ None).await?,
RolloutSearchMatches::new()
);
assert_eq!(
@@ -73,3 +76,230 @@ async fn compressed_search_preserves_error_handling() -> io::Result<()> {
);
Ok(())
}
#[tokio::test]
async fn fallback_reports_plain_and_compressed_candidates() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let plain = root.join("rollout-plain.jsonl");
let compressed = root.join("rollout-compressed.jsonl");
std::fs::write(&plain, line)?;
std::fs::write(
compressed.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
let (sender, mut receiver) = mpsc::channel(4);
let matches = search_rollout_matches_with_progress(
&home.path().join("missing-rg"),
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await?;
let mut progress = HashSet::new();
while let Some(candidate) = receiver.recv().await {
progress.insert(candidate.path);
}
assert_eq!(
(matches, progress),
(
HashSet::from([plain.clone(), compressed.clone()]),
HashSet::from([plain, compressed])
)
);
Ok(())
}
#[cfg(unix)]
mod streaming {
use super::*;
use pretty_assertions::assert_eq;
use std::time::Duration;
#[cfg(target_os = "linux")]
use tokio::sync::oneshot;
use tokio::time::timeout;
fn gated_rg(home: &Path) -> io::Result<PathBuf> {
let root = home.join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let rg = home.join("gated-rg");
codex_utils_cargo_bin::write_executable(
&rg,
r#"#!/bin/sh
for root do :; done
printf '%s\n' "$$" > "$root/rg.pid"
printf '%s\0001\n' "$root/first
file.jsonl"
while ! test -f "$root/release"; do sleep 0.02; done
printf '%s\0001\n' "$root/second.jsonl"
"#,
)?;
Ok(rg)
}
/// A path discovered by ripgrep reaches the consumer while the child is still running.
#[tokio::test]
async fn reports_paths_before_ripgrep_exits() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
)
.await
});
assert_eq!(
timeout(Duration::from_secs(5), paths.recv()).await?,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
assert!(!search.is_finished());
tokio::fs::write(root.join("release"), []).await?;
let result = timeout(Duration::from_secs(5), search).await???;
assert_eq!(
result,
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")])
);
Ok(())
}
/// Full progress queues never prevent ripgrep or the final search from completing.
#[tokio::test]
async fn full_progress_queue_does_not_stall_search() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
tokio::fs::write(root.join("release"), []).await?;
let (sender, mut paths) = mpsc::channel(1);
let matches = timeout(
Duration::from_secs(5),
search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(sender),
Box::pin(pending()),
),
)
.await??;
assert_eq!(
(matches, paths.recv().await, paths.recv().await,),
(
HashSet::from([root.join("first\nfile.jsonl"), root.join("second.jsonl")]),
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
}),
None,
)
);
Ok(())
}
/// Cancelling a held search reaps ripgrep and closes its progress stream.
#[cfg(target_os = "linux")]
#[tokio::test]
async fn cancellation_reaps_ripgrep() -> anyhow::Result<()> {
let home = TempDir::new()?;
let rg = gated_rg(home.path())?;
let root = home.path().join(SESSIONS_SUBDIR);
let codex_home = home.path().to_path_buf();
let (sender, mut paths) = mpsc::channel(2);
let (cancel, cancelled) = oneshot::channel::<()>();
let search = tokio::spawn(async move {
search_rollout_matches_with_progress(
&rg,
&codex_home,
/*archived*/ false,
"needle",
Some(sender),
Box::pin(async {
let _ = cancelled.await;
}),
)
.await
});
let candidate = timeout(Duration::from_secs(5), paths.recv()).await?;
if candidate.is_none() {
let outcome = timeout(Duration::from_secs(5), search).await?;
anyhow::bail!("search ended before reporting a candidate: {outcome:?}");
}
assert_eq!(
candidate,
Some(RolloutSearchCandidate {
path: root.join("first\nfile.jsonl"),
})
);
let pid = tokio::fs::read_to_string(root.join("rg.pid")).await?;
let child = Path::new("/proc").join(pid.trim());
assert!(child.exists());
cancel.send(()).expect("search is still running");
let error = timeout(Duration::from_secs(5), search).await??.unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
assert_eq!(paths.recv().await, None);
assert!(!child.exists());
Ok(())
}
}
#[cfg(unix)]
#[tokio::test]
async fn search_finds_archive_published_while_ripgrep_is_running() -> io::Result<()> {
let home = TempDir::new()?;
let root = home.path().join(SESSIONS_SUBDIR);
std::fs::create_dir(&root)?;
let line = br#"{"timestamp":"2025-01-01T00:00:00Z","type":"event_msg","payload":{"type":"agent_message","message":"needle"}}"#;
let target = root.join("rollout-target.jsonl");
std::fs::write(&target, line)?;
let rg = home.path().join("rg");
codex_utils_cargo_bin::write_executable(
&rg,
"#!/bin/sh\nfor root do :; done\nprintf '%s\\0001\\n' \"$root/ready\"\nread release < \"$root/release\"\nexit 1\n",
)?;
let fifo = root.join("release");
assert!(
std::process::Command::new("mkfifo")
.arg(&fifo)
.status()?
.success()
);
let (tx, mut rx) = mpsc::channel(4);
let search = search_rollout_matches_with_progress(
&rg,
home.path(),
/*archived*/ false,
"needle",
Some(tx),
Box::pin(pending()),
);
tokio::pin!(search);
tokio::select! {
result = &mut search => panic!("search completed before archive publication: {result:?}"),
candidate = rx.recv() => assert_eq!(candidate.expect("rg ready").path, root.join("ready")),
}
std::fs::write(
target.with_extension("jsonl.zst"),
zstd::stream::encode_all(&line[..], 3)?,
)?;
std::fs::remove_file(&target)?;
let release = tokio::task::spawn_blocking(move || std::fs::write(fifo, b"release\n"));
let matches = search.await?;
release.await.map_err(io::Error::other)??;
assert!(matches.contains(&target));
Ok(())
}
+2
View File
@@ -66,6 +66,7 @@ pub use realtime::ListRealtimeParams;
pub use realtime::RealtimePage;
pub use store::PendingDaybreakPreference;
pub use store::PersistContext;
pub use store::ThreadSearchObserver;
pub use store::ThreadStore;
pub use store::ThreadStoreFuture;
pub use store::ThreadStoreResumePreparation;
@@ -124,6 +125,7 @@ pub use types::ThreadPage;
pub use types::ThreadPersistenceMetadata;
pub use types::ThreadRelationFilter;
pub use types::ThreadSearchPage;
pub use types::ThreadSearchSourceKind;
pub use types::ThreadSortKey;
pub use types::TimelinePage;
pub use types::TurnPage;
+10
View File
@@ -840,6 +840,16 @@ impl ThreadStore for LocalThreadStore {
Box::pin(async move { search_threads::search_threads(self, params).await })
}
fn search_threads_observed(
&self,
params: SearchThreadsParams,
observer: crate::ThreadSearchObserver,
) -> ThreadStoreFuture<'_, ThreadSearchPage> {
Box::pin(search_threads::search_threads_observed(
self, params, observer,
))
}
fn search_thread_occurrences(
&self,
params: SearchThreadOccurrencesParams,
@@ -0,0 +1,135 @@
//! Optional progress from the same rollout discovery and content verification used by final search.
use super::LocalThreadStore;
use super::SearchThreadsParams;
use super::StoredThreadSearchResult;
use super::ThreadSearchItem;
use super::eligible_search_item;
use super::materialize_search_results;
use super::rollout_search_path;
use super::verified_snippet;
use codex_protocol::ThreadId;
use codex_rollout::RolloutSearchCandidate;
use codex_rollout::RolloutSnippetMatcher;
use codex_rollout::state_db::list_threads_db;
use std::collections::HashMap;
use std::collections::HashSet;
use tokio::sync::mpsc;
const PROGRESS_RESULT_LIMIT: usize = 8;
const CANDIDATE_LIMIT: usize = 32;
const CANDIDATE_BATCH: usize = 16;
pub(super) struct SearchProgressContext<'a, 'term> {
pub(super) store: &'a LocalThreadStore,
pub(super) params: &'a SearchThreadsParams,
pub(super) state_db: Option<&'a codex_state::StateRuntime>,
pub(super) cursor: Option<&'a codex_rollout::Cursor>,
pub(super) sort_key: codex_rollout::ThreadSortKey,
pub(super) sort_direction: codex_rollout::SortDirection,
pub(super) matcher: &'a mut RolloutSnippetMatcher<'term>,
}
impl SearchProgressContext<'_, '_> {
pub(super) async fn report_search_candidates(
self,
mut candidates: mpsc::Receiver<RolloutSearchCandidate>,
progress: mpsc::Sender<Vec<StoredThreadSearchResult>>,
) {
let Some(db) = self.state_db else { return };
let mut emitted = HashSet::<ThreadId>::new();
let mut seen = HashSet::new();
let mut scanned = 0;
let mut batch = Vec::new();
while scanned < CANDIDATE_LIMIT
&& emitted.len() < self.params.page_size.min(PROGRESS_RESULT_LIMIT)
{
if candidates
.recv_many(&mut batch, CANDIDATE_BATCH.min(CANDIDATE_LIMIT - scanned))
.await
== 0
{
break;
}
scanned += batch.len();
let mut ids = Vec::with_capacity(batch.len());
let hits = batch
.drain(..)
.filter_map(|candidate| {
let path = rollout_search_path(&candidate.path);
let id = codex_rollout::thread_id_from_rollout_path(&path)?;
if !seen.insert(path.clone()) || emitted.contains(&id) {
return None;
}
ids.push(id);
Some((id, path))
})
.collect::<Vec<_>>();
if hits.is_empty() {
continue;
}
let relation = self
.params
.include_attached_threads
.then_some(codex_state::ThreadRelationFilter::IncludeAttached);
let Some(page) = list_threads_db(
Some(db),
db.sqlite(),
ids.len(),
self.cursor,
self.sort_key,
self.sort_direction,
&self.params.allowed_sources,
/*model_providers*/ None,
/*cwd_filters*/ None,
Some(&ids),
relation,
self.params.archived,
/*section*/ None,
/*project_id*/ None,
/*search_term*/ None,
)
.await
else {
continue;
};
let page: codex_rollout::ThreadsPage = page.into();
let mut items = page
.items
.into_iter()
.filter_map(|item| item.thread_id.map(|id| (id, item)))
.collect::<HashMap<_, _>>();
let mut originals = None;
for (id, path) in hits {
let Some(item) = items.remove(&id) else {
continue;
};
if rollout_search_path(&item.path) != path
|| !eligible_search_item(self.store, self.params, &item, &ids, &mut originals)
.await
{
continue;
}
let Ok(Some(snippet)) = verified_snippet(self.matcher, &item.path).await else {
continue;
};
let data = materialize_search_results(
self.store,
Some(db),
self.params.archived,
vec![ThreadSearchItem { item, snippet }],
)
.await;
if data.is_empty() {
continue;
}
if progress.try_send(data).is_ok() {
emitted.insert(id);
}
if emitted.len() >= self.params.page_size.min(PROGRESS_RESULT_LIMIT) {
break;
}
}
}
}
}
+198 -42
View File
@@ -6,8 +6,11 @@ use codex_install_context::InstallContext;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutSnippetMatcher;
use codex_rollout::parse_cursor;
use codex_rollout::search_rollout_matches;
use codex_rollout::search_rollout_matches_with_progress;
use codex_utils_absolute_path::normalize_windows_device_path;
use futures::FutureExt;
use futures::future::pending;
use tokio::sync::mpsc;
use super::LocalThreadStore;
use super::helpers::resolve_thread_names;
@@ -19,6 +22,7 @@ use crate::ListThreadsParams;
use crate::SearchThreadsParams;
use crate::SortDirection;
use crate::StoredThreadSearchResult;
use crate::ThreadSearchObserver;
use crate::ThreadSearchPage;
use crate::ThreadSortKey;
use crate::ThreadStoreError;
@@ -28,6 +32,9 @@ use crate::ThreadStoreResult;
#[path = "search_threads_tests.rs"]
mod tests;
#[path = "search_progress.rs"]
mod progress;
struct ThreadSearchItem {
item: codex_rollout::ThreadItem,
snippet: String,
@@ -37,6 +44,23 @@ pub(super) async fn search_threads(
store: &LocalThreadStore,
params: SearchThreadsParams,
) -> ThreadStoreResult<ThreadSearchPage> {
search_threads_observed(
store,
params,
ThreadSearchObserver {
progress: None,
cancelled: Box::pin(pending()),
},
)
.await
}
pub(super) async fn search_threads_observed(
store: &LocalThreadStore,
params: SearchThreadsParams,
observer: ThreadSearchObserver,
) -> ThreadStoreResult<ThreadSearchPage> {
let cancelled = observer.cancelled.shared();
let search_term = params.search_term.as_str();
if search_term.is_empty() {
return Err(ThreadStoreError::InvalidRequest {
@@ -66,7 +90,11 @@ pub(super) async fn search_threads(
SortDirection::Asc => codex_rollout::SortDirection::Asc,
SortDirection::Desc => codex_rollout::SortDirection::Desc,
};
let state_db = store.state_db().await;
let state_db = tokio::select! {
biased;
() = cancelled.clone() => return Err(search_cancelled()),
db = store.state_db() => db,
};
let rollout_config = RolloutConfig {
codex_home: store.config.codex_home.clone(),
sqlite: store.config.sqlite.clone(),
@@ -74,17 +102,99 @@ pub(super) async fn search_threads(
model_provider_id: store.config.default_model_provider_id.clone(),
generate_memories: false,
};
let mut matcher = RolloutSnippetMatcher::new(search_term);
if observer.progress.is_some() {
matcher
.compile()
.map_err(|err| ThreadStoreError::InvalidRequest {
message: format!("invalid rollout search term: {err}"),
})?;
}
let rg_command = InstallContext::current().rg_command();
let matching_rollouts = search_rollout_matches(
rg_command.as_path(),
store.config.codex_home.as_path(),
params.archived,
search_term,
)
.await
.map_err(|err| ThreadStoreError::Internal {
let matching_rollouts = if let Some(progress) = observer.progress {
let (paths, candidates) = mpsc::channel(16);
let discovery = search_rollout_matches_with_progress(
&rg_command,
&store.config.codex_home,
params.archived,
search_term,
Some(paths),
Box::pin(cancelled.clone()),
);
tokio::pin!(discovery);
{
let context = progress::SearchProgressContext {
store,
params: &params,
state_db: state_db.as_deref(),
cursor: cursor.as_ref(),
sort_key,
sort_direction,
matcher: &mut matcher,
};
let reporter = context.report_search_candidates(candidates, progress);
tokio::pin!(reporter);
tokio::select! {
biased;
matches = &mut discovery => matches,
() = &mut reporter => discovery.await,
}
}
} else {
search_rollout_matches_with_progress(
&rg_command,
&store.config.codex_home,
params.archived,
search_term,
/*progress*/ None,
Box::pin(cancelled.clone()),
)
.await
};
let matching_rollouts = matching_rollouts.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to search rollout contents: {err}"),
})?;
tokio::select! {
biased;
() = cancelled => Err(search_cancelled()),
result = finish_search(
SearchContext { store, params: &params, state_db, rollout_config: &rollout_config, cursor,
sort_key, sort_direction, matcher: &mut matcher }, matching_rollouts,
) => result,
}
}
fn search_cancelled() -> ThreadStoreError {
ThreadStoreError::Internal {
message: "thread search cancelled".to_string(),
}
}
struct SearchContext<'a, 'term> {
store: &'a LocalThreadStore,
params: &'a SearchThreadsParams,
state_db: Option<codex_rollout::StateDbHandle>,
rollout_config: &'a RolloutConfig,
cursor: Option<codex_rollout::Cursor>,
sort_key: codex_rollout::ThreadSortKey,
sort_direction: codex_rollout::SortDirection,
matcher: &'a mut RolloutSnippetMatcher<'term>,
}
async fn finish_search(
context: SearchContext<'_, '_>,
matching_rollouts: codex_rollout::RolloutSearchMatches,
) -> ThreadStoreResult<ThreadSearchPage> {
let SearchContext {
store,
params,
state_db,
rollout_config,
cursor,
sort_key,
sort_direction,
matcher,
} = context;
if matching_rollouts.is_empty() {
return Ok(ThreadSearchPage {
items: Vec::new(),
@@ -114,14 +224,13 @@ pub(super) async fn search_threads(
};
let mut remaining_rollouts = matching_rollouts
.into_iter()
.map(|(path, snippet)| (rollout_search_path(&path), snippet))
.collect::<HashMap<_, _>>();
.map(|path| rollout_search_path(&path))
.collect::<std::collections::HashSet<_>>();
let mut snippet_matcher = RolloutSnippetMatcher::new(search_term);
loop {
let page = list_rollout_threads(
state_db.clone(),
&rollout_config,
rollout_config,
store.config.default_model_provider_id.as_str(),
&scan_params,
page_cursor.as_ref(),
@@ -138,34 +247,17 @@ pub(super) async fn search_threads(
let mut original_paths = None;
for item in page.items {
let logical_path = rollout_search_path(item.path.as_path());
let Some(mut snippet) = remaining_rollouts.remove(logical_path.as_path()) else {
if !remaining_rollouts.remove(logical_path.as_path()) {
continue;
};
if let Some(account) = &params.account {
let Some(thread_id) = item.thread_id else {
continue;
};
if !account
.includes(
&store.config.codex_home,
thread_id,
Some(&item.path),
&page_thread_ids,
&mut original_paths,
)
.await
{
continue;
}
}
if snippet.is_none() {
snippet = snippet_matcher
.first_match(&item.path)
.await
.map_err(|err| ThreadStoreError::Internal {
message: format!("failed to read rollout search match: {err}"),
})?;
if !eligible_search_item(store, params, &item, &page_thread_ids, &mut original_paths)
.await
{
continue;
}
// Discovery can decode an archive, but its earlier match is not authoritative:
// the archive or the catalog-selected rollout can change before this page is built.
let snippet = verified_snippet(matcher, &item.path).await?;
let Some(snippet) = snippet else { continue };
matching_items.push(ThreadSearchItem { item, snippet });
if matching_items.len() > params.page_size {
@@ -198,12 +290,76 @@ pub(super) async fn search_threads(
.and_then(|cursor| serde_json::to_value(cursor).ok())
.and_then(|value| value.as_str().map(str::to_owned));
let items =
materialize_search_results(store, state_db.as_deref(), params.archived, matching_items)
.await;
Ok(ThreadSearchPage { items, next_cursor })
}
async fn verified_snippet(
matcher: &mut RolloutSnippetMatcher<'_>,
path: &Path,
) -> ThreadStoreResult<Option<String>> {
match matcher.first_match(path).await {
Ok(snippet) => Ok(snippet),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(ThreadStoreError::Internal {
message: format!("failed to read rollout search match: {err}"),
}),
}
}
async fn eligible_search_item(
store: &LocalThreadStore,
params: &SearchThreadsParams,
item: &codex_rollout::ThreadItem,
ids: &[codex_protocol::ThreadId],
originals: &mut Option<HashMap<codex_protocol::ThreadId, PathBuf>>,
) -> bool {
if !matches_source(params, item) {
return false;
}
match (&params.account, item.thread_id) {
(Some(account), Some(id)) => {
account
.includes(
&store.config.codex_home,
id,
Some(&item.path),
ids,
originals,
)
.await
}
(Some(_), None) => false,
(None, _) => true,
}
}
fn matches_source(params: &SearchThreadsParams, item: &codex_rollout::ThreadItem) -> bool {
params.source_kinds.as_ref().is_none_or(|kinds| {
kinds.iter().any(|kind| {
kind.matches(
item.source
.as_ref()
.unwrap_or(&codex_protocol::protocol::SessionSource::Unknown),
)
})
})
}
async fn materialize_search_results(
store: &LocalThreadStore,
state_db: Option<&codex_state::StateRuntime>,
archived: bool,
matching_items: Vec<ThreadSearchItem>,
) -> Vec<StoredThreadSearchResult> {
let mut items = matching_items
.into_iter()
.filter_map(|item| {
stored_thread_from_rollout_item(
item.item,
params.archived,
archived,
store.config.default_model_provider_id.as_str(),
)
.map(|thread| StoredThreadSearchResult {
@@ -219,7 +375,7 @@ pub(super) async fn search_threads(
.map(|item| item.thread.thread_id)
.collect::<Vec<_>>();
let mut section_metadata =
resolve_thread_section_metadata(state_db.as_ref(), &sectioned_thread_ids).await;
resolve_thread_section_metadata(state_db, &sectioned_thread_ids).await;
for item in &mut items {
if let Some((section_position, section_entered_at)) =
section_metadata.remove(&item.thread.thread_id)
@@ -231,7 +387,7 @@ pub(super) async fn search_threads(
}
set_thread_search_result_names(store, &mut items).await;
Ok(ThreadSearchPage { items, next_cursor })
items
}
fn rollout_search_path(path: &Path) -> PathBuf {
@@ -5,6 +5,7 @@ use std::path::PathBuf;
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::ThreadItem;
use pretty_assertions::assert_eq;
@@ -21,6 +22,7 @@ use crate::ListThreadsParams;
use crate::SearchThreadsParams;
use crate::SortDirection;
use crate::ThreadRelationFilter;
use crate::ThreadSearchSourceKind;
use crate::ThreadSortKey;
use crate::ThreadStore;
@@ -52,6 +54,65 @@ fn time_sort_cursors_include_thread_id_tie_breaker() {
}
}
#[tokio::test]
async fn authoritative_search_rechecks_stale_compressed_discovery() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let store = LocalThreadStore::new(config.clone(), /*state_db*/ None);
let id = Uuid::new_v4();
let path = write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
id,
"changed message",
Some("test-provider"),
ThreadHistoryMode::Legacy,
)
.expect("write selected rollout");
let contents = std::fs::read(&path).expect("read rollout");
let compressed =
zstd::stream::encode_all(contents.as_slice(), /*level*/ 3).expect("compress rollout");
std::fs::write(path.with_extension("jsonl.zst"), compressed).expect("write compressed rollout");
std::fs::remove_file(&path).expect("remove plain rollout");
let rollout_config = codex_rollout::RolloutConfig {
codex_home: config.codex_home.clone(),
sqlite: config.sqlite.clone(),
cwd: config.codex_home,
model_provider_id: config.default_model_provider_id,
generate_memories: false,
};
let params = SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
source_kinds: None,
archived: false,
search_term: "needle".to_string(),
};
let mut matcher = codex_rollout::RolloutSnippetMatcher::new("needle");
let page = super::finish_search(
super::SearchContext {
store: &store,
params: &params,
state_db: None,
rollout_config: &rollout_config,
cursor: None,
sort_key: codex_rollout::ThreadSortKey::CreatedAt,
sort_direction: codex_rollout::SortDirection::Desc,
matcher: &mut matcher,
},
std::collections::HashSet::from([path]),
)
.await
.expect("finish search");
assert_eq!((page.items.len(), page.next_cursor), (0, None));
}
#[tokio::test]
async fn search_matches_selected_rollout_across_path_spellings_and_compression() {
for compressed in [false, true] {
@@ -155,6 +216,7 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression()
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
source_kinds: None,
archived: false,
search_term: search_term.to_string(),
})
@@ -177,6 +239,89 @@ async fn search_matches_selected_rollout_across_path_spellings_and_compression()
}
}
#[tokio::test]
async fn source_kind_filter_applies_before_search_pagination() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let state_db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("initialize state database");
let mut review_ids = Vec::new();
for (age, source) in [
(0, SessionSource::SubAgent(SubAgentSource::Review)),
(1, SessionSource::Cli),
(2, SessionSource::SubAgent(SubAgentSource::Review)),
] {
let uuid = Uuid::new_v4();
let id = ThreadId::from_u128(uuid.as_u128());
let path = write_session_file_with(
home.path(),
home.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
uuid,
"needle message",
Some("test-provider"),
ThreadHistoryMode::Legacy,
)
.expect("write rollout");
let mut metadata = codex_state::ThreadMetadataBuilder::new(
id,
path,
Utc::now() - chrono::TimeDelta::days(age),
source.clone(),
)
.build("test-provider");
metadata.first_user_message = Some("needle message".to_string());
state_db
.upsert_thread(&metadata)
.await
.expect("index rollout");
if matches!(source, SessionSource::SubAgent(SubAgentSource::Review)) {
review_ids.push(id);
}
}
let store = LocalThreadStore::new(config, Some(state_db));
let mut params = SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 1,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: Vec::new(),
source_kinds: Some(vec![ThreadSearchSourceKind::SubAgentReview]),
archived: false,
search_term: "needle".to_string(),
};
let first = store
.search_threads(params.clone())
.await
.expect("first page");
params.cursor = first.next_cursor.clone();
let second = store.search_threads(params).await.expect("second page");
assert_eq!(
(
first
.items
.into_iter()
.map(|item| item.thread.thread_id)
.collect::<Vec<_>>(),
first.next_cursor.is_some(),
second
.items
.into_iter()
.map(|item| item.thread.thread_id)
.collect::<Vec<_>>(),
second.next_cursor,
),
(vec![review_ids[0]], true, vec![review_ids[1]], None),
);
}
#[tokio::test]
async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn std::error::Error>>
{
@@ -226,6 +371,7 @@ async fn searches_are_independent_across_catalog_refills() -> Result<(), Box<dyn
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
source_kinds: None,
archived: false,
search_term: query.to_owned(),
})
@@ -331,6 +477,7 @@ async fn title_and_content_search_return_attached_parent_without_state_db() {
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
source_kinds: None,
archived: false,
search_term: "searchable message".to_string(),
include_attached_threads: true,
@@ -406,3 +553,61 @@ async fn inclusive_title_search_finds_unindexed_matches_when_filesystem_scanning
);
}
}
#[tokio::test]
async fn streaming_search_never_reports_undiscovered_rows_from_a_shared_database() {
let home = TempDir::new().unwrap();
let other = TempDir::new().unwrap();
let config = test_config(home.path());
let db = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.unwrap();
let uuid = Uuid::new_v4();
let path = write_session_file_with(
other.path(),
other.path().join("sessions/2025/01/03"),
"2025-01-03T12-00-00",
uuid,
"private needle",
Some("test-provider"),
ThreadHistoryMode::Legacy,
)
.unwrap();
let metadata = codex_state::ThreadMetadataBuilder::new(
ThreadId::from_u128(uuid.as_u128()),
path,
Utc::now(),
SessionSource::Cli,
)
.build("test-provider");
db.upsert_thread(&metadata).await.unwrap();
let store = LocalThreadStore::new(config, Some(db));
let params = SearchThreadsParams {
include_attached_threads: false,
account: None,
page_size: 10,
cursor: None,
sort_key: ThreadSortKey::CreatedAt,
sort_direction: SortDirection::Desc,
allowed_sources: vec![SessionSource::Cli],
source_kinds: None,
archived: false,
search_term: "needle".into(),
};
let (tx, mut rx) = tokio::sync::mpsc::channel(4);
let page = store
.search_threads_observed(
params,
crate::ThreadSearchObserver {
progress: Some(tx),
cancelled: Box::pin(std::future::pending()),
},
)
.await
.unwrap();
assert!(page.items.is_empty());
assert!(rx.recv().await.is_none());
}
+28
View File
@@ -1,11 +1,13 @@
use codex_protocol::ThreadId;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::RolloutItem;
use futures::future::BoxFuture;
use std::any::Any;
use std::fmt;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use tokio::sync::mpsc;
use crate::AddThreadAttachmentOutcome;
use crate::AddThreadAttachmentParams;
@@ -48,6 +50,7 @@ use crate::StoredProject;
use crate::StoredProjectsPage;
use crate::StoredThread;
use crate::StoredThreadHistory;
use crate::StoredThreadSearchResult;
use crate::StoredThreadSection;
use crate::StoredThreadSectionsPage;
use crate::ThreadAttachmentOwnerPage;
@@ -66,6 +69,14 @@ use crate::UpdatedProject;
/// Future returned by [`ThreadStore`] operations.
pub type ThreadStoreFuture<'a, T> = Pin<Box<dyn Future<Output = ThreadStoreResult<T>> + Send + 'a>>;
/// Request lifetime and optional progress sink for a thread search.
pub struct ThreadSearchObserver {
/// Verified matches discovered before the authoritative page is complete.
pub progress: Option<mpsc::Sender<Vec<StoredThreadSearchResult>>>,
/// Resolves when the requester no longer needs the result.
pub cancelled: BoxFuture<'static, ()>,
}
/// Data a [`ThreadStore`] has already loaded while preparing a thread for resume.
///
/// Core does not inspect this value. It gives the value back to the store when reopening the
@@ -569,6 +580,23 @@ pub trait ThreadStore: Any + Send + Sync {
})
}
/// Searches threads with cancellation and optional progress where supported.
fn search_threads_observed(
&self,
params: SearchThreadsParams,
observer: ThreadSearchObserver,
) -> ThreadStoreFuture<'_, ThreadSearchPage> {
Box::pin(async move {
tokio::select! {
biased;
() = observer.cancelled => Err(ThreadStoreError::Internal {
message: "thread search cancelled".to_string(),
}),
result = self.search_threads(params) => result,
}
})
}
/// Searches visible message occurrences within one paginated thread.
fn search_thread_occurrences(
&self,
+47
View File
@@ -18,6 +18,7 @@ use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::HistoryPosition;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode as MemoryMode;
use codex_protocol::protocol::ThreadSource;
@@ -394,6 +395,49 @@ pub struct ListThreadsParams {
pub use_state_db_only: bool,
}
/// Source categories accepted by thread content search.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ThreadSearchSourceKind {
Cli,
VsCode,
Exec,
AppServer,
SubAgent,
SubAgentReview,
SubAgentCompact,
SubAgentThreadSpawn,
SubAgentOther,
Unknown,
}
impl ThreadSearchSourceKind {
/// Determine whether a stored session source belongs to this category.
pub fn matches(self, source: &SessionSource) -> bool {
match self {
Self::Cli => matches!(source, SessionSource::Cli),
Self::VsCode => matches!(source, SessionSource::VSCode),
Self::Exec => matches!(source, SessionSource::Exec),
Self::AppServer => matches!(source, SessionSource::Mcp),
Self::SubAgent => matches!(source, SessionSource::SubAgent(_)),
Self::SubAgentReview => {
matches!(source, SessionSource::SubAgent(SubAgentSource::Review))
}
Self::SubAgentCompact => {
matches!(source, SessionSource::SubAgent(SubAgentSource::Compact))
}
Self::SubAgentThreadSpawn => matches!(
source,
SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. })
),
Self::SubAgentOther => {
matches!(source, SessionSource::SubAgent(SubAgentSource::Other(_)))
}
Self::Unknown => matches!(source, SessionSource::Unknown),
}
}
}
/// Parameters for searching thread content.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SearchThreadsParams {
@@ -412,6 +456,9 @@ pub struct SearchThreadsParams {
pub sort_direction: SortDirection,
/// Allowed session sources. Empty means implementation default.
pub allowed_sources: Vec<SessionSource>,
/// Exact source categories to apply before pagination. None accepts any allowed source.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_kinds: Option<Vec<ThreadSearchSourceKind>>,
/// Whether archived threads should be searched instead of active threads.
pub archived: bool,
/// Visible thread content to search for.
@@ -213,6 +213,7 @@ pub(super) fn server_notification_thread_target(
| ServerNotification::ExternalAgentConfigImportCompleted(_)
| ServerNotification::DeprecationNotice(_)
| ServerNotification::ConfigWarning(_)
| ServerNotification::ThreadSearchPartial(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
| ServerNotification::CommandExecOutputDelta(_)
+4 -3
View File
@@ -2115,6 +2115,7 @@ impl App {
self.skill_load_warnings.startup_complete = true;
}
AppEvent::StartFileSearch(query) => {
app_server.task_search.cancel();
self.file_search.on_user_query(query.clone());
if let Some(thread_id) = self.active_thread_id
&& app_server.task_tools_available(thread_id)
@@ -2130,12 +2131,12 @@ impl App {
.map(std::path::Path::to_path_buf)
})
.unwrap_or_else(|| self.config.cwd.to_path_buf());
crate::task_mentions::spawn_search(
app_server.request_handle(),
let handle = app_server.request_handle();
app_server.task_search.spawn(
handle,
query,
thread_id,
cwd,
app_server.task_search_generation(),
self.app_event_tx.clone(),
);
}
@@ -776,6 +776,7 @@ impl App {
thread_id: ThreadId,
selection: Selection,
) -> Result<()> {
app_server.task_search.cancel();
let Selection {
mode,
cached_session,
@@ -2438,12 +2438,12 @@ async fn external_transport_registers_dynamic_tools_and_finds_task_mentions() ->
.await?;
let (sender, mut events) = tokio::sync::mpsc::unbounded_channel();
crate::task_mentions::spawn_search(
restarted_app_server.request_handle(),
let handle = restarted_app_server.request_handle();
restarted_app_server.task_search.spawn(
handle,
"bbd".to_string(),
startup.session.thread_id,
app.config.cwd.to_path_buf(),
restarted_app_server.task_search_generation(),
crate::app_event_sender::AppEventSender::new(sender),
);
+1
View File
@@ -125,6 +125,7 @@ impl App {
app_server: &mut AppServerSession,
destination: Option<ThreadId>,
) {
app_server.task_search.cancel();
let thread_ids: Vec<_> = self
.thread_event_channels
.iter()
+4 -8
View File
@@ -177,7 +177,6 @@ use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::AtomicU64;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Semaphore;
@@ -410,7 +409,7 @@ pub(crate) struct AppServerSession {
pub(crate) unread_revisions: HashMap<ThreadId, String>,
task_tool_threads: HashSet<ThreadId>,
task_tool_capabilities_dir: Option<AbsolutePathBuf>,
task_search_generation: Arc<AtomicU64>,
pub(crate) task_search: crate::task_mentions::TaskSearch,
remote_cwd_override: Option<PathBuf>,
thread_params_mode: ThreadParamsMode,
/// Explicit CLI provider selection, distinct from resolved configuration defaults.
@@ -663,7 +662,7 @@ impl AppServerSession {
unread_revisions: HashMap::new(),
task_tool_threads: HashSet::new(),
task_tool_capabilities_dir: None,
task_search_generation: Arc::new(AtomicU64::new(0)),
task_search: crate::task_mentions::TaskSearch::default(),
remote_cwd_override: None,
thread_params_mode,
model_provider_override: None,
@@ -897,10 +896,6 @@ impl AppServerSession {
}
}
pub(crate) fn task_search_generation(&self) -> Arc<AtomicU64> {
Arc::clone(&self.task_search_generation)
}
pub(crate) fn codex_home_path(
&self,
local_codex_home: &AbsolutePathBuf,
@@ -2356,7 +2351,8 @@ impl AppServerSession {
client.resolve_server_request(request_id, result).await
}
pub(crate) async fn shutdown(self) -> std::io::Result<()> {
pub(crate) async fn shutdown(mut self) -> std::io::Result<()> {
self.task_search.cancel();
match self.connection_owner {
Some(client) => client.shutdown().await,
None => Ok(()),
+11 -1
View File
@@ -83,7 +83,8 @@
//! the Chinese closing book-title bracket (`》`), takes the place of a newly inserted separator.
//! Existing whitespace stays part of the draft.
//!
//! `Esc` records the active token as dismissed. A completed value that begins with `@` or `$` is
//! `Esc` records the active token as dismissed and resets any pending unified mention search.
//! A completed value that begins with `@` or `$` is
//! also re-dismissed before popup synchronization, because separator affinity can still identify
//! the completed token to the left of the cursor. Synchronization keeps the popup hidden only while
//! the query, complete token text, and ordinal among matching occurrences remain the same.
@@ -2514,6 +2515,10 @@ impl ChatComposer {
self.popups.dismissed_mention_token =
Some(DismissedToken::new(&self.draft.textarea, range, query));
}
if self.popups.current_file_query.take().is_some() {
self.app_event_tx
.send(AppEvent::StartFileSearch(String::new()));
}
self.popups.active = ActivePopup::None;
(InputResult::None, true)
}
@@ -8055,6 +8060,11 @@ mod tests {
composer.on_file_search_result("foo".to_string(), Vec::new());
let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE));
assert!(matches!(composer.popups.active, ActivePopup::None));
assert!(matches!(
rx.try_recv(),
Ok(AppEvent::StartFileSearch(query)) if query.is_empty()
));
assert!(composer.popups.current_file_query.is_none());
composer.draft.textarea.set_cursor("@foo @foo".len());
composer.sync_popups();
+1
View File
@@ -419,6 +419,7 @@ impl ChatWidget {
| ServerNotification::ExternalAgentConfigImportProgress(_)
| ServerNotification::ExternalAgentConfigImportCompleted(_)
| ServerNotification::FsChanged(_)
| ServerNotification::ThreadSearchPartial(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
| ServerNotification::ThreadRealtimeItemAdded(_)
+182 -105
View File
@@ -1,9 +1,6 @@
//! Same-host task discovery and bounded task-reference prompt context.
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_app_server_client::AppServerRequestHandle;
@@ -13,6 +10,8 @@ use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SortDirection;
use codex_app_server_protocol::ThreadListParams;
use codex_app_server_protocol::ThreadListResponse;
use codex_app_server_protocol::ThreadSearchCancelParams;
use codex_app_server_protocol::ThreadSearchCancelResponse;
use codex_app_server_protocol::ThreadSearchParams;
use codex_app_server_protocol::ThreadSearchResponse;
use codex_app_server_protocol::ThreadSearchResult;
@@ -22,6 +21,8 @@ use codex_app_server_protocol::ThreadSource;
use codex_app_server_protocol::UserInput;
use codex_protocol::ThreadId;
use serde_json::json;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
use crate::app_event::AppEvent;
use crate::app_event_sender::AppEventSender;
@@ -30,6 +31,8 @@ use crate::thread_label::ThreadLabelStyle;
const MAX_SEARCH_RESULTS: u32 = 50;
const SEARCH_DEBOUNCE: Duration = Duration::from_millis(/*millis*/ 100);
const SEARCH_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 10);
const CANCEL_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 1);
pub(crate) const MAX_REFERENCED_TASKS: usize = 16;
pub(crate) const MAX_TASK_TITLE_CHARS: usize = 160;
const MAX_REFERENCED_THREAD_ID_BYTES: usize = 768;
@@ -44,104 +47,128 @@ pub(crate) struct TaskMention {
pub(crate) snippet: String,
}
pub(crate) fn spawn_search(
handle: AppServerRequestHandle,
query: String,
current_thread_id: ThreadId,
cwd: PathBuf,
generation: Arc<AtomicU64>,
app_event_tx: AppEventSender,
) {
let label_style =
ThreadLabelStyle::from_is_cloud(matches!(handle, AppServerRequestHandle::Cloud(_)));
let request_generation = generation.fetch_add(1, Ordering::Relaxed) + 1;
if query.trim().is_empty() {
app_event_tx.send(AppEvent::TaskSearchResult {
thread_id: current_thread_id,
query,
matches: Vec::new(),
});
return;
/// Owns the pending task search for a session and cancels it when replaced or dropped.
#[derive(Default)]
pub(crate) struct TaskSearch {
pending: Option<CancellationToken>,
}
impl Drop for TaskSearch {
fn drop(&mut self) {
self.cancel();
}
}
impl TaskSearch {
pub(crate) fn cancel(&mut self) {
if let Some(pending) = self.pending.take() {
pending.cancel();
}
}
tokio::spawn(async move {
tokio::time::sleep(SEARCH_DEBOUNCE).await;
if generation.load(Ordering::Relaxed) != request_generation {
pub(crate) fn spawn(
&mut self,
handle: AppServerRequestHandle,
query: String,
current_thread_id: ThreadId,
cwd: PathBuf,
app_event_tx: AppEventSender,
) {
let label_style =
ThreadLabelStyle::from_is_cloud(matches!(handle, AppServerRequestHandle::Cloud(_)));
self.cancel();
if query.trim().is_empty() {
app_event_tx.send(AppEvent::TaskSearchResult {
thread_id: current_thread_id,
query,
matches: Vec::new(),
});
return;
}
let timeout = Duration::from_secs(/*secs*/ 10);
let (response, titled_threads) = tokio::join!(
tokio::time::timeout(
timeout,
handle.request_typed::<ThreadSearchResponse>(ClientRequest::ThreadSearch {
request_id: RequestId::String(format!("task-search-{request_generation}")),
params: ThreadSearchParams {
include_attached_threads: false,
current_account_only: true,
cursor: None,
limit: Some(MAX_SEARCH_RESULTS),
sort_key: Some(ThreadSearchSortKey::RecencyAt),
sort_direction: Some(SortDirection::Desc),
source_kinds: Some(Vec::new()),
archived: Some(false),
search_term: query.trim().to_string(),
let cancellation = CancellationToken::new();
self.pending = Some(cancellation.clone());
tokio::spawn(async move {
tokio::select! {
biased;
() = cancellation.cancelled() => return,
() = tokio::time::sleep(SEARCH_DEBOUNCE) => {}
}
let request_id = RequestId::String(format!("task-search-{}", Uuid::new_v4()));
let (response, titled_threads) = tokio::join!(
search_with_cancellation(
&handle,
&cancellation,
ClientRequest::ThreadSearch {
request_id: request_id.clone(),
params: ThreadSearchParams {
include_attached_threads: false,
current_account_only: true,
cursor: None,
limit: Some(MAX_SEARCH_RESULTS),
sort_key: Some(ThreadSearchSortKey::RecencyAt),
sort_direction: Some(SortDirection::Desc),
source_kinds: Some(Vec::new()),
archived: Some(false),
search_term: query.trim().to_string(),
},
},
})
),
tokio::time::timeout(
timeout,
handle.request_typed::<ThreadListResponse>(ClientRequest::ThreadList {
request_id: RequestId::String(format!(
"task-title-search-{request_generation}"
)),
params: ThreadListParams {
include_attached_threads: false,
current_account_only: true,
included_thread_ids: None,
excluded_thread_ids: None,
attached_to_thread_id: None,
originators: None,
cursor: None,
limit: Some(MAX_SEARCH_RESULTS),
sort_key: Some(ThreadSortKey::UpdatedAt),
sort_direction: Some(SortDirection::Desc),
model_providers: Some(Vec::new()),
source_kinds: Some(Vec::new()),
archived: Some(false),
section_id: None,
project_id: None,
cwd: None,
use_state_db_only: true,
search_term: None,
parent_thread_id: None,
ancestor_thread_id: None,
},
})
)
);
request_id,
),
async {
tokio::select! {
biased;
() = cancellation.cancelled() => None,
result = tokio::time::timeout(SEARCH_TIMEOUT,
handle.request_typed::<ThreadListResponse>(ClientRequest::ThreadList {
request_id: RequestId::String(format!("task-title-search-{}", Uuid::new_v4())),
params: ThreadListParams {
include_attached_threads: false,
current_account_only: true,
included_thread_ids: None,
excluded_thread_ids: None,
attached_to_thread_id: None,
originators: None,
cursor: None,
limit: Some(MAX_SEARCH_RESULTS),
sort_key: Some(ThreadSortKey::UpdatedAt),
sort_direction: Some(SortDirection::Desc),
model_providers: Some(Vec::new()),
source_kinds: Some(Vec::new()),
archived: Some(false),
section_id: None,
project_id: None,
cwd: None,
use_state_db_only: true,
search_term: None,
parent_thread_id: None,
ancestor_thread_id: None,
},
})) => result.ok().and_then(Result::ok),
}
}
);
if generation.load(Ordering::Relaxed) != request_generation {
return;
}
if cancellation.is_cancelled() {
return;
}
let response = response.ok().and_then(Result::ok);
let titled_threads = titled_threads.ok().and_then(Result::ok);
if response.is_none() && titled_threads.is_none() {
return;
}
let mut results = response.map(|response| response.data).unwrap_or_default();
if let Some(response) = titled_threads {
for thread in response.data {
if results.iter().all(|result| result.thread.id != thread.id) {
results.push(ThreadSearchResult {
thread,
snippet: String::new(),
});
if response.is_none() && titled_threads.is_none() {
return;
}
let mut results = response.map(|response| response.data).unwrap_or_default();
if let Some(response) = titled_threads {
for thread in response.data {
if results.iter().all(|result| result.thread.id != thread.id) {
results.push(ThreadSearchResult {
thread,
snippet: String::new(),
});
}
}
}
}
let mut matches = results
let mut matches = results
.into_iter()
.filter_map(|result| {
let thread = result.thread;
@@ -174,22 +201,72 @@ pub(crate) fn spawn_search(
})
.take(MAX_SEARCH_RESULTS as usize)
.collect::<Vec<_>>();
let cwd = cwd.to_string_lossy();
let windows_cwd = cwd.as_bytes().get(1) == Some(&b':') || cwd.starts_with(r"\\");
matches.sort_by_key(|task| {
let Some((prefix, suffix)) = task.cwd.get(..cwd.len()).zip(task.cwd.get(cwd.len()..))
else {
return true;
};
!(prefix == cwd.as_ref() || windows_cwd && prefix.eq_ignore_ascii_case(cwd.as_ref()))
|| !(suffix.is_empty() || suffix.starts_with(['/', '\\']))
let cwd = cwd.to_string_lossy();
let windows_cwd = cwd.as_bytes().get(1) == Some(&b':') || cwd.starts_with(r"\\");
matches.sort_by_key(|task| {
let Some((prefix, suffix)) =
task.cwd.get(..cwd.len()).zip(task.cwd.get(cwd.len()..))
else {
return true;
};
!(prefix == cwd.as_ref()
|| windows_cwd && prefix.eq_ignore_ascii_case(cwd.as_ref()))
|| !(suffix.is_empty() || suffix.starts_with(['/', '\\']))
});
if cancellation.is_cancelled() {
return;
}
app_event_tx.send(AppEvent::TaskSearchResult {
thread_id: current_thread_id,
query,
matches,
});
});
app_event_tx.send(AppEvent::TaskSearchResult {
thread_id: current_thread_id,
query,
matches,
}
}
async fn search_with_cancellation(
handle: &AppServerRequestHandle,
cancellation: &CancellationToken,
request: ClientRequest,
request_id: RequestId,
) -> Option<ThreadSearchResponse> {
// Native Cloud supports title listing, but does not implement thread/search or its
// cancellation method. Avoid queuing an unsupported detached Cloud command.
if matches!(handle, AppServerRequestHandle::Cloud(_)) {
return None;
}
tokio::select! {
biased;
() = cancellation.cancelled() => {},
result = tokio::time::timeout(SEARCH_TIMEOUT, handle.request_typed::<ThreadSearchResponse>(request)) => {
if let Ok(result) = result {
return result.ok();
}
}
}
// Dropping the request future cancels an in-process search through the facade. Remote
// requests need an explicit connection-scoped cancellation message. Send it in the
// background so a slow cancellation reply cannot delay already available title results.
if matches!(handle, AppServerRequestHandle::Remote(_)) {
let handle = handle.clone();
tokio::spawn(async move {
let _ = tokio::time::timeout(
CANCEL_TIMEOUT,
handle.request_typed::<ThreadSearchCancelResponse>(
ClientRequest::ThreadSearchCancel {
request_id: RequestId::String(format!(
"task-search-cancel-{}",
Uuid::new_v4()
)),
params: ThreadSearchCancelParams { request_id },
},
),
)
.await;
});
});
}
None
}
pub(crate) fn valid_thread_path(path: &str) -> Option<&str> {
+127
View File
@@ -3,7 +3,134 @@ use crate::mention_codec::LinkedMention;
use crate::mention_codec::decode_history_mentions_with_at_mentions;
use crate::mention_codec::encode_history_mentions_at_elements;
use codex_protocol::user_input::TextElement;
use futures::SinkExt;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use tokio_tungstenite::tungstenite::Message;
#[tokio::test]
async fn remote_task_search_cancels_without_delaying_results() {
for timeout in [false, true] {
check_remote_task_search_cancellation(timeout).await;
}
}
async fn check_remote_task_search_cancellation(timeout: bool) {
use codex_app_server_client::RemoteAppServerClient;
use codex_app_server_client::RemoteAppServerConnectArgs;
use codex_app_server_client::RemoteAppServerEndpoint;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let websocket_url = format!("ws://{}", listener.local_addr().unwrap());
let (search_started, started) = tokio::sync::oneshot::channel();
let (cancel_sent, cancelled) = tokio::sync::oneshot::channel();
let rpc = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap();
let mut search_started = Some(search_started);
let mut search_id = None;
while let Some(Ok(Message::Text(text))) = socket.next().await {
let request: serde_json::Value = serde_json::from_str(&text).unwrap();
let result = match request["method"].as_str() {
Some("initialize") => {
json!({"userAgent":"task-search-test", "codexHome":"/unused"})
}
Some("thread/list") => json!({
"data": [{
"id": "task-123", "sessionId": "task-123", "preview": "needle",
"name": "needle title", "ephemeral": false, "modelProvider": "openai",
"createdAt": 0, "updatedAt": 0, "status": {"type": "notLoaded"},
"cwd": std::env::temp_dir(), "cliVersion": "test", "source": "cli",
"turns": []
}],
"nextCursor": null
}),
Some("thread/search") => {
search_id = Some(request["id"].clone());
search_started.take().unwrap().send(()).unwrap();
continue;
}
Some("thread/search/cancel") => {
assert_eq!(Some(&request["params"]["requestId"]), search_id.as_ref());
cancel_sent.send(()).unwrap();
// Leave cancellation unanswered to verify it cannot delay title results.
std::future::pending::<()>().await;
return;
}
_ => continue,
};
socket
.send(Message::Text(
json!({"jsonrpc":"2.0", "id":request["id"], "result":result})
.to_string()
.into(),
))
.await
.unwrap();
}
panic!("remote disconnected before cancelling the search");
});
let remote = RemoteAppServerClient::connect(RemoteAppServerConnectArgs {
required_human_approval: false,
endpoint: RemoteAppServerEndpoint::WebSocket {
websocket_url,
auth_token: None,
},
client_name: "task-search-test".into(),
client_version: "test".into(),
experimental_api: true,
mcp_server_openai_form_elicitation: false,
opt_out_notification_methods: Vec::new(),
channel_capacity: 8,
})
.await
.unwrap();
let handle = AppServerRequestHandle::Remote(remote.request_handle());
let (tx, mut events) = tokio::sync::mpsc::unbounded_channel();
let current = ThreadId::from_u128(/*value*/ 1);
let mut search = TaskSearch::default();
search.spawn(
handle.clone(),
"needle".into(),
current,
PathBuf::from("/tmp"),
AppEventSender::new(tx.clone()),
);
tokio::time::timeout(Duration::from_secs(/*secs*/ 5), started)
.await
.unwrap()
.unwrap();
if timeout {
tokio::time::pause();
let deadline = tokio::time::Instant::now() + SEARCH_TIMEOUT;
tokio::time::advance(SEARCH_TIMEOUT).await;
let event = events.recv().await.unwrap();
assert!(
matches!(event, AppEvent::TaskSearchResult { query, matches, .. }
if query == "needle" && matches.len() == 1 && matches[0].title == "needle title")
);
assert!(tokio::time::Instant::now() < deadline + CANCEL_TIMEOUT / 2);
tokio::time::resume();
} else {
search.spawn(
handle,
String::new(),
current,
PathBuf::from("/tmp"),
AppEventSender::new(tx),
);
let event = events.recv().await.unwrap();
assert!(
matches!(event, AppEvent::TaskSearchResult { query, matches, .. } if query.is_empty() && matches.is_empty())
);
}
tokio::time::timeout(Duration::from_secs(/*secs*/ 5), cancelled)
.await
.unwrap()
.unwrap();
rpc.abort();
let _ = remote.shutdown().await;
}
#[test]
fn task_mention_history_round_trips_multiword_and_escaped_titles() {
@@ -77,6 +77,7 @@ from .v2_all import ThreadRealtimeStartedNotification
from .v2_all import ThreadRealtimeTranscriptDeltaNotification
from .v2_all import ThreadRealtimeTranscriptDoneNotification
from .v2_all import ThreadRevertedNotification
from .v2_all import ThreadSearchPartialNotification
from .v2_all import ThreadSectionChangedNotification
from .v2_all import ThreadSettingsUpdatedNotification
from .v2_all import ThreadStartedNotification
@@ -163,6 +164,7 @@ KnownNotificationPayload: TypeAlias = (
| ThreadRealtimeTranscriptDeltaNotification
| ThreadRealtimeTranscriptDoneNotification
| ThreadRevertedNotification
| ThreadSearchPartialNotification
| ThreadSectionChangedNotification
| ThreadSettingsUpdatedNotification
| ThreadStartedNotification
@@ -252,6 +254,7 @@ NOTIFICATION_MODELS: dict[str, type[KnownNotificationPayload]] = {
"thread/realtime/transcript/delta": ThreadRealtimeTranscriptDeltaNotification,
"thread/realtime/transcript/done": ThreadRealtimeTranscriptDoneNotification,
"thread/reverted": ThreadRevertedNotification,
"thread/search/partial": ThreadSearchPartialNotification,
"thread/section/changed": ThreadSectionChangedNotification,
"thread/settings/updated": ThreadSettingsUpdatedNotification,
"thread/started": ThreadStartedNotification,
@@ -14413,6 +14413,31 @@ class ItemAutoApprovalReviewCompletedServerNotification(BaseModel):
params: ItemGuardianApprovalReviewCompletedNotification
class ThreadSearchPartialNotification(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
data: list[ThreadSearchResult]
request_id: Annotated[RequestId, Field(alias="requestId")]
class ThreadSearchPartialServerNotification(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
emitted_at_ms: Annotated[
int | None,
Field(
alias="emittedAtMs",
description="Unix timestamp (in milliseconds) when app-server emitted this notification.",
),
] = None
method: Annotated[
Literal["thread/search/partial"], Field(title="Thread/search/partialNotificationMethod")
]
params: ThreadSearchPartialNotification
class ServerNotification(
RootModel[
ErrorServerNotification
@@ -14453,6 +14478,7 @@ class ServerNotification(
| ItemPlanDeltaServerNotification
| CommandExecOutputDeltaServerNotification
| ProcessOutputDeltaServerNotification
| ThreadSearchPartialServerNotification
| ProcessExitedServerNotification
| ItemCommandExecutionOutputDeltaServerNotification
| ItemCommandExecutionTerminalInteractionServerNotification
@@ -14542,6 +14568,7 @@ class ServerNotification(
| ItemPlanDeltaServerNotification
| CommandExecOutputDeltaServerNotification
| ProcessOutputDeltaServerNotification
| ThreadSearchPartialServerNotification
| ProcessExitedServerNotification
| ItemCommandExecutionOutputDeltaServerNotification
| ItemCommandExecutionTerminalInteractionServerNotification