diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index 3e9f143d09..f77b82340c 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -3234,17 +3234,28 @@ async fn thread_list_archived_filter() -> Result<()> { assert_eq!(data.len(), 1); assert_eq!(data[0].id, active_id); - let ThreadListResponse { data, .. } = list_threads( - &mut mcp, - /*cursor*/ None, - Some(10), - Some(vec!["mock_provider".to_string()]), - /*source_kinds*/ None, - Some(true), - ) - .await?; - assert_eq!(data.len(), 1); - assert_eq!(data[0].id, archived_id); + for (sort_key, provider, expected_ids) in [ + (None, "mock_provider", vec![archived_id.as_str()]), + ( + Some(ThreadSortKey::UpdatedAt), + "mock_provider", + vec![archived_id.as_str()], + ), + (Some(ThreadSortKey::UpdatedAt), "other_provider", vec![]), + ] { + let page = list_threads_with_sort( + &mut mcp, + /*cursor*/ None, + Some(10), + Some(vec![provider.to_string()]), + /*source_kinds*/ None, + sort_key, + /*archived*/ Some(true), + ) + .await?; + let ids: Vec<_> = page.data.iter().map(|thread| thread.id.as_str()).collect(); + assert_eq!(ids, expected_ids); + } Ok(()) } diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index 2c3ac1451a..da7f9cd3b0 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -96,12 +96,12 @@ async fn full_history_load_preserves_records_and_errors_across_representations() } #[tokio::test] -async fn read_session_meta_line_stops_before_invalid_utf8_tail() -> anyhow::Result<()> { +async fn metadata_and_preview_reads_stop_before_invalid_utf8_tail() -> anyhow::Result<()> { let home = TempDir::new()?; let uuid = Uuid::from_u128(16); let thread_id = ThreadId::from_string(&uuid.to_string())?; let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid); - write_rollout(&rollout_path, thread_id, "ignored tail")?; + write_rollout(&rollout_path, thread_id, "first message")?; let mut contents = fs::read(&rollout_path)?; contents.resize(contents.len() + 64 * 1024, 0xff); fs::write(&rollout_path, contents)?; @@ -109,16 +109,22 @@ async fn read_session_meta_line_stops_before_invalid_utf8_tail() -> anyhow::Resu let session_meta = read_session_meta_line(&rollout_path).await?; assert_eq!(session_meta.meta.id, thread_id); + let summary = crate::list::read_thread_item_from_rollout(rollout_path).await; + assert_eq!( + summary.and_then(|item| item.first_user_message), + Some("first message".to_string()) + ); Ok(()) } #[tokio::test] -async fn read_session_meta_line_stops_before_invalid_utf8_compressed_tail() -> anyhow::Result<()> { +async fn metadata_and_preview_reads_stop_before_invalid_utf8_compressed_tail() -> anyhow::Result<()> +{ let home = TempDir::new()?; let uuid = Uuid::from_u128(17); let thread_id = ThreadId::from_string(&uuid.to_string())?; let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid); - write_rollout(&rollout_path, thread_id, "ignored compressed tail")?; + write_rollout(&rollout_path, thread_id, "first message")?; let mut contents = fs::read(&rollout_path)?; contents.resize(contents.len() + 64 * 1024, 0xff); fs::write(&rollout_path, contents)?; @@ -129,6 +135,11 @@ async fn read_session_meta_line_stops_before_invalid_utf8_compressed_tail() -> a assert_eq!(session_meta.meta.id, thread_id); assert!(!rollout_path.exists()); assert!(compressed_rollout_path(&rollout_path).exists()); + let summary = crate::list::read_thread_item_from_rollout(rollout_path).await; + assert_eq!( + summary.and_then(|item| item.first_user_message), + Some("first message".to_string()) + ); Ok(()) } diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index 2120ea389a..50a4e8515b 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -1,6 +1,7 @@ #![allow(warnings, clippy::all)] use codex_utils_path as path_utils; +use memchr::memmem::Finder; use std::cmp::Reverse; use std::collections::HashMap; use std::ffi::OsStr; @@ -10,6 +11,8 @@ use std::num::NonZero; use std::ops::ControlFlow; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::LazyLock; use time::OffsetDateTime; use time::PrimitiveDateTime; use time::format_description::FormatItem; @@ -149,6 +152,10 @@ const MAX_SCAN_FILES: usize = 10000; const HEAD_RECORD_LIMIT: usize = 10; const USER_EVENT_SCAN_LIMIT: usize = 200; +// A Unicode escape can spell part of the event tag. Either marker requires a full decode. +static EVENT_MSG_MARKERS: LazyLock<[Finder<'static>; 2]> = + LazyLock::new(|| [Finder::new(br#""event_msg""#), Finder::new(br"\u")]); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ThreadSortKey { CreatedAt, @@ -260,11 +267,10 @@ struct FilesByCreatedAtVisitor<'a> { page_size: usize, anchor_state: AnchorState, more_matches_available: bool, - config: ThreadListConfig<'a>, - provider_matcher: Option<&'a ProviderMatcher<'a>>, + filters: Arc, } -impl<'a> RolloutFileVisitor for FilesByCreatedAtVisitor<'a> { +impl RolloutFileVisitor for FilesByCreatedAtVisitor<'_> { async fn visit( &mut self, ts: OffsetDateTime, @@ -289,12 +295,9 @@ impl<'a> RolloutFileVisitor for FilesByCreatedAtVisitor<'a> { .and_then(format_rfc3339); if let Some(item) = build_thread_item( path, - self.config.allowed_sources, - self.provider_matcher, - self.config.cwd_filters, + Arc::clone(&self.filters), updated_at, PreviewFilter::RequirePreview, - self.config.include_attached_threads, ) .await { @@ -391,32 +394,22 @@ pub async fn get_threads_in_root( let anchor = cursor.cloned(); - let provider_matcher = config - .model_providers - .and_then(|filters| ProviderMatcher::new(filters, config.default_provider)); + let filters = Arc::new(SummaryFilters { + allowed_sources: config.allowed_sources.to_vec(), + provider_matcher: config + .model_providers + .and_then(|providers| ProviderMatcher::new(providers, config.default_provider)), + cwd_filters: config.cwd_filters.map(<[PathBuf]>::to_vec), + include_attached_threads: config.include_attached_threads, + }); let result = match config.layout { ThreadListLayout::NestedByDate => { - traverse_directories_for_paths( - root.clone(), - page_size, - anchor, - sort_key, - config, - provider_matcher.as_ref(), - ) - .await? + traverse_directories_for_paths(root.clone(), page_size, anchor, sort_key, filters) + .await? } ThreadListLayout::Flat => { - traverse_flat_paths( - root.clone(), - page_size, - anchor, - sort_key, - config, - provider_matcher.as_ref(), - ) - .await? + traverse_flat_paths(root.clone(), page_size, anchor, sort_key, filters).await? } }; Ok(result) @@ -431,29 +424,14 @@ async fn traverse_directories_for_paths( page_size: usize, anchor: Option, sort_key: ThreadSortKey, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { match sort_key { ThreadSortKey::CreatedAt => { - traverse_directories_for_paths_created( - root, - page_size, - anchor, - config, - provider_matcher, - ) - .await + traverse_directories_for_paths_created(root, page_size, anchor, filters).await } ThreadSortKey::UpdatedAt | ThreadSortKey::RecencyAt => { - traverse_directories_for_paths_updated( - root, - page_size, - anchor, - config, - provider_matcher, - ) - .await + traverse_directories_for_paths_updated(root, page_size, anchor, filters).await } } } @@ -463,15 +441,14 @@ async fn traverse_flat_paths( page_size: usize, anchor: Option, sort_key: ThreadSortKey, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { match sort_key { ThreadSortKey::CreatedAt => { - traverse_flat_paths_created(root, page_size, anchor, config, provider_matcher).await + traverse_flat_paths_created(root, page_size, anchor, filters).await } ThreadSortKey::UpdatedAt | ThreadSortKey::RecencyAt => { - traverse_flat_paths_updated(root, page_size, anchor, config, provider_matcher).await + traverse_flat_paths_updated(root, page_size, anchor, filters).await } } } @@ -486,8 +463,7 @@ async fn traverse_directories_for_paths_created( root: PathBuf, page_size: usize, anchor: Option, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { let mut items: Vec = Vec::with_capacity(page_size); let mut scanned_files = 0usize; @@ -497,8 +473,7 @@ async fn traverse_directories_for_paths_created( page_size, anchor_state: AnchorState::new(anchor), more_matches_available, - config, - provider_matcher, + filters, }; walk_rollout_files(&root, &mut scanned_files, &mut visitor).await?; more_matches_available = visitor.more_matches_available; @@ -535,8 +510,7 @@ async fn traverse_directories_for_paths_updated( root: PathBuf, page_size: usize, anchor: Option, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { let mut items: Vec = Vec::with_capacity(page_size); let mut scanned_files = 0usize; @@ -562,12 +536,9 @@ async fn traverse_directories_for_paths_updated( let updated_at_fallback = format_rfc3339(ts); if let Some(item) = build_thread_item( candidate.path, - config.allowed_sources, - provider_matcher, - config.cwd_filters, + Arc::clone(&filters), updated_at_fallback, PreviewFilter::RequirePreview, - config.include_attached_threads, ) .await { @@ -599,8 +570,7 @@ async fn traverse_flat_paths_created( root: PathBuf, page_size: usize, anchor: Option, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { let mut items: Vec = Vec::with_capacity(page_size); let mut scanned_files = 0usize; @@ -622,12 +592,9 @@ async fn traverse_flat_paths_created( .and_then(format_rfc3339); if let Some(item) = build_thread_item( path, - config.allowed_sources, - provider_matcher, - config.cwd_filters, + Arc::clone(&filters), updated_at, PreviewFilter::IncludeEmpty, - config.include_attached_threads, ) .await { @@ -659,8 +626,7 @@ async fn traverse_flat_paths_updated( root: PathBuf, page_size: usize, anchor: Option, - config: ThreadListConfig<'_>, - provider_matcher: Option<&ProviderMatcher<'_>>, + filters: Arc, ) -> io::Result { let mut items: Vec = Vec::with_capacity(page_size); let mut scanned_files = 0usize; @@ -686,12 +652,9 @@ async fn traverse_flat_paths_updated( let updated_at_fallback = format_rfc3339(ts); if let Some(item) = build_thread_item( candidate.path, - config.allowed_sources, - provider_matcher, - config.cwd_filters, + Arc::clone(&filters), updated_at_fallback, PreviewFilter::IncludeEmpty, - config.include_attached_threads, ) .await { @@ -781,24 +744,14 @@ enum PreviewFilter { async fn build_thread_item( path: PathBuf, - allowed_sources: &[SessionSource], - provider_matcher: Option<&ProviderMatcher<'_>>, - cwd_filters: Option<&[PathBuf]>, + filters: Arc, updated_at: Option, preview_filter: PreviewFilter, - include_attached_threads: bool, ) -> Option { - // Read head and detect preview-bearing events; goal previews can appear before - // the first normal user message. - let filters = SummaryFilters { - allowed_sources, - provider_matcher, - cwd_filters, - include_attached_threads, - }; - let summary = read_head_summary(&path, HEAD_RECORD_LIMIT, filters) - .await - .ok()??; + let summary = + compression::read_rollout_lines(&path, move |lines| read_head_summary(lines, &filters)) + .await + .ok()??; if summary.saw_session_meta && (matches!(preview_filter, PreviewFilter::IncludeEmpty) || summary.preview.is_some() @@ -867,12 +820,14 @@ async fn build_thread_item( pub async fn read_thread_item_from_rollout(path: PathBuf) -> Option { build_thread_item( path, - &[], - /*provider_matcher*/ None, - /*cwd_filters*/ None, + Arc::new(SummaryFilters { + allowed_sources: Vec::new(), + provider_matcher: None, + cwd_filters: None, + include_attached_threads: true, + }), /*updated_at*/ None, PreviewFilter::RequirePreview, - /*include_attached_threads*/ true, ) .await } @@ -929,20 +884,20 @@ async fn walk_rollout_files( Ok(()) } -struct ProviderMatcher<'a> { - filters: &'a [String], +struct ProviderMatcher { + filters: Vec, matches_default_provider: bool, } -impl<'a> ProviderMatcher<'a> { - fn new(filters: &'a [String], default_provider: &'a str) -> Option { +impl ProviderMatcher { + fn new(filters: &[String], default_provider: &str) -> Option { if filters.is_empty() { return None; } let matches_default_provider = filters.iter().any(|provider| provider == default_provider); Some(Self { - filters, + filters: filters.to_vec(), matches_default_provider, }) } @@ -955,14 +910,14 @@ impl<'a> ProviderMatcher<'a> { } } -struct SummaryFilters<'a> { - allowed_sources: &'a [SessionSource], - provider_matcher: Option<&'a ProviderMatcher<'a>>, - cwd_filters: Option<&'a [PathBuf]>, +struct SummaryFilters { + allowed_sources: Vec, + provider_matcher: Option, + cwd_filters: Option>, include_attached_threads: bool, } -impl SummaryFilters<'_> { +impl SummaryFilters { /// Filters only the rollout's first session metadata; copied fork metadata cannot override it. fn matches(&self, summary: &HeadTailSummary) -> bool { (self.include_attached_threads || summary.attached_to_thread_id.is_none()) @@ -973,8 +928,9 @@ impl SummaryFilters<'_> { .is_some_and(|source| self.allowed_sources.contains(source))) && self .provider_matcher + .as_ref() .is_none_or(|matcher| matcher.matches(summary.model_provider.as_deref())) - && self.cwd_filters.is_none_or(|filters| { + && self.cwd_filters.as_deref().is_none_or(|filters| { summary.cwd.as_ref().is_some_and(|cwd| { filters .iter() @@ -984,28 +940,37 @@ impl SummaryFilters<'_> { } } -async fn read_head_summary( - path: &Path, - head_limit: usize, - filters: SummaryFilters<'_>, +fn read_head_summary( + lines: &mut dyn Iterator>, + filters: &SummaryFilters, ) -> io::Result> { - let mut lines = compression::open_rollout_line_reader(path).await?; let mut summary = HeadTailSummary::default(); let mut lines_scanned = 0usize; + let event_msg_markers = LazyLock::force(&EVENT_MSG_MARKERS); - while lines_scanned < head_limit + while lines_scanned < HEAD_RECORD_LIMIT || (summary.saw_session_meta && (summary.preview.is_none() || summary.first_user_message.is_none()) - && lines_scanned < head_limit + USER_EVENT_SCAN_LIMIT) + && lines_scanned < HEAD_RECORD_LIMIT + USER_EVENT_SCAN_LIMIT) { - let line_opt = lines.next_line().await?; - let Some(line) = line_opt else { break }; + let Some(line) = lines.next().transpose()? else { + break; + }; let trimmed = line.trim(); if trimmed.is_empty() { continue; } lines_scanned += 1; + // After the first session header, only events can change the summary. + if summary.saw_session_meta + && !event_msg_markers + .iter() + .any(|marker| marker.find(trimmed.as_bytes()).is_some()) + { + continue; + } + let parsed = crate::parse_rollout_line(trimmed); let rollout_line = match parsed { Ok(rollout_line) => rollout_line, diff --git a/codex-rs/rollout/src/tests.rs b/codex-rs/rollout/src/tests.rs index dfc4970978..23c8da15ff 100644 --- a/codex-rs/rollout/src/tests.rs +++ b/codex-rs/rollout/src/tests.rs @@ -8,6 +8,7 @@ use std::fs::File; use std::fs::FileTimes; use std::io::Write; use std::path::Path; +use std::path::PathBuf; use chrono::TimeZone; use pretty_assertions::assert_eq; @@ -619,7 +620,7 @@ fn write_session_file_with_delayed_user_event( ts_str: &str, uuid: Uuid, meta_lines_before_user: usize, -) -> std::io::Result<()> { +) -> std::io::Result { let format: &[FormatItem] = format_description!("[year]-[month]-[day]T[hour]-[minute]-[second]"); let dt = PrimitiveDateTime::parse(ts_str, format) @@ -634,7 +635,7 @@ fn write_session_file_with_delayed_user_event( let filename = format!("rollout-{ts_str}-{uuid}.jsonl"); let file_path = dir.join(filename); - let mut file = File::create(file_path)?; + let mut file = File::create(&file_path)?; for i in 0..meta_lines_before_user { let id = if i == 0 { @@ -669,7 +670,7 @@ fn write_session_file_with_delayed_user_event( let times = FileTimes::new().set_modified(dt.into()); file.set_times(times)?; - Ok(()) + Ok(file_path) } fn write_session_file_with_meta_payload( @@ -1177,25 +1178,45 @@ async fn test_list_threads_scans_past_head_for_user_event() { let uuid = Uuid::from_u128(99); let ts = "2025-05-01T10-30-00"; - write_session_file_with_delayed_user_event(home, ts, uuid, /*meta_lines_before_user*/ 12) - .unwrap(); - - let provider_filter = provider_vec(&[TEST_PROVIDER]); - let page = get_threads( - home, - /*page_size*/ 10, - /*cursor*/ None, - ThreadSortKey::CreatedAt, - INTERACTIVE_SESSION_SOURCES.as_slice(), - Some(provider_filter.as_slice()), - /*cwd_filters*/ None, - TEST_PROVIDER, + let path = write_session_file_with_delayed_user_event( + home, ts, uuid, /*meta_lines_before_user*/ 12, ) - .await .unwrap(); - assert_eq!(page.items.len(), 1); - assert_eq!(page.items[0].thread_id, Some(thread_id_from_uuid(uuid))); + let provider_filter = provider_vec(&[TEST_PROVIDER]); + let rollout = fs::read_to_string(&path).unwrap(); + let (headers, user_event) = rollout.trim_end().rsplit_once('\n').unwrap(); + let malformed_event = r#"{"type":"event_msg"}"#; + // Persisted events can use an escaped outer tag or a numeric inner tag. + for event in [ + user_event.to_owned(), + user_event.replace(r#""event_msg""#, r#""event\u005fmsg""#), + user_event.replace(r#""user_message""#, "20"), + ] { + fs::write( + &path, + format!("{headers}\ninvalid json\n{malformed_event}\n{event}\n"), + ) + .unwrap(); + let page = get_threads( + home, + /*page_size*/ 10, + /*cursor*/ None, + ThreadSortKey::CreatedAt, + INTERACTIVE_SESSION_SOURCES.as_slice(), + Some(provider_filter.as_slice()), + /*cwd_filters*/ None, + TEST_PROVIDER, + ) + .await + .unwrap(); + + let [item] = page.items.as_slice() else { + panic!("expected one thread, got {}", page.items.len()); + }; + assert_eq!(item.thread_id, Some(thread_id_from_uuid(uuid))); + assert_eq!(item.first_user_message.as_deref(), Some("Hello from user")); + } } #[tokio::test]