Optimize filesystem thread listing

This commit is contained in:
Friel
2026-06-21 20:12:04 +00:00
parent 70a6aa2634
commit 6feb350a5e
6 changed files with 1151 additions and 378 deletions
+33
View File
@@ -40,6 +40,26 @@ pub(crate) async fn file_modified_time(path: &Path) -> io::Result<Option<time::O
Ok(modified.map(time::OffsetDateTime::from))
}
/// Blocking equivalent of [`file_modified_time`] for filesystem walkers.
pub(crate) fn file_modified_time_blocking(path: &Path) -> io::Result<Option<time::OffsetDateTime>> {
for attempt in 0..MAX_NOT_FOUND_RETRIES {
if let Some(path) = path::existing_rollout_path_blocking(path) {
match std::fs::metadata(path) {
Ok(meta) => {
let modified = meta.modified().ok();
return Ok(modified.map(time::OffsetDateTime::from));
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
if attempt + 1 < MAX_NOT_FOUND_RETRIES {
std::thread::sleep(OPEN_ROLLOUT_LINE_READER_RETRY_DELAY);
}
}
Ok(None)
}
/// Opens a rollout line reader that transparently handles plain `.jsonl` and `.jsonl.zst` files.
///
/// If the requested path disappears during a representation transition, this briefly retries
@@ -955,6 +975,19 @@ mod path {
}
None
}
pub(super) fn existing_rollout_path_blocking(path: &Path) -> Option<PathBuf> {
let plain_path = plain_rollout_path(path);
if matches!(std::fs::metadata(plain_path.as_path()), Ok(metadata) if metadata.is_file()) {
return Some(plain_path);
}
let compressed_path = compressed_rollout_path(plain_path.as_path());
if matches!(std::fs::metadata(compressed_path.as_path()), Ok(metadata) if metadata.is_file())
{
return Some(compressed_path);
}
None
}
}
mod file_name {
+19
View File
@@ -353,6 +353,25 @@ async fn worker_skips_existing_compressed_archived_rollouts() -> anyhow::Result<
Ok(())
}
#[test]
fn blocking_file_modified_time_resolves_compressed_sibling() -> anyhow::Result<()> {
let home = TempDir::new()?;
let uuid = Uuid::from_u128(12);
let thread_id = ThreadId::from_string(&uuid.to_string())?;
let rollout_path = archived_rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
write_rollout(&rollout_path, thread_id, "compressed mtime")?;
compress_now(&rollout_path)?;
let compressed_path = compressed_rollout_path(&rollout_path);
set_old_mtime(&compressed_path)?;
let expected = fs::metadata(&compressed_path)?.modified()?;
assert_eq!(
file_modified_time_blocking(&rollout_path)?,
Some(time::OffsetDateTime::from(expected))
);
Ok(())
}
#[tokio::test]
async fn worker_skips_when_fresh_run_marker_exists() -> anyhow::Result<()> {
let home = TempDir::new()?;
+2
View File
@@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionSource;
pub(crate) mod compression;
pub(crate) mod config;
pub(crate) mod list;
#[cfg(test)]
pub(crate) mod list_test_support;
pub(crate) mod metadata;
pub(crate) mod policy;
pub(crate) mod recorder;
File diff suppressed because it is too large Load Diff
+80
View File
@@ -0,0 +1,80 @@
//! Test-only operation counts for filesystem thread listing.
use std::future::Future;
use std::sync::Arc;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ThreadListWork {
pub rollout_opens: usize,
pub session_meta_records: usize,
pub full_head_summaries: usize,
}
#[derive(Default)]
struct ThreadListWorkRecorder {
rollout_opens: AtomicUsize,
session_meta_records: AtomicUsize,
full_head_summaries: AtomicUsize,
}
#[derive(Clone)]
pub(crate) struct ThreadListWorkScope(Option<Arc<ThreadListWorkRecorder>>);
impl ThreadListWorkScope {
pub(crate) async fn scope<F>(self, future: F) -> F::Output
where
F: Future,
{
match self.0 {
Some(recorder) => THREAD_LIST_WORK.scope(recorder, future).await,
None => future.await,
}
}
}
impl ThreadListWorkRecorder {
fn snapshot(&self) -> ThreadListWork {
ThreadListWork {
rollout_opens: self.rollout_opens.load(Ordering::Relaxed),
session_meta_records: self.session_meta_records.load(Ordering::Relaxed),
full_head_summaries: self.full_head_summaries.load(Ordering::Relaxed),
}
}
}
tokio::task_local! {
static THREAD_LIST_WORK: Arc<ThreadListWorkRecorder>;
}
pub(crate) async fn record_thread_list_work<F>(future: F) -> (F::Output, ThreadListWork)
where
F: Future,
{
let recorder = Arc::new(ThreadListWorkRecorder::default());
let output = THREAD_LIST_WORK.scope(Arc::clone(&recorder), future).await;
(output, recorder.snapshot())
}
pub(crate) fn capture_thread_list_work() -> ThreadListWorkScope {
ThreadListWorkScope(THREAD_LIST_WORK.try_with(Arc::clone).ok())
}
pub(crate) fn record_session_meta() {
let _ = THREAD_LIST_WORK.try_with(|work| {
work.session_meta_records.fetch_add(1, Ordering::Relaxed);
});
}
pub(crate) fn record_rollout_open() {
let _ = THREAD_LIST_WORK.try_with(|work| {
work.rollout_opens.fetch_add(1, Ordering::Relaxed);
});
}
pub(crate) fn record_full_head_summary() {
let _ = THREAD_LIST_WORK.try_with(|work| {
work.full_head_summaries.fetch_add(1, Ordering::Relaxed);
});
}
+364
View File
@@ -7,6 +7,7 @@ use std::fs::File;
use std::fs::FileTimes;
use std::io::Write;
use std::path::Path;
use std::time::Instant;
use chrono::TimeZone;
use pretty_assertions::assert_eq;
@@ -22,6 +23,7 @@ use crate::INTERACTIVE_SESSION_SOURCES;
use crate::find_thread_path_by_id_str;
use crate::list::Cursor;
use crate::list::ThreadItem;
use crate::list::ThreadListLayout;
use crate::list::ThreadSortKey;
use crate::list::ThreadsPage;
use crate::list::get_threads;
@@ -37,6 +39,7 @@ use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadGoal;
use codex_protocol::protocol::ThreadGoalStatus;
use codex_protocol::protocol::ThreadGoalUpdatedEvent;
@@ -521,6 +524,367 @@ fn write_session_file_with_meta_payload(
Ok(())
}
fn write_thread_list_benchmark_fixture(home: &Path) {
let timestamp = "2025-01-01T12-00-00";
for index in 0..5_000_u128 {
write_session_file(
home,
timestamp,
Uuid::from_u128(10_000 + index),
/*num_records*/ 0,
Some(SessionSource::SubAgent(SubAgentSource::Review)),
)
.unwrap();
}
for index in 0..100_u128 {
write_session_file(
home,
timestamp,
Uuid::from_u128(index + 1),
/*num_records*/ 0,
Some(SessionSource::Cli),
)
.unwrap();
}
}
fn write_interactive_thread_list_fixture(home: &Path, count: u128, compressed: bool) {
let timestamp = "2025-01-01T12-00-00";
for index in 0..count {
let uuid = Uuid::from_u128(index + 1);
write_session_file(
home,
timestamp,
uuid,
/*num_records*/ 0,
Some(SessionSource::Cli),
)
.unwrap();
if compressed {
compress_session_file(home, timestamp, uuid);
}
}
}
fn compress_session_file(home: &Path, timestamp: &str, uuid: Uuid) {
let path = home
.join("sessions")
.join("2025")
.join("01")
.join("01")
.join(format!("rollout-{timestamp}-{uuid}.jsonl"));
let compressed_path = crate::compression::compressed_rollout_path(path.as_path());
let input = File::open(path.as_path()).unwrap();
let output = File::create(compressed_path).unwrap();
zstd::stream::copy_encode(input, output, 1).unwrap();
fs::remove_file(path).unwrap();
}
async fn benchmark_thread_list_fixture(
label: &str,
home: &Path,
expected_work: crate::list_test_support::ThreadListWork,
) {
for sort_key in [ThreadSortKey::CreatedAt, ThreadSortKey::UpdatedAt] {
let mut durations = Vec::with_capacity(20);
let mut last_work = None;
for _ in 0..20 {
let started = Instant::now();
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
home,
/*page_size*/ 50,
/*cursor*/ None,
sort_key,
INTERACTIVE_SESSION_SOURCES.as_slice(),
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
assert_eq!(result.unwrap().items.len(), 50);
durations.push(started.elapsed());
last_work = Some(work);
}
durations.sort_unstable();
let p50 = durations[durations.len() / 2];
let p95 = durations[durations.len() * 95 / 100];
let max = *durations.last().unwrap();
let last_work = last_work.unwrap();
eprintln!(
"{label} {sort_key:?} benchmark: p50={p50:?} p95={p95:?} max={max:?} work={last_work:?}"
);
assert_eq!(last_work, expected_work);
}
}
#[tokio::test]
async fn thread_list_filters_source_before_reading_summary() {
let temp = TempDir::new().unwrap();
write_session_file(
temp.path(),
"2025-01-01T12-00-00",
Uuid::from_u128(9_001),
/*num_records*/ 3,
Some(SessionSource::SubAgent(SubAgentSource::Review)),
)
.unwrap();
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
temp.path(),
/*page_size*/ 50,
/*cursor*/ None,
ThreadSortKey::UpdatedAt,
INTERACTIVE_SESSION_SOURCES.as_slice(),
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
assert_eq!(result.unwrap().items, Vec::new());
assert_eq!(work.rollout_opens, 1);
assert_eq!(work.session_meta_records, 1);
assert_eq!(work.full_head_summaries, 0);
}
#[test]
fn thread_candidate_collection_caps_directory_entries() {
for layout in [ThreadListLayout::NestedByDate, ThreadListLayout::Flat] {
let temp = TempDir::new().unwrap();
let root = match layout {
ThreadListLayout::NestedByDate => temp.path().join("2025").join("01").join("01"),
ThreadListLayout::Flat => temp.path().to_path_buf(),
};
fs::create_dir_all(root.as_path()).unwrap();
for index in 0..100_u128 {
File::create(root.join(format!("unrelated-{index}.tmp"))).unwrap();
}
let collection_root = match layout {
ThreadListLayout::NestedByDate => temp.path(),
ThreadListLayout::Flat => root.as_path(),
};
assert_eq!(
crate::list::collect_thread_candidate_stats_for_test(
collection_root,
layout,
/*entry_limit*/ 64,
)
.unwrap(),
(0, 64, true)
);
}
}
#[tokio::test]
async fn thread_list_skips_metadata_pre_read_without_metadata_filters() {
let temp = TempDir::new().unwrap();
write_session_file(
temp.path(),
"2025-01-01T12-00-00",
Uuid::from_u128(9_001),
/*num_records*/ 3,
Some(SessionSource::Cli),
)
.unwrap();
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
temp.path(),
/*page_size*/ 50,
/*cursor*/ None,
ThreadSortKey::UpdatedAt,
&[],
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
assert_eq!(result.unwrap().items.len(), 1);
assert_eq!(work.rollout_opens, 1);
assert_eq!(work.session_meta_records, 0);
assert_eq!(work.full_head_summaries, 1);
}
#[tokio::test]
async fn updated_at_thread_list_builds_remaining_page_plus_lookahead() {
let temp = TempDir::new().unwrap();
write_interactive_thread_list_fixture(
temp.path(),
/*count*/ 100,
/*compressed*/ false,
);
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
temp.path(),
/*page_size*/ 25,
/*cursor*/ None,
ThreadSortKey::UpdatedAt,
INTERACTIVE_SESSION_SOURCES.as_slice(),
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
assert_eq!(result.unwrap().items.len(), 25);
assert_eq!(
work,
crate::list_test_support::ThreadListWork {
rollout_opens: 26,
session_meta_records: 26,
full_head_summaries: 26,
}
);
}
#[tokio::test]
async fn created_at_thread_list_stops_before_older_day() {
let temp = TempDir::new().unwrap();
for index in 0..500_u128 {
write_session_file(
temp.path(),
"2025-01-01T12-00-00",
Uuid::from_u128(10_000 + index),
/*num_records*/ 0,
Some(SessionSource::SubAgent(SubAgentSource::Review)),
)
.unwrap();
}
for index in 0..100_u128 {
write_session_file(
temp.path(),
"2025-01-02T12-00-00",
Uuid::from_u128(index + 1),
/*num_records*/ 0,
Some(SessionSource::Cli),
)
.unwrap();
}
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
temp.path(),
/*page_size*/ 50,
/*cursor*/ None,
ThreadSortKey::CreatedAt,
INTERACTIVE_SESSION_SOURCES.as_slice(),
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
let page = result.unwrap();
let thread_ids = page
.items
.iter()
.map(|item| item.thread_id)
.collect::<Vec<_>>();
let expected_thread_ids = (51_u128..=100)
.rev()
.map(|id| Some(thread_id_from_uuid(Uuid::from_u128(id))))
.collect::<Vec<_>>();
assert_eq!(thread_ids, expected_thread_ids);
assert_eq!(page.num_scanned_files, 100);
assert_eq!(work.rollout_opens, 51);
assert_eq!(work.session_meta_records, 51);
assert_eq!(work.full_head_summaries, 51);
}
#[tokio::test]
async fn thread_list_bounds_summary_work_for_rejected_sources() {
let temp = TempDir::new().unwrap();
write_thread_list_benchmark_fixture(temp.path());
for sort_key in [ThreadSortKey::CreatedAt, ThreadSortKey::UpdatedAt] {
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
temp.path(),
/*page_size*/ 50,
/*cursor*/ None,
sort_key,
INTERACTIVE_SESSION_SOURCES.as_slice(),
/*model_providers*/ None,
/*cwd_filters*/ None,
TEST_PROVIDER,
))
.await;
let page = result.unwrap();
let thread_ids = page
.items
.iter()
.map(|item| item.thread_id)
.collect::<Vec<_>>();
let expected_thread_ids = (51_u128..=100)
.rev()
.map(|id| Some(thread_id_from_uuid(Uuid::from_u128(id))))
.collect::<Vec<_>>();
assert_eq!(thread_ids, expected_thread_ids);
assert_eq!(work.rollout_opens, 5_051);
assert_eq!(work.session_meta_records, 5_051);
assert_eq!(work.full_head_summaries, 51);
}
}
#[tokio::test]
#[ignore = "release benchmark"]
async fn thread_list_benchmark_5_100_rollouts() {
let temp = TempDir::new().unwrap();
write_thread_list_benchmark_fixture(temp.path());
benchmark_thread_list_fixture(
"5,100-rollout stress",
temp.path(),
crate::list_test_support::ThreadListWork {
rollout_opens: 5_051,
session_meta_records: 5_051,
full_head_summaries: 51,
},
)
.await;
}
#[tokio::test]
#[ignore = "release benchmark"]
async fn thread_list_benchmark_all_interactive_rollouts() {
let temp = TempDir::new().unwrap();
write_interactive_thread_list_fixture(
temp.path(),
/*count*/ 200,
/*compressed*/ false,
);
benchmark_thread_list_fixture(
"200 interactive rollouts",
temp.path(),
crate::list_test_support::ThreadListWork {
rollout_opens: 51,
session_meta_records: 51,
full_head_summaries: 51,
},
)
.await;
}
#[tokio::test]
#[ignore = "release benchmark"]
async fn thread_list_benchmark_compressed_rollouts() {
let temp = TempDir::new().unwrap();
write_interactive_thread_list_fixture(
temp.path(),
/*count*/ 200,
/*compressed*/ true,
);
benchmark_thread_list_fixture(
"200 compressed interactive rollouts",
temp.path(),
crate::list_test_support::ThreadListWork {
rollout_opens: 51,
session_meta_records: 51,
full_head_summaries: 51,
},
)
.await;
}
#[tokio::test]
async fn test_list_conversations_latest_first() {
let temp = TempDir::new().unwrap();