mirror of
https://github.com/openai/codex.git
synced 2026-11-22 23:00:42 +00:00
Optimize filesystem thread listing
This commit is contained in:
@@ -40,6 +40,26 @@ pub(crate) async fn file_modified_time(path: &Path) -> io::Result<Option<time::O
|
||||
Ok(modified.map(time::OffsetDateTime::from))
|
||||
}
|
||||
|
||||
/// Blocking equivalent of [`file_modified_time`] for filesystem walkers.
|
||||
pub(crate) fn file_modified_time_blocking(path: &Path) -> io::Result<Option<time::OffsetDateTime>> {
|
||||
for attempt in 0..MAX_NOT_FOUND_RETRIES {
|
||||
if let Some(path) = path::existing_rollout_path_blocking(path) {
|
||||
match std::fs::metadata(path) {
|
||||
Ok(meta) => {
|
||||
let modified = meta.modified().ok();
|
||||
return Ok(modified.map(time::OffsetDateTime::from));
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if attempt + 1 < MAX_NOT_FOUND_RETRIES {
|
||||
std::thread::sleep(OPEN_ROLLOUT_LINE_READER_RETRY_DELAY);
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Opens a rollout line reader that transparently handles plain `.jsonl` and `.jsonl.zst` files.
|
||||
///
|
||||
/// If the requested path disappears during a representation transition, this briefly retries
|
||||
@@ -955,6 +975,19 @@ mod path {
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn existing_rollout_path_blocking(path: &Path) -> Option<PathBuf> {
|
||||
let plain_path = plain_rollout_path(path);
|
||||
if matches!(std::fs::metadata(plain_path.as_path()), Ok(metadata) if metadata.is_file()) {
|
||||
return Some(plain_path);
|
||||
}
|
||||
let compressed_path = compressed_rollout_path(plain_path.as_path());
|
||||
if matches!(std::fs::metadata(compressed_path.as_path()), Ok(metadata) if metadata.is_file())
|
||||
{
|
||||
return Some(compressed_path);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
mod file_name {
|
||||
|
||||
@@ -353,6 +353,25 @@ async fn worker_skips_existing_compressed_archived_rollouts() -> anyhow::Result<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_file_modified_time_resolves_compressed_sibling() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let uuid = Uuid::from_u128(12);
|
||||
let thread_id = ThreadId::from_string(&uuid.to_string())?;
|
||||
let rollout_path = archived_rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
|
||||
write_rollout(&rollout_path, thread_id, "compressed mtime")?;
|
||||
compress_now(&rollout_path)?;
|
||||
let compressed_path = compressed_rollout_path(&rollout_path);
|
||||
set_old_mtime(&compressed_path)?;
|
||||
let expected = fs::metadata(&compressed_path)?.modified()?;
|
||||
|
||||
assert_eq!(
|
||||
file_modified_time_blocking(&rollout_path)?,
|
||||
Some(time::OffsetDateTime::from(expected))
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_skips_when_fresh_run_marker_exists() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
|
||||
@@ -7,6 +7,8 @@ use codex_protocol::protocol::SessionSource;
|
||||
pub(crate) mod compression;
|
||||
pub(crate) mod config;
|
||||
pub(crate) mod list;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod list_test_support;
|
||||
pub(crate) mod metadata;
|
||||
pub(crate) mod policy;
|
||||
pub(crate) mod recorder;
|
||||
|
||||
+653
-378
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,80 @@
|
||||
//! Test-only operation counts for filesystem thread listing.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct ThreadListWork {
|
||||
pub rollout_opens: usize,
|
||||
pub session_meta_records: usize,
|
||||
pub full_head_summaries: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ThreadListWorkRecorder {
|
||||
rollout_opens: AtomicUsize,
|
||||
session_meta_records: AtomicUsize,
|
||||
full_head_summaries: AtomicUsize,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ThreadListWorkScope(Option<Arc<ThreadListWorkRecorder>>);
|
||||
|
||||
impl ThreadListWorkScope {
|
||||
pub(crate) async fn scope<F>(self, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
match self.0 {
|
||||
Some(recorder) => THREAD_LIST_WORK.scope(recorder, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ThreadListWorkRecorder {
|
||||
fn snapshot(&self) -> ThreadListWork {
|
||||
ThreadListWork {
|
||||
rollout_opens: self.rollout_opens.load(Ordering::Relaxed),
|
||||
session_meta_records: self.session_meta_records.load(Ordering::Relaxed),
|
||||
full_head_summaries: self.full_head_summaries.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static THREAD_LIST_WORK: Arc<ThreadListWorkRecorder>;
|
||||
}
|
||||
|
||||
pub(crate) async fn record_thread_list_work<F>(future: F) -> (F::Output, ThreadListWork)
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
let recorder = Arc::new(ThreadListWorkRecorder::default());
|
||||
let output = THREAD_LIST_WORK.scope(Arc::clone(&recorder), future).await;
|
||||
(output, recorder.snapshot())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_thread_list_work() -> ThreadListWorkScope {
|
||||
ThreadListWorkScope(THREAD_LIST_WORK.try_with(Arc::clone).ok())
|
||||
}
|
||||
|
||||
pub(crate) fn record_session_meta() {
|
||||
let _ = THREAD_LIST_WORK.try_with(|work| {
|
||||
work.session_meta_records.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn record_rollout_open() {
|
||||
let _ = THREAD_LIST_WORK.try_with(|work| {
|
||||
work.rollout_opens.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn record_full_head_summary() {
|
||||
let _ = THREAD_LIST_WORK.try_with(|work| {
|
||||
work.full_head_summaries.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
}
|
||||
@@ -7,6 +7,7 @@ use std::fs::File;
|
||||
use std::fs::FileTimes;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::TimeZone;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -22,6 +23,7 @@ use crate::INTERACTIVE_SESSION_SOURCES;
|
||||
use crate::find_thread_path_by_id_str;
|
||||
use crate::list::Cursor;
|
||||
use crate::list::ThreadItem;
|
||||
use crate::list::ThreadListLayout;
|
||||
use crate::list::ThreadSortKey;
|
||||
use crate::list::ThreadsPage;
|
||||
use crate::list::get_threads;
|
||||
@@ -37,6 +39,7 @@ use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::ThreadGoal;
|
||||
use codex_protocol::protocol::ThreadGoalStatus;
|
||||
use codex_protocol::protocol::ThreadGoalUpdatedEvent;
|
||||
@@ -521,6 +524,367 @@ fn write_session_file_with_meta_payload(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_thread_list_benchmark_fixture(home: &Path) {
|
||||
let timestamp = "2025-01-01T12-00-00";
|
||||
for index in 0..5_000_u128 {
|
||||
write_session_file(
|
||||
home,
|
||||
timestamp,
|
||||
Uuid::from_u128(10_000 + index),
|
||||
/*num_records*/ 0,
|
||||
Some(SessionSource::SubAgent(SubAgentSource::Review)),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for index in 0..100_u128 {
|
||||
write_session_file(
|
||||
home,
|
||||
timestamp,
|
||||
Uuid::from_u128(index + 1),
|
||||
/*num_records*/ 0,
|
||||
Some(SessionSource::Cli),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn write_interactive_thread_list_fixture(home: &Path, count: u128, compressed: bool) {
|
||||
let timestamp = "2025-01-01T12-00-00";
|
||||
for index in 0..count {
|
||||
let uuid = Uuid::from_u128(index + 1);
|
||||
write_session_file(
|
||||
home,
|
||||
timestamp,
|
||||
uuid,
|
||||
/*num_records*/ 0,
|
||||
Some(SessionSource::Cli),
|
||||
)
|
||||
.unwrap();
|
||||
if compressed {
|
||||
compress_session_file(home, timestamp, uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compress_session_file(home: &Path, timestamp: &str, uuid: Uuid) {
|
||||
let path = home
|
||||
.join("sessions")
|
||||
.join("2025")
|
||||
.join("01")
|
||||
.join("01")
|
||||
.join(format!("rollout-{timestamp}-{uuid}.jsonl"));
|
||||
let compressed_path = crate::compression::compressed_rollout_path(path.as_path());
|
||||
let input = File::open(path.as_path()).unwrap();
|
||||
let output = File::create(compressed_path).unwrap();
|
||||
zstd::stream::copy_encode(input, output, 1).unwrap();
|
||||
fs::remove_file(path).unwrap();
|
||||
}
|
||||
|
||||
async fn benchmark_thread_list_fixture(
|
||||
label: &str,
|
||||
home: &Path,
|
||||
expected_work: crate::list_test_support::ThreadListWork,
|
||||
) {
|
||||
for sort_key in [ThreadSortKey::CreatedAt, ThreadSortKey::UpdatedAt] {
|
||||
let mut durations = Vec::with_capacity(20);
|
||||
let mut last_work = None;
|
||||
for _ in 0..20 {
|
||||
let started = Instant::now();
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
home,
|
||||
/*page_size*/ 50,
|
||||
/*cursor*/ None,
|
||||
sort_key,
|
||||
INTERACTIVE_SESSION_SOURCES.as_slice(),
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
assert_eq!(result.unwrap().items.len(), 50);
|
||||
durations.push(started.elapsed());
|
||||
last_work = Some(work);
|
||||
}
|
||||
durations.sort_unstable();
|
||||
let p50 = durations[durations.len() / 2];
|
||||
let p95 = durations[durations.len() * 95 / 100];
|
||||
let max = *durations.last().unwrap();
|
||||
let last_work = last_work.unwrap();
|
||||
eprintln!(
|
||||
"{label} {sort_key:?} benchmark: p50={p50:?} p95={p95:?} max={max:?} work={last_work:?}"
|
||||
);
|
||||
assert_eq!(last_work, expected_work);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_list_filters_source_before_reading_summary() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_session_file(
|
||||
temp.path(),
|
||||
"2025-01-01T12-00-00",
|
||||
Uuid::from_u128(9_001),
|
||||
/*num_records*/ 3,
|
||||
Some(SessionSource::SubAgent(SubAgentSource::Review)),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
temp.path(),
|
||||
/*page_size*/ 50,
|
||||
/*cursor*/ None,
|
||||
ThreadSortKey::UpdatedAt,
|
||||
INTERACTIVE_SESSION_SOURCES.as_slice(),
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap().items, Vec::new());
|
||||
assert_eq!(work.rollout_opens, 1);
|
||||
assert_eq!(work.session_meta_records, 1);
|
||||
assert_eq!(work.full_head_summaries, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_candidate_collection_caps_directory_entries() {
|
||||
for layout in [ThreadListLayout::NestedByDate, ThreadListLayout::Flat] {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let root = match layout {
|
||||
ThreadListLayout::NestedByDate => temp.path().join("2025").join("01").join("01"),
|
||||
ThreadListLayout::Flat => temp.path().to_path_buf(),
|
||||
};
|
||||
fs::create_dir_all(root.as_path()).unwrap();
|
||||
for index in 0..100_u128 {
|
||||
File::create(root.join(format!("unrelated-{index}.tmp"))).unwrap();
|
||||
}
|
||||
let collection_root = match layout {
|
||||
ThreadListLayout::NestedByDate => temp.path(),
|
||||
ThreadListLayout::Flat => root.as_path(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
crate::list::collect_thread_candidate_stats_for_test(
|
||||
collection_root,
|
||||
layout,
|
||||
/*entry_limit*/ 64,
|
||||
)
|
||||
.unwrap(),
|
||||
(0, 64, true)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_list_skips_metadata_pre_read_without_metadata_filters() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_session_file(
|
||||
temp.path(),
|
||||
"2025-01-01T12-00-00",
|
||||
Uuid::from_u128(9_001),
|
||||
/*num_records*/ 3,
|
||||
Some(SessionSource::Cli),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
temp.path(),
|
||||
/*page_size*/ 50,
|
||||
/*cursor*/ None,
|
||||
ThreadSortKey::UpdatedAt,
|
||||
&[],
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap().items.len(), 1);
|
||||
assert_eq!(work.rollout_opens, 1);
|
||||
assert_eq!(work.session_meta_records, 0);
|
||||
assert_eq!(work.full_head_summaries, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn updated_at_thread_list_builds_remaining_page_plus_lookahead() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_interactive_thread_list_fixture(
|
||||
temp.path(),
|
||||
/*count*/ 100,
|
||||
/*compressed*/ false,
|
||||
);
|
||||
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
temp.path(),
|
||||
/*page_size*/ 25,
|
||||
/*cursor*/ None,
|
||||
ThreadSortKey::UpdatedAt,
|
||||
INTERACTIVE_SESSION_SOURCES.as_slice(),
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap().items.len(), 25);
|
||||
assert_eq!(
|
||||
work,
|
||||
crate::list_test_support::ThreadListWork {
|
||||
rollout_opens: 26,
|
||||
session_meta_records: 26,
|
||||
full_head_summaries: 26,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn created_at_thread_list_stops_before_older_day() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
for index in 0..500_u128 {
|
||||
write_session_file(
|
||||
temp.path(),
|
||||
"2025-01-01T12-00-00",
|
||||
Uuid::from_u128(10_000 + index),
|
||||
/*num_records*/ 0,
|
||||
Some(SessionSource::SubAgent(SubAgentSource::Review)),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for index in 0..100_u128 {
|
||||
write_session_file(
|
||||
temp.path(),
|
||||
"2025-01-02T12-00-00",
|
||||
Uuid::from_u128(index + 1),
|
||||
/*num_records*/ 0,
|
||||
Some(SessionSource::Cli),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
temp.path(),
|
||||
/*page_size*/ 50,
|
||||
/*cursor*/ None,
|
||||
ThreadSortKey::CreatedAt,
|
||||
INTERACTIVE_SESSION_SOURCES.as_slice(),
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
let page = result.unwrap();
|
||||
let thread_ids = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| item.thread_id)
|
||||
.collect::<Vec<_>>();
|
||||
let expected_thread_ids = (51_u128..=100)
|
||||
.rev()
|
||||
.map(|id| Some(thread_id_from_uuid(Uuid::from_u128(id))))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(thread_ids, expected_thread_ids);
|
||||
assert_eq!(page.num_scanned_files, 100);
|
||||
assert_eq!(work.rollout_opens, 51);
|
||||
assert_eq!(work.session_meta_records, 51);
|
||||
assert_eq!(work.full_head_summaries, 51);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_list_bounds_summary_work_for_rejected_sources() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_thread_list_benchmark_fixture(temp.path());
|
||||
|
||||
for sort_key in [ThreadSortKey::CreatedAt, ThreadSortKey::UpdatedAt] {
|
||||
let (result, work) = crate::list_test_support::record_thread_list_work(get_threads(
|
||||
temp.path(),
|
||||
/*page_size*/ 50,
|
||||
/*cursor*/ None,
|
||||
sort_key,
|
||||
INTERACTIVE_SESSION_SOURCES.as_slice(),
|
||||
/*model_providers*/ None,
|
||||
/*cwd_filters*/ None,
|
||||
TEST_PROVIDER,
|
||||
))
|
||||
.await;
|
||||
let page = result.unwrap();
|
||||
let thread_ids = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| item.thread_id)
|
||||
.collect::<Vec<_>>();
|
||||
let expected_thread_ids = (51_u128..=100)
|
||||
.rev()
|
||||
.map(|id| Some(thread_id_from_uuid(Uuid::from_u128(id))))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(thread_ids, expected_thread_ids);
|
||||
assert_eq!(work.rollout_opens, 5_051);
|
||||
assert_eq!(work.session_meta_records, 5_051);
|
||||
assert_eq!(work.full_head_summaries, 51);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "release benchmark"]
|
||||
async fn thread_list_benchmark_5_100_rollouts() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_thread_list_benchmark_fixture(temp.path());
|
||||
benchmark_thread_list_fixture(
|
||||
"5,100-rollout stress",
|
||||
temp.path(),
|
||||
crate::list_test_support::ThreadListWork {
|
||||
rollout_opens: 5_051,
|
||||
session_meta_records: 5_051,
|
||||
full_head_summaries: 51,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "release benchmark"]
|
||||
async fn thread_list_benchmark_all_interactive_rollouts() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_interactive_thread_list_fixture(
|
||||
temp.path(),
|
||||
/*count*/ 200,
|
||||
/*compressed*/ false,
|
||||
);
|
||||
benchmark_thread_list_fixture(
|
||||
"200 interactive rollouts",
|
||||
temp.path(),
|
||||
crate::list_test_support::ThreadListWork {
|
||||
rollout_opens: 51,
|
||||
session_meta_records: 51,
|
||||
full_head_summaries: 51,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "release benchmark"]
|
||||
async fn thread_list_benchmark_compressed_rollouts() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
write_interactive_thread_list_fixture(
|
||||
temp.path(),
|
||||
/*count*/ 200,
|
||||
/*compressed*/ true,
|
||||
);
|
||||
benchmark_thread_list_fixture(
|
||||
"200 compressed interactive rollouts",
|
||||
temp.path(),
|
||||
crate::list_test_support::ThreadListWork {
|
||||
rollout_opens: 51,
|
||||
session_meta_records: 51,
|
||||
full_head_summaries: 51,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_conversations_latest_first() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user