mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Filter rollout filenames before checking compressed siblings (#53997)
## Why Filesystem lookups by thread or rollout ID check whether unrelated compressed rollouts have an uncompressed sibling. These checks are unnecessary when the filename does not match the requested ID. ## What changed Apply the thread or rollout ID filter before resolving plain/compressed file precedence. Preserve preference for uncompressed files and selection of the newest rollout for a thread. ## Testing Extend lookup tests to cover unrelated compressed files, uncompressed-file precedence, missing rollout IDs, and rollout ID tie-breaking for files with the same timestamp. GitOrigin-RevId: 5bdb56ebec8031fceb5f6adbfcf644ad010e52f2
This commit is contained in:
@@ -777,6 +777,14 @@ async fn find_thread_path_by_id_handles_compressed_rollout_filenames() -> anyhow
|
||||
compress_now(&rollout_path)?;
|
||||
let compressed_path = compressed_rollout_path(&rollout_path);
|
||||
|
||||
for id in 1000..1064 {
|
||||
let unrelated_path = compressed_rollout_path(&rollout_path.with_file_name(format!(
|
||||
"rollout-2025-01-03T12-00-00-{}.jsonl",
|
||||
Uuid::from_u128(id)
|
||||
)));
|
||||
fs::write(unrelated_path, zstd::stream::encode_all(&b""[..], 0)?)?;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
crate::find_thread_path_by_id_str(
|
||||
home.path(),
|
||||
@@ -784,8 +792,32 @@ async fn find_thread_path_by_id_handles_compressed_rollout_filenames() -> anyhow
|
||||
/*state_db_ctx*/ None
|
||||
)
|
||||
.await?,
|
||||
Some(compressed_path.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
crate::find_rollout_path_by_rollout_id(home.path(), thread_id).await?,
|
||||
Some(compressed_path)
|
||||
);
|
||||
|
||||
write_rollout(&rollout_path, thread_id, "plain filename lookup")?;
|
||||
assert_eq!(
|
||||
crate::find_thread_path_by_id_str(
|
||||
home.path(),
|
||||
&uuid.to_string(),
|
||||
/*state_db_ctx*/ None
|
||||
)
|
||||
.await?,
|
||||
Some(rollout_path.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
crate::find_rollout_path_by_rollout_id(home.path(), thread_id).await?,
|
||||
Some(rollout_path)
|
||||
);
|
||||
let missing_rollout_id = ThreadId::from_string(&Uuid::from_u128(2000).to_string())?;
|
||||
assert_eq!(
|
||||
crate::find_rollout_path_by_rollout_id(home.path(), missing_rollout_id).await?,
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
crate::find_thread_path_by_id_str(home.path(), "not-a-uuid", /*state_db_ctx*/ None).await?,
|
||||
None
|
||||
|
||||
@@ -1417,31 +1417,33 @@ async fn find_thread_path_by_id_from_filenames(
|
||||
return Ok(None);
|
||||
};
|
||||
let mut newest = None;
|
||||
visit_rollout_filenames::<()>(root, |file_name, path| {
|
||||
if file_name.thread_id() != target {
|
||||
return ControlFlow::Continue(());
|
||||
}
|
||||
let rollout_id = Uuid::parse_str(&file_name.rollout_id().to_string()).ok();
|
||||
let Some(rollout_id) = rollout_id else {
|
||||
return ControlFlow::Continue(());
|
||||
};
|
||||
// Rollout filenames only encode timestamps to second precision, so use the UUIDv7
|
||||
// rollout ID as a deterministic tie-breaker when multiple files are created in the same
|
||||
// second.
|
||||
let candidate = (file_name.timestamp(), rollout_id, path);
|
||||
if newest.as_ref().is_none_or(|(timestamp, id, _)| {
|
||||
candidate.0 > *timestamp || (candidate.0 == *timestamp && candidate.1 > *id)
|
||||
}) {
|
||||
newest = Some(candidate);
|
||||
}
|
||||
ControlFlow::Continue(())
|
||||
})
|
||||
visit_rollout_filenames::<()>(
|
||||
root,
|
||||
|file_name| file_name.thread_id() == target,
|
||||
|file_name, path| {
|
||||
let rollout_id = Uuid::parse_str(&file_name.rollout_id().to_string()).ok();
|
||||
let Some(rollout_id) = rollout_id else {
|
||||
return ControlFlow::Continue(());
|
||||
};
|
||||
// Rollout filenames only encode timestamps to second precision, so use the UUIDv7
|
||||
// rollout ID as a deterministic tie-breaker when multiple files are created in the same
|
||||
// second.
|
||||
let candidate = (file_name.timestamp(), rollout_id, path);
|
||||
if newest.as_ref().is_none_or(|(timestamp, id, _)| {
|
||||
candidate.0 > *timestamp || (candidate.0 == *timestamp && candidate.1 > *id)
|
||||
}) {
|
||||
newest = Some(candidate);
|
||||
}
|
||||
ControlFlow::Continue(())
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(newest.map(|(_timestamp, _id, path)| path))
|
||||
}
|
||||
|
||||
async fn visit_rollout_filenames<T>(
|
||||
root: &Path,
|
||||
matches: impl Fn(&RolloutFileName) -> bool,
|
||||
mut visitor: impl FnMut(RolloutFileName, PathBuf) -> ControlFlow<T>,
|
||||
) -> io::Result<Option<T>> {
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
@@ -1461,10 +1463,19 @@ async fn visit_rollout_filenames<T>(
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
|
||||
// Check filename matches before RolloutFile::from_path(), which calls
|
||||
// Path::exists() to check for a compressed rollout's plain sibling.
|
||||
let Some(file_name) = path
|
||||
.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.and_then(RolloutFileName::parse)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(file_name) = RolloutFileName::parse(rollout_file.plain_file_name()) else {
|
||||
if !matches(&file_name) {
|
||||
continue;
|
||||
}
|
||||
let Some(rollout_file) = compression::RolloutFile::from_path(path) else {
|
||||
continue;
|
||||
};
|
||||
if let ControlFlow::Break(found) = visitor(file_name, rollout_file.into_path()) {
|
||||
@@ -1479,13 +1490,11 @@ async fn find_rollout_path_by_rollout_id_from_filenames(
|
||||
root: &Path,
|
||||
rollout_id: RolloutId,
|
||||
) -> io::Result<Option<PathBuf>> {
|
||||
visit_rollout_filenames(root, |file_name, path| {
|
||||
if file_name.rollout_id() == rollout_id {
|
||||
ControlFlow::Break(path)
|
||||
} else {
|
||||
ControlFlow::Continue(())
|
||||
}
|
||||
})
|
||||
visit_rollout_filenames(
|
||||
root,
|
||||
|file_name| file_name.rollout_id() == rollout_id,
|
||||
|_file_name, path| ControlFlow::Break(path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
|
||||
@@ -227,12 +227,17 @@ async fn filesystem_lookup_distinguishes_thread_ids_from_rollout_ids() {
|
||||
"sessions/2025/01/04/rollout-{reverted_ts}-{thread_uuid}_{rollout_uuid}.jsonl"
|
||||
));
|
||||
fs::rename(reverted_source, reverted_path.as_path()).unwrap();
|
||||
let subsequent_rollout_uuid = Uuid::from_u128(403);
|
||||
let subsequent_path = home.join(format!(
|
||||
"sessions/2025/01/04/rollout-{reverted_ts}-{thread_uuid}_{subsequent_rollout_uuid}.jsonl"
|
||||
));
|
||||
fs::copy(&reverted_path, &subsequent_path).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
find_thread_path_by_id_str(home, &thread_uuid.to_string(), /*state_db_ctx*/ None)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(reverted_path.clone())
|
||||
Some(subsequent_path)
|
||||
);
|
||||
assert_eq!(
|
||||
find_rollout_path_by_rollout_id(home, thread_id_from_uuid(rollout_uuid))
|
||||
|
||||
Reference in New Issue
Block a user