diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rs index 603fe3383c..12b98375aa 100644 --- a/codex-rs/rollout/src/compression.rs +++ b/codex-rs/rollout/src/compression.rs @@ -388,6 +388,49 @@ mod worker { time_budget_exhausted: bool, } + impl CompressionStats { + fn scanned_file(&mut self, trigger: RolloutCompressionTrigger) { + self.scanned = self.scanned.saturating_add(1); + metrics::file(trigger, "scanned"); + } + + fn measured_file( + &mut self, + trigger: RolloutCompressionTrigger, + duration: Duration, + measurement: CompressionMeasurement, + ) { + let outcome = measurement.outcome; + match outcome { + CompressionOutcome::Compressed => { + self.compressed = self.compressed.saturating_add(1) + } + CompressionOutcome::SkippedNotCold + | CompressionOutcome::SkippedBusy + | CompressionOutcome::SkippedChanged + | CompressionOutcome::SkippedAlreadyCompressed => { + self.skipped = self.skipped.saturating_add(1) + } + } + metrics::file(trigger, outcome.tag()); + metrics::file_duration(trigger, outcome.tag(), duration); + if let Some(source_bytes) = measurement.source_bytes { + metrics::source_bytes(trigger, outcome.tag(), source_bytes); + } + if let Some(compressed_bytes) = measurement.compressed_bytes { + metrics::compressed_bytes(trigger, outcome.tag(), compressed_bytes); + if let Some(source_bytes) = measurement.source_bytes { + metrics::compression_ratio( + trigger, + outcome.tag(), + source_bytes, + compressed_bytes, + ); + } + } + } + } + pub(super) struct CompressionRunMarker { path: PathBuf, remove_on_drop: bool, @@ -670,6 +713,25 @@ mod worker { metrics::file(trigger, "skipped_unreadable_meta"); continue; }; + // Recent files cannot be compressed. Only open the header for a + // possible candidate; the job rechecks its age before encoding. + let checked_at = Instant::now(); + if let Ok(metadata) = entry.metadata().await + && let Ok(modified) = metadata.modified() + && !is_cold(modified) + { + stats.scanned_file(trigger); + stats.measured_file( + trigger, + checked_at.elapsed(), + CompressionMeasurement::new( + CompressionOutcome::SkippedNotCold, + Some(metadata.len()), + /*compressed_bytes*/ None, + ), + ); + continue; + } let thread_id = match crate::read_session_meta_line(path.as_path()).await { Ok(metadata) => metadata.meta.id, Err(err) => { @@ -716,8 +778,7 @@ mod worker { continue; } }; - stats.scanned = stats.scanned.saturating_add(1); - metrics::file(trigger, "scanned"); + stats.scanned_file(trigger); while jobs.len() >= MAX_CONCURRENT_COMPRESSION_JOBS { collect_next_compression_job(&mut jobs, stats, trigger).await; } @@ -807,34 +868,7 @@ mod worker { }; match result { Ok((_, duration, Ok(measurement))) => { - let outcome = measurement.outcome; - match outcome { - CompressionOutcome::Compressed => { - stats.compressed = stats.compressed.saturating_add(1); - } - CompressionOutcome::SkippedNotCold - | CompressionOutcome::SkippedBusy - | CompressionOutcome::SkippedChanged - | CompressionOutcome::SkippedAlreadyCompressed => { - stats.skipped = stats.skipped.saturating_add(1); - } - } - metrics::file(trigger, outcome.tag()); - metrics::file_duration(trigger, outcome.tag(), duration); - if let Some(source_bytes) = measurement.source_bytes { - metrics::source_bytes(trigger, outcome.tag(), source_bytes); - } - if let Some(compressed_bytes) = measurement.compressed_bytes { - metrics::compressed_bytes(trigger, outcome.tag(), compressed_bytes); - if let Some(source_bytes) = measurement.source_bytes { - metrics::compression_ratio( - trigger, - outcome.tag(), - source_bytes, - compressed_bytes, - ); - } - } + stats.measured_file(trigger, duration, measurement); } Ok((path, duration, Err(err))) => { stats.failed = stats.failed.saturating_add(1); @@ -1004,15 +1038,16 @@ mod worker { #[cfg(unix)] change_time: (metadata.ctime(), metadata.ctime_nsec()), }; - let age = SystemTime::now() - .duration_since(modified) - .unwrap_or(Duration::ZERO); - if age < MIN_ROLLOUT_AGE { + if !is_cold(modified) { return Ok(ColdFileState::NotCold(Some(state))); } Ok(ColdFileState::Cold(state)) } + fn is_cold(modified: SystemTime) -> bool { + modified.elapsed().unwrap_or_default() >= MIN_ROLLOUT_AGE + } + fn same_file_state(path: &Path, expected: &FileState) -> io::Result { match std::fs::metadata(path) { Ok(metadata) => { diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index 2c1db321e6..d3b32445a4 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -442,6 +442,12 @@ async fn worker_compresses_old_active_and_archived_rollouts() -> anyhow::Result< let fresh_id = ThreadId::from_string(&fresh_uuid.to_string())?; let fresh_path = rollout_path(home.path(), "2025-01-05T12-00-00", fresh_uuid); write_rollout(&fresh_path, fresh_id, "fresh active")?; + // Reading this recently modified file would advance its old access time on relatime mounts. + fs::OpenOptions::new() + .write(true) + .open(&fresh_path)? + .set_times(FileTimes::new().set_accessed(SystemTime::UNIX_EPOCH))?; + let fresh_accessed = fs::metadata(&fresh_path)?.accessed()?; let stale_temp = active_path.with_file_name("rollout-stale.jsonl.zst.tmp"); fs::write(&stale_temp, "stale temp")?; @@ -469,6 +475,7 @@ async fn worker_compresses_old_active_and_archived_rollouts() -> anyhow::Result< assert!(compressed_rollout_path(&archived_path).exists()); assert!(fresh_path.exists()); assert!(!compressed_rollout_path(&fresh_path).exists()); + assert_eq!(fs::metadata(&fresh_path)?.accessed()?, fresh_accessed); assert!(!stale_temp.exists()); assert!(fresh_temp.exists()); let materialized_path = home.path().join("materialized.jsonl"); @@ -560,6 +567,7 @@ async fn worker_skips_unreadable_metadata_without_blocking_other_compression() - let unreadable_path = rollout_path(home.path(), "2025-01-03T12-00-01", Uuid::from_u128(21)); fs::write(unreadable_path.as_path(), "{not json}\n")?; + set_old_mtime(&unreadable_path)?; worker::run(home.path().to_path_buf(), Startup).await?;