Preserve daemon diagnostics and include updater logs in reports (#49843)

## Why

Detached launches truncate stderr logs, losing details from previous failures. Updater handoffs can also leave existing writers with stale offsets after truncation.

## What changed

- Record daemon lifecycle and update stages with timings, outcomes, and typed OS errors, omitting error messages and context.
- Preserve the last 256 KiB of nonempty stderr logs before truncation and use append handles to prevent gaps during handoff. Preservation failures do not block launch.
- Attach current and previous updater log tails when log inclusion is enabled for a report, with a 512 KiB total limit and legacy filename fallback. Skip missing, empty, and nonregular files.

## Testing

Add regression coverage for diagnostic error redaction, bounded log preservation, append behavior during handoff, attachment selection, and updater logs in captured report uploads.

GitOrigin-RevId: 918317022ab9dcfd1b13d676872706c77e9ffbb2
This commit is contained in:
Eric Traut
2026-10-01 03:24:18 +00:00
committed by copyberry
parent e53e932dc8
commit 08b07eef12
15 changed files with 587 additions and 32 deletions
@@ -1,4 +1,5 @@
mod pid;
mod stderr_log;
#[cfg(windows)]
pub(crate) mod windows;
+24 -12
View File
@@ -172,6 +172,7 @@ impl PidBackend {
}
let pid = record.pid;
crate::diagnostics::event("shutdown_requested", serde_json::json!({ "pid": pid }));
let started_at = tokio::time::Instant::now();
let force_after = Duration::from_secs(grace_seconds.into());
let deadline = started_at + force_after + STOP_FORCE_TIMEOUT;
@@ -227,6 +228,7 @@ impl PidBackend {
break;
}
if !forced && started_at.elapsed() >= force_after {
crate::diagnostics::event("shutdown_forced", serde_json::json!({ "pid": pid }));
#[cfg(unix)]
self.force_terminate_process(pid)?;
#[cfg(windows)]
@@ -353,18 +355,28 @@ impl PidBackend {
#[cfg(any(unix, windows))]
async fn open_stderr_log(&self) -> Result<fs::File> {
let stderr_log_file = stderr_log_file_for_pid_file(&self.pid_file);
fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&stderr_log_file)
.await
.with_context(|| {
format!(
"failed to open stderr log for pid-managed app server {}",
stderr_log_file.display()
)
})
super::stderr_log::preserve(&stderr_log_file).await;
let result = async {
fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&stderr_log_file)
.await?;
// The updater predecessor can still write after a Windows handoff
// truncates this file. Append handles prevent stale offsets and gaps.
fs::OpenOptions::new()
.append(true)
.open(&stderr_log_file)
.await
}
.await;
result.with_context(|| {
format!(
"failed to open stderr log for pid-managed app server {}",
stderr_log_file.display()
)
})
}
#[cfg(any(unix, windows))]
@@ -188,10 +188,12 @@ impl PidBackend {
}
}
let started = std::time::Instant::now();
#[cfg(windows)]
let child = super::super::windows::spawn_without_inheriting_stdio(&mut command);
#[cfg(not(windows))]
let child = command.spawn().map_err(anyhow::Error::from);
let child = crate::diagnostics::result("process_spawn", started, child);
let child = match child {
Ok(child) => child,
Err(err) => {
@@ -214,7 +216,9 @@ impl PidBackend {
let pid = child
.id()
.context("spawned app-server process has no pid")?;
let record = match async {
crate::diagnostics::event("process_spawned", serde_json::json!({ "pid": pid }));
let started = std::time::Instant::now();
let record = async {
let process_start_time = read_process_start_time(pid).await?;
#[cfg(any(target_os = "linux", target_os = "macos"))]
let process_identity = super::identity::read_process_details(pid)
@@ -230,8 +234,8 @@ impl PidBackend {
executable_identity: launched_identity,
})
}
.await
{
.await;
let record = match crate::diagnostics::result("process_identity", started, record) {
Ok(record) => record,
Err(err) => {
let _ = self.terminate_process(pid);
@@ -246,7 +250,14 @@ impl PidBackend {
};
let contents = serde_json::to_vec(&record).context("failed to serialize pid record")?;
let temp_pid_file = self.pid_file.with_extension("pid.tmp");
if let Err(err) = fs::write(&temp_pid_file, &contents).await {
let started = std::time::Instant::now();
if let Err(err) = crate::diagnostics::result(
"pid_write",
started,
fs::write(&temp_pid_file, &contents)
.await
.map_err(anyhow::Error::from),
) {
let _ = self.terminate_process(pid);
if replacement.is_none() {
let _ = fs::remove_file(&self.pid_file).await;
@@ -255,7 +266,14 @@ impl PidBackend {
format!("failed to write pid temp file {}", temp_pid_file.display())
});
}
if let Err(err) = fs::rename(&temp_pid_file, &self.pid_file).await {
let started = std::time::Instant::now();
if let Err(err) = crate::diagnostics::result(
"pid_publish",
started,
fs::rename(&temp_pid_file, &self.pid_file)
.await
.map_err(anyhow::Error::from),
) {
let _ = self.terminate_process(pid);
let _ = fs::remove_file(&temp_pid_file).await;
if replacement.is_none() {
@@ -756,6 +756,7 @@ async fn failed_updater_handoff_preserves_predecessor_record() {
#[tokio::test]
async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
use futures::FutureExt;
use tokio::io::AsyncWriteExt;
use windows_sys::Win32::Security::ImpersonateAnonymousToken;
use windows_sys::Win32::Security::RevertToSelf;
use windows_sys::Win32::System::Threading::GetCurrentThread;
@@ -770,13 +771,23 @@ async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
.acquire_reservation_lock()
.await
.expect("reservation lock");
let mut predecessor_stderr = backend.open_stderr_log().await.expect("predecessor log");
predecessor_stderr
.write_all(&vec![b'x'; 512 * 1024])
.await
.unwrap();
predecessor_stderr.flush().await.unwrap();
let successor_stderr = backend.open_stderr_log().await.expect("successor log");
let ready = backend.pid_file.with_extension("ready");
let mut child = tokio::process::Command::new("powershell.exe")
.args([
"-NoProfile",
"-NonInteractive",
"-Command",
"Start-Sleep 60",
"[Console]::Error.WriteLine('successor startup'); [IO.File]::WriteAllText($env:CODEX_TEST_UPDATER_READY, ''); Start-Sleep 60",
])
.env("CODEX_TEST_UPDATER_READY", &ready)
.stderr(Stdio::from(successor_stderr.into_std().await))
.creation_flags(windows_sys::Win32::System::Threading::CREATE_NO_WINDOW)
.kill_on_drop(true)
.spawn()
@@ -801,12 +812,21 @@ async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
tokio::fs::write(&backend.pid_file, serde_json::to_vec(&successor).unwrap())
.await
.unwrap();
let ready = backend.pid_file.with_extension("ready");
tokio::fs::write(&ready, b"").await.unwrap();
backend
.finish_updater_start(&successor, Some(&predecessor))
.await
.expect("ready successor");
predecessor_stderr
.write_all(b"predecessor handoff complete\n")
.await
.unwrap();
predecessor_stderr.flush().await.unwrap();
let log_path = stderr_log_file_for_pid_file(&backend.pid_file);
let log = tokio::fs::read(&log_path).await.unwrap();
assert!(log.len() < 256 * 1024);
let log = String::from_utf8(log).expect("no stale-offset gap");
assert!(log.contains("successor startup"));
assert!(log.contains("predecessor handoff complete"));
assert!(!ready.exists());
assert_eq!(
backend.read_pid_file_state().await.unwrap(),
@@ -934,3 +954,47 @@ fn inaccessible_pid_preserves_identity_check_error() {
.join()
.expect("anonymous identity check");
}
#[tokio::test]
async fn stderr_preservation_failure_does_not_prevent_opening_new_log() -> anyhow::Result<()> {
let home = TempDir::new()?;
let pid_file = home.path().join("daemon.pid");
let path = stderr_log_file_for_pid_file(&pid_file);
let backend = PidBackend::new(
home.path().join("codex"),
pid_file,
/*remote_control_enabled*/ false,
);
tokio::fs::write(&path, b"previous failure").await?;
tokio::fs::create_dir(path.with_extension("log.previous")).await?;
let _log = backend.open_stderr_log().await?;
assert_eq!(tokio::fs::read(&path).await?, b"");
Ok(())
}
#[tokio::test]
async fn stderr_handoff_appends_after_truncation_without_gaps() -> anyhow::Result<()> {
use tokio::io::AsyncWriteExt;
let home = TempDir::new()?;
let backend = PidBackend::new_update_loop(
home.path().join("codex"),
home.path().join("updater.pid"),
/*restore_release*/ None,
);
let mut predecessor = backend.open_stderr_log().await?;
predecessor.write_all(&vec![b'x'; 512 * 1024]).await?;
predecessor.flush().await?;
let mut successor = backend.open_stderr_log().await?;
successor.write_all(b"successor startup failure\n").await?;
successor.flush().await?;
predecessor
.write_all(b"predecessor handoff complete\n")
.await?;
predecessor.flush().await?;
assert_eq!(
tokio::fs::read(stderr_log_file_for_pid_file(&backend.pid_file)).await?,
b"successor startup failure\npredecessor handoff complete\n"
);
Ok(())
}
@@ -0,0 +1,61 @@
//! Saves a bounded previous-launch stderr tail before a detached launch truncates its log.
//! Preservation is best effort; missing or empty logs leave the last useful tail intact.
use std::io;
use std::path::Path;
use std::time::Instant;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncSeekExt;
const MAX_PREVIOUS_LOG_BYTES: u64 = 256 * 1024;
pub(super) async fn preserve(path: &Path) {
let started = Instant::now();
let result = async {
match tokio::fs::symlink_metadata(path).await {
Ok(metadata) if !metadata.is_file() => return Ok(()),
Ok(_) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
}
let mut file = match tokio::fs::File::open(path).await {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
let metadata = file.metadata().await?;
let len = metadata.len();
if !metadata.is_file() || len == 0 {
return Ok(());
}
file.seek(io::SeekFrom::Start(
len.saturating_sub(MAX_PREVIOUS_LOG_BYTES),
))
.await?;
let mut bytes = Vec::new();
file.take(MAX_PREVIOUS_LOG_BYTES)
.read_to_end(&mut bytes)
.await?;
let previous = path.with_extension("log.previous");
let temporary = path.with_extension("log.previous.tmp");
let result = async {
tokio::fs::write(&temporary, bytes).await?;
tokio::fs::rename(&temporary, previous).await
}
.await;
if result.is_err() {
let _ = tokio::fs::remove_file(&temporary).await;
}
result?;
anyhow::Ok(())
}
.await;
if let Err(error) = result {
let _ = crate::diagnostics::result::<()>("stderr_preservation", started, Err(error));
}
}
#[cfg(test)]
#[path = "stderr_log_tests.rs"]
mod tests;
@@ -0,0 +1,30 @@
use super::MAX_PREVIOUS_LOG_BYTES;
use super::preserve;
use pretty_assertions::assert_eq;
#[tokio::test]
async fn preserves_bounded_tail_and_keeps_it_after_empty_launch() -> anyhow::Result<()> {
let home = tempfile::tempdir()?;
let path = home.path().join("daemon.stderr.log");
let previous = path.with_extension("log.previous");
preserve(&path).await;
assert!(!previous.exists());
let mut bytes = vec![b'x'; MAX_PREVIOUS_LOG_BYTES as usize + 100];
bytes.extend_from_slice(b"latest failure");
tokio::fs::write(&path, &bytes).await?;
preserve(&path).await;
assert_eq!(
tokio::fs::read(&previous).await?,
bytes[bytes.len() - MAX_PREVIOUS_LOG_BYTES as usize..]
);
tokio::fs::write(&path, b"").await?;
preserve(&path).await;
assert_eq!(
tokio::fs::read(&previous).await?,
bytes[bytes.len() - MAX_PREVIOUS_LOG_BYTES as usize..]
);
tokio::fs::write(&path, b"next launch").await?;
preserve(&path).await;
assert_eq!(tokio::fs::read(&previous).await?, b"next launch");
Ok(())
}