mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Preserve daemon diagnostics and include updater logs in reports (#49843)
## Why Detached launches truncate stderr logs, losing details from previous failures. Updater handoffs can also leave existing writers with stale offsets after truncation. ## What changed - Record daemon lifecycle and update stages with timings, outcomes, and typed OS errors, omitting error messages and context. - Preserve the last 256 KiB of nonempty stderr logs before truncation and use append handles to prevent gaps during handoff. Preservation failures do not block launch. - Attach current and previous updater log tails when log inclusion is enabled for a report, with a 512 KiB total limit and legacy filename fallback. Skip missing, empty, and nonregular files. ## Testing Add regression coverage for diagnostic error redaction, bounded log preservation, append behavior during handoff, attachment selection, and updater logs in captured report uploads. GitOrigin-RevId: 918317022ab9dcfd1b13d676872706c77e9ffbb2
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
mod pid;
|
||||
mod stderr_log;
|
||||
#[cfg(windows)]
|
||||
pub(crate) mod windows;
|
||||
|
||||
|
||||
@@ -172,6 +172,7 @@ impl PidBackend {
|
||||
}
|
||||
|
||||
let pid = record.pid;
|
||||
crate::diagnostics::event("shutdown_requested", serde_json::json!({ "pid": pid }));
|
||||
let started_at = tokio::time::Instant::now();
|
||||
let force_after = Duration::from_secs(grace_seconds.into());
|
||||
let deadline = started_at + force_after + STOP_FORCE_TIMEOUT;
|
||||
@@ -227,6 +228,7 @@ impl PidBackend {
|
||||
break;
|
||||
}
|
||||
if !forced && started_at.elapsed() >= force_after {
|
||||
crate::diagnostics::event("shutdown_forced", serde_json::json!({ "pid": pid }));
|
||||
#[cfg(unix)]
|
||||
self.force_terminate_process(pid)?;
|
||||
#[cfg(windows)]
|
||||
@@ -353,18 +355,28 @@ impl PidBackend {
|
||||
#[cfg(any(unix, windows))]
|
||||
async fn open_stderr_log(&self) -> Result<fs::File> {
|
||||
let stderr_log_file = stderr_log_file_for_pid_file(&self.pid_file);
|
||||
fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&stderr_log_file)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"failed to open stderr log for pid-managed app server {}",
|
||||
stderr_log_file.display()
|
||||
)
|
||||
})
|
||||
super::stderr_log::preserve(&stderr_log_file).await;
|
||||
let result = async {
|
||||
fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&stderr_log_file)
|
||||
.await?;
|
||||
// The updater predecessor can still write after a Windows handoff
|
||||
// truncates this file. Append handles prevent stale offsets and gaps.
|
||||
fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&stderr_log_file)
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
result.with_context(|| {
|
||||
format!(
|
||||
"failed to open stderr log for pid-managed app server {}",
|
||||
stderr_log_file.display()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(any(unix, windows))]
|
||||
|
||||
@@ -188,10 +188,12 @@ impl PidBackend {
|
||||
}
|
||||
}
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
#[cfg(windows)]
|
||||
let child = super::super::windows::spawn_without_inheriting_stdio(&mut command);
|
||||
#[cfg(not(windows))]
|
||||
let child = command.spawn().map_err(anyhow::Error::from);
|
||||
let child = crate::diagnostics::result("process_spawn", started, child);
|
||||
let child = match child {
|
||||
Ok(child) => child,
|
||||
Err(err) => {
|
||||
@@ -214,7 +216,9 @@ impl PidBackend {
|
||||
let pid = child
|
||||
.id()
|
||||
.context("spawned app-server process has no pid")?;
|
||||
let record = match async {
|
||||
crate::diagnostics::event("process_spawned", serde_json::json!({ "pid": pid }));
|
||||
let started = std::time::Instant::now();
|
||||
let record = async {
|
||||
let process_start_time = read_process_start_time(pid).await?;
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
let process_identity = super::identity::read_process_details(pid)
|
||||
@@ -230,8 +234,8 @@ impl PidBackend {
|
||||
executable_identity: launched_identity,
|
||||
})
|
||||
}
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
let record = match crate::diagnostics::result("process_identity", started, record) {
|
||||
Ok(record) => record,
|
||||
Err(err) => {
|
||||
let _ = self.terminate_process(pid);
|
||||
@@ -246,7 +250,14 @@ impl PidBackend {
|
||||
};
|
||||
let contents = serde_json::to_vec(&record).context("failed to serialize pid record")?;
|
||||
let temp_pid_file = self.pid_file.with_extension("pid.tmp");
|
||||
if let Err(err) = fs::write(&temp_pid_file, &contents).await {
|
||||
let started = std::time::Instant::now();
|
||||
if let Err(err) = crate::diagnostics::result(
|
||||
"pid_write",
|
||||
started,
|
||||
fs::write(&temp_pid_file, &contents)
|
||||
.await
|
||||
.map_err(anyhow::Error::from),
|
||||
) {
|
||||
let _ = self.terminate_process(pid);
|
||||
if replacement.is_none() {
|
||||
let _ = fs::remove_file(&self.pid_file).await;
|
||||
@@ -255,7 +266,14 @@ impl PidBackend {
|
||||
format!("failed to write pid temp file {}", temp_pid_file.display())
|
||||
});
|
||||
}
|
||||
if let Err(err) = fs::rename(&temp_pid_file, &self.pid_file).await {
|
||||
let started = std::time::Instant::now();
|
||||
if let Err(err) = crate::diagnostics::result(
|
||||
"pid_publish",
|
||||
started,
|
||||
fs::rename(&temp_pid_file, &self.pid_file)
|
||||
.await
|
||||
.map_err(anyhow::Error::from),
|
||||
) {
|
||||
let _ = self.terminate_process(pid);
|
||||
let _ = fs::remove_file(&temp_pid_file).await;
|
||||
if replacement.is_none() {
|
||||
|
||||
@@ -756,6 +756,7 @@ async fn failed_updater_handoff_preserves_predecessor_record() {
|
||||
#[tokio::test]
|
||||
async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
|
||||
use futures::FutureExt;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use windows_sys::Win32::Security::ImpersonateAnonymousToken;
|
||||
use windows_sys::Win32::Security::RevertToSelf;
|
||||
use windows_sys::Win32::System::Threading::GetCurrentThread;
|
||||
@@ -770,13 +771,23 @@ async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
|
||||
.acquire_reservation_lock()
|
||||
.await
|
||||
.expect("reservation lock");
|
||||
let mut predecessor_stderr = backend.open_stderr_log().await.expect("predecessor log");
|
||||
predecessor_stderr
|
||||
.write_all(&vec![b'x'; 512 * 1024])
|
||||
.await
|
||||
.unwrap();
|
||||
predecessor_stderr.flush().await.unwrap();
|
||||
let successor_stderr = backend.open_stderr_log().await.expect("successor log");
|
||||
let ready = backend.pid_file.with_extension("ready");
|
||||
let mut child = tokio::process::Command::new("powershell.exe")
|
||||
.args([
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
"Start-Sleep 60",
|
||||
"[Console]::Error.WriteLine('successor startup'); [IO.File]::WriteAllText($env:CODEX_TEST_UPDATER_READY, ''); Start-Sleep 60",
|
||||
])
|
||||
.env("CODEX_TEST_UPDATER_READY", &ready)
|
||||
.stderr(Stdio::from(successor_stderr.into_std().await))
|
||||
.creation_flags(windows_sys::Win32::System::Threading::CREATE_NO_WINDOW)
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
@@ -801,12 +812,21 @@ async fn updater_readiness_and_post_publication_failure_preserve_ownership() {
|
||||
tokio::fs::write(&backend.pid_file, serde_json::to_vec(&successor).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let ready = backend.pid_file.with_extension("ready");
|
||||
tokio::fs::write(&ready, b"").await.unwrap();
|
||||
backend
|
||||
.finish_updater_start(&successor, Some(&predecessor))
|
||||
.await
|
||||
.expect("ready successor");
|
||||
predecessor_stderr
|
||||
.write_all(b"predecessor handoff complete\n")
|
||||
.await
|
||||
.unwrap();
|
||||
predecessor_stderr.flush().await.unwrap();
|
||||
let log_path = stderr_log_file_for_pid_file(&backend.pid_file);
|
||||
let log = tokio::fs::read(&log_path).await.unwrap();
|
||||
assert!(log.len() < 256 * 1024);
|
||||
let log = String::from_utf8(log).expect("no stale-offset gap");
|
||||
assert!(log.contains("successor startup"));
|
||||
assert!(log.contains("predecessor handoff complete"));
|
||||
assert!(!ready.exists());
|
||||
assert_eq!(
|
||||
backend.read_pid_file_state().await.unwrap(),
|
||||
@@ -934,3 +954,47 @@ fn inaccessible_pid_preserves_identity_check_error() {
|
||||
.join()
|
||||
.expect("anonymous identity check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stderr_preservation_failure_does_not_prevent_opening_new_log() -> anyhow::Result<()> {
|
||||
let home = TempDir::new()?;
|
||||
let pid_file = home.path().join("daemon.pid");
|
||||
let path = stderr_log_file_for_pid_file(&pid_file);
|
||||
let backend = PidBackend::new(
|
||||
home.path().join("codex"),
|
||||
pid_file,
|
||||
/*remote_control_enabled*/ false,
|
||||
);
|
||||
tokio::fs::write(&path, b"previous failure").await?;
|
||||
tokio::fs::create_dir(path.with_extension("log.previous")).await?;
|
||||
let _log = backend.open_stderr_log().await?;
|
||||
assert_eq!(tokio::fs::read(&path).await?, b"");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stderr_handoff_appends_after_truncation_without_gaps() -> anyhow::Result<()> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let home = TempDir::new()?;
|
||||
let backend = PidBackend::new_update_loop(
|
||||
home.path().join("codex"),
|
||||
home.path().join("updater.pid"),
|
||||
/*restore_release*/ None,
|
||||
);
|
||||
let mut predecessor = backend.open_stderr_log().await?;
|
||||
predecessor.write_all(&vec![b'x'; 512 * 1024]).await?;
|
||||
predecessor.flush().await?;
|
||||
let mut successor = backend.open_stderr_log().await?;
|
||||
successor.write_all(b"successor startup failure\n").await?;
|
||||
successor.flush().await?;
|
||||
predecessor
|
||||
.write_all(b"predecessor handoff complete\n")
|
||||
.await?;
|
||||
predecessor.flush().await?;
|
||||
assert_eq!(
|
||||
tokio::fs::read(stderr_log_file_for_pid_file(&backend.pid_file)).await?,
|
||||
b"successor startup failure\npredecessor handoff complete\n"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! Saves a bounded previous-launch stderr tail before a detached launch truncates its log.
|
||||
//! Preservation is best effort; missing or empty logs leave the last useful tail intact.
|
||||
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::io::AsyncSeekExt;
|
||||
|
||||
const MAX_PREVIOUS_LOG_BYTES: u64 = 256 * 1024;
|
||||
|
||||
pub(super) async fn preserve(path: &Path) {
|
||||
let started = Instant::now();
|
||||
let result = async {
|
||||
match tokio::fs::symlink_metadata(path).await {
|
||||
Ok(metadata) if !metadata.is_file() => return Ok(()),
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let mut file = match tokio::fs::File::open(path).await {
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let metadata = file.metadata().await?;
|
||||
let len = metadata.len();
|
||||
if !metadata.is_file() || len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
file.seek(io::SeekFrom::Start(
|
||||
len.saturating_sub(MAX_PREVIOUS_LOG_BYTES),
|
||||
))
|
||||
.await?;
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_PREVIOUS_LOG_BYTES)
|
||||
.read_to_end(&mut bytes)
|
||||
.await?;
|
||||
let previous = path.with_extension("log.previous");
|
||||
let temporary = path.with_extension("log.previous.tmp");
|
||||
let result = async {
|
||||
tokio::fs::write(&temporary, bytes).await?;
|
||||
tokio::fs::rename(&temporary, previous).await
|
||||
}
|
||||
.await;
|
||||
if result.is_err() {
|
||||
let _ = tokio::fs::remove_file(&temporary).await;
|
||||
}
|
||||
result?;
|
||||
anyhow::Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
let _ = crate::diagnostics::result::<()>("stderr_preservation", started, Err(error));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "stderr_log_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,30 @@
|
||||
use super::MAX_PREVIOUS_LOG_BYTES;
|
||||
use super::preserve;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test]
|
||||
async fn preserves_bounded_tail_and_keeps_it_after_empty_launch() -> anyhow::Result<()> {
|
||||
let home = tempfile::tempdir()?;
|
||||
let path = home.path().join("daemon.stderr.log");
|
||||
let previous = path.with_extension("log.previous");
|
||||
preserve(&path).await;
|
||||
assert!(!previous.exists());
|
||||
let mut bytes = vec![b'x'; MAX_PREVIOUS_LOG_BYTES as usize + 100];
|
||||
bytes.extend_from_slice(b"latest failure");
|
||||
tokio::fs::write(&path, &bytes).await?;
|
||||
preserve(&path).await;
|
||||
assert_eq!(
|
||||
tokio::fs::read(&previous).await?,
|
||||
bytes[bytes.len() - MAX_PREVIOUS_LOG_BYTES as usize..]
|
||||
);
|
||||
tokio::fs::write(&path, b"").await?;
|
||||
preserve(&path).await;
|
||||
assert_eq!(
|
||||
tokio::fs::read(&previous).await?,
|
||||
bytes[bytes.len() - MAX_PREVIOUS_LOG_BYTES as usize..]
|
||||
);
|
||||
tokio::fs::write(&path, b"next launch").await?;
|
||||
preserve(&path).await;
|
||||
assert_eq!(tokio::fs::read(&previous).await?, b"next launch");
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user