mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Persist Guardian review failures for reports across restarts (#51651)
## Why Failed Guardian review evidence was process-local, so restarting Codex lost the context needed for later diagnostic reports. ## What changed - Persist failed-review records in SQLite before reviewer cleanup, retaining at most 64 records globally, eight per thread, and 8 MiB of evidence. Delete persisted evidence when its reviewed thread is deleted. - Recover and deduplicate persisted and in-memory records for the reported task tree when uploading `auto-review-failures.jsonl`. - Bound persistence attempts to 250 ms and retain an in-memory fallback when storage is unavailable. Keep capture gated by user opt-in and non-ephemeral sessions, and export evidence only when logs are included. ## Testing Add coverage for persistence across reopen and process restart, retention limits, cascading deletion, busy-storage fallback, and report uploads with and without logs, including SQLite log failures. GitOrigin-RevId: ea2da1818d4a5da3da29f211042ca45e834342bc
This commit is contained in:
@@ -1,15 +1,20 @@
|
||||
//! Bounded process-local failed-review evidence, shared by root and child reviewers.
|
||||
//! Bounded durable failed-review evidence, with an in-memory fallback for storage failures.
|
||||
//! Captures are exported only by a feedback request that includes logs.
|
||||
|
||||
use crate::FeedbackAttachment;
|
||||
use codex_protocol::ThreadId;
|
||||
pub use codex_state::GuardianReviewRecord;
|
||||
use codex_state::MAX_GUARDIAN_REVIEW_BYTES as MAX_BYTES;
|
||||
use codex_state::MAX_GUARDIAN_REVIEW_RECORDS as MAX_RECORDS;
|
||||
use codex_state::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD as MAX_RECORDS_PER_THREAD;
|
||||
use codex_state::StateRuntime;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
const PERSIST_TIMEOUT: Duration = Duration::from_millis(250);
|
||||
|
||||
const MAX_RECORDS: usize = 64;
|
||||
const MAX_RECORDS_PER_THREAD: usize = 8;
|
||||
const MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
static RECORDS: Mutex<ReviewRecords> = Mutex::new(ReviewRecords {
|
||||
records: VecDeque::new(),
|
||||
bytes: 0,
|
||||
@@ -25,47 +30,51 @@ pub struct GuardianReviewFailures {
|
||||
pub process_discarded_records: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[derive(Clone, Default)]
|
||||
struct ReviewRecords {
|
||||
records: VecDeque<(ThreadId, Vec<u8>)>,
|
||||
records: VecDeque<GuardianReviewRecord>,
|
||||
bytes: usize,
|
||||
discarded_records: usize,
|
||||
}
|
||||
|
||||
impl ReviewRecords {
|
||||
fn push(&mut self, thread_id: ThreadId, record: Vec<u8>) {
|
||||
if record.len() + 1 > MAX_BYTES {
|
||||
fn push(&mut self, record: GuardianReviewRecord) {
|
||||
if record.record.len() + 1 > MAX_BYTES {
|
||||
self.discarded_records = self.discarded_records.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
if self
|
||||
.records
|
||||
.iter()
|
||||
.filter(|(id, _)| *id == thread_id)
|
||||
.filter(|entry| entry.thread_id == record.thread_id)
|
||||
.count()
|
||||
>= MAX_RECORDS_PER_THREAD
|
||||
&& let Some(index) = self.records.iter().position(|(id, _)| *id == thread_id)
|
||||
&& let Some((_, removed)) = self.records.remove(index)
|
||||
&& let Some(index) = self
|
||||
.records
|
||||
.iter()
|
||||
.position(|entry| entry.thread_id == record.thread_id)
|
||||
&& let Some(removed) = self.records.remove(index)
|
||||
{
|
||||
self.bytes -= removed.len() + 1;
|
||||
self.bytes -= removed.record.len() + 1;
|
||||
self.discarded_records = self.discarded_records.saturating_add(1);
|
||||
}
|
||||
while self.records.len() >= MAX_RECORDS || self.bytes + record.len() + 1 > MAX_BYTES {
|
||||
if let Some((_, removed)) = self.records.pop_front() {
|
||||
self.bytes -= removed.len() + 1;
|
||||
while self.records.len() >= MAX_RECORDS || self.bytes + record.record.len() + 1 > MAX_BYTES
|
||||
{
|
||||
if let Some(removed) = self.records.pop_front() {
|
||||
self.bytes -= removed.record.len() + 1;
|
||||
self.discarded_records = self.discarded_records.saturating_add(1);
|
||||
}
|
||||
}
|
||||
self.bytes += record.len() + 1;
|
||||
self.records.push_back((thread_id, record));
|
||||
self.bytes += record.record.len() + 1;
|
||||
self.records.push_back(record);
|
||||
}
|
||||
|
||||
fn snapshot(&self, thread_ids: &[ThreadId]) -> GuardianReviewFailures {
|
||||
let thread_ids = thread_ids.iter().copied().collect::<HashSet<_>>();
|
||||
let mut buffer = Vec::new();
|
||||
for (thread_id, record) in &self.records {
|
||||
if thread_ids.contains(thread_id) {
|
||||
buffer.extend_from_slice(record);
|
||||
for record in &self.records {
|
||||
if thread_ids.contains(&record.thread_id) {
|
||||
buffer.extend_from_slice(&record.record);
|
||||
buffer.push(b'\n');
|
||||
}
|
||||
}
|
||||
@@ -79,8 +88,10 @@ impl ReviewRecords {
|
||||
.records
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|(id, _)| thread_ids.contains(id) && seen.insert(*id))
|
||||
.map(|(id, _)| *id)
|
||||
.filter(|record| {
|
||||
thread_ids.contains(&record.thread_id) && seen.insert(record.thread_id)
|
||||
})
|
||||
.map(|record| record.thread_id)
|
||||
.collect();
|
||||
GuardianReviewFailures {
|
||||
attachment,
|
||||
@@ -90,20 +101,55 @@ impl ReviewRecords {
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain one complete, serialized failed-review JSON record for later opt-in feedback.
|
||||
pub fn record_guardian_review_failure(thread_id: ThreadId, record: Vec<u8>) {
|
||||
/// Retain a failed review in memory and persist it before its session is cleaned up.
|
||||
pub async fn record_guardian_review_failure(
|
||||
state_db: Option<&StateRuntime>,
|
||||
record: GuardianReviewRecord,
|
||||
) {
|
||||
RECORDS
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(thread_id, record);
|
||||
.push(record.clone());
|
||||
if let Some(state_db) = state_db
|
||||
&& !matches!(
|
||||
tokio::time::timeout(
|
||||
PERSIST_TIMEOUT,
|
||||
state_db.record_guardian_review_failure(&record)
|
||||
)
|
||||
.await,
|
||||
Ok(Ok(()))
|
||||
)
|
||||
{
|
||||
tracing::warn!("failed to persist Guardian feedback; retaining an in-memory fallback");
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot failures for the reported task tree without including other tasks' records.
|
||||
pub fn guardian_review_failures(thread_ids: &[ThreadId]) -> GuardianReviewFailures {
|
||||
RECORDS
|
||||
/// Recover retained failures before selecting rollout attachments, scoped to the reported task tree.
|
||||
pub async fn guardian_review_failures(
|
||||
state_db: Option<&StateRuntime>,
|
||||
thread_ids: &[ThreadId],
|
||||
) -> GuardianReviewFailures {
|
||||
let fallback = RECORDS
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.snapshot(thread_ids)
|
||||
.clone();
|
||||
let Some(state_db) = state_db else {
|
||||
return fallback.snapshot(thread_ids);
|
||||
};
|
||||
let Ok(mut records) = state_db.list_guardian_review_records().await else {
|
||||
tracing::warn!("failed to load persisted Guardian feedback");
|
||||
return fallback.snapshot(thread_ids);
|
||||
};
|
||||
records.extend(fallback.records);
|
||||
records.sort_unstable_by(|left, right| left.id.cmp(&right.id));
|
||||
records.dedup_by(|left, right| left.id == right.id);
|
||||
let mut retained = ReviewRecords::default();
|
||||
for record in records {
|
||||
retained.push(record);
|
||||
}
|
||||
// This legacy diagnostic describes only the in-memory fallback, not durable evictions.
|
||||
retained.discarded_records = fallback.discarded_records;
|
||||
retained.snapshot(thread_ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user