Persist Guardian review failures for reports across restarts (#51651)

## Why

Failed Guardian review evidence was process-local, so restarting Codex lost the context needed for later diagnostic reports.

## What changed

- Persist failed-review records in SQLite before reviewer cleanup, retaining at most 64 records globally, eight per thread, and 8 MiB of evidence. Delete persisted evidence when its reviewed thread is deleted.
- Recover and deduplicate persisted and in-memory records for the reported task tree when uploading `auto-review-failures.jsonl`.
- Bound persistence attempts to 250 ms and retain an in-memory fallback when storage is unavailable. Keep capture gated by user opt-in and non-ephemeral sessions, and export evidence only when logs are included.

## Testing

Add coverage for persistence across reopen and process restart, retention limits, cascading deletion, busy-storage fallback, and report uploads with and without logs, including SQLite log failures.

GitOrigin-RevId: ea2da1818d4a5da3da29f211042ca45e834342bc
This commit is contained in:
jif
2026-10-07 11:18:05 +00:00
committed by copyberry
parent 37eaae6eeb
commit 30bdfec59d
17 changed files with 442 additions and 74 deletions
+4
View File
@@ -34,6 +34,10 @@ pub use model::QueuedUserSubmissionRecord;
pub use model::RolloutMigrationCursor;
pub use model::RolloutMigrationSkippedRollout;
pub use model::RolloutMigrationState;
pub use runtime::GuardianReviewRecord;
pub use runtime::MAX_GUARDIAN_REVIEW_BYTES;
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS;
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD;
/// Preferred entrypoint: owns configuration and metrics.
pub use runtime::StateRuntime;
pub use sqlite::SqliteConfig;
+5
View File
@@ -42,6 +42,7 @@ use tracing::warn;
mod backfill;
mod external_agent_config_imports;
mod goals;
mod guardian_feedback;
mod logs;
mod logs_maintenance;
mod memories;
@@ -68,6 +69,10 @@ pub use goals::GoalAccountingMode;
pub use goals::GoalAccountingOutcome;
pub use goals::GoalStore;
pub use goals::GoalUpdate;
pub use guardian_feedback::GuardianReviewRecord;
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_BYTES;
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_RECORDS;
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD;
pub use memories::MemoryStore;
pub use queued_items::SqliteQueueStore;
pub use recovery::backup_runtime_db_for_fresh_start;
@@ -0,0 +1,92 @@
//! Bounded failed-review evidence, retained independently of short-lived reviewers.
//! Insertion and eviction share one budget; deleting the reviewed thread cascades to its evidence.
use super::StateRuntime;
use codex_protocol::ThreadId;
use sqlx::Row;
use uuid::Uuid;
pub const MAX_GUARDIAN_REVIEW_RECORDS: usize = 64;
pub const MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD: usize = 8;
pub const MAX_GUARDIAN_REVIEW_BYTES: usize = 8 * 1024 * 1024;
/// One complete serialized review, ordered by its capture-time UUIDv7.
#[derive(Clone)]
pub struct GuardianReviewRecord {
pub id: String,
pub thread_id: ThreadId,
pub record: Vec<u8>,
}
impl GuardianReviewRecord {
pub fn new(thread_id: ThreadId, record: Vec<u8>) -> Self {
Self {
id: Uuid::now_v7().to_string(),
thread_id,
record,
}
}
}
impl StateRuntime {
/// Retain opt-in evidence for a persisted actor; the reviewer need not remain alive.
pub async fn record_guardian_review_failure(
&self,
record: &GuardianReviewRecord,
) -> anyhow::Result<()> {
anyhow::ensure!(
record.record.len() < MAX_GUARDIAN_REVIEW_BYTES,
"Guardian feedback record exceeds its size limit"
);
let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?;
sqlx::query(
"INSERT INTO guardian_review_feedback (id, thread_id, record) VALUES (?, ?, ?)",
)
.bind(&record.id)
.bind(record.thread_id.to_string())
.bind(&record.record)
.execute(&mut *transaction)
.await?;
sqlx::query(
"DELETE FROM guardian_review_feedback WHERE id IN \
(SELECT id FROM guardian_review_feedback WHERE thread_id = ? \
ORDER BY id DESC LIMIT -1 OFFSET ?)",
)
.bind(record.thread_id.to_string())
.bind(MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD as i64)
.execute(&mut *transaction)
.await?;
sqlx::query(
"DELETE FROM guardian_review_feedback WHERE id IN \
(SELECT id FROM (SELECT id, ROW_NUMBER() OVER (ORDER BY id DESC) AS n, \
SUM(length(record) + 1) OVER (ORDER BY id DESC) AS bytes \
FROM guardian_review_feedback) WHERE n > ? OR bytes > ?)",
)
.bind(MAX_GUARDIAN_REVIEW_RECORDS as i64)
.bind(MAX_GUARDIAN_REVIEW_BYTES as i64)
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
Ok(())
}
/// Read the bounded retained history, oldest first. Callers must scope exports to a task.
pub async fn list_guardian_review_records(&self) -> anyhow::Result<Vec<GuardianReviewRecord>> {
sqlx::query("SELECT id, thread_id, record FROM guardian_review_feedback ORDER BY id")
.fetch_all(self.pool.as_ref())
.await?
.into_iter()
.map(|row| {
Ok(GuardianReviewRecord {
id: row.try_get("id")?,
thread_id: ThreadId::from_string(row.try_get("thread_id")?)?,
record: row.try_get("record")?,
})
})
.collect()
}
}
#[cfg(test)]
#[path = "guardian_feedback_tests.rs"]
mod tests;
@@ -0,0 +1,99 @@
//! Storage-level coverage for bounded review retention and actor deletion.
use super::*;
use crate::SqliteConfig;
use crate::runtime::test_support::test_thread_metadata;
use crate::runtime::test_support::unique_temp_dir;
use pretty_assertions::assert_eq;
#[tokio::test]
async fn retention_survives_reopen_and_bounds_count_and_bytes() -> anyhow::Result<()> {
let home = unique_temp_dir();
let sqlite = SqliteConfig::new_for_testing(home.clone().try_into()?);
let state = StateRuntime::init(sqlite.clone(), "test".to_string()).await?;
let mut ids = Vec::new();
for _ in 0..=MAX_GUARDIAN_REVIEW_RECORDS {
let id = ThreadId::new();
state
.upsert_thread(&test_thread_metadata(&home, id, home.clone()))
.await?;
let record = GuardianReviewRecord::new(id, b"{}".to_vec());
state.record_guardian_review_failure(&record).await?;
ids.push(id);
}
state.close().await;
let state = StateRuntime::init(sqlite, "test".to_string()).await?;
assert_eq!(
state
.list_guardian_review_records()
.await?
.iter()
.map(|record| record.thread_id)
.collect::<Vec<_>>(),
ids[1..],
);
let id = ids[0];
for index in 0..12 {
state
.record_guardian_review_failure(&GuardianReviewRecord::new(
id,
index.to_string().into_bytes(),
))
.await?;
}
assert_eq!(
state
.list_guardian_review_records()
.await?
.into_iter()
.filter(|record| record.thread_id == id)
.map(|record| record.record)
.collect::<Vec<_>>(),
(4..12)
.map(|index| index.to_string().into_bytes())
.collect::<Vec<_>>(),
);
let payload = vec![b' '; MAX_GUARDIAN_REVIEW_BYTES / 3];
for _ in 0..4 {
state
.record_guardian_review_failure(&GuardianReviewRecord::new(id, payload.clone()))
.await?;
}
assert_eq!(
state
.list_guardian_review_records()
.await?
.into_iter()
.map(|record| (record.thread_id, record.record))
.collect::<Vec<_>>(),
vec![(id, payload.clone()), (id, payload)],
);
state.close().await;
std::fs::remove_dir_all(home)?;
Ok(())
}
#[tokio::test]
async fn deleting_an_actor_removes_evidence_and_rejects_late_captures() -> anyhow::Result<()> {
let home = unique_temp_dir();
let sqlite = SqliteConfig::new_for_testing(home.clone().try_into()?);
let state = StateRuntime::init(sqlite.clone(), "test".to_string()).await?;
let id = ThreadId::new();
state
.upsert_thread(&test_thread_metadata(&home, id, home.clone()))
.await?;
let record = GuardianReviewRecord::new(id, b"private review context".to_vec());
state.record_guardian_review_failure(&record).await?;
// Older binaries delete the actor without knowing about the review table.
sqlx::query("DELETE FROM threads WHERE id = ?")
.bind(id.to_string())
.execute(state.pool.as_ref())
.await?;
state.close().await;
let state = StateRuntime::init(sqlite, "test".to_string()).await?;
assert!(state.list_guardian_review_records().await?.is_empty());
assert!(state.record_guardian_review_failure(&record).await.is_err());
state.close().await;
std::fs::remove_dir_all(home)?;
Ok(())
}