mirror of
https://github.com/openai/codex.git
synced 2026-10-20 17:38:45 +00:00
Persist Guardian review failures for reports across restarts (#51651)
## Why Failed Guardian review evidence was process-local, so restarting Codex lost the context needed for later diagnostic reports. ## What changed - Persist failed-review records in SQLite before reviewer cleanup, retaining at most 64 records globally, eight per thread, and 8 MiB of evidence. Delete persisted evidence when its reviewed thread is deleted. - Recover and deduplicate persisted and in-memory records for the reported task tree when uploading `auto-review-failures.jsonl`. - Bound persistence attempts to 250 ms and retain an in-memory fallback when storage is unavailable. Keep capture gated by user opt-in and non-ephemeral sessions, and export evidence only when logs are included. ## Testing Add coverage for persistence across reopen and process restart, retention limits, cascading deletion, busy-storage fallback, and report uploads with and without logs, including SQLite log failures. GitOrigin-RevId: ea2da1818d4a5da3da29f211042ca45e834342bc
This commit is contained in:
@@ -34,6 +34,10 @@ pub use model::QueuedUserSubmissionRecord;
|
||||
pub use model::RolloutMigrationCursor;
|
||||
pub use model::RolloutMigrationSkippedRollout;
|
||||
pub use model::RolloutMigrationState;
|
||||
pub use runtime::GuardianReviewRecord;
|
||||
pub use runtime::MAX_GUARDIAN_REVIEW_BYTES;
|
||||
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS;
|
||||
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD;
|
||||
/// Preferred entrypoint: owns configuration and metrics.
|
||||
pub use runtime::StateRuntime;
|
||||
pub use sqlite::SqliteConfig;
|
||||
|
||||
@@ -42,6 +42,7 @@ use tracing::warn;
|
||||
mod backfill;
|
||||
mod external_agent_config_imports;
|
||||
mod goals;
|
||||
mod guardian_feedback;
|
||||
mod logs;
|
||||
mod logs_maintenance;
|
||||
mod memories;
|
||||
@@ -68,6 +69,10 @@ pub use goals::GoalAccountingMode;
|
||||
pub use goals::GoalAccountingOutcome;
|
||||
pub use goals::GoalStore;
|
||||
pub use goals::GoalUpdate;
|
||||
pub use guardian_feedback::GuardianReviewRecord;
|
||||
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_BYTES;
|
||||
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_RECORDS;
|
||||
pub use guardian_feedback::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD;
|
||||
pub use memories::MemoryStore;
|
||||
pub use queued_items::SqliteQueueStore;
|
||||
pub use recovery::backup_runtime_db_for_fresh_start;
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
//! Bounded failed-review evidence, retained independently of short-lived reviewers.
|
||||
//! Insertion and eviction share one budget; deleting the reviewed thread cascades to its evidence.
|
||||
|
||||
use super::StateRuntime;
|
||||
use codex_protocol::ThreadId;
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const MAX_GUARDIAN_REVIEW_RECORDS: usize = 64;
|
||||
pub const MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD: usize = 8;
|
||||
pub const MAX_GUARDIAN_REVIEW_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// One complete serialized review, ordered by its capture-time UUIDv7.
|
||||
#[derive(Clone)]
|
||||
pub struct GuardianReviewRecord {
|
||||
pub id: String,
|
||||
pub thread_id: ThreadId,
|
||||
pub record: Vec<u8>,
|
||||
}
|
||||
|
||||
impl GuardianReviewRecord {
|
||||
pub fn new(thread_id: ThreadId, record: Vec<u8>) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7().to_string(),
|
||||
thread_id,
|
||||
record,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StateRuntime {
|
||||
/// Retain opt-in evidence for a persisted actor; the reviewer need not remain alive.
|
||||
pub async fn record_guardian_review_failure(
|
||||
&self,
|
||||
record: &GuardianReviewRecord,
|
||||
) -> anyhow::Result<()> {
|
||||
anyhow::ensure!(
|
||||
record.record.len() < MAX_GUARDIAN_REVIEW_BYTES,
|
||||
"Guardian feedback record exceeds its size limit"
|
||||
);
|
||||
let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO guardian_review_feedback (id, thread_id, record) VALUES (?, ?, ?)",
|
||||
)
|
||||
.bind(&record.id)
|
||||
.bind(record.thread_id.to_string())
|
||||
.bind(&record.record)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"DELETE FROM guardian_review_feedback WHERE id IN \
|
||||
(SELECT id FROM guardian_review_feedback WHERE thread_id = ? \
|
||||
ORDER BY id DESC LIMIT -1 OFFSET ?)",
|
||||
)
|
||||
.bind(record.thread_id.to_string())
|
||||
.bind(MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD as i64)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"DELETE FROM guardian_review_feedback WHERE id IN \
|
||||
(SELECT id FROM (SELECT id, ROW_NUMBER() OVER (ORDER BY id DESC) AS n, \
|
||||
SUM(length(record) + 1) OVER (ORDER BY id DESC) AS bytes \
|
||||
FROM guardian_review_feedback) WHERE n > ? OR bytes > ?)",
|
||||
)
|
||||
.bind(MAX_GUARDIAN_REVIEW_RECORDS as i64)
|
||||
.bind(MAX_GUARDIAN_REVIEW_BYTES as i64)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
transaction.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the bounded retained history, oldest first. Callers must scope exports to a task.
|
||||
pub async fn list_guardian_review_records(&self) -> anyhow::Result<Vec<GuardianReviewRecord>> {
|
||||
sqlx::query("SELECT id, thread_id, record FROM guardian_review_feedback ORDER BY id")
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok(GuardianReviewRecord {
|
||||
id: row.try_get("id")?,
|
||||
thread_id: ThreadId::from_string(row.try_get("thread_id")?)?,
|
||||
record: row.try_get("record")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "guardian_feedback_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,99 @@
|
||||
//! Storage-level coverage for bounded review retention and actor deletion.
|
||||
|
||||
use super::*;
|
||||
use crate::SqliteConfig;
|
||||
use crate::runtime::test_support::test_thread_metadata;
|
||||
use crate::runtime::test_support::unique_temp_dir;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_survives_reopen_and_bounds_count_and_bytes() -> anyhow::Result<()> {
|
||||
let home = unique_temp_dir();
|
||||
let sqlite = SqliteConfig::new_for_testing(home.clone().try_into()?);
|
||||
let state = StateRuntime::init(sqlite.clone(), "test".to_string()).await?;
|
||||
let mut ids = Vec::new();
|
||||
for _ in 0..=MAX_GUARDIAN_REVIEW_RECORDS {
|
||||
let id = ThreadId::new();
|
||||
state
|
||||
.upsert_thread(&test_thread_metadata(&home, id, home.clone()))
|
||||
.await?;
|
||||
let record = GuardianReviewRecord::new(id, b"{}".to_vec());
|
||||
state.record_guardian_review_failure(&record).await?;
|
||||
ids.push(id);
|
||||
}
|
||||
state.close().await;
|
||||
let state = StateRuntime::init(sqlite, "test".to_string()).await?;
|
||||
assert_eq!(
|
||||
state
|
||||
.list_guardian_review_records()
|
||||
.await?
|
||||
.iter()
|
||||
.map(|record| record.thread_id)
|
||||
.collect::<Vec<_>>(),
|
||||
ids[1..],
|
||||
);
|
||||
let id = ids[0];
|
||||
for index in 0..12 {
|
||||
state
|
||||
.record_guardian_review_failure(&GuardianReviewRecord::new(
|
||||
id,
|
||||
index.to_string().into_bytes(),
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
assert_eq!(
|
||||
state
|
||||
.list_guardian_review_records()
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|record| record.thread_id == id)
|
||||
.map(|record| record.record)
|
||||
.collect::<Vec<_>>(),
|
||||
(4..12)
|
||||
.map(|index| index.to_string().into_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
let payload = vec![b' '; MAX_GUARDIAN_REVIEW_BYTES / 3];
|
||||
for _ in 0..4 {
|
||||
state
|
||||
.record_guardian_review_failure(&GuardianReviewRecord::new(id, payload.clone()))
|
||||
.await?;
|
||||
}
|
||||
assert_eq!(
|
||||
state
|
||||
.list_guardian_review_records()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|record| (record.thread_id, record.record))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![(id, payload.clone()), (id, payload)],
|
||||
);
|
||||
state.close().await;
|
||||
std::fs::remove_dir_all(home)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleting_an_actor_removes_evidence_and_rejects_late_captures() -> anyhow::Result<()> {
|
||||
let home = unique_temp_dir();
|
||||
let sqlite = SqliteConfig::new_for_testing(home.clone().try_into()?);
|
||||
let state = StateRuntime::init(sqlite.clone(), "test".to_string()).await?;
|
||||
let id = ThreadId::new();
|
||||
state
|
||||
.upsert_thread(&test_thread_metadata(&home, id, home.clone()))
|
||||
.await?;
|
||||
let record = GuardianReviewRecord::new(id, b"private review context".to_vec());
|
||||
state.record_guardian_review_failure(&record).await?;
|
||||
// Older binaries delete the actor without knowing about the review table.
|
||||
sqlx::query("DELETE FROM threads WHERE id = ?")
|
||||
.bind(id.to_string())
|
||||
.execute(state.pool.as_ref())
|
||||
.await?;
|
||||
state.close().await;
|
||||
let state = StateRuntime::init(sqlite, "test".to_string()).await?;
|
||||
assert!(state.list_guardian_review_records().await?.is_empty());
|
||||
assert!(state.record_guardian_review_failure(&record).await.is_err());
|
||||
state.close().await;
|
||||
std::fs::remove_dir_all(home)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user