mirror of
https://github.com/openai/codex.git
synced 2026-11-10 21:00:47 +00:00
Charlie Marsh
dddc3c0a79
## Why Migrating selected threads reads rollout headers even for unrelated threads. Use the state database to skip files whose indexed owner and canonical filename agree that they belong to an unselected thread. ## What changed - Read thread IDs and rollout paths without creating or migrating the state database, and fall back to header inspection if the index cannot be read immediately. - Keep inspecting selected, unindexed, misnamed, and conflicting paths so rollout headers still determine ownership. - Recognize stable thread IDs in reverted rollout filenames and match plain and compressed paths when filtering. ## Testing Add a regression test covering pruning of unrelated indexed rollouts, including archived and compressed files, while retaining header inspection for selected and ambiguously named files. GitOrigin-RevId: d76827174c00f5bc9e38ad9b69dadeee91f4415f
173 lines
6.9 KiB
Rust
173 lines
6.9 KiB
Rust
//! SQLite-backed state for rollout metadata.
|
|
//!
|
|
//! This crate is intentionally small and focused: it extracts rollout metadata
|
|
//! from JSONL rollouts and mirrors it into a local SQLite database. Backfill
|
|
//! orchestration and rollout scanning live in `codex-core`.
|
|
|
|
const _: () = assert!(
|
|
libsqlite3_sys::SQLITE_VERSION_NUMBER >= 3_051_003,
|
|
"bundled SQLite must include the WAL-reset corruption fix",
|
|
);
|
|
|
|
mod audit;
|
|
mod delegated_preview;
|
|
mod extract;
|
|
pub mod log_db;
|
|
mod migrations;
|
|
mod model;
|
|
mod nfs_migration;
|
|
mod paths;
|
|
mod runtime;
|
|
mod sqlite;
|
|
mod telemetry;
|
|
|
|
pub use log_db::LogWriteFailureReporter;
|
|
pub use model::CreatedProject;
|
|
pub use model::LogEntry;
|
|
pub use model::LogQuery;
|
|
pub use model::LogRow;
|
|
pub use model::Phase2JobClaimOutcome;
|
|
pub use model::Project;
|
|
pub use model::ProjectRoot;
|
|
pub use model::ProjectSortKey;
|
|
pub use model::ProjectsPage;
|
|
pub use model::QueuedUserSubmissionRecord;
|
|
pub use model::RolloutMigrationCursor;
|
|
pub use model::RolloutMigrationSkippedRollout;
|
|
pub use model::RolloutMigrationState;
|
|
pub use nfs_migration::NfsDbMigration;
|
|
pub use nfs_migration::NfsDbMigrationOutcome;
|
|
pub use runtime::GuardianReviewRecord;
|
|
pub use runtime::MAX_GUARDIAN_REVIEW_BYTES;
|
|
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS;
|
|
pub use runtime::MAX_GUARDIAN_REVIEW_RECORDS_PER_THREAD;
|
|
/// Preferred entrypoint: owns configuration and metrics.
|
|
pub use runtime::StateRuntime;
|
|
pub use sqlite::SqliteConfig;
|
|
pub use sqlite::SqlitePoolWriteExt;
|
|
|
|
pub use audit::ThreadStateAuditRow;
|
|
pub use audit::read_thread_rollout_paths;
|
|
pub use audit::read_thread_state_audit_rows;
|
|
pub use delegated_preview::delegated_output_preview;
|
|
pub use extract::GUARDIAN_THREAD_PREVIEW;
|
|
pub use extract::GUARDIAN_THREAD_TITLE;
|
|
/// Low-level storage engine: useful for focused tests.
|
|
///
|
|
/// Most consumers should prefer [`StateRuntime`].
|
|
pub use extract::apply_rollout_item;
|
|
pub use extract::is_guardian_review_source;
|
|
pub use extract::rollout_item_affects_thread_metadata;
|
|
pub use model::AddThreadAttachmentOutcome;
|
|
pub use model::Anchor;
|
|
pub use model::BackfillState;
|
|
pub use model::BackfillStats;
|
|
pub use model::BackfillStatus;
|
|
pub use model::DirectionalThreadSpawnEdgeStatus;
|
|
pub use model::ExtractionOutcome;
|
|
pub use model::RemoveThreadAttachmentOutcome;
|
|
pub use model::SortDirection;
|
|
pub use model::SortKey;
|
|
pub use model::Stage1JobClaim;
|
|
pub use model::Stage1JobClaimOutcome;
|
|
pub use model::Stage1Output;
|
|
pub use model::Stage1StartupClaimParams;
|
|
pub use model::ThreadAttachment;
|
|
pub use model::ThreadAttachmentArchiveFilter;
|
|
pub use model::ThreadAttachmentOwner;
|
|
pub use model::ThreadAttachmentOwnerPage;
|
|
pub use model::ThreadAttachmentPage;
|
|
pub use model::ThreadGoal;
|
|
pub use model::ThreadGoalStatus;
|
|
pub use model::ThreadMetadata;
|
|
pub use model::ThreadMetadataBuilder;
|
|
pub use model::ThreadRelationFilter;
|
|
pub use model::ThreadSection;
|
|
pub use model::ThreadSectionAppearance;
|
|
pub use model::ThreadSectionsPage;
|
|
pub use model::ThreadsPage;
|
|
pub use runtime::ExternalAgentConfigImportDetailsRecord;
|
|
pub use runtime::ExternalAgentConfigImportFailureRecord;
|
|
pub use runtime::ExternalAgentConfigImportHistoryRecord;
|
|
pub use runtime::ExternalAgentConfigImportSuccessRecord;
|
|
pub use runtime::GoalAccountingMode;
|
|
pub use runtime::GoalAccountingOutcome;
|
|
pub use runtime::GoalStore;
|
|
pub use runtime::GoalUpdate;
|
|
pub use runtime::MemoryStore;
|
|
pub use runtime::ReadStateOperation;
|
|
pub use runtime::ReadStateUpdate;
|
|
pub use runtime::RemoteControlEnrollmentRecord;
|
|
pub use runtime::SqliteIntegrityCheck;
|
|
pub use runtime::SqliteQueueStore;
|
|
pub use runtime::ThreadFilterOptions;
|
|
pub use runtime::ThreadReadState;
|
|
pub use runtime::backup_runtime_db_for_fresh_start;
|
|
pub use runtime::collect_runtime_db_backups;
|
|
pub use runtime::is_sqlite_corruption_error;
|
|
pub use runtime::open_thread_history_db;
|
|
pub use runtime::recovery::RuntimeDbBackup;
|
|
pub use runtime::runtime_db_path_for_corruption_error;
|
|
pub use runtime::sqlite_error_detail_is_lock;
|
|
pub use runtime::sqlite_integrity_check;
|
|
pub use sqlite::RuntimeDbPath;
|
|
pub use telemetry::DbTelemetry;
|
|
pub use telemetry::DbTelemetryHandle;
|
|
pub use telemetry::install_process_db_telemetry;
|
|
pub use telemetry::record_backfill_gate;
|
|
pub use telemetry::record_fallback;
|
|
|
|
/// Maximum number of pending user submissions permitted for one thread.
|
|
pub const MAX_QUEUE_ITEMS: usize = 100;
|
|
|
|
/// Maximum serialized size of one persisted thread-attachment payload.
|
|
pub const MAX_THREAD_ATTACHMENT_PAYLOAD_BYTES: usize = 64 * 1024;
|
|
|
|
/// Maximum byte length of a persisted attachment type.
|
|
pub const MAX_THREAD_ATTACHMENT_TYPE_BYTES: usize = 256;
|
|
|
|
/// Maximum byte length of a persisted stable attachment identity key.
|
|
pub const MAX_THREAD_ATTACHMENT_IDENTITY_KEY_BYTES: usize = 256;
|
|
|
|
/// Maximum number of attachments returned in one page.
|
|
pub const MAX_THREAD_ATTACHMENT_LIST_PAGE_SIZE: usize = 100;
|
|
|
|
/// Maximum number of active attachments retained for one thread.
|
|
pub const MAX_THREAD_ATTACHMENTS_PER_THREAD: usize = 100;
|
|
|
|
/// Stable UUIDv7 identifying the built-in pinned thread section.
|
|
pub const PINNED_THREAD_SECTION_ID: &str = "01984de2-8f74-7c91-a3b2-5c5e937cf318";
|
|
|
|
/// User-facing name of the built-in pinned thread section.
|
|
pub const PINNED_THREAD_SECTION_NAME: &str = "Pinned";
|
|
|
|
/// Environment variable for overriding the SQLite state database home directory.
|
|
pub const SQLITE_HOME_ENV: &str = "CODEX_SQLITE_HOME";
|
|
|
|
/// Errors encountered during DB operations. Tags: [stage]
|
|
pub const DB_ERROR_METRIC: &str = "codex.db.error";
|
|
/// Metrics on backfill process. Tags: [status]
|
|
pub const DB_METRIC_BACKFILL: &str = "codex.db.backfill";
|
|
/// Metrics on backfill duration. Tags: [status]
|
|
pub const DB_METRIC_BACKFILL_DURATION_MS: &str = "codex.db.backfill.duration_ms";
|
|
/// Confirmed SQLite quick-check corruption findings. Tags: [db]
|
|
pub const DB_CORRUPTION_METRIC: &str = "codex.sqlite.corruption.count";
|
|
/// SQLite initialization attempts. Tags: [status, phase, db, error]
|
|
pub const DB_INIT_METRIC: &str = "codex.sqlite.init.count";
|
|
/// SQLite initialization latency. Tags: [status, phase, db, error]
|
|
pub const DB_INIT_DURATION_METRIC: &str = "codex.sqlite.init.duration_ms";
|
|
/// Rollout fallback attempts. Tags: [caller, reason]
|
|
pub const DB_FALLBACK_METRIC: &str = "codex.sqlite.fallback.count";
|
|
/// SQLite log batch write attempts. Tags: [status, error]
|
|
pub const LOG_WRITE_METRIC: &str = "codex.sqlite.logs.write.count";
|
|
/// SQLite log batch write latency. Tags: [status, error]
|
|
pub const LOG_WRITE_DURATION_METRIC: &str = "codex.sqlite.logs.write.duration_ms";
|
|
/// Estimated bytes in each SQLite log batch. Tags: [status, error]
|
|
pub const LOG_WRITE_BYTES_METRIC: &str = "codex.sqlite.logs.write.bytes";
|
|
/// Number of entries in each SQLite log batch. Tags: [status, error]
|
|
pub const LOG_WRITE_ENTRIES_METRIC: &str = "codex.sqlite.logs.write.entries";
|
|
/// Largest estimated entry size in each SQLite log batch. Tags: [status, error]
|
|
pub const LOG_WRITE_MAX_ENTRY_BYTES_METRIC: &str = "codex.sqlite.logs.write.max_entry_bytes";
|
|
/// SQLite log entries discarded before they can be queued. Tags: [reason]
|
|
pub const LOG_QUEUE_DROPPED_METRIC: &str = "codex.sqlite.logs.queue.dropped";
|