openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-20 17:38:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: wal config

Browse Source
This commit is contained in:
jif-oai
2026-06-04 12:44:46 +01:00
parent ad2012d645
commit 00e9f068e2
12 changed files with 277 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/cli/src/main.rs
+5 -2
View File
@@ -1911,8 +1911,11 @@ async fn run_debug_clear_memories_command(
.await?;
let memories_path = memories_db_path(config.sqlite_home.as_path());
let cleared_memories_db =
StateRuntime::clear_memory_data_in_sqlite_home(config.sqlite_home.as_path()).await?;
let cleared_memories_db = StateRuntime::clear_memory_data_in_sqlite_home_with_journal_mode(
config.sqlite_home.as_path(),
config.sqlite_journal_mode,
)
.await?;
clear_memory_roots_contents(&config.codex_home).await?;
codex-rs/config/src/config_toml.rs
+18
View File
@@ -310,6 +310,9 @@ pub struct ConfigToml {
/// Defaults to `$CODEX_SQLITE_HOME` when set. Otherwise uses `$CODEX_HOME`.
pub sqlite_home: Option<AbsolutePathBuf>,
/// SQLite runtime settings.
pub sqlite: Option<SqliteToml>,
/// Directory where Codex writes log files. Setting this value explicitly
/// also enables the TUI text log in this directory.
/// Defaults to `$CODEX_HOME/log`.
@@ -538,6 +541,21 @@ pub enum ThreadStoreToml {
},
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct SqliteToml {
/// Journal mode used when opening Codex runtime SQLite databases.
pub journal_mode: Option<SqliteJournalModeToml>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum SqliteJournalModeToml {
#[default]
Wal,
Delete,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
pub struct AutoReviewToml {
/// Additional policy instructions inserted into the guardian prompt.
codex-rs/core/config.schema.json
+29
View File
@@ -2656,6 +2656,27 @@
},
"type": "object"
},
"SqliteJournalModeToml": {
"enum": [
"wal",
"delete"
],
"type": "string"
},
"SqliteToml": {
"additionalProperties": false,
"properties": {
"journal_mode": {
"allOf": [
{
"$ref": "#/definitions/SqliteJournalModeToml"
}
],
"description": "Journal mode used when opening Codex runtime SQLite databases."
}
},
"type": "object"
},
"ThreadStoreToml": {
"oneOf": [
{
@@ -5143,6 +5164,14 @@
],
"description": "User-level skill config entries keyed by SKILL.md path."
},
"sqlite": {
"allOf": [
{
"$ref": "#/definitions/SqliteToml"
}
],
"description": "SQLite runtime settings."
},
"sqlite_home": {
"allOf": [
{
codex-rs/core/src/config/config_tests.rs
+25
View File
@@ -19,6 +19,8 @@ use codex_config::config_toml::RealtimeToml;
use codex_config::config_toml::RealtimeTransport;
use codex_config::config_toml::RealtimeWsMode;
use codex_config::config_toml::RealtimeWsVersion;
use codex_config::config_toml::SqliteJournalModeToml;
use codex_config::config_toml::SqliteToml;
use codex_config::config_toml::ToolsToml;
use codex_config::loader::project_trust_key;
use codex_config::permissions_toml::FilesystemPermissionToml;
@@ -4665,6 +4667,29 @@ async fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Re
Ok(())
}
#[tokio::test]
async fn sqlite_journal_mode_can_disable_wal() -> std::io::Result<()> {
let codex_home = TempDir::new()?;
let config = Config::load_from_base_config_with_overrides(
ConfigToml {
sqlite: Some(SqliteToml {
journal_mode: Some(SqliteJournalModeToml::Delete),
}),
..Default::default()
},
ConfigOverrides::default(),
codex_home.abs(),
)
.await?;
assert_eq!(
config.sqlite_journal_mode,
codex_state::RuntimeSqliteJournalMode::Delete
);
Ok(())
}
#[tokio::test]
async fn workspace_write_includes_configured_writable_root_once_without_memories_root()
-> std::io::Result<()> {
codex-rs/core/src/config/mod.rs
+20
View File
@@ -29,6 +29,7 @@ use codex_config::config_toml::DEFAULT_PROJECT_DOC_MAX_BYTES;
use codex_config::config_toml::ProjectConfig;
use codex_config::config_toml::RealtimeAudioConfig;
use codex_config::config_toml::RealtimeConfig;
use codex_config::config_toml::SqliteJournalModeToml;
use codex_config::config_toml::ThreadStoreToml;
use codex_config::config_toml::validate_model_providers;
use codex_config::loader::load_config_layers_state;
@@ -252,6 +253,15 @@ fn resolve_sqlite_home_env(resolved_cwd: &Path) -> Option<PathBuf> {
}
}
fn resolve_sqlite_journal_mode(
mode: SqliteJournalModeToml,
) -> codex_state::RuntimeSqliteJournalMode {
match mode {
SqliteJournalModeToml::Wal => codex_state::RuntimeSqliteJournalMode::Wal,
SqliteJournalModeToml::Delete => codex_state::RuntimeSqliteJournalMode::Delete,
}
}
fn resolve_cli_auth_credentials_store_mode(
configured: AuthCredentialsStoreMode,
package_version: &str,
@@ -842,6 +852,9 @@ pub struct Config {
/// Directory where Codex stores the SQLite state DB.
pub sqlite_home: PathBuf,
/// Journal mode used when opening Codex runtime SQLite databases.
pub sqlite_journal_mode: codex_state::RuntimeSqliteJournalMode,
/// Directory where Codex writes log files (defaults to `$CODEX_HOME/log`).
pub log_dir: PathBuf,
@@ -3276,6 +3289,12 @@ impl Config {
.map(AbsolutePathBuf::to_path_buf)
.or_else(|| resolve_sqlite_home_env(&resolved_cwd))
.unwrap_or_else(|| codex_home.to_path_buf());
let sqlite_journal_mode = cfg
.sqlite
.as_ref()
.and_then(|sqlite| sqlite.journal_mode)
.map(resolve_sqlite_journal_mode)
.unwrap_or_default();
let original_permission_profile = permission_profile.clone();
apply_requirement_constrained_value(
"approval_policy",
@@ -3477,6 +3496,7 @@ impl Config {
agent_interrupt_message_enabled,
codex_home,
sqlite_home,
sqlite_journal_mode,
log_dir,
config_lock_export_dir: cfg
.debug
codex-rs/core/src/rollout.rs
+4
View File
@@ -32,6 +32,10 @@ impl codex_rollout::RolloutConfigView for Config {
self.sqlite_home.as_path()
}
fn sqlite_journal_mode(&self) -> codex_state::RuntimeSqliteJournalMode {
self.sqlite_journal_mode
}
fn cwd(&self) -> &std::path::Path {
self.cwd.as_path()
}
codex-rs/rollout/src/config.rs
+11
View File
@@ -5,6 +5,9 @@ use std::sync::Arc;
pub trait RolloutConfigView {
fn codex_home(&self) -> &Path;
fn sqlite_home(&self) -> &Path;
fn sqlite_journal_mode(&self) -> codex_state::RuntimeSqliteJournalMode {
codex_state::RuntimeSqliteJournalMode::default()
}
fn cwd(&self) -> &Path;
fn model_provider_id(&self) -> &str;
fn generate_memories(&self) -> bool;
@@ -64,6 +67,10 @@ impl<T: RolloutConfigView + ?Sized> RolloutConfigView for &T {
(*self).sqlite_home()
}
fn sqlite_journal_mode(&self) -> codex_state::RuntimeSqliteJournalMode {
(*self).sqlite_journal_mode()
}
fn cwd(&self) -> &Path {
(*self).cwd()
}
@@ -86,6 +93,10 @@ impl<T: RolloutConfigView + ?Sized> RolloutConfigView for Arc<T> {
self.as_ref().sqlite_home()
}
fn sqlite_journal_mode(&self) -> codex_state::RuntimeSqliteJournalMode {
self.as_ref().sqlite_journal_mode()
}
fn cwd(&self) -> &Path {
self.as_ref().cwd()
}
codex-rs/rollout/src/state_db.rs
+27 -19
View File
@@ -1,4 +1,3 @@
use crate::config::RolloutConfig;
use crate::config::RolloutConfigView;
use crate::list::Cursor;
use crate::list::SortDirection;
@@ -40,11 +39,11 @@ const STARTUP_BACKFILL_WAIT_TIMEOUT: Duration = Duration::from_secs(2);
/// runtime, applies rollout metadata backfills as needed, and returns the
/// initialized handle.
pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
let config = RolloutConfig::from_view(config);
match try_init_with_roots(
config.codex_home,
config.sqlite_home,
config.model_provider_id,
config.codex_home().to_path_buf(),
config.sqlite_home().to_path_buf(),
config.sqlite_journal_mode(),
config.model_provider_id().to_string(),
)
.await
{
@@ -61,11 +60,11 @@ pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
/// Prefer [`init`] unless the caller needs to surface the exact failure after
/// tracing or UI setup has completed.
pub async fn try_init(config: &impl RolloutConfigView) -> anyhow::Result<StateDbHandle> {
let config = RolloutConfig::from_view(config);
try_init_with_roots(
config.codex_home,
config.sqlite_home,
config.model_provider_id,
config.codex_home().to_path_buf(),
config.sqlite_home().to_path_buf(),
config.sqlite_journal_mode(),
config.model_provider_id().to_string(),
)
.await
}
@@ -73,11 +72,13 @@ pub async fn try_init(config: &impl RolloutConfigView) -> anyhow::Result<StateDb
async fn try_init_with_roots(
codex_home: PathBuf,
sqlite_home: PathBuf,
sqlite_journal_mode: codex_state::RuntimeSqliteJournalMode,
default_model_provider_id: String,
) -> anyhow::Result<StateDbHandle> {
try_init_with_roots_inner(
codex_home,
sqlite_home,
sqlite_journal_mode,
default_model_provider_id,
/*backfill_lease_seconds*/ None,
)
@@ -88,12 +89,14 @@ async fn try_init_with_roots(
async fn try_init_with_roots_and_backfill_lease(
codex_home: PathBuf,
sqlite_home: PathBuf,
sqlite_journal_mode: codex_state::RuntimeSqliteJournalMode,
default_model_provider_id: String,
backfill_lease_seconds: i64,
) -> anyhow::Result<StateDbHandle> {
try_init_with_roots_inner(
codex_home,
sqlite_home,
sqlite_journal_mode,
default_model_provider_id,
Some(backfill_lease_seconds),
)
@@ -103,18 +106,22 @@ async fn try_init_with_roots_and_backfill_lease(
async fn try_init_with_roots_inner(
codex_home: PathBuf,
sqlite_home: PathBuf,
sqlite_journal_mode: codex_state::RuntimeSqliteJournalMode,
default_model_provider_id: String,
backfill_lease_seconds: Option<i64>,
) -> anyhow::Result<StateDbHandle> {
let runtime =
codex_state::StateRuntime::init(sqlite_home.clone(), default_model_provider_id.clone())
.await
.map_err(|err| {
anyhow::anyhow!(
"failed to initialize state runtime at {}: {err}",
sqlite_home.display()
)
})?;
let runtime = codex_state::StateRuntime::init_with_journal_mode(
sqlite_home.clone(),
default_model_provider_id.clone(),
sqlite_journal_mode,
)
.await
.map_err(|err| {
anyhow::anyhow!(
"failed to initialize state runtime at {}: {err}",
sqlite_home.display()
)
})?;
let backfill_gate_started = Instant::now();
let backfill_gate_result = wait_for_backfill_gate(
runtime.as_ref(),
@@ -220,9 +227,10 @@ pub async fn get_state_db(config: &impl RolloutConfigView) -> Option<StateDbHand
);
return None;
}
let runtime = match codex_state::StateRuntime::init(
let runtime = match codex_state::StateRuntime::init_with_journal_mode(
config.sqlite_home().to_path_buf(),
config.model_provider_id().to_string(),
config.sqlite_journal_mode(),
)
.await
{
codex-rs/rollout/src/state_db_tests.rs
+2
View File
@@ -49,6 +49,7 @@ async fn try_init_waits_for_concurrent_startup_backfill() -> anyhow::Result<()>
let initialized = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
codex_state::RuntimeSqliteJournalMode::default(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
@@ -74,6 +75,7 @@ async fn try_init_times_out_waiting_for_stuck_startup_backfill() -> anyhow::Resu
let result = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
codex_state::RuntimeSqliteJournalMode::default(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
codex-rs/state/src/lib.rs
+1
View File
@@ -18,6 +18,7 @@ pub use model::LogQuery;
pub use model::LogRow;
pub use model::Phase2JobClaimOutcome;
/// Preferred entrypoint: owns configuration and metrics.
pub use runtime::RuntimeSqliteJournalMode;
pub use runtime::StateRuntime;
pub use audit::ThreadStateAuditRow;
codex-rs/state/src/runtime.rs
+134 -21
View File
@@ -45,7 +45,7 @@ use sqlx::SqlitePool;
use sqlx::migrate::Migrator;
use sqlx::sqlite::SqliteAutoVacuum;
use sqlx::sqlite::SqliteConnectOptions;
use sqlx::sqlite::SqliteJournalMode;
use sqlx::sqlite::SqliteJournalMode as SqlxSqliteJournalMode;
use sqlx::sqlite::SqlitePoolOptions;
use sqlx::sqlite::SqliteSynchronous;
use std::collections::BTreeSet;
@@ -84,6 +84,24 @@ pub use threads::ThreadFilterOptions;
const LOG_PARTITION_SIZE_LIMIT_BYTES: i64 = 10 * 1024 * 1024;
const LOG_PARTITION_ROW_LIMIT: i64 = 1_000;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum RuntimeSqliteJournalMode {
/// SQLite write-ahead logging, the default Codex runtime behavior.
#[default]
Wal,
/// SQLite rollback journal mode. Useful when WAL sidecar files are unsafe on the backing FS.
Delete,
}
impl RuntimeSqliteJournalMode {
fn to_sqlx(self) -> SqlxSqliteJournalMode {
match self {
Self::Wal => SqlxSqliteJournalMode::Wal,
Self::Delete => SqlxSqliteJournalMode::Delete,
}
}
}
#[derive(Clone, Copy)]
struct RuntimeDbSpec {
label: &'static str,
@@ -157,9 +175,23 @@ impl StateRuntime {
/// keeping logs in a dedicated file to reduce lock contention with the
/// rest of the state store.
pub async fn init(codex_home: PathBuf, default_provider: String) -> anyhow::Result<Arc<Self>> {
Self::init_with_journal_mode(
codex_home,
default_provider,
RuntimeSqliteJournalMode::default(),
)
.await
}
pub async fn init_with_journal_mode(
codex_home: PathBuf,
default_provider: String,
journal_mode: RuntimeSqliteJournalMode,
) -> anyhow::Result<Arc<Self>> {
Self::init_inner(
codex_home,
default_provider,
journal_mode,
/*telemetry_override*/ None,
)
.await
@@ -171,12 +203,19 @@ impl StateRuntime {
default_provider: String,
telemetry_override: &dyn DbTelemetry,
) -> anyhow::Result<Arc<Self>> {
Self::init_inner(codex_home, default_provider, Some(telemetry_override)).await
Self::init_inner(
codex_home,
default_provider,
RuntimeSqliteJournalMode::default(),
Some(telemetry_override),
)
.await
}
async fn init_inner(
codex_home: PathBuf,
default_provider: String,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<Arc<Self>> {
tokio::fs::create_dir_all(&codex_home).await?;
@@ -188,32 +227,48 @@ impl StateRuntime {
let logs_path = LOGS_DB.path(codex_home.as_path());
let goals_path = GOALS_DB.path(codex_home.as_path());
let memories_path = MEMORIES_DB.path(codex_home.as_path());
let pool = match open_state_sqlite(&state_path, &state_migrator, telemetry_override).await {
let pool = match open_state_sqlite(
&state_path,
&state_migrator,
journal_mode,
telemetry_override,
)
.await
{
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open state db at {}: {err}", state_path.display());
return Err(err);
}
};
let logs_pool = match open_logs_sqlite(&logs_path, &logs_migrator, telemetry_override).await
{
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open logs db at {}: {err}", logs_path.display());
return Err(err);
}
};
let goals_pool =
match open_goals_sqlite(&goals_path, &goals_migrator, telemetry_override).await {
let logs_pool =
match open_logs_sqlite(&logs_path, &logs_migrator, journal_mode, telemetry_override)
.await
{
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open goals db at {}: {err}", goals_path.display());
warn!("failed to open logs db at {}: {err}", logs_path.display());
return Err(err);
}
};
let goals_pool = match open_goals_sqlite(
&goals_path,
&goals_migrator,
journal_mode,
telemetry_override,
)
.await
{
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open goals db at {}: {err}", goals_path.display());
return Err(err);
}
};
let memories_pool = match open_memories_sqlite(
&memories_path,
&memories_migrator,
journal_mode,
telemetry_override,
)
.await
@@ -284,6 +339,17 @@ impl StateRuntime {
}
pub async fn clear_memory_data_in_sqlite_home(sqlite_home: &Path) -> anyhow::Result<bool> {
Self::clear_memory_data_in_sqlite_home_with_journal_mode(
sqlite_home,
RuntimeSqliteJournalMode::default(),
)
.await
}
pub async fn clear_memory_data_in_sqlite_home_with_journal_mode(
sqlite_home: &Path,
journal_mode: RuntimeSqliteJournalMode,
) -> anyhow::Result<bool> {
let memories_path = MEMORIES_DB.path(sqlite_home);
if !tokio::fs::try_exists(&memories_path).await? {
return Ok(false);
@@ -293,6 +359,7 @@ impl StateRuntime {
let pool = open_memories_sqlite(
&memories_path,
&memories_migrator,
journal_mode,
/*telemetry_override*/ None,
)
.await?;
@@ -302,11 +369,14 @@ impl StateRuntime {
}
}
fn base_sqlite_options(path: &Path) -> SqliteConnectOptions {
fn base_sqlite_options(
path: &Path,
journal_mode: RuntimeSqliteJournalMode,
) -> SqliteConnectOptions {
SqliteConnectOptions::new()
.filename(path)
.create_if_missing(true)
.journal_mode(SqliteJournalMode::Wal)
.journal_mode(journal_mode.to_sqlx())
.synchronous(SqliteSynchronous::Normal)
.busy_timeout(Duration::from_secs(5))
.log_statements(LevelFilter::Off)
@@ -315,45 +385,58 @@ fn base_sqlite_options(path: &Path) -> SqliteConnectOptions {
async fn open_state_sqlite(
path: &Path,
migrator: &Migrator,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
// New state DBs should use incremental auto-vacuum, but retrofitting an
// existing DB requires a full VACUUM. Do not attempt that during process
// startup: it is maintenance work that can contend with foreground writers.
open_sqlite(path, migrator, STATE_DB, telemetry_override).await
open_sqlite(path, migrator, STATE_DB, journal_mode, telemetry_override).await
}
async fn open_logs_sqlite(
path: &Path,
migrator: &Migrator,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
open_sqlite(path, migrator, LOGS_DB, telemetry_override).await
open_sqlite(path, migrator, LOGS_DB, journal_mode, telemetry_override).await
}
async fn open_goals_sqlite(
path: &Path,
migrator: &Migrator,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
open_sqlite(path, migrator, GOALS_DB, telemetry_override).await
open_sqlite(path, migrator, GOALS_DB, journal_mode, telemetry_override).await
}
async fn open_memories_sqlite(
path: &Path,
migrator: &Migrator,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
open_sqlite(path, migrator, MEMORIES_DB, telemetry_override).await
open_sqlite(
path,
migrator,
MEMORIES_DB,
journal_mode,
telemetry_override,
)
.await
}
async fn open_sqlite(
path: &Path,
migrator: &Migrator,
spec: RuntimeDbSpec,
journal_mode: RuntimeSqliteJournalMode,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
let options = base_sqlite_options(path).auto_vacuum(SqliteAutoVacuum::Incremental);
let options =
base_sqlite_options(path, journal_mode).auto_vacuum(SqliteAutoVacuum::Incremental);
let started = Instant::now();
let pool_result = SqlitePoolOptions::new()
.max_connections(5)
@@ -461,6 +544,7 @@ pub async fn sqlite_integrity_check(path: &Path) -> anyhow::Result<Vec<String>>
#[cfg(test)]
mod tests {
use super::RuntimeSqliteJournalMode;
use super::StateRuntime;
use super::open_state_sqlite;
use super::runtime_state_migrator;
@@ -611,6 +695,7 @@ mod tests {
let tolerant_pool = open_state_sqlite(
state_path.as_path(),
&tolerant_migrator,
RuntimeSqliteJournalMode::default(),
/*telemetry_override*/ None,
)
.await
@@ -620,6 +705,34 @@ mod tests {
let _ = tokio::fs::remove_dir_all(codex_home).await;
}
#[tokio::test]
async fn open_state_sqlite_honors_delete_journal_mode() {
let codex_home = unique_temp_dir();
tokio::fs::create_dir_all(&codex_home)
.await
.expect("create codex home");
let state_path = state_db_path(codex_home.as_path());
let migrator = runtime_state_migrator();
let pool = open_state_sqlite(
state_path.as_path(),
&migrator,
RuntimeSqliteJournalMode::Delete,
/*telemetry_override*/ None,
)
.await
.expect("open state db with delete journal mode");
let journal_mode: String = sqlx::query_scalar("PRAGMA journal_mode")
.fetch_one(&pool)
.await
.expect("query journal mode");
assert_eq!(journal_mode, "delete");
pool.close().await;
let _ = tokio::fs::remove_dir_all(codex_home).await;
}
#[tokio::test]
async fn init_records_successful_sqlite_init_phases_to_explicit_telemetry() {
let codex_home = unique_temp_dir();
codex-rs/thread-manager-sample/src/main.rs
+1
View File
@@ -230,6 +230,7 @@ fn new_config(model: Option<String>, arg0_paths: Arg0DispatchPaths) -> anyhow::R
agent_roles: BTreeMap::new(),
memories: MemoriesConfig::default(),
sqlite_home: codex_home.to_path_buf(),
sqlite_journal_mode: Default::default(),
log_dir: codex_home.join("log").to_path_buf(),
config_lock_export_dir: None,
config_lock_allow_codex_version_mismatch: false,