From d7c289ec258b5810ee0a5532039b645cf57cc14d Mon Sep 17 00:00:00 2001 From: felixxia-oai Date: Wed, 18 Nov 2026 21:40:15 +0000 Subject: [PATCH] Add opt-in in-memory message boards for Guardian v3 (#62373) ## What changed Set `features.guardianv3.message_board_in_memory = true` to give each new or cleared root thread an in-memory SQLite board. The housekeeper and reviewer share the board until the last handle closes. The reviewer can read evidence and update its own subscriptions. On-disk storage remains the default. With in-memory storage enabled, the attached reviewer is unavailable on resumed roots because evidence intake starts at the end of history and a fresh board has no earlier evidence. ## Testing Exercise evidence catch-up and later post delivery with both storage modes. Check that only the on-disk mode creates a database file, the reviewer cannot update channels, and a cold resume skips evidence intake in memory mode. GitOrigin-RevId: 338f44720fca510b2baf40fd4b3a4b30245d4fc2 --- codex-rs/core/config.schema.json | 4 + codex-rs/ext/guardian-v3/src/cutoff_tests.rs | 138 +++++++++++++------ codex-rs/ext/guardian-v3/src/host.rs | 51 ++++++- codex-rs/features/src/feature_configs.rs | 3 + codex-rs/features/src/tests.rs | 3 + codex-rs/message-board/src/local.rs | 33 +++++ codex-rs/message-board/src/local/options.rs | 7 +- codex-rs/state/src/sqlite.rs | 18 +++ 8 files changed, 208 insertions(+), 49 deletions(-) diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 98cfa7fef8..a706695448 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -2518,6 +2518,10 @@ "minimum": 0.0, "type": "integer" }, + "message_board_in_memory": { + "description": "Keep each fresh root's Guardian board in memory until the runtime closes. Defaults to false.", + "type": "boolean" + }, "reviewer": { "$ref": "#/definitions/GuardianV3ReviewerConfigToml" } diff --git a/codex-rs/ext/guardian-v3/src/cutoff_tests.rs b/codex-rs/ext/guardian-v3/src/cutoff_tests.rs index 2104e00982..ba6b03bdc8 100644 --- a/codex-rs/ext/guardian-v3/src/cutoff_tests.rs +++ b/codex-rs/ext/guardian-v3/src/cutoff_tests.rs @@ -168,19 +168,32 @@ async fn retries_unavailable_history_and_remembers_rejected_evidence() -> anyhow Ok(()) } +#[test_case::test_case(false; "on_disk")] +#[test_case::test_case(true; "in_memory")] #[tokio::test] -async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyhow::Result<()> { +async fn catch_up_waits_for_committed_delivery_and_allows_later_posts( + in_memory: bool, +) -> anyhow::Result<()> { use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadReadyInput; use codex_extension_api::ThreadStopInput; use codex_message_board::ReadPostRequest; + use codex_protocol::error::CodexErrorDetails; use codex_protocol::protocol::SessionSource; use std::num::NonZeroU32; core_test_support::skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let home = Arc::new(tempfile::tempdir()?); + let settings = if in_memory { + "[features.guardianv3]\nenabled = true\nmessage_board_in_memory = true" + } else { + "[features.guardianv3]\nenabled = true" + }; + std::fs::write(home.path().join("config.toml"), settings)?; let test = test_codex() + .with_home(home) .with_history_mode(ThreadHistoryMode::Paginated) .with_model_info_override("gpt-5.6-luna", |_| {}) .with_config(|config| { @@ -191,7 +204,7 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho let root = test.session_configured.thread_id; let host = Arc::new(PausedClockHost { inner: board_host(root), - pause_next: AtomicBool::new(true), + pause_next: AtomicBool::new(false), started: Notify::new(), release: Notify::new(), }); @@ -209,31 +222,87 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho let session = ExtensionData::new(root.to_string()); let thread = test.codex.thread_extension_data(); let lifecycle = ®istry.thread_lifecycle_contributors()[0]; + let ready = |history_initialization| ThreadReadyInput { + config: &test.config, + session_source: &SessionSource::Exec, + history_initialization, + history_mode: ThreadHistoryMode::Paginated, + session_store: &session, + thread_store: thread, + }; + if in_memory { + lifecycle + .on_thread_ready(ready(HistoryInitialization::ColdResume)) + .await; + assert!(thread.get::().is_none()); + assert!( + thread + .get::>() + .is_some() + ); + lifecycle + .on_thread_stop(ThreadStopInput { + thread_store: thread, + session_store: &session, + }) + .await; + } lifecycle - .on_thread_ready(ThreadReadyInput { - config: &test.config, - session_source: &SessionSource::Exec, - history_initialization: codex_history::HistoryInitialization::New, - history_mode: ThreadHistoryMode::Paginated, - session_store: &session, - thread_store: thread, - }) + .on_thread_ready(ready(HistoryInitialization::New)) .await; let evidence = thread.get::().context("root intake")?; - let board = crate::local_board(test.config.sqlite_config().clone(), root.into()) - .read_write() - .open(board_host(root)) - .await?; - for channel in ["release", "releases-copy"] { - board - .upsert_channel( - root, - UpsertChannelRequest { - channel_name: channel.into(), - description: "Release policy".to_owned().try_into()?, - }, - ) + let worker = thread + .get::() + .context("root worker")?; + let board = &worker.reader; + let denied = board + .upsert_channel( + root, + UpsertChannelRequest { + channel_name: "reviews".into(), + description: "Read-only".to_owned().try_into()?, + }, + ) + .await + .err() + .context("reviewer unexpectedly updated a channel")?; + assert!( + matches!(denied.details(), CodexErrorDetails::Io(error) if error.kind() == std::io::ErrorKind::PermissionDenied) + ); + let append_user_message = async |order, text: &str| -> anyhow::Result { + let item = RolloutItem::ResponseItem(ResponseItemEnvelope { + item: user_message_item(text), + metadata: Some(CodexHarnessMetadata { + user_input_order: Some(order), + ..Default::default() + }), + }); + let original = serde_json::to_value(&item)?; + test.thread_store + .append_items(AppendThreadItemsParams { + thread_id: root, + items: vec![item], + }) .await?; + Ok(original) + }; + // Create the channels through the housekeeper and subscribe through the reviewer's handle. + let _seed_route = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("seed", r#"{"routes":[["release","releases-copy"]],"descriptions":[{"channel":"release","description":"Release policy"},{"channel":"releases-copy","description":"Release copy"}]}"#), + ev_completed("seed"), + ]), + ).await; + append_user_message(1, "Set up the release channels.").await?; + evidence.request_catch_up().await?.await?; + let database = test + .config + .sqlite_config() + .home() + .join("message_board_guardian_1.sqlite"); + assert_eq!(tokio::fs::try_exists(database).await?, !in_memory); + for channel in ["release", "releases-copy"] { board .set_subscription( root, @@ -257,24 +326,9 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho ]), ) .await; - let append_user_message = async |order, text: &str| -> anyhow::Result { - let item = RolloutItem::ResponseItem(ResponseItemEnvelope { - item: user_message_item(text), - metadata: Some(CodexHarnessMetadata { - user_input_order: Some(order), - ..Default::default() - }), - }); - let original = serde_json::to_value(&item)?; - test.thread_store - .append_items(AppendThreadItemsParams { - thread_id: root, - items: vec![item], - }) - .await?; - Ok(original) - }; - let first_original = append_user_message(1, "Stop the release.").await?; + host.pause_next + .store(true, std::sync::atomic::Ordering::SeqCst); + let first_original = append_user_message(2, "Stop the release.").await?; let wait = Duration::from_secs(15); let mut first = evidence.request_catch_up().await?; tokio::time::timeout(wait, host.started.notified()).await?; @@ -289,7 +343,7 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho ]), ) .await; - let second_original = append_user_message(2, "New revocation.").await?; + let second_original = append_user_message(3, "New revocation.").await?; let second = tokio::time::timeout(wait, evidence.request_catch_up()).await??; host.release.notify_one(); let first_position = first.await?; diff --git a/codex-rs/ext/guardian-v3/src/host.rs b/codex-rs/ext/guardian-v3/src/host.rs index 15c121bbbe..fdd06fd737 100644 --- a/codex-rs/ext/guardian-v3/src/host.rs +++ b/codex-rs/ext/guardian-v3/src/host.rs @@ -18,7 +18,9 @@ use codex_extension_api::TurnLifecycleContributor; use codex_extension_api::TurnStopInput; use codex_features::GuardianV3HousekeeperConfigToml; use codex_guardian_reviewer::ReviewerTasks; +use codex_history::HistoryInitialization; use codex_message_board::AgentMessageBoard; +use codex_message_board::LocalAgentMessageBoard; use codex_message_board::MessageBoardHost; use codex_protocol::AgentPath; use codex_protocol::ThreadId; @@ -31,6 +33,7 @@ use crate::EvidenceCursor; use crate::Housekeeper; use crate::cutoff::RootEvidence; use crate::local_board; +use crate::local_board_options; pub fn install( registry: &mut ExtensionRegistryBuilder, @@ -41,7 +44,14 @@ pub fn install( let settings = config.guardian_v3.as_deref(); let host = Arc::new(Host { manager, - sqlite: config.sqlite_config().clone(), + storage: if settings + .and_then(|settings| settings.message_board_in_memory) + .unwrap_or(false) + { + BoardStorage::Memory + } else { + BoardStorage::File(config.sqlite_config().clone()) + }, housekeeper: Arc::new( settings .and_then(|settings| settings.housekeeper.clone()) @@ -58,9 +68,14 @@ pub fn install( registry.turn_lifecycle_contributor(host); } +enum BoardStorage { + File(SqliteConfig), + Memory, +} + struct Host { manager: Weak, - sqlite: SqliteConfig, + storage: BoardStorage, housekeeper: Arc, retrieval: ReviewRetrieval, board_host: Box Arc + Send + Sync>, @@ -78,6 +93,17 @@ impl Host { root: ThreadId, input: &ThreadReadyInput<'_, Config>, ) -> anyhow::Result<()> { + // On resume the intake cursor starts at the tail. A new in-memory board + // cannot serve earlier evidence, so leave its attached reviewer unavailable. + if matches!(self.storage, BoardStorage::Memory) { + anyhow::ensure!( + matches!( + input.history_initialization, + HistoryInitialization::New | HistoryInitialization::Cleared + ), + "in-memory Guardian evidence requires a fresh root" + ); + } let history = self .manager .upgrade() @@ -90,11 +116,24 @@ impl Host { input.history_initialization, ) .await?; - let storage = local_board(self.sqlite.clone(), root.into()); let board_host = (self.board_host)(root); - let board = storage.read_write().open(board_host.clone()).await?; - let reader: Arc = - Arc::new(storage.read_only().open(board_host).await?); + let board = match &self.storage { + BoardStorage::File(sqlite) => { + local_board(sqlite.clone(), root.into()) + .read_write() + .open(board_host) + .await? + } + BoardStorage::Memory => { + LocalAgentMessageBoard::open_in_memory( + &local_board_options(), + root.into(), + board_host, + ) + .await? + } + }; + let reader: Arc = Arc::new(board.read_only_handle()); let lifetime = Arc::new(ReviewerTasks::default()); let (wake, wakes) = mpsc::channel(1); let (queue, mut captures) = RootEvidence::new(lifetime.clone(), history, cursor.clone()); diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs index 58bfef29ac..19c4ed6f3a 100644 --- a/codex-rs/features/src/feature_configs.rs +++ b/codex-rs/features/src/feature_configs.rs @@ -315,6 +315,9 @@ where #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct GuardianV3ConfigToml { + /// Keep each fresh root's Guardian board in memory until the runtime closes. Defaults to false. + #[serde(skip_serializing_if = "Option::is_none")] + pub message_board_in_memory: Option, #[serde(skip_serializing_if = "Option::is_none")] pub enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 1122399aeb..bb13583dba 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -255,6 +255,8 @@ fn guardian_v3_feature_config_resolves_base_and_profile_toggles() { fn guardian_v3_prompt_settings_do_not_implicitly_enable_the_feature() { let features: FeaturesToml = toml::from_str( r#" +[guardianv3] +message_board_in_memory = true [guardianv3.housekeeper] routing_instructions = "Group related deployment evidence." batch_timeout_ms = 15000 @@ -269,6 +271,7 @@ retrieval_instructions = "Search before assessing deployments." features.guardianv3, Some(FeatureToml::Config(crate::GuardianV3ConfigToml { enabled: None, + message_board_in_memory: Some(true), housekeeper: Some(crate::GuardianV3HousekeeperConfigToml { routing_instructions: Some("Group related deployment evidence.".to_owned()), batch_timeout_ms: std::num::NonZeroU32::new(15_000), diff --git a/codex-rs/message-board/src/local.rs b/codex-rs/message-board/src/local.rs index 194a6b45d6..412b082dc3 100644 --- a/codex-rs/message-board/src/local.rs +++ b/codex-rs/message-board/src/local.rs @@ -97,6 +97,39 @@ impl LocalAgentMessageBoard { }) } + /// Opens a fresh read-write SQLite board with no backing file. Clones share its contents + /// until the last handle closes. A later open always creates an independent board. + pub async fn open_in_memory( + options: &LocalBoardOptions, + identity: SessionId, + host: Arc, + ) -> Result { + options.validate()?; + let pool = SqliteConfig::open_in_memory_pool() + .await + .map_err(storage_error)?; + sqlx::raw_sql(pool::SCHEMA) + .execute(&pool) + .await + .map_err(storage_error)?; + Ok(Self { + identity, + storage_namespace: Arc::from(options.storage_namespace.as_str()), + pool: Arc::new(pool), + host, + access: Some(Access::ReadWrite), + nested_json_pointer: None, + }) + } + + /// Reads the same live board and can change only the caller's subscriptions. + pub fn read_only_handle(&self) -> Self { + Self { + access: Some(Access::ReadOnly), + ..self.clone() + } + } + pub async fn create_channel( &self, caller: ThreadId, diff --git a/codex-rs/message-board/src/local/options.rs b/codex-rs/message-board/src/local/options.rs index 3959dc9665..a8f8587421 100644 --- a/codex-rs/message-board/src/local/options.rs +++ b/codex-rs/message-board/src/local/options.rs @@ -23,7 +23,7 @@ impl Default for LocalBoardOptions { } impl LocalBoardOptions { - pub(super) fn database_file(&self) -> Result { + pub(super) fn validate(&self) -> Result<()> { if self.storage_namespace.is_empty() || self.storage_namespace.len() > 64 || !self.storage_namespace.bytes().all(|byte| { @@ -34,6 +34,11 @@ impl LocalBoardOptions { "board storage namespaces require 1–64 lowercase ASCII letters, digits, underscores or hyphens", )); } + Ok(()) + } + + pub(super) fn database_file(&self) -> Result { + self.validate()?; if self.storage_namespace == DEFAULT_STORAGE_NAMESPACE { return Ok("agent_message_board_1.sqlite".into()); } diff --git a/codex-rs/state/src/sqlite.rs b/codex-rs/state/src/sqlite.rs index b449f87c54..733cf1de86 100644 --- a/codex-rs/state/src/sqlite.rs +++ b/codex-rs/state/src/sqlite.rs @@ -447,6 +447,24 @@ impl SqliteConfig { Ok(pool) } + /// Open a private database that lasts for the lifetime of this pool. + pub async fn open_in_memory_pool() -> Result { + // Keep the sole connection alive. Disabling the checkout ping also keeps + // a cancelled acquire from discarding the connection and its database. + SqlitePoolOptions::new() + .max_connections(1) + .max_lifetime(None) + .idle_timeout(None) + .test_before_acquire(false) + .connect_with( + SqliteConnectOptions::new() + .in_memory(true) + .pragma("temp_store", "MEMORY") + .log_statements(LevelFilter::Off), + ) + .await + } + /// Open a writable Codex SQLite database, creating it if necessary. /// Keep this configuration alive to retain the cleanup lock for local logs. pub async fn open_read_write_pool(&self, path: &Path) -> anyhow::Result {