mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add opt-in in-memory message boards for Guardian v3 (#62373)
## What changed Set `features.guardianv3.message_board_in_memory = true` to give each new or cleared root thread an in-memory SQLite board. The housekeeper and reviewer share the board until the last handle closes. The reviewer can read evidence and update its own subscriptions. On-disk storage remains the default. With in-memory storage enabled, the attached reviewer is unavailable on resumed roots because evidence intake starts at the end of history and a fresh board has no earlier evidence. ## Testing Exercise evidence catch-up and later post delivery with both storage modes. Check that only the on-disk mode creates a database file, the reviewer cannot update channels, and a cold resume skips evidence intake in memory mode. GitOrigin-RevId: 338f44720fca510b2baf40fd4b3a4b30245d4fc2
This commit is contained in:
@@ -2518,6 +2518,10 @@
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"message_board_in_memory": {
|
||||
"description": "Keep each fresh root's Guardian board in memory until the runtime closes. Defaults to false.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"reviewer": {
|
||||
"$ref": "#/definitions/GuardianV3ReviewerConfigToml"
|
||||
}
|
||||
|
||||
@@ -168,19 +168,32 @@ async fn retries_unavailable_history_and_remembers_rejected_evidence() -> anyhow
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case::test_case(false; "on_disk")]
|
||||
#[test_case::test_case(true; "in_memory")]
|
||||
#[tokio::test]
|
||||
async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyhow::Result<()> {
|
||||
async fn catch_up_waits_for_committed_delivery_and_allows_later_posts(
|
||||
in_memory: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::ThreadReadyInput;
|
||||
use codex_extension_api::ThreadStopInput;
|
||||
use codex_message_board::ReadPostRequest;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let home = Arc::new(tempfile::tempdir()?);
|
||||
let settings = if in_memory {
|
||||
"[features.guardianv3]\nenabled = true\nmessage_board_in_memory = true"
|
||||
} else {
|
||||
"[features.guardianv3]\nenabled = true"
|
||||
};
|
||||
std::fs::write(home.path().join("config.toml"), settings)?;
|
||||
let test = test_codex()
|
||||
.with_home(home)
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.with_model_info_override("gpt-5.6-luna", |_| {})
|
||||
.with_config(|config| {
|
||||
@@ -191,7 +204,7 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho
|
||||
let root = test.session_configured.thread_id;
|
||||
let host = Arc::new(PausedClockHost {
|
||||
inner: board_host(root),
|
||||
pause_next: AtomicBool::new(true),
|
||||
pause_next: AtomicBool::new(false),
|
||||
started: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
@@ -209,31 +222,87 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho
|
||||
let session = ExtensionData::new(root.to_string());
|
||||
let thread = test.codex.thread_extension_data();
|
||||
let lifecycle = ®istry.thread_lifecycle_contributors()[0];
|
||||
let ready = |history_initialization| ThreadReadyInput {
|
||||
config: &test.config,
|
||||
session_source: &SessionSource::Exec,
|
||||
history_initialization,
|
||||
history_mode: ThreadHistoryMode::Paginated,
|
||||
session_store: &session,
|
||||
thread_store: thread,
|
||||
};
|
||||
if in_memory {
|
||||
lifecycle
|
||||
.on_thread_ready(ready(HistoryInitialization::ColdResume))
|
||||
.await;
|
||||
assert!(thread.get::<RootEvidence>().is_none());
|
||||
assert!(
|
||||
thread
|
||||
.get::<Arc<dyn codex_core::guardian_review::AttachedReviewer>>()
|
||||
.is_some()
|
||||
);
|
||||
lifecycle
|
||||
.on_thread_stop(ThreadStopInput {
|
||||
thread_store: thread,
|
||||
session_store: &session,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
lifecycle
|
||||
.on_thread_ready(ThreadReadyInput {
|
||||
config: &test.config,
|
||||
session_source: &SessionSource::Exec,
|
||||
history_initialization: codex_history::HistoryInitialization::New,
|
||||
history_mode: ThreadHistoryMode::Paginated,
|
||||
session_store: &session,
|
||||
thread_store: thread,
|
||||
})
|
||||
.on_thread_ready(ready(HistoryInitialization::New))
|
||||
.await;
|
||||
let evidence = thread.get::<RootEvidence>().context("root intake")?;
|
||||
let board = crate::local_board(test.config.sqlite_config().clone(), root.into())
|
||||
.read_write()
|
||||
.open(board_host(root))
|
||||
.await?;
|
||||
for channel in ["release", "releases-copy"] {
|
||||
board
|
||||
.upsert_channel(
|
||||
root,
|
||||
UpsertChannelRequest {
|
||||
channel_name: channel.into(),
|
||||
description: "Release policy".to_owned().try_into()?,
|
||||
},
|
||||
)
|
||||
let worker = thread
|
||||
.get::<crate::host::RootWorker>()
|
||||
.context("root worker")?;
|
||||
let board = &worker.reader;
|
||||
let denied = board
|
||||
.upsert_channel(
|
||||
root,
|
||||
UpsertChannelRequest {
|
||||
channel_name: "reviews".into(),
|
||||
description: "Read-only".to_owned().try_into()?,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.context("reviewer unexpectedly updated a channel")?;
|
||||
assert!(
|
||||
matches!(denied.details(), CodexErrorDetails::Io(error) if error.kind() == std::io::ErrorKind::PermissionDenied)
|
||||
);
|
||||
let append_user_message = async |order, text: &str| -> anyhow::Result<serde_json::Value> {
|
||||
let item = RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item: user_message_item(text),
|
||||
metadata: Some(CodexHarnessMetadata {
|
||||
user_input_order: Some(order),
|
||||
..Default::default()
|
||||
}),
|
||||
});
|
||||
let original = serde_json::to_value(&item)?;
|
||||
test.thread_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: root,
|
||||
items: vec![item],
|
||||
})
|
||||
.await?;
|
||||
Ok(original)
|
||||
};
|
||||
// Create the channels through the housekeeper and subscribe through the reviewer's handle.
|
||||
let _seed_route = mount_sse_once(
|
||||
&server,
|
||||
sse(vec![
|
||||
ev_assistant_message("seed", r#"{"routes":[["release","releases-copy"]],"descriptions":[{"channel":"release","description":"Release policy"},{"channel":"releases-copy","description":"Release copy"}]}"#),
|
||||
ev_completed("seed"),
|
||||
]),
|
||||
).await;
|
||||
append_user_message(1, "Set up the release channels.").await?;
|
||||
evidence.request_catch_up().await?.await?;
|
||||
let database = test
|
||||
.config
|
||||
.sqlite_config()
|
||||
.home()
|
||||
.join("message_board_guardian_1.sqlite");
|
||||
assert_eq!(tokio::fs::try_exists(database).await?, !in_memory);
|
||||
for channel in ["release", "releases-copy"] {
|
||||
board
|
||||
.set_subscription(
|
||||
root,
|
||||
@@ -257,24 +326,9 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let append_user_message = async |order, text: &str| -> anyhow::Result<serde_json::Value> {
|
||||
let item = RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item: user_message_item(text),
|
||||
metadata: Some(CodexHarnessMetadata {
|
||||
user_input_order: Some(order),
|
||||
..Default::default()
|
||||
}),
|
||||
});
|
||||
let original = serde_json::to_value(&item)?;
|
||||
test.thread_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: root,
|
||||
items: vec![item],
|
||||
})
|
||||
.await?;
|
||||
Ok(original)
|
||||
};
|
||||
let first_original = append_user_message(1, "Stop the release.").await?;
|
||||
host.pause_next
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let first_original = append_user_message(2, "Stop the release.").await?;
|
||||
let wait = Duration::from_secs(15);
|
||||
let mut first = evidence.request_catch_up().await?;
|
||||
tokio::time::timeout(wait, host.started.notified()).await?;
|
||||
@@ -289,7 +343,7 @@ async fn catch_up_waits_for_committed_delivery_and_allows_later_posts() -> anyho
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let second_original = append_user_message(2, "New revocation.").await?;
|
||||
let second_original = append_user_message(3, "New revocation.").await?;
|
||||
let second = tokio::time::timeout(wait, evidence.request_catch_up()).await??;
|
||||
host.release.notify_one();
|
||||
let first_position = first.await?;
|
||||
|
||||
@@ -18,7 +18,9 @@ use codex_extension_api::TurnLifecycleContributor;
|
||||
use codex_extension_api::TurnStopInput;
|
||||
use codex_features::GuardianV3HousekeeperConfigToml;
|
||||
use codex_guardian_reviewer::ReviewerTasks;
|
||||
use codex_history::HistoryInitialization;
|
||||
use codex_message_board::AgentMessageBoard;
|
||||
use codex_message_board::LocalAgentMessageBoard;
|
||||
use codex_message_board::MessageBoardHost;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
@@ -31,6 +33,7 @@ use crate::EvidenceCursor;
|
||||
use crate::Housekeeper;
|
||||
use crate::cutoff::RootEvidence;
|
||||
use crate::local_board;
|
||||
use crate::local_board_options;
|
||||
|
||||
pub fn install(
|
||||
registry: &mut ExtensionRegistryBuilder<Config>,
|
||||
@@ -41,7 +44,14 @@ pub fn install(
|
||||
let settings = config.guardian_v3.as_deref();
|
||||
let host = Arc::new(Host {
|
||||
manager,
|
||||
sqlite: config.sqlite_config().clone(),
|
||||
storage: if settings
|
||||
.and_then(|settings| settings.message_board_in_memory)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
BoardStorage::Memory
|
||||
} else {
|
||||
BoardStorage::File(config.sqlite_config().clone())
|
||||
},
|
||||
housekeeper: Arc::new(
|
||||
settings
|
||||
.and_then(|settings| settings.housekeeper.clone())
|
||||
@@ -58,9 +68,14 @@ pub fn install(
|
||||
registry.turn_lifecycle_contributor(host);
|
||||
}
|
||||
|
||||
enum BoardStorage {
|
||||
File(SqliteConfig),
|
||||
Memory,
|
||||
}
|
||||
|
||||
struct Host {
|
||||
manager: Weak<ThreadManager>,
|
||||
sqlite: SqliteConfig,
|
||||
storage: BoardStorage,
|
||||
housekeeper: Arc<GuardianV3HousekeeperConfigToml>,
|
||||
retrieval: ReviewRetrieval,
|
||||
board_host: Box<dyn Fn(ThreadId) -> Arc<dyn MessageBoardHost> + Send + Sync>,
|
||||
@@ -78,6 +93,17 @@ impl Host {
|
||||
root: ThreadId,
|
||||
input: &ThreadReadyInput<'_, Config>,
|
||||
) -> anyhow::Result<()> {
|
||||
// On resume the intake cursor starts at the tail. A new in-memory board
|
||||
// cannot serve earlier evidence, so leave its attached reviewer unavailable.
|
||||
if matches!(self.storage, BoardStorage::Memory) {
|
||||
anyhow::ensure!(
|
||||
matches!(
|
||||
input.history_initialization,
|
||||
HistoryInitialization::New | HistoryInitialization::Cleared
|
||||
),
|
||||
"in-memory Guardian evidence requires a fresh root"
|
||||
);
|
||||
}
|
||||
let history = self
|
||||
.manager
|
||||
.upgrade()
|
||||
@@ -90,11 +116,24 @@ impl Host {
|
||||
input.history_initialization,
|
||||
)
|
||||
.await?;
|
||||
let storage = local_board(self.sqlite.clone(), root.into());
|
||||
let board_host = (self.board_host)(root);
|
||||
let board = storage.read_write().open(board_host.clone()).await?;
|
||||
let reader: Arc<dyn AgentMessageBoard> =
|
||||
Arc::new(storage.read_only().open(board_host).await?);
|
||||
let board = match &self.storage {
|
||||
BoardStorage::File(sqlite) => {
|
||||
local_board(sqlite.clone(), root.into())
|
||||
.read_write()
|
||||
.open(board_host)
|
||||
.await?
|
||||
}
|
||||
BoardStorage::Memory => {
|
||||
LocalAgentMessageBoard::open_in_memory(
|
||||
&local_board_options(),
|
||||
root.into(),
|
||||
board_host,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
let reader: Arc<dyn AgentMessageBoard> = Arc::new(board.read_only_handle());
|
||||
let lifetime = Arc::new(ReviewerTasks::default());
|
||||
let (wake, wakes) = mpsc::channel(1);
|
||||
let (queue, mut captures) = RootEvidence::new(lifetime.clone(), history, cursor.clone());
|
||||
|
||||
@@ -315,6 +315,9 @@ where
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GuardianV3ConfigToml {
|
||||
/// Keep each fresh root's Guardian board in memory until the runtime closes. Defaults to false.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub message_board_in_memory: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub enabled: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
|
||||
@@ -255,6 +255,8 @@ fn guardian_v3_feature_config_resolves_base_and_profile_toggles() {
|
||||
fn guardian_v3_prompt_settings_do_not_implicitly_enable_the_feature() {
|
||||
let features: FeaturesToml = toml::from_str(
|
||||
r#"
|
||||
[guardianv3]
|
||||
message_board_in_memory = true
|
||||
[guardianv3.housekeeper]
|
||||
routing_instructions = "Group related deployment evidence."
|
||||
batch_timeout_ms = 15000
|
||||
@@ -269,6 +271,7 @@ retrieval_instructions = "Search before assessing deployments."
|
||||
features.guardianv3,
|
||||
Some(FeatureToml::Config(crate::GuardianV3ConfigToml {
|
||||
enabled: None,
|
||||
message_board_in_memory: Some(true),
|
||||
housekeeper: Some(crate::GuardianV3HousekeeperConfigToml {
|
||||
routing_instructions: Some("Group related deployment evidence.".to_owned()),
|
||||
batch_timeout_ms: std::num::NonZeroU32::new(15_000),
|
||||
|
||||
@@ -97,6 +97,39 @@ impl LocalAgentMessageBoard {
|
||||
})
|
||||
}
|
||||
|
||||
/// Opens a fresh read-write SQLite board with no backing file. Clones share its contents
|
||||
/// until the last handle closes. A later open always creates an independent board.
|
||||
pub async fn open_in_memory(
|
||||
options: &LocalBoardOptions,
|
||||
identity: SessionId,
|
||||
host: Arc<dyn MessageBoardHost>,
|
||||
) -> Result<Self> {
|
||||
options.validate()?;
|
||||
let pool = SqliteConfig::open_in_memory_pool()
|
||||
.await
|
||||
.map_err(storage_error)?;
|
||||
sqlx::raw_sql(pool::SCHEMA)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.map_err(storage_error)?;
|
||||
Ok(Self {
|
||||
identity,
|
||||
storage_namespace: Arc::from(options.storage_namespace.as_str()),
|
||||
pool: Arc::new(pool),
|
||||
host,
|
||||
access: Some(Access::ReadWrite),
|
||||
nested_json_pointer: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads the same live board and can change only the caller's subscriptions.
|
||||
pub fn read_only_handle(&self) -> Self {
|
||||
Self {
|
||||
access: Some(Access::ReadOnly),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_channel(
|
||||
&self,
|
||||
caller: ThreadId,
|
||||
|
||||
@@ -23,7 +23,7 @@ impl Default for LocalBoardOptions {
|
||||
}
|
||||
|
||||
impl LocalBoardOptions {
|
||||
pub(super) fn database_file(&self) -> Result<String> {
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
if self.storage_namespace.is_empty()
|
||||
|| self.storage_namespace.len() > 64
|
||||
|| !self.storage_namespace.bytes().all(|byte| {
|
||||
@@ -34,6 +34,11 @@ impl LocalBoardOptions {
|
||||
"board storage namespaces require 1–64 lowercase ASCII letters, digits, underscores or hyphens",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn database_file(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
if self.storage_namespace == DEFAULT_STORAGE_NAMESPACE {
|
||||
return Ok("agent_message_board_1.sqlite".into());
|
||||
}
|
||||
|
||||
@@ -447,6 +447,24 @@ impl SqliteConfig {
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
/// Open a private database that lasts for the lifetime of this pool.
|
||||
pub async fn open_in_memory_pool() -> Result<SqlitePool, Error> {
|
||||
// Keep the sole connection alive. Disabling the checkout ping also keeps
|
||||
// a cancelled acquire from discarding the connection and its database.
|
||||
SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.max_lifetime(None)
|
||||
.idle_timeout(None)
|
||||
.test_before_acquire(false)
|
||||
.connect_with(
|
||||
SqliteConnectOptions::new()
|
||||
.in_memory(true)
|
||||
.pragma("temp_store", "MEMORY")
|
||||
.log_statements(LevelFilter::Off),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Open a writable Codex SQLite database, creating it if necessary.
|
||||
/// Keep this configuration alive to retain the cleanup lock for local logs.
|
||||
pub async fn open_read_write_pool(&self, path: &Path) -> anyhow::Result<SqlitePool> {
|
||||
|
||||
Reference in New Issue
Block a user