diff --git a/codex-rs/ext/guardian-v3/BUILD.bazel b/codex-rs/ext/guardian-v3/BUILD.bazel index 457c6a644b..18d47f5768 100644 --- a/codex-rs/ext/guardian-v3/BUILD.bazel +++ b/codex-rs/ext/guardian-v3/BUILD.bazel @@ -7,4 +7,9 @@ codex_rust_crate( integration_compile_data_extra = ["//codex-rs/message-board:tests/common/mod.rs"], run_tests_with_wine_exec = True, test_tags = ["no-sandbox"], + unit_test_srcs = [ + "src/cutoff_tests.rs", + "tests/common/board_host.rs", + "//codex-rs/message-board:tests/common/mod.rs", + ], ) diff --git a/codex-rs/ext/guardian-v3/Cargo.toml b/codex-rs/ext/guardian-v3/Cargo.toml index 44f5e92ca9..cec534640f 100644 --- a/codex-rs/ext/guardian-v3/Cargo.toml +++ b/codex-rs/ext/guardian-v3/Cargo.toml @@ -6,7 +6,6 @@ version.workspace = true [lib] doctest = false -test = false [lints] workspace = true diff --git a/codex-rs/ext/guardian-v3/src/cutoff.rs b/codex-rs/ext/guardian-v3/src/cutoff.rs new file mode 100644 index 0000000000..70f0fba205 --- /dev/null +++ b/codex-rs/ext/guardian-v3/src/cutoff.rs @@ -0,0 +1,176 @@ +//! Capture and enqueue each history suffix in one order. The sole board writer opens +//! each requested view after routing that suffix and before processing its successor. +use std::sync::Arc; + +use anyhow::Context; +use codex_guardian_reviewer::ReviewerTasks; +use codex_message_board::AgentMessageBoard; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::protocol::HistoryPosition; +use codex_thread_store::ThreadStore; +use tokio::sync::Mutex; +use tokio::sync::mpsc; +use tokio::sync::oneshot; + +use crate::EvidenceCursor; +use crate::EvidenceSnapshot; +use crate::ReviewerSubscriptions; + +#[cfg(test)] +#[path = "cutoff_tests.rs"] +mod tests; + +/// One approval's fixed source boundary and matching discovery, retrieval and delivery view. +pub struct ReviewCutoff { + pub source: HistoryPosition, + pub subscriptions: ReviewerSubscriptions, +} + +/// Installed on the root thread while its evidence intake is active. +pub struct RootEvidence { + lifetime: Arc, + history: Arc, + // Advances on enqueue. The housekeeper independently tracks successful routing. + // A rejected durable record keeps its reason for subsequent reviews of this root. + enqueued: Mutex>, + pending: mpsc::Sender, +} + +struct Freeze { + caller: ThreadId, + path: AgentPath, + reply: oneshot::Sender>, +} + +pub(super) struct Capture { + pub snapshot: EvidenceSnapshot, + reviewer: Option, +} + +impl Capture { + pub async fn finish(self, reader: &Arc) { + if let Some(Freeze { + caller, + path, + reply, + }) = self.reviewer + && !reply.is_closed() + { + let source = self.snapshot.cutoff(); + let result = ReviewerSubscriptions::capture(reader.clone(), caller, path) + .await + .map(|subscriptions| ReviewCutoff { + source, + subscriptions, + }); + let _ = reply.send(result); + } + } +} + +impl RootEvidence { + pub(super) fn new( + lifetime: Arc, + history: Arc, + cursor: EvidenceCursor, + ) -> (Arc, mpsc::Receiver) { + let (pending, requests) = mpsc::channel(1); + ( + Arc::new(Self { + lifetime, + history, + enqueued: Mutex::new(Ok(cursor)), + pending, + }), + requests, + ) + } + + pub(super) async fn capture_on_activity(self: Arc, mut wakes: mpsc::Receiver<()>) { + let capture = async { + while wakes.recv().await.is_some() { + if self.enqueue(/*reviewer*/ None).await.is_err() { + if self.enqueued.lock().await.is_err() { + break; + } + tracing::warn!("Guardian background capture failed; waiting for the next wake"); + } + } + }; + tokio::select! { + biased; + _ = self.lifetime.cancellation.cancelled() => (), + _ = self.pending.closed() => (), + () = capture => (), + } + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "capture and enqueue must agree on source order across concurrent approvals" + )] + async fn enqueue(&self, reviewer: Option) -> anyhow::Result<()> { + let mut enqueued = self.enqueued.lock().await; + let cursor = enqueued + .as_mut() + .map_err(|reason| anyhow::anyhow!("{reason}"))?; + anyhow::ensure!(!self.pending.is_closed(), "Guardian housekeeper stopped"); + self.history.flush_thread(cursor.thread_id()).await?; + let history = cursor.read_history(self.history.as_ref()).await?; + let snapshot = match cursor.snapshot(history) { + Ok(snapshot) => snapshot, + Err(error) => { + tracing::warn!(root_thread_id = %cursor.thread_id(), %error, "Guardian intake rejected stored evidence"); + *enqueued = Err(error.to_string()); + return Err(error); + } + }; + // A freeze must wait behind earlier writes even when the source has not advanced. + if reviewer.is_none() && !snapshot.source_advanced() { + return Ok(()); + } + // Cancellation before reserving leaves the cursor unchanged. After reservation, + // admission and advancing the captured range cannot be interrupted. + let permit = self + .pending + .reserve() + .await + .context("Guardian housekeeper stopped")?; + cursor.acknowledge(&snapshot, snapshot.records().len())?; + for &source_ordinal in &snapshot.skipped_incomplete_deliveries { + tracing::warn!( + name: "guardian_v3.evidence_skipped", + root_thread_id = %cursor.thread_id(), + source_ordinal, + reason = "incomplete_delivered_message", + "Skipped incomplete Guardian evidence" + ); + } + permit.send(Capture { snapshot, reviewer }); + Ok(()) + } +} + +impl RootEvidence { + /// Freeze once per review, using caller identity and path from the acting thread's host. + pub async fn freeze(&self, caller: ThreadId, path: AgentPath) -> anyhow::Result { + let _task = self.lifetime.tasks.token(); + let request = async { + let (reply, ready) = oneshot::channel(); + self.enqueue(Some(Freeze { + caller, + path, + reply, + })) + .await?; + ready + .await + .context("Guardian stopped before the review cutoff")? + }; + tokio::select! { biased; + _ = self.lifetime.cancellation.cancelled() => anyhow::bail!("Guardian intake stopped"), + result = request => result, + } + } +} diff --git a/codex-rs/ext/guardian-v3/src/cutoff_tests.rs b/codex-rs/ext/guardian-v3/src/cutoff_tests.rs new file mode 100644 index 0000000000..8b619f6502 --- /dev/null +++ b/codex-rs/ext/guardian-v3/src/cutoff_tests.rs @@ -0,0 +1,322 @@ +#![cfg(test)] + +use super::Freeze; +use super::RootEvidence; +use crate::EvidenceCursor; +use anyhow::Context; +use board_host::PausedClockHost; +use board_host::board_host; +use codex_history::CodexHarnessMetadata; +use codex_history::HistoryInitialization; +use codex_history::ResponseItemEnvelope; +use codex_history::RetainedContextEvent; +use codex_history::RetainedUserMessage; +use codex_history::RolloutItem; +use codex_history::UserInputOrigin; +use codex_message_board::PageRequest; +use codex_message_board::PostQuery; +use codex_protocol::AgentPath; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::protocol::ThreadHistoryMode; +use codex_thread_store::AppendThreadItemsParams; +use codex_thread_store::PersistContext; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::responses::user_message_item; +use core_test_support::test_codex::test_codex; +use core_test_support::tracing::NamedTraceEvents; +use futures::FutureExt; +use pretty_assertions::assert_eq; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::time::Duration; +use tokio::sync::Notify; +use tokio::sync::mpsc::error::TryRecvError; +use tokio::sync::oneshot; + +#[path = "../tests/common/board_host.rs"] +mod board_host; + +#[tokio::test] +async fn skips_a_partial_delivery_once_across_wakes_and_freezes() -> anyhow::Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let test = test_codex() + .with_history_mode(ThreadHistoryMode::Paginated) + .build_with_auto_env(&server) + .await?; + let root = test.session_configured.thread_id; + let history = &test.thread_store; + let cursor = EvidenceCursor::at_thread_ready( + history.as_ref(), + root, + ThreadHistoryMode::Paginated, + HistoryInitialization::New, + ) + .await?; + let (evidence, mut writer) = RootEvidence::new(Default::default(), history.clone(), cursor); + + // The initial rollout header advances the writer even without evidence to route. + history + .persist_thread(root, PersistContext::Standard) + .await?; + evidence.enqueue(/*reviewer*/ None).await?; + let first = writer.try_recv()?; + assert!(first.snapshot.records().is_empty()); + + let items = [(1, false, "partial"), (2, true, "next complete message")] + .into_iter() + .map(|(acceptance_order, complete, text)| { + RolloutItem::RetainedContext(RetainedContextEvent::DeliveredAssistantMessage { + message: RetainedUserMessage { + origin: UserInputOrigin::User, + turn_id: "turn-1".into(), + message_id: None, + text: text.into(), + complete, + phase: None, + }, + acceptance_order, + retained_source: None, + }) + }) + .collect(); + history + .append_items(AppendThreadItemsParams { + thread_id: root, + items, + }) + .await?; + let mut warnings = NamedTraceEvents::subscribe("guardian_v3.evidence_skipped"); + evidence.enqueue(/*reviewer*/ None).await?; + let routed = writer.try_recv()?.snapshot; + let [next] = routed.records() else { + anyhow::bail!("expected only the complete delivery") + }; + let skipped = first.snapshot.cutoff().end_ordinal_exclusive; + assert_eq!(routed.skipped_incomplete_deliveries, [skipped]); + assert_eq!(next.id.ordinal, skipped + 1); + warnings.next().await?; + + evidence.enqueue(/*reviewer*/ None).await?; + assert!(matches!(writer.try_recv(), Err(TryRecvError::Empty))); + + // A review at that very same position still needs a response from the writer. + let freeze = evidence.freeze(root, AgentPath::root()); + tokio::pin!(freeze); + let review = tokio::select! { + _ = &mut freeze => anyhow::bail!("freeze completed before the writer replied"), + request = writer.recv() => request.context("writer closed")?, + }; + assert!(futures::poll!(&mut freeze).is_pending()); + assert_eq!(review.snapshot.cutoff(), routed.cutoff()); + assert!(warnings.next().now_or_never().is_none()); + test.codex.shutdown_and_wait().await?; + Ok(()) +} + +#[tokio::test] +async fn retries_unavailable_history_and_remembers_rejected_evidence() -> anyhow::Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let test = test_codex() + .with_history_mode(ThreadHistoryMode::Paginated) + .build_with_auto_env(&server) + .await?; + let root = test.session_configured.thread_id; + let history = &test.thread_store; + history + .persist_thread(root, PersistContext::Standard) + .await?; + let cursor = EvidenceCursor::from_now(history.as_ref(), root).await?; + let (evidence, _writer) = RootEvidence::new(Default::default(), history.clone(), cursor); + + let path = test + .session_configured + .rollout_path + .as_deref() + .context("rollout path")?; + let hidden = path.with_extension("unavailable"); + tokio::fs::rename(path, &hidden).await?; + let unavailable = evidence.enqueue(/*reviewer*/ None).await; + tokio::fs::rename(hidden, path).await?; + assert!(unavailable.is_err()); + evidence.enqueue(/*reviewer*/ None).await?; + + history + .append_items(AppendThreadItemsParams { + thread_id: root, + items: vec![RolloutItem::ResponseItem(ResponseItemEnvelope::new( + user_message_item("Missing provenance"), + ))], + }) + .await?; + let rejected = evidence + .enqueue(/*reviewer*/ None) + .await + .err() + .context("expected rejected evidence")?; + let reason = "user evidence has no host acceptance or context provenance"; + assert_eq!(rejected.to_string(), reason); + test.codex.shutdown_and_wait().await?; + // A fresh flush would now fail. The review must still get the original rejection. + assert!(history.flush_thread(root).await.is_err()); + let error = evidence + .freeze(root, AgentPath::root()) + .await + .err() + .context("expected a rejected cutoff")?; + assert_eq!(error.to_string(), reason); + Ok(()) +} + +#[tokio::test] +async fn overlapping_reviews_only_expose_a_revocation_in_the_second_view() -> anyhow::Result<()> { + use codex_extension_api::ExtensionData; + use codex_extension_api::ExtensionRegistryBuilder; + use codex_extension_api::ThreadReadyInput; + use codex_extension_api::ThreadStopInput; + use codex_protocol::protocol::SessionSource; + + core_test_support::skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let test = test_codex() + .with_history_mode(ThreadHistoryMode::Paginated) + .with_model_info_override("gpt-5.6-luna", |_| {}) + .with_config(|config| { + config.approvals_reviewer = ApprovalsReviewer::User; + }) + .build_with_auto_env(&server) + .await?; + let root = test.session_configured.thread_id; + let host = Arc::new(PausedClockHost { + inner: board_host(root), + pause_next: AtomicBool::new(true), + started: Notify::new(), + release: Notify::new(), + }); + let mut registry = ExtensionRegistryBuilder::default(); + crate::install( + &mut registry, + Arc::downgrade(&test.thread_manager), + test.config.sqlite_config().clone(), + { + let host = host.clone(); + move |_| host.clone() + }, + ); + let registry = registry.build(); + let session = ExtensionData::new(root.to_string()); + let thread = test.codex.thread_extension_data(); + let lifecycle = ®istry.thread_lifecycle_contributors()[0]; + lifecycle + .on_thread_ready(ThreadReadyInput { + config: &test.config, + session_source: &SessionSource::Exec, + history_initialization: codex_history::HistoryInitialization::New, + history_mode: ThreadHistoryMode::Paginated, + session_store: &session, + thread_store: thread, + }) + .await; + let evidence = thread.get::().context("root intake")?; + let _first_route = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message( + "route1", + r#"{"routes":[["release"]],"descriptions":{"release":"Release policy"}}"#, + ), + ev_completed("route1"), + ]), + ) + .await; + let append_user_message = async |order, text: &str| { + test.thread_store + .append_items(AppendThreadItemsParams { + thread_id: root, + items: vec![RolloutItem::ResponseItem(ResponseItemEnvelope { + item: user_message_item(text), + metadata: Some(CodexHarnessMetadata { + user_input_order: Some(order), + ..Default::default() + }), + })], + }) + .await + }; + append_user_message(1, "Stop the release.").await?; + // The first review has captured its history once its board write reaches the gate. + let wait = Duration::from_secs(15); + let first = evidence.freeze(root, AgentPath::root()); + tokio::pin!(first); + tokio::select! { + result = &mut first => { + result?; + anyhow::bail!("first review finished before its board write") + } + started = tokio::time::timeout(wait, host.started.notified()) => started?, + } + + let _second_route = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message( + "route2", + r#"{"routes":[["revocations"]],"descriptions":{"revocations":"Updated authorization"}}"#, + ), + ev_completed("route2"), + ]), + ) + .await; + append_user_message(2, "New revocation.").await?; + // Queue the second review behind the first write before allowing either view to open. + let (reply, second) = oneshot::channel(); + let enqueue_second = evidence.enqueue(Some(Freeze { + caller: root, + path: AgentPath::root(), + reply, + })); + tokio::time::timeout(wait, enqueue_second) + .await + .context("second review never entered the writer queue")??; + host.release.notify_one(); + let first = first.await?; + let second = second.await??; + assert!(second.source.end_ordinal_exclusive > first.source.end_ordinal_exclusive); + for (cutoff, query, expected) in [ + (&first, "Stop the release", 1), + (&first, "New revocation", 0), + (&second, "Stop the release", 1), + (&second, "New revocation", 1), + ] { + let posts = cutoff + .subscriptions + .view() + .search_posts( + root, + PostQuery { + channel_name: None, + query: Some(query.into()), + author: None, + after_message_id: None, + page: PageRequest::default(), + max_chars_per_post: NonZeroU32::MAX, + }, + ) + .await?; + assert_eq!(posts.results.len(), expected); + } + lifecycle + .on_thread_stop(ThreadStopInput { + thread_store: thread, + session_store: &session, + }) + .await; + test.codex.shutdown_and_wait().await?; + Ok(()) +} diff --git a/codex-rs/ext/guardian-v3/src/evidence.rs b/codex-rs/ext/guardian-v3/src/evidence.rs index 5e42cf97d7..f37ecc2b58 100644 --- a/codex-rs/ext/guardian-v3/src/evidence.rs +++ b/codex-rs/ext/guardian-v3/src/evidence.rs @@ -13,8 +13,10 @@ use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::HistoryPosition; use codex_protocol::protocol::ThreadHistoryMode; +use codex_thread_store::OriginalHistoryRead; use codex_thread_store::OriginalHistoryRequest; use codex_thread_store::ThreadStore; +use codex_thread_store::ThreadStoreResult; use codex_utils_string::to_json_string_bounded; use serde::Deserialize; use serde::Serialize; @@ -64,9 +66,15 @@ pub struct EvidenceSnapshot { start: HistoryPosition, end: HistoryPosition, records: Arc<[OriginalEvidence]>, + // Report these positions only when the intake cursor accepts this capture. + pub(super) skipped_incomplete_deliveries: Vec, } impl EvidenceSnapshot { + pub(super) fn source_advanced(&self) -> bool { + self.start != self.end + } + pub fn thread_id(&self) -> ThreadId { self.thread_id } @@ -135,11 +143,25 @@ impl EvidenceCursor { /// Read the unread suffix through a fixed cutoff without advancing progress. pub async fn capture(&self, store: &dyn ThreadStore) -> anyhow::Result { - let history = store + self.snapshot(self.read_history(store).await?) + } + + pub(super) async fn read_history( + &self, + store: &dyn ThreadStore, + ) -> ThreadStoreResult { + store .read_original_history(self.thread_id, OriginalHistoryRequest::After(self.position)) - .await?; + .await + } + + pub(super) fn snapshot( + &self, + history: OriginalHistoryRead, + ) -> anyhow::Result { let end = history.end; let mut records = Vec::new(); + let mut skipped_incomplete_deliveries = Vec::new(); for record in history.items { // The first capture of a previously empty rollout includes its validated header. if record.end.end_ordinal_exclusive == 1 @@ -147,13 +169,16 @@ impl EvidenceCursor { { continue; } - if let Some(original) = review_original(record.item)? { + let ordinal = record + .end + .end_ordinal_exclusive + .checked_sub(1) + .ok_or_else(|| anyhow::anyhow!("evidence record has no source ordinal"))?; + if let Some(original) = + review_original(record.item, ordinal, &mut skipped_incomplete_deliveries)? + { records.push(OriginalEvidence { - id: EvidenceId { - ordinal: record.end.end_ordinal_exclusive.checked_sub(1).ok_or_else( - || anyhow::anyhow!("evidence record has no source ordinal"), - )?, - }, + id: EvidenceId { ordinal }, original, end: record.end, }); @@ -164,6 +189,7 @@ impl EvidenceCursor { start: self.position, end, records: records.into(), + skipped_incomplete_deliveries, }) } @@ -307,7 +333,11 @@ fn selected(envelope: &ResponseItemEnvelope) -> anyhow::Result { } /// Select original evidence at intake and strip host-only metadata before routing. -fn review_original(mut item: RolloutItem) -> anyhow::Result> { +fn review_original( + mut item: RolloutItem, + ordinal: u64, + skipped_incomplete_deliveries: &mut Vec, +) -> anyhow::Result> { let include = match &item { RolloutItem::ResponseItem(envelope) => selected(envelope)?, RolloutItem::RetainedContext(RetainedContextEvent::VerifiedAnswer { answer, .. }) => { @@ -318,8 +348,10 @@ fn review_original(mut item: RolloutItem) -> anyhow::Result> message, .. }) => { - ensure!(message.complete, "incomplete delivered assistant message"); - true + if !message.complete { + skipped_incomplete_deliveries.push(ordinal); + } + message.complete } RolloutItem::SessionMeta(_) | RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_)) => { anyhow::bail!("evidence history was replaced"); diff --git a/codex-rs/ext/guardian-v3/src/host.rs b/codex-rs/ext/guardian-v3/src/host.rs index 11178ae453..975b944ec4 100644 --- a/codex-rs/ext/guardian-v3/src/host.rs +++ b/codex-rs/ext/guardian-v3/src/host.rs @@ -16,16 +16,16 @@ use codex_extension_api::TurnAbortInput; use codex_extension_api::TurnLifecycleContributor; use codex_extension_api::TurnStopInput; use codex_guardian_reviewer::ReviewerTasks; -use codex_history::HistoryInitialization; +use codex_message_board::AgentMessageBoard; use codex_message_board::MessageBoardHost; use codex_protocol::ThreadId; use codex_protocol::items::TurnItem; use codex_state::SqliteConfig; -use codex_thread_store::ThreadStoreError; use tokio::sync::mpsc; use crate::EvidenceCursor; use crate::Housekeeper; +use crate::cutoff::RootEvidence; use crate::local_board; pub fn install( @@ -59,7 +59,7 @@ impl Host { &self, root: ThreadId, input: &ThreadReadyInput<'_, Config>, - ) -> anyhow::Result { + ) -> anyhow::Result<()> { let history = self .manager .upgrade() @@ -73,20 +73,29 @@ impl Host { ) .await?; let storage = local_board(self.sqlite.clone(), root.into()); - let board = storage.read_write().open((self.board_host)(root)).await?; + let board_host = (self.board_host)(root); + let board = storage.read_write().open(board_host.clone()).await?; + let reader: Arc = + Arc::new(storage.read_only().open(board_host).await?); let lifetime = Arc::new(ReviewerTasks::default()); - let (wake, mut requests) = mpsc::channel(1); + let (wake, wakes) = mpsc::channel(1); + let (queue, mut captures) = RootEvidence::new(lifetime.clone(), history, cursor.clone()); + lifetime + .tasks + .spawn(queue.clone().capture_on_activity(wakes)); let scope = lifetime.clone(); let manager = self.manager.clone(); - let mut awaiting_first_write = matches!( - input.history_initialization, - HistoryInitialization::New | HistoryInitialization::Cleared - ); lifetime.tasks.spawn(async move { let cancellation = &scope.cancellation; let work = async { + let Some(mut capture) = cancellation + .run_until_cancelled(captures.recv()) + .await + .flatten() + else { + return Ok(()); + }; let startup = async { - requests.recv().await.context("Guardian worker closed")?; let manager = manager.upgrade().context("Guardian host stopped")?; let parent = manager.get_thread(root).await?; Housekeeper::new(Arc::new(board), cursor, &parent, &manager).await @@ -96,30 +105,26 @@ impl Host { }; let mut housekeeper = housekeeper?; loop { - let Some(flushed) = cancellation.run_until_cancelled(history.flush_thread(root)).await else { - break; - }; - flushed?; - let capture = housekeeper.cursor().capture(history.as_ref()); - let Some(captured) = cancellation.run_until_cancelled(capture).await else { - break; - }; - match captured { - Ok(snapshot) => { - awaiting_first_write = false; - while housekeeper - .route_next(&snapshot, cancellation) - .await? != 0 - {} - } - // A wake can precede the first durable record. Keep the cursor until the next wake. - Err(error) if awaiting_first_write && matches!(error.downcast_ref::(), - Some(ThreadStoreError::ThreadNotFound { thread_id }) if *thread_id == root) => {} - Err(error) => return Err(error), - } - if cancellation.run_until_cancelled(requests.recv()).await.flatten().is_none() { + while housekeeper + .route_next(&capture.snapshot, cancellation) + .await? + != 0 + {} + if cancellation + .run_until_cancelled(capture.finish(&reader)) + .await + .is_none() + { break; } + let Some(next) = cancellation + .run_until_cancelled(captures.recv()) + .await + .flatten() + else { + break; + }; + capture = next; } anyhow::Ok(()) }; @@ -128,7 +133,9 @@ impl Host { tracing::warn!(%root, "Guardian housekeeping stopped"); } }); - Ok(RootWorker { wake, lifetime }) + input.thread_store.insert_arc(queue); + input.thread_store.insert(RootWorker { wake, lifetime }); + Ok(()) } } @@ -147,17 +154,15 @@ impl ThreadLifecycleContributor for Host { let Ok(root) = ThreadId::from_string(input.thread_store.level_id()) else { return; }; - match self.activate(root, &input).await { - Ok(worker) => { - input.thread_store.insert(worker); - } - Err(error) => tracing::warn!(%root, error = %error, "Guardian intake unavailable"), + if let Err(error) = self.activate(root, &input).await { + tracing::warn!(%root, error = %error, "Guardian intake unavailable"); } }) } fn on_thread_stop<'a>(&'a self, input: ThreadStopInput<'a>) -> ExtensionFuture<'a, ()> { Box::pin(async move { if let Some(worker) = input.thread_store.remove::() { + input.thread_store.remove::(); worker.lifetime.cancellation.cancel(); worker.lifetime.tasks.close(); worker.lifetime.tasks.wait().await; diff --git a/codex-rs/ext/guardian-v3/src/lib.rs b/codex-rs/ext/guardian-v3/src/lib.rs index ac6cb09998..87c85b22d9 100644 --- a/codex-rs/ext/guardian-v3/src/lib.rs +++ b/codex-rs/ext/guardian-v3/src/lib.rs @@ -8,6 +8,9 @@ use codex_message_board::LocalBoardOptions; use codex_protocol::SessionId; use codex_state::SqliteConfig; +mod cutoff; +pub use cutoff::ReviewCutoff; +pub use cutoff::RootEvidence; mod delivery; mod evidence; mod host; diff --git a/codex-rs/ext/guardian-v3/tests/common/board_host.rs b/codex-rs/ext/guardian-v3/tests/common/board_host.rs new file mode 100644 index 0000000000..18949ae766 --- /dev/null +++ b/codex-rs/ext/guardian-v3/tests/common/board_host.rs @@ -0,0 +1,68 @@ +//! A local board host whose next write can be held until the test releases it. + +use chrono::DateTime; +use chrono::Utc; +use codex_message_board::MessageBoardHost; +use codex_message_board::NotificationDelivery; +use codex_message_board::PostPreview; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::error::Result as CodexResult; +use futures::future::BoxFuture; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI64; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use tokio::sync::Notify; + +#[path = "../../../../message-board/tests/common/mod.rs"] +mod shared; + +pub(crate) fn board_host(root: ThreadId) -> Arc { + Arc::new(shared::Host { + clock: AtomicI64::default(), + fail_clock: AtomicBool::new(false), + agent_path_calls: AtomicUsize::default(), + members: [(root, AgentPath::root())].into(), + active: AtomicBool::new(true), + fail_notifications: AtomicBool::new(false), + notifications: Mutex::default(), + }) +} + +pub(crate) struct PausedClockHost { + pub(crate) inner: Arc, + pub(crate) pause_next: AtomicBool, + pub(crate) started: Notify, + pub(crate) release: Notify, +} + +impl MessageBoardHost for PausedClockHost { + fn agent_path(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult> { + self.inner.agent_path(caller) + } + + fn resolve_agent(&self, path: AgentPath) -> BoxFuture<'_, CodexResult> { + self.inner.resolve_agent(path) + } + + fn current_time(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult>> { + Box::pin(async move { + if self.pause_next.swap(false, Ordering::SeqCst) { + self.started.notify_one(); + self.release.notified().await; + } + self.inner.current_time(caller).await + }) + } + + fn notify( + &self, + recipient: ThreadId, + post: PostPreview, + ) -> BoxFuture<'_, CodexResult> { + self.inner.notify(recipient, post) + } +} diff --git a/codex-rs/ext/guardian-v3/tests/evidence.rs b/codex-rs/ext/guardian-v3/tests/evidence.rs index e39d331299..7e38340bb2 100644 --- a/codex-rs/ext/guardian-v3/tests/evidence.rs +++ b/codex-rs/ext/guardian-v3/tests/evidence.rs @@ -185,6 +185,7 @@ Approved action: {"action":{"command":"write /workspace/report.txt"},"outcome":" ) .await?; assert!(!store.live_rollout_path(owner).await?.exists()); + assert!(fresh.capture(&store).await.is_err()); // A resume must establish its tail even if the rollout is missing. assert!( EvidenceCursor::at_thread_ready( diff --git a/codex-rs/ext/guardian-v3/tests/housekeeping.rs b/codex-rs/ext/guardian-v3/tests/housekeeping.rs index 3f761818f3..e95f45c3b8 100644 --- a/codex-rs/ext/guardian-v3/tests/housekeeping.rs +++ b/codex-rs/ext/guardian-v3/tests/housekeeping.rs @@ -2,7 +2,6 @@ #![allow(clippy::expect_used)] -use chrono::DateTime; use chrono::Utc; use codex_guardian_v3::EvidenceCursor; use codex_guardian_v3::HOUSEKEEPER_INSTRUCTIONS; @@ -13,18 +12,13 @@ use codex_history::ResponseItemEnvelope; use codex_history::RolloutItem; use codex_message_board::AgentMessageBoard; use codex_message_board::ChannelQuery; -use codex_message_board::MessageBoardHost; -use codex_message_board::NotificationDelivery; use codex_message_board::PageRequest; -use codex_message_board::PostPreview; use codex_message_board::PostQuery; use codex_message_board::ReadPostRequest; use codex_message_board::SortDirection; use codex_message_board::UpsertChannelRequest; -use codex_protocol::AgentPath; use codex_protocol::ThreadId; use codex_protocol::config_types::ApprovalsReviewer; -use codex_protocol::error::Result as CodexResult; use codex_protocol::openai_models::TruncationPolicyConfig; use codex_protocol::protocol::ThreadHistoryMode; use codex_state::SqliteConfig; @@ -34,24 +28,24 @@ use core_test_support::context_snapshot; use core_test_support::context_snapshot::ContextSnapshotOptions; use core_test_support::responses::*; use core_test_support::test_codex::test_codex; -use futures::future::BoxFuture; use pretty_assertions::assert_eq; use serde_json::json; use std::num::NonZeroU32; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicI64; -use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; use tokio::sync::Notify; use tokio_util::sync::CancellationToken; -#[path = "../../../message-board/tests/common/mod.rs"] +#[path = "common/board_host.rs"] mod board_host; +use board_host::PausedClockHost; +use board_host::board_host; + enum RoutingCase { Success, RetryExhausted, @@ -480,53 +474,6 @@ async fn routes_captured_originals_and_updates_descriptions( Ok(()) } -fn board_host(root: ThreadId) -> Arc { - Arc::new(board_host::Host { - clock: AtomicI64::default(), - fail_clock: AtomicBool::new(false), - agent_path_calls: AtomicUsize::default(), - members: [(root, AgentPath::root())].into(), - active: AtomicBool::new(true), - fail_notifications: AtomicBool::new(false), - notifications: Mutex::default(), - }) -} - -struct PausedClockHost { - inner: Arc, - pause_next: AtomicBool, - started: Notify, - release: Notify, -} - -impl MessageBoardHost for PausedClockHost { - fn agent_path(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult> { - self.inner.agent_path(caller) - } - - fn resolve_agent(&self, path: AgentPath) -> BoxFuture<'_, CodexResult> { - self.inner.resolve_agent(path) - } - - fn current_time(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult>> { - Box::pin(async move { - if self.pause_next.swap(false, Ordering::SeqCst) { - self.started.notify_one(); - self.release.notified().await; - } - self.inner.current_time(caller).await - }) - } - - fn notify( - &self, - recipient: ThreadId, - post: PostPreview, - ) -> BoxFuture<'_, CodexResult> { - self.inner.notify(recipient, post) - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn root_shutdown_waits_for_the_accepted_evidence_append() -> anyhow::Result<()> { use codex_extension_api::ExtensionData;