mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Freeze Guardian review views at ordered evidence cutoffs (#61929)
## Why Overlapping reviews need board views that match their captured history boundaries, so later evidence does not appear in an earlier review. ## What changed - Add `RootEvidence::freeze` to capture and enqueue history in source order, returning a `ReviewCutoff` with the source position and reviewer subscriptions. - Open each review's board view after routing its captured evidence and before routing the next capture. - Retry transient history failures on subsequent activity, preserve evidence validation failures for later reviews, and skip incomplete delivered messages with a warning once per accepted capture. ## Testing Add tests for overlapping reviews with a later revocation, retryable history failures, persistent evidence rejection, and incomplete deliveries across activity wakes and freezes. GitOrigin-RevId: b87040e99bf0e56626d25a8acdd036cf3eed32ce
This commit is contained in:
@@ -7,4 +7,9 @@ codex_rust_crate(
|
||||
integration_compile_data_extra = ["//codex-rs/message-board:tests/common/mod.rs"],
|
||||
run_tests_with_wine_exec = True,
|
||||
test_tags = ["no-sandbox"],
|
||||
unit_test_srcs = [
|
||||
"src/cutoff_tests.rs",
|
||||
"tests/common/board_host.rs",
|
||||
"//codex-rs/message-board:tests/common/mod.rs",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -6,7 +6,6 @@ version.workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
test = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Capture and enqueue each history suffix in one order. The sole board writer opens
|
||||
//! each requested view after routing that suffix and before processing its successor.
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Context;
|
||||
use codex_guardian_reviewer::ReviewerTasks;
|
||||
use codex_message_board::AgentMessageBoard;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_thread_store::ThreadStore;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
use crate::EvidenceCursor;
|
||||
use crate::EvidenceSnapshot;
|
||||
use crate::ReviewerSubscriptions;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "cutoff_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
/// One approval's fixed source boundary and matching discovery, retrieval and delivery view.
|
||||
pub struct ReviewCutoff {
|
||||
pub source: HistoryPosition,
|
||||
pub subscriptions: ReviewerSubscriptions,
|
||||
}
|
||||
|
||||
/// Installed on the root thread while its evidence intake is active.
|
||||
pub struct RootEvidence {
|
||||
lifetime: Arc<ReviewerTasks>,
|
||||
history: Arc<dyn ThreadStore>,
|
||||
// Advances on enqueue. The housekeeper independently tracks successful routing.
|
||||
// A rejected durable record keeps its reason for subsequent reviews of this root.
|
||||
enqueued: Mutex<Result<EvidenceCursor, String>>,
|
||||
pending: mpsc::Sender<Capture>,
|
||||
}
|
||||
|
||||
struct Freeze {
|
||||
caller: ThreadId,
|
||||
path: AgentPath,
|
||||
reply: oneshot::Sender<anyhow::Result<ReviewCutoff>>,
|
||||
}
|
||||
|
||||
pub(super) struct Capture {
|
||||
pub snapshot: EvidenceSnapshot,
|
||||
reviewer: Option<Freeze>,
|
||||
}
|
||||
|
||||
impl Capture {
|
||||
pub async fn finish(self, reader: &Arc<dyn AgentMessageBoard>) {
|
||||
if let Some(Freeze {
|
||||
caller,
|
||||
path,
|
||||
reply,
|
||||
}) = self.reviewer
|
||||
&& !reply.is_closed()
|
||||
{
|
||||
let source = self.snapshot.cutoff();
|
||||
let result = ReviewerSubscriptions::capture(reader.clone(), caller, path)
|
||||
.await
|
||||
.map(|subscriptions| ReviewCutoff {
|
||||
source,
|
||||
subscriptions,
|
||||
});
|
||||
let _ = reply.send(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RootEvidence {
|
||||
pub(super) fn new(
|
||||
lifetime: Arc<ReviewerTasks>,
|
||||
history: Arc<dyn ThreadStore>,
|
||||
cursor: EvidenceCursor,
|
||||
) -> (Arc<Self>, mpsc::Receiver<Capture>) {
|
||||
let (pending, requests) = mpsc::channel(1);
|
||||
(
|
||||
Arc::new(Self {
|
||||
lifetime,
|
||||
history,
|
||||
enqueued: Mutex::new(Ok(cursor)),
|
||||
pending,
|
||||
}),
|
||||
requests,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn capture_on_activity(self: Arc<Self>, mut wakes: mpsc::Receiver<()>) {
|
||||
let capture = async {
|
||||
while wakes.recv().await.is_some() {
|
||||
if self.enqueue(/*reviewer*/ None).await.is_err() {
|
||||
if self.enqueued.lock().await.is_err() {
|
||||
break;
|
||||
}
|
||||
tracing::warn!("Guardian background capture failed; waiting for the next wake");
|
||||
}
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = self.lifetime.cancellation.cancelled() => (),
|
||||
_ = self.pending.closed() => (),
|
||||
() = capture => (),
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "capture and enqueue must agree on source order across concurrent approvals"
|
||||
)]
|
||||
async fn enqueue(&self, reviewer: Option<Freeze>) -> anyhow::Result<()> {
|
||||
let mut enqueued = self.enqueued.lock().await;
|
||||
let cursor = enqueued
|
||||
.as_mut()
|
||||
.map_err(|reason| anyhow::anyhow!("{reason}"))?;
|
||||
anyhow::ensure!(!self.pending.is_closed(), "Guardian housekeeper stopped");
|
||||
self.history.flush_thread(cursor.thread_id()).await?;
|
||||
let history = cursor.read_history(self.history.as_ref()).await?;
|
||||
let snapshot = match cursor.snapshot(history) {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
tracing::warn!(root_thread_id = %cursor.thread_id(), %error, "Guardian intake rejected stored evidence");
|
||||
*enqueued = Err(error.to_string());
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
// A freeze must wait behind earlier writes even when the source has not advanced.
|
||||
if reviewer.is_none() && !snapshot.source_advanced() {
|
||||
return Ok(());
|
||||
}
|
||||
// Cancellation before reserving leaves the cursor unchanged. After reservation,
|
||||
// admission and advancing the captured range cannot be interrupted.
|
||||
let permit = self
|
||||
.pending
|
||||
.reserve()
|
||||
.await
|
||||
.context("Guardian housekeeper stopped")?;
|
||||
cursor.acknowledge(&snapshot, snapshot.records().len())?;
|
||||
for &source_ordinal in &snapshot.skipped_incomplete_deliveries {
|
||||
tracing::warn!(
|
||||
name: "guardian_v3.evidence_skipped",
|
||||
root_thread_id = %cursor.thread_id(),
|
||||
source_ordinal,
|
||||
reason = "incomplete_delivered_message",
|
||||
"Skipped incomplete Guardian evidence"
|
||||
);
|
||||
}
|
||||
permit.send(Capture { snapshot, reviewer });
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RootEvidence {
|
||||
/// Freeze once per review, using caller identity and path from the acting thread's host.
|
||||
pub async fn freeze(&self, caller: ThreadId, path: AgentPath) -> anyhow::Result<ReviewCutoff> {
|
||||
let _task = self.lifetime.tasks.token();
|
||||
let request = async {
|
||||
let (reply, ready) = oneshot::channel();
|
||||
self.enqueue(Some(Freeze {
|
||||
caller,
|
||||
path,
|
||||
reply,
|
||||
}))
|
||||
.await?;
|
||||
ready
|
||||
.await
|
||||
.context("Guardian stopped before the review cutoff")?
|
||||
};
|
||||
tokio::select! { biased;
|
||||
_ = self.lifetime.cancellation.cancelled() => anyhow::bail!("Guardian intake stopped"),
|
||||
result = request => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::Freeze;
|
||||
use super::RootEvidence;
|
||||
use crate::EvidenceCursor;
|
||||
use anyhow::Context;
|
||||
use board_host::PausedClockHost;
|
||||
use board_host::board_host;
|
||||
use codex_history::CodexHarnessMetadata;
|
||||
use codex_history::HistoryInitialization;
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_history::RetainedContextEvent;
|
||||
use codex_history::RetainedUserMessage;
|
||||
use codex_history::RolloutItem;
|
||||
use codex_history::UserInputOrigin;
|
||||
use codex_message_board::PageRequest;
|
||||
use codex_message_board::PostQuery;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_thread_store::AppendThreadItemsParams;
|
||||
use codex_thread_store::PersistContext;
|
||||
use core_test_support::responses::ev_assistant_message;
|
||||
use core_test_support::responses::ev_completed;
|
||||
use core_test_support::responses::mount_sse_once;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::responses::start_mock_server;
|
||||
use core_test_support::responses::user_message_item;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::tracing::NamedTraceEvents;
|
||||
use futures::FutureExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::mpsc::error::TryRecvError;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
#[path = "../tests/common/board_host.rs"]
|
||||
mod board_host;
|
||||
|
||||
#[tokio::test]
|
||||
async fn skips_a_partial_delivery_once_across_wakes_and_freezes() -> anyhow::Result<()> {
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let test = test_codex()
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let root = test.session_configured.thread_id;
|
||||
let history = &test.thread_store;
|
||||
let cursor = EvidenceCursor::at_thread_ready(
|
||||
history.as_ref(),
|
||||
root,
|
||||
ThreadHistoryMode::Paginated,
|
||||
HistoryInitialization::New,
|
||||
)
|
||||
.await?;
|
||||
let (evidence, mut writer) = RootEvidence::new(Default::default(), history.clone(), cursor);
|
||||
|
||||
// The initial rollout header advances the writer even without evidence to route.
|
||||
history
|
||||
.persist_thread(root, PersistContext::Standard)
|
||||
.await?;
|
||||
evidence.enqueue(/*reviewer*/ None).await?;
|
||||
let first = writer.try_recv()?;
|
||||
assert!(first.snapshot.records().is_empty());
|
||||
|
||||
let items = [(1, false, "partial"), (2, true, "next complete message")]
|
||||
.into_iter()
|
||||
.map(|(acceptance_order, complete, text)| {
|
||||
RolloutItem::RetainedContext(RetainedContextEvent::DeliveredAssistantMessage {
|
||||
message: RetainedUserMessage {
|
||||
origin: UserInputOrigin::User,
|
||||
turn_id: "turn-1".into(),
|
||||
message_id: None,
|
||||
text: text.into(),
|
||||
complete,
|
||||
phase: None,
|
||||
},
|
||||
acceptance_order,
|
||||
retained_source: None,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
history
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: root,
|
||||
items,
|
||||
})
|
||||
.await?;
|
||||
let mut warnings = NamedTraceEvents::subscribe("guardian_v3.evidence_skipped");
|
||||
evidence.enqueue(/*reviewer*/ None).await?;
|
||||
let routed = writer.try_recv()?.snapshot;
|
||||
let [next] = routed.records() else {
|
||||
anyhow::bail!("expected only the complete delivery")
|
||||
};
|
||||
let skipped = first.snapshot.cutoff().end_ordinal_exclusive;
|
||||
assert_eq!(routed.skipped_incomplete_deliveries, [skipped]);
|
||||
assert_eq!(next.id.ordinal, skipped + 1);
|
||||
warnings.next().await?;
|
||||
|
||||
evidence.enqueue(/*reviewer*/ None).await?;
|
||||
assert!(matches!(writer.try_recv(), Err(TryRecvError::Empty)));
|
||||
|
||||
// A review at that very same position still needs a response from the writer.
|
||||
let freeze = evidence.freeze(root, AgentPath::root());
|
||||
tokio::pin!(freeze);
|
||||
let review = tokio::select! {
|
||||
_ = &mut freeze => anyhow::bail!("freeze completed before the writer replied"),
|
||||
request = writer.recv() => request.context("writer closed")?,
|
||||
};
|
||||
assert!(futures::poll!(&mut freeze).is_pending());
|
||||
assert_eq!(review.snapshot.cutoff(), routed.cutoff());
|
||||
assert!(warnings.next().now_or_never().is_none());
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retries_unavailable_history_and_remembers_rejected_evidence() -> anyhow::Result<()> {
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let test = test_codex()
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let root = test.session_configured.thread_id;
|
||||
let history = &test.thread_store;
|
||||
history
|
||||
.persist_thread(root, PersistContext::Standard)
|
||||
.await?;
|
||||
let cursor = EvidenceCursor::from_now(history.as_ref(), root).await?;
|
||||
let (evidence, _writer) = RootEvidence::new(Default::default(), history.clone(), cursor);
|
||||
|
||||
let path = test
|
||||
.session_configured
|
||||
.rollout_path
|
||||
.as_deref()
|
||||
.context("rollout path")?;
|
||||
let hidden = path.with_extension("unavailable");
|
||||
tokio::fs::rename(path, &hidden).await?;
|
||||
let unavailable = evidence.enqueue(/*reviewer*/ None).await;
|
||||
tokio::fs::rename(hidden, path).await?;
|
||||
assert!(unavailable.is_err());
|
||||
evidence.enqueue(/*reviewer*/ None).await?;
|
||||
|
||||
history
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: root,
|
||||
items: vec![RolloutItem::ResponseItem(ResponseItemEnvelope::new(
|
||||
user_message_item("Missing provenance"),
|
||||
))],
|
||||
})
|
||||
.await?;
|
||||
let rejected = evidence
|
||||
.enqueue(/*reviewer*/ None)
|
||||
.await
|
||||
.err()
|
||||
.context("expected rejected evidence")?;
|
||||
let reason = "user evidence has no host acceptance or context provenance";
|
||||
assert_eq!(rejected.to_string(), reason);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
// A fresh flush would now fail. The review must still get the original rejection.
|
||||
assert!(history.flush_thread(root).await.is_err());
|
||||
let error = evidence
|
||||
.freeze(root, AgentPath::root())
|
||||
.await
|
||||
.err()
|
||||
.context("expected a rejected cutoff")?;
|
||||
assert_eq!(error.to_string(), reason);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlapping_reviews_only_expose_a_revocation_in_the_second_view() -> anyhow::Result<()> {
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::ThreadReadyInput;
|
||||
use codex_extension_api::ThreadStopInput;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let test = test_codex()
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.with_model_info_override("gpt-5.6-luna", |_| {})
|
||||
.with_config(|config| {
|
||||
config.approvals_reviewer = ApprovalsReviewer::User;
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let root = test.session_configured.thread_id;
|
||||
let host = Arc::new(PausedClockHost {
|
||||
inner: board_host(root),
|
||||
pause_next: AtomicBool::new(true),
|
||||
started: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
let mut registry = ExtensionRegistryBuilder::default();
|
||||
crate::install(
|
||||
&mut registry,
|
||||
Arc::downgrade(&test.thread_manager),
|
||||
test.config.sqlite_config().clone(),
|
||||
{
|
||||
let host = host.clone();
|
||||
move |_| host.clone()
|
||||
},
|
||||
);
|
||||
let registry = registry.build();
|
||||
let session = ExtensionData::new(root.to_string());
|
||||
let thread = test.codex.thread_extension_data();
|
||||
let lifecycle = ®istry.thread_lifecycle_contributors()[0];
|
||||
lifecycle
|
||||
.on_thread_ready(ThreadReadyInput {
|
||||
config: &test.config,
|
||||
session_source: &SessionSource::Exec,
|
||||
history_initialization: codex_history::HistoryInitialization::New,
|
||||
history_mode: ThreadHistoryMode::Paginated,
|
||||
session_store: &session,
|
||||
thread_store: thread,
|
||||
})
|
||||
.await;
|
||||
let evidence = thread.get::<RootEvidence>().context("root intake")?;
|
||||
let _first_route = mount_sse_once(
|
||||
&server,
|
||||
sse(vec![
|
||||
ev_assistant_message(
|
||||
"route1",
|
||||
r#"{"routes":[["release"]],"descriptions":{"release":"Release policy"}}"#,
|
||||
),
|
||||
ev_completed("route1"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let append_user_message = async |order, text: &str| {
|
||||
test.thread_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: root,
|
||||
items: vec![RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item: user_message_item(text),
|
||||
metadata: Some(CodexHarnessMetadata {
|
||||
user_input_order: Some(order),
|
||||
..Default::default()
|
||||
}),
|
||||
})],
|
||||
})
|
||||
.await
|
||||
};
|
||||
append_user_message(1, "Stop the release.").await?;
|
||||
// The first review has captured its history once its board write reaches the gate.
|
||||
let wait = Duration::from_secs(15);
|
||||
let first = evidence.freeze(root, AgentPath::root());
|
||||
tokio::pin!(first);
|
||||
tokio::select! {
|
||||
result = &mut first => {
|
||||
result?;
|
||||
anyhow::bail!("first review finished before its board write")
|
||||
}
|
||||
started = tokio::time::timeout(wait, host.started.notified()) => started?,
|
||||
}
|
||||
|
||||
let _second_route = mount_sse_once(
|
||||
&server,
|
||||
sse(vec![
|
||||
ev_assistant_message(
|
||||
"route2",
|
||||
r#"{"routes":[["revocations"]],"descriptions":{"revocations":"Updated authorization"}}"#,
|
||||
),
|
||||
ev_completed("route2"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
append_user_message(2, "New revocation.").await?;
|
||||
// Queue the second review behind the first write before allowing either view to open.
|
||||
let (reply, second) = oneshot::channel();
|
||||
let enqueue_second = evidence.enqueue(Some(Freeze {
|
||||
caller: root,
|
||||
path: AgentPath::root(),
|
||||
reply,
|
||||
}));
|
||||
tokio::time::timeout(wait, enqueue_second)
|
||||
.await
|
||||
.context("second review never entered the writer queue")??;
|
||||
host.release.notify_one();
|
||||
let first = first.await?;
|
||||
let second = second.await??;
|
||||
assert!(second.source.end_ordinal_exclusive > first.source.end_ordinal_exclusive);
|
||||
for (cutoff, query, expected) in [
|
||||
(&first, "Stop the release", 1),
|
||||
(&first, "New revocation", 0),
|
||||
(&second, "Stop the release", 1),
|
||||
(&second, "New revocation", 1),
|
||||
] {
|
||||
let posts = cutoff
|
||||
.subscriptions
|
||||
.view()
|
||||
.search_posts(
|
||||
root,
|
||||
PostQuery {
|
||||
channel_name: None,
|
||||
query: Some(query.into()),
|
||||
author: None,
|
||||
after_message_id: None,
|
||||
page: PageRequest::default(),
|
||||
max_chars_per_post: NonZeroU32::MAX,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(posts.results.len(), expected);
|
||||
}
|
||||
lifecycle
|
||||
.on_thread_stop(ThreadStopInput {
|
||||
thread_store: thread,
|
||||
session_store: &session,
|
||||
})
|
||||
.await;
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -13,8 +13,10 @@ use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_thread_store::OriginalHistoryRead;
|
||||
use codex_thread_store::OriginalHistoryRequest;
|
||||
use codex_thread_store::ThreadStore;
|
||||
use codex_thread_store::ThreadStoreResult;
|
||||
use codex_utils_string::to_json_string_bounded;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
@@ -64,9 +66,15 @@ pub struct EvidenceSnapshot {
|
||||
start: HistoryPosition,
|
||||
end: HistoryPosition,
|
||||
records: Arc<[OriginalEvidence]>,
|
||||
// Report these positions only when the intake cursor accepts this capture.
|
||||
pub(super) skipped_incomplete_deliveries: Vec<u64>,
|
||||
}
|
||||
|
||||
impl EvidenceSnapshot {
|
||||
pub(super) fn source_advanced(&self) -> bool {
|
||||
self.start != self.end
|
||||
}
|
||||
|
||||
pub fn thread_id(&self) -> ThreadId {
|
||||
self.thread_id
|
||||
}
|
||||
@@ -135,11 +143,25 @@ impl EvidenceCursor {
|
||||
|
||||
/// Read the unread suffix through a fixed cutoff without advancing progress.
|
||||
pub async fn capture(&self, store: &dyn ThreadStore) -> anyhow::Result<EvidenceSnapshot> {
|
||||
let history = store
|
||||
self.snapshot(self.read_history(store).await?)
|
||||
}
|
||||
|
||||
pub(super) async fn read_history(
|
||||
&self,
|
||||
store: &dyn ThreadStore,
|
||||
) -> ThreadStoreResult<OriginalHistoryRead> {
|
||||
store
|
||||
.read_original_history(self.thread_id, OriginalHistoryRequest::After(self.position))
|
||||
.await?;
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(
|
||||
&self,
|
||||
history: OriginalHistoryRead,
|
||||
) -> anyhow::Result<EvidenceSnapshot> {
|
||||
let end = history.end;
|
||||
let mut records = Vec::new();
|
||||
let mut skipped_incomplete_deliveries = Vec::new();
|
||||
for record in history.items {
|
||||
// The first capture of a previously empty rollout includes its validated header.
|
||||
if record.end.end_ordinal_exclusive == 1
|
||||
@@ -147,13 +169,16 @@ impl EvidenceCursor {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(original) = review_original(record.item)? {
|
||||
let ordinal = record
|
||||
.end
|
||||
.end_ordinal_exclusive
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| anyhow::anyhow!("evidence record has no source ordinal"))?;
|
||||
if let Some(original) =
|
||||
review_original(record.item, ordinal, &mut skipped_incomplete_deliveries)?
|
||||
{
|
||||
records.push(OriginalEvidence {
|
||||
id: EvidenceId {
|
||||
ordinal: record.end.end_ordinal_exclusive.checked_sub(1).ok_or_else(
|
||||
|| anyhow::anyhow!("evidence record has no source ordinal"),
|
||||
)?,
|
||||
},
|
||||
id: EvidenceId { ordinal },
|
||||
original,
|
||||
end: record.end,
|
||||
});
|
||||
@@ -164,6 +189,7 @@ impl EvidenceCursor {
|
||||
start: self.position,
|
||||
end,
|
||||
records: records.into(),
|
||||
skipped_incomplete_deliveries,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -307,7 +333,11 @@ fn selected(envelope: &ResponseItemEnvelope) -> anyhow::Result<bool> {
|
||||
}
|
||||
|
||||
/// Select original evidence at intake and strip host-only metadata before routing.
|
||||
fn review_original(mut item: RolloutItem) -> anyhow::Result<Option<RolloutItem>> {
|
||||
fn review_original(
|
||||
mut item: RolloutItem,
|
||||
ordinal: u64,
|
||||
skipped_incomplete_deliveries: &mut Vec<u64>,
|
||||
) -> anyhow::Result<Option<RolloutItem>> {
|
||||
let include = match &item {
|
||||
RolloutItem::ResponseItem(envelope) => selected(envelope)?,
|
||||
RolloutItem::RetainedContext(RetainedContextEvent::VerifiedAnswer { answer, .. }) => {
|
||||
@@ -318,8 +348,10 @@ fn review_original(mut item: RolloutItem) -> anyhow::Result<Option<RolloutItem>>
|
||||
message,
|
||||
..
|
||||
}) => {
|
||||
ensure!(message.complete, "incomplete delivered assistant message");
|
||||
true
|
||||
if !message.complete {
|
||||
skipped_incomplete_deliveries.push(ordinal);
|
||||
}
|
||||
message.complete
|
||||
}
|
||||
RolloutItem::SessionMeta(_) | RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_)) => {
|
||||
anyhow::bail!("evidence history was replaced");
|
||||
|
||||
@@ -16,16 +16,16 @@ use codex_extension_api::TurnAbortInput;
|
||||
use codex_extension_api::TurnLifecycleContributor;
|
||||
use codex_extension_api::TurnStopInput;
|
||||
use codex_guardian_reviewer::ReviewerTasks;
|
||||
use codex_history::HistoryInitialization;
|
||||
use codex_message_board::AgentMessageBoard;
|
||||
use codex_message_board::MessageBoardHost;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_state::SqliteConfig;
|
||||
use codex_thread_store::ThreadStoreError;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::EvidenceCursor;
|
||||
use crate::Housekeeper;
|
||||
use crate::cutoff::RootEvidence;
|
||||
use crate::local_board;
|
||||
|
||||
pub fn install(
|
||||
@@ -59,7 +59,7 @@ impl Host {
|
||||
&self,
|
||||
root: ThreadId,
|
||||
input: &ThreadReadyInput<'_, Config>,
|
||||
) -> anyhow::Result<RootWorker> {
|
||||
) -> anyhow::Result<()> {
|
||||
let history = self
|
||||
.manager
|
||||
.upgrade()
|
||||
@@ -73,20 +73,29 @@ impl Host {
|
||||
)
|
||||
.await?;
|
||||
let storage = local_board(self.sqlite.clone(), root.into());
|
||||
let board = storage.read_write().open((self.board_host)(root)).await?;
|
||||
let board_host = (self.board_host)(root);
|
||||
let board = storage.read_write().open(board_host.clone()).await?;
|
||||
let reader: Arc<dyn AgentMessageBoard> =
|
||||
Arc::new(storage.read_only().open(board_host).await?);
|
||||
let lifetime = Arc::new(ReviewerTasks::default());
|
||||
let (wake, mut requests) = mpsc::channel(1);
|
||||
let (wake, wakes) = mpsc::channel(1);
|
||||
let (queue, mut captures) = RootEvidence::new(lifetime.clone(), history, cursor.clone());
|
||||
lifetime
|
||||
.tasks
|
||||
.spawn(queue.clone().capture_on_activity(wakes));
|
||||
let scope = lifetime.clone();
|
||||
let manager = self.manager.clone();
|
||||
let mut awaiting_first_write = matches!(
|
||||
input.history_initialization,
|
||||
HistoryInitialization::New | HistoryInitialization::Cleared
|
||||
);
|
||||
lifetime.tasks.spawn(async move {
|
||||
let cancellation = &scope.cancellation;
|
||||
let work = async {
|
||||
let Some(mut capture) = cancellation
|
||||
.run_until_cancelled(captures.recv())
|
||||
.await
|
||||
.flatten()
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let startup = async {
|
||||
requests.recv().await.context("Guardian worker closed")?;
|
||||
let manager = manager.upgrade().context("Guardian host stopped")?;
|
||||
let parent = manager.get_thread(root).await?;
|
||||
Housekeeper::new(Arc::new(board), cursor, &parent, &manager).await
|
||||
@@ -96,30 +105,26 @@ impl Host {
|
||||
};
|
||||
let mut housekeeper = housekeeper?;
|
||||
loop {
|
||||
let Some(flushed) = cancellation.run_until_cancelled(history.flush_thread(root)).await else {
|
||||
break;
|
||||
};
|
||||
flushed?;
|
||||
let capture = housekeeper.cursor().capture(history.as_ref());
|
||||
let Some(captured) = cancellation.run_until_cancelled(capture).await else {
|
||||
break;
|
||||
};
|
||||
match captured {
|
||||
Ok(snapshot) => {
|
||||
awaiting_first_write = false;
|
||||
while housekeeper
|
||||
.route_next(&snapshot, cancellation)
|
||||
.await? != 0
|
||||
{}
|
||||
}
|
||||
// A wake can precede the first durable record. Keep the cursor until the next wake.
|
||||
Err(error) if awaiting_first_write && matches!(error.downcast_ref::<ThreadStoreError>(),
|
||||
Some(ThreadStoreError::ThreadNotFound { thread_id }) if *thread_id == root) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
if cancellation.run_until_cancelled(requests.recv()).await.flatten().is_none() {
|
||||
while housekeeper
|
||||
.route_next(&capture.snapshot, cancellation)
|
||||
.await?
|
||||
!= 0
|
||||
{}
|
||||
if cancellation
|
||||
.run_until_cancelled(capture.finish(&reader))
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
break;
|
||||
}
|
||||
let Some(next) = cancellation
|
||||
.run_until_cancelled(captures.recv())
|
||||
.await
|
||||
.flatten()
|
||||
else {
|
||||
break;
|
||||
};
|
||||
capture = next;
|
||||
}
|
||||
anyhow::Ok(())
|
||||
};
|
||||
@@ -128,7 +133,9 @@ impl Host {
|
||||
tracing::warn!(%root, "Guardian housekeeping stopped");
|
||||
}
|
||||
});
|
||||
Ok(RootWorker { wake, lifetime })
|
||||
input.thread_store.insert_arc(queue);
|
||||
input.thread_store.insert(RootWorker { wake, lifetime });
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,17 +154,15 @@ impl ThreadLifecycleContributor<Config> for Host {
|
||||
let Ok(root) = ThreadId::from_string(input.thread_store.level_id()) else {
|
||||
return;
|
||||
};
|
||||
match self.activate(root, &input).await {
|
||||
Ok(worker) => {
|
||||
input.thread_store.insert(worker);
|
||||
}
|
||||
Err(error) => tracing::warn!(%root, error = %error, "Guardian intake unavailable"),
|
||||
if let Err(error) = self.activate(root, &input).await {
|
||||
tracing::warn!(%root, error = %error, "Guardian intake unavailable");
|
||||
}
|
||||
})
|
||||
}
|
||||
fn on_thread_stop<'a>(&'a self, input: ThreadStopInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
if let Some(worker) = input.thread_store.remove::<RootWorker>() {
|
||||
input.thread_store.remove::<RootEvidence>();
|
||||
worker.lifetime.cancellation.cancel();
|
||||
worker.lifetime.tasks.close();
|
||||
worker.lifetime.tasks.wait().await;
|
||||
|
||||
@@ -8,6 +8,9 @@ use codex_message_board::LocalBoardOptions;
|
||||
use codex_protocol::SessionId;
|
||||
use codex_state::SqliteConfig;
|
||||
|
||||
mod cutoff;
|
||||
pub use cutoff::ReviewCutoff;
|
||||
pub use cutoff::RootEvidence;
|
||||
mod delivery;
|
||||
mod evidence;
|
||||
mod host;
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//! A local board host whose next write can be held until the test releases it.
|
||||
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_message_board::MessageBoardHost;
|
||||
use codex_message_board::NotificationDelivery;
|
||||
use codex_message_board::PostPreview;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use futures::future::BoxFuture;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::AtomicI64;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[path = "../../../../message-board/tests/common/mod.rs"]
|
||||
mod shared;
|
||||
|
||||
pub(crate) fn board_host(root: ThreadId) -> Arc<shared::Host> {
|
||||
Arc::new(shared::Host {
|
||||
clock: AtomicI64::default(),
|
||||
fail_clock: AtomicBool::new(false),
|
||||
agent_path_calls: AtomicUsize::default(),
|
||||
members: [(root, AgentPath::root())].into(),
|
||||
active: AtomicBool::new(true),
|
||||
fail_notifications: AtomicBool::new(false),
|
||||
notifications: Mutex::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) struct PausedClockHost {
|
||||
pub(crate) inner: Arc<shared::Host>,
|
||||
pub(crate) pause_next: AtomicBool,
|
||||
pub(crate) started: Notify,
|
||||
pub(crate) release: Notify,
|
||||
}
|
||||
|
||||
impl MessageBoardHost for PausedClockHost {
|
||||
fn agent_path(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult<AgentPath>> {
|
||||
self.inner.agent_path(caller)
|
||||
}
|
||||
|
||||
fn resolve_agent(&self, path: AgentPath) -> BoxFuture<'_, CodexResult<ThreadId>> {
|
||||
self.inner.resolve_agent(path)
|
||||
}
|
||||
|
||||
fn current_time(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult<DateTime<Utc>>> {
|
||||
Box::pin(async move {
|
||||
if self.pause_next.swap(false, Ordering::SeqCst) {
|
||||
self.started.notify_one();
|
||||
self.release.notified().await;
|
||||
}
|
||||
self.inner.current_time(caller).await
|
||||
})
|
||||
}
|
||||
|
||||
fn notify(
|
||||
&self,
|
||||
recipient: ThreadId,
|
||||
post: PostPreview,
|
||||
) -> BoxFuture<'_, CodexResult<NotificationDelivery>> {
|
||||
self.inner.notify(recipient, post)
|
||||
}
|
||||
}
|
||||
@@ -185,6 +185,7 @@ Approved action: {"action":{"command":"write /workspace/report.txt"},"outcome":"
|
||||
)
|
||||
.await?;
|
||||
assert!(!store.live_rollout_path(owner).await?.exists());
|
||||
assert!(fresh.capture(&store).await.is_err());
|
||||
// A resume must establish its tail even if the rollout is missing.
|
||||
assert!(
|
||||
EvidenceCursor::at_thread_ready(
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
#![allow(clippy::expect_used)]
|
||||
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_guardian_v3::EvidenceCursor;
|
||||
use codex_guardian_v3::HOUSEKEEPER_INSTRUCTIONS;
|
||||
@@ -13,18 +12,13 @@ use codex_history::ResponseItemEnvelope;
|
||||
use codex_history::RolloutItem;
|
||||
use codex_message_board::AgentMessageBoard;
|
||||
use codex_message_board::ChannelQuery;
|
||||
use codex_message_board::MessageBoardHost;
|
||||
use codex_message_board::NotificationDelivery;
|
||||
use codex_message_board::PageRequest;
|
||||
use codex_message_board::PostPreview;
|
||||
use codex_message_board::PostQuery;
|
||||
use codex_message_board::ReadPostRequest;
|
||||
use codex_message_board::SortDirection;
|
||||
use codex_message_board::UpsertChannelRequest;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::openai_models::TruncationPolicyConfig;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_state::SqliteConfig;
|
||||
@@ -34,24 +28,24 @@ use core_test_support::context_snapshot;
|
||||
use core_test_support::context_snapshot::ContextSnapshotOptions;
|
||||
use core_test_support::responses::*;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use futures::future::BoxFuture;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::AtomicI64;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[path = "../../../message-board/tests/common/mod.rs"]
|
||||
#[path = "common/board_host.rs"]
|
||||
mod board_host;
|
||||
|
||||
use board_host::PausedClockHost;
|
||||
use board_host::board_host;
|
||||
|
||||
enum RoutingCase {
|
||||
Success,
|
||||
RetryExhausted,
|
||||
@@ -480,53 +474,6 @@ async fn routes_captured_originals_and_updates_descriptions(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn board_host(root: ThreadId) -> Arc<board_host::Host> {
|
||||
Arc::new(board_host::Host {
|
||||
clock: AtomicI64::default(),
|
||||
fail_clock: AtomicBool::new(false),
|
||||
agent_path_calls: AtomicUsize::default(),
|
||||
members: [(root, AgentPath::root())].into(),
|
||||
active: AtomicBool::new(true),
|
||||
fail_notifications: AtomicBool::new(false),
|
||||
notifications: Mutex::default(),
|
||||
})
|
||||
}
|
||||
|
||||
struct PausedClockHost {
|
||||
inner: Arc<board_host::Host>,
|
||||
pause_next: AtomicBool,
|
||||
started: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
impl MessageBoardHost for PausedClockHost {
|
||||
fn agent_path(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult<AgentPath>> {
|
||||
self.inner.agent_path(caller)
|
||||
}
|
||||
|
||||
fn resolve_agent(&self, path: AgentPath) -> BoxFuture<'_, CodexResult<ThreadId>> {
|
||||
self.inner.resolve_agent(path)
|
||||
}
|
||||
|
||||
fn current_time(&self, caller: ThreadId) -> BoxFuture<'_, CodexResult<DateTime<Utc>>> {
|
||||
Box::pin(async move {
|
||||
if self.pause_next.swap(false, Ordering::SeqCst) {
|
||||
self.started.notify_one();
|
||||
self.release.notified().await;
|
||||
}
|
||||
self.inner.current_time(caller).await
|
||||
})
|
||||
}
|
||||
|
||||
fn notify(
|
||||
&self,
|
||||
recipient: ThreadId,
|
||||
post: PostPreview,
|
||||
) -> BoxFuture<'_, CodexResult<NotificationDelivery>> {
|
||||
self.inner.notify(recipient, post)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn root_shutdown_waits_for_the_accepted_evidence_append() -> anyhow::Result<()> {
|
||||
use codex_extension_api::ExtensionData;
|
||||
|
||||
Reference in New Issue
Block a user