mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
core: fence Guardian session cleanup by generation
This commit is contained in:
@@ -172,8 +172,6 @@ use prompt::render_guardian_transcript_entries;
|
||||
#[cfg(test)]
|
||||
use review::GuardianReviewOutcome;
|
||||
#[cfg(test)]
|
||||
use review::run_guardian_review_session as run_guardian_review_session_for_test;
|
||||
#[cfg(test)]
|
||||
use review_session::build_guardian_review_session_config as build_guardian_review_session_config_for_test;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -18,6 +18,7 @@ use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::TurnAbortReason;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -40,9 +41,11 @@ use super::approval_request::guardian_reviewed_action;
|
||||
use super::metrics::emit_guardian_review_metrics;
|
||||
use super::prompt::guardian_output_schema;
|
||||
use super::prompt::parse_guardian_assessment;
|
||||
use super::review_session::GuardianReviewSessionGeneration;
|
||||
use super::review_session::GuardianReviewSessionOutcome;
|
||||
use super::review_session::GuardianReviewSessionParams;
|
||||
use super::review_session::build_guardian_review_session_config;
|
||||
use super::task_owner::GuardianReviewActivity;
|
||||
|
||||
const GUARDIAN_REJECTION_INSTRUCTIONS: &str = concat!(
|
||||
"The agent must not attempt to achieve the same outcome via workaround, ",
|
||||
@@ -58,6 +61,8 @@ const GUARDIAN_TIMEOUT_INSTRUCTIONS: &str = concat!(
|
||||
"You may retry once, or ask the user for guidance or explicit approval.",
|
||||
);
|
||||
|
||||
const GUARDIAN_REVIEW_CANCEL_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
pub(crate) fn new_guardian_review_id() -> String {
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
@@ -93,6 +98,12 @@ pub(super) enum GuardianReviewOutcome {
|
||||
Error(GuardianReviewError),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct GuardianReviewRunContext {
|
||||
pub(super) activity: GuardianReviewActivity,
|
||||
pub(super) generation: GuardianReviewSessionGeneration,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum GuardianReviewError {
|
||||
PromptBuild { message: String },
|
||||
@@ -258,10 +269,11 @@ async fn run_guardian_review(
|
||||
request: GuardianApprovalRequest,
|
||||
retry_reason: Option<String>,
|
||||
approval_request_source: GuardianApprovalRequestSource,
|
||||
external_cancel: Option<CancellationToken>,
|
||||
review_context: GuardianReviewRunContext,
|
||||
) -> ReviewDecision {
|
||||
let target_item_id = guardian_request_target_item_id(&request).map(str::to_string);
|
||||
let assessment_turn_id = guardian_request_turn_id(&request, &turn.sub_id).to_string();
|
||||
let cancellation_token = review_context.activity.cancellation_token();
|
||||
let action_summary = guardian_assessment_action(&request);
|
||||
let reviewed_action = guardian_reviewed_action(&request);
|
||||
let review_tracking = GuardianReviewTrackContext::new(
|
||||
@@ -293,10 +305,7 @@ async fn run_guardian_review(
|
||||
)
|
||||
.await;
|
||||
|
||||
if external_cancel
|
||||
.as_ref()
|
||||
.is_some_and(CancellationToken::is_cancelled)
|
||||
{
|
||||
if cancellation_token.is_cancelled() {
|
||||
let completed_at_ms = now_unix_timestamp_ms();
|
||||
track_guardian_review(
|
||||
session.as_ref(),
|
||||
@@ -335,15 +344,42 @@ async fn run_guardian_review(
|
||||
|
||||
let schema = guardian_output_schema();
|
||||
let terminal_action = action_summary.clone();
|
||||
let (outcome, analytics_result) = Box::pin(run_guardian_review_session(
|
||||
let reset_generation = review_context.generation.clone();
|
||||
let mut session_review = Box::pin(run_guardian_review_session(
|
||||
session.clone(),
|
||||
turn.clone(),
|
||||
review_context.clone(),
|
||||
request,
|
||||
retry_reason.clone(),
|
||||
schema,
|
||||
external_cancel,
|
||||
))
|
||||
.await;
|
||||
Some(cancellation_token.clone()),
|
||||
));
|
||||
let (outcome, analytics_result) = tokio::select! {
|
||||
biased;
|
||||
_ = cancellation_token.cancelled() => {
|
||||
if review_context.activity.is_committed() {
|
||||
session_review.await
|
||||
} else {
|
||||
let analytics_result = match tokio::time::timeout(
|
||||
GUARDIAN_REVIEW_CANCEL_DRAIN_TIMEOUT,
|
||||
&mut session_review,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((_, analytics_result)) => analytics_result,
|
||||
Err(_) => {
|
||||
session.guardian_review_session.shutdown_generation_in_background(
|
||||
reset_generation,
|
||||
&session.services.runtime_handle,
|
||||
);
|
||||
GuardianReviewAnalyticsResult::without_session()
|
||||
}
|
||||
};
|
||||
(GuardianReviewOutcome::Error(GuardianReviewError::Cancelled), analytics_result)
|
||||
}
|
||||
}
|
||||
result = &mut session_review => result,
|
||||
};
|
||||
|
||||
let completed_at_ms = now_unix_timestamp_ms();
|
||||
let (assessment, count_denial_for_circuit_breaker) = match outcome {
|
||||
@@ -554,7 +590,6 @@ async fn run_guardian_review(
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
if count_denial_for_circuit_breaker {
|
||||
record_guardian_denial(&session, &turn, &assessment_turn_id).await;
|
||||
} else {
|
||||
@@ -583,6 +618,7 @@ async fn run_guardian_review_in_task(
|
||||
let cancel_activity = review_activity.clone();
|
||||
let request_cancel = owner_cancel.clone();
|
||||
let runtime_handle = session.services.runtime_handle.clone();
|
||||
let review_generation = session.guardian_review_session.generation();
|
||||
let review = run_guardian_review(
|
||||
session,
|
||||
Arc::clone(&turn),
|
||||
@@ -590,7 +626,10 @@ async fn run_guardian_review_in_task(
|
||||
request,
|
||||
retry_reason,
|
||||
approval_request_source,
|
||||
Some(review_activity.cancellation_token()),
|
||||
GuardianReviewRunContext {
|
||||
activity: review_activity.clone(),
|
||||
generation: review_generation,
|
||||
},
|
||||
);
|
||||
let Some(review) = turn.guardian_reviews.spawn(&runtime_handle, async move {
|
||||
tokio::pin!(review);
|
||||
@@ -702,6 +741,7 @@ pub(crate) fn spawn_approval_request_review(
|
||||
pub(super) async fn run_guardian_review_session(
|
||||
session: Arc<Session>,
|
||||
turn: Arc<TurnContext>,
|
||||
review_context: GuardianReviewRunContext,
|
||||
request: GuardianApprovalRequest,
|
||||
retry_reason: Option<String>,
|
||||
schema: serde_json::Value,
|
||||
@@ -780,10 +820,10 @@ pub(super) async fn run_guardian_review_session(
|
||||
}
|
||||
};
|
||||
|
||||
let (session_outcome, session_analytics_result) = Box::pin(
|
||||
session
|
||||
.guardian_review_session
|
||||
.run_review(GuardianReviewSessionParams {
|
||||
let (session_outcome, session_analytics_result) =
|
||||
Box::pin(session.guardian_review_session.run_review(
|
||||
review_context.generation,
|
||||
GuardianReviewSessionParams {
|
||||
parent_session: Arc::clone(&session),
|
||||
parent_turn: turn.clone(),
|
||||
spawn_config: guardian_config,
|
||||
@@ -795,9 +835,15 @@ pub(super) async fn run_guardian_review_session(
|
||||
reasoning_summary: turn.reasoning_summary,
|
||||
personality: turn.personality,
|
||||
external_cancel,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
review_activity: review_context.activity.clone(),
|
||||
},
|
||||
))
|
||||
.await;
|
||||
let session_outcome = match session_outcome {
|
||||
GuardianReviewSessionOutcome::Aborted => GuardianReviewSessionOutcome::Aborted,
|
||||
outcome if review_context.activity.try_commit() => outcome,
|
||||
_ => GuardianReviewSessionOutcome::Aborted,
|
||||
};
|
||||
|
||||
match session_outcome {
|
||||
GuardianReviewSessionOutcome::Completed(Ok(last_agent_message)) => match last_agent_message
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use arc_swap::ArcSwap;
|
||||
use codex_analytics::GuardianReviewAnalyticsResult;
|
||||
use codex_analytics::GuardianReviewSessionKind;
|
||||
use codex_protocol::ThreadId;
|
||||
@@ -54,6 +55,7 @@ use super::prompt::GuardianTranscriptCursor;
|
||||
use super::prompt::build_guardian_prompt_items_with_parent_turn;
|
||||
use super::prompt::guardian_policy_prompt;
|
||||
use super::prompt::guardian_policy_prompt_with_config;
|
||||
use super::task_owner::GuardianReviewActivity;
|
||||
|
||||
const GUARDIAN_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
#[derive(Debug)]
|
||||
@@ -77,15 +79,20 @@ pub(crate) struct GuardianReviewSessionParams {
|
||||
pub(crate) reasoning_summary: ReasoningSummaryConfig,
|
||||
pub(crate) personality: Option<Personality>,
|
||||
pub(crate) external_cancel: Option<CancellationToken>,
|
||||
pub(crate) review_activity: GuardianReviewActivity,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct GuardianReviewSessionManager {
|
||||
state: Arc<Mutex<GuardianReviewSessionState>>,
|
||||
state: ArcSwap<Mutex<GuardianReviewSessionState>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GuardianReviewSessionGeneration(Arc<Mutex<GuardianReviewSessionState>>);
|
||||
|
||||
#[derive(Default)]
|
||||
struct GuardianReviewSessionState {
|
||||
closed: bool,
|
||||
trunk: Option<Arc<GuardianReviewSession>>,
|
||||
ephemeral_reviews: Vec<Arc<GuardianReviewSession>>,
|
||||
}
|
||||
@@ -276,8 +283,31 @@ impl Drop for EphemeralReviewCleanup {
|
||||
}
|
||||
|
||||
impl GuardianReviewSessionManager {
|
||||
fn state(&self) -> Arc<Mutex<GuardianReviewSessionState>> {
|
||||
self.state.load_full()
|
||||
}
|
||||
|
||||
pub(crate) fn generation(&self) -> GuardianReviewSessionGeneration {
|
||||
GuardianReviewSessionGeneration(self.state())
|
||||
}
|
||||
|
||||
async fn take_review_sessions(
|
||||
state: &Mutex<GuardianReviewSessionState>,
|
||||
) -> (
|
||||
Option<Arc<GuardianReviewSession>>,
|
||||
Vec<Arc<GuardianReviewSession>>,
|
||||
) {
|
||||
let mut state = state.lock().await;
|
||||
state.closed = true;
|
||||
(
|
||||
state.trunk.take(),
|
||||
std::mem::take(&mut state.ephemeral_reviews),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn trunk_rollout_path(&self) -> Option<PathBuf> {
|
||||
let trunk = self.state.lock().await.trunk.clone()?;
|
||||
let state = self.state();
|
||||
let trunk = state.lock().await.trunk.clone()?;
|
||||
trunk.codex.session.ensure_rollout_materialized().await;
|
||||
match trunk.codex.session.current_rollout_path().await {
|
||||
Ok(path) => path,
|
||||
@@ -289,13 +319,10 @@ impl GuardianReviewSessionManager {
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) {
|
||||
let (review_session, ephemeral_reviews) = {
|
||||
let mut state = self.state.lock().await;
|
||||
(
|
||||
state.trunk.take(),
|
||||
std::mem::take(&mut state.ephemeral_reviews),
|
||||
)
|
||||
};
|
||||
let state = self
|
||||
.state
|
||||
.swap(Arc::new(Mutex::new(GuardianReviewSessionState::default())));
|
||||
let (review_session, ephemeral_reviews) = Self::take_review_sessions(&state).await;
|
||||
if let Some(review_session) = review_session {
|
||||
review_session.shutdown().await;
|
||||
}
|
||||
@@ -304,14 +331,53 @@ impl GuardianReviewSessionManager {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn shutdown_in_background(&self, runtime_handle: &tokio::runtime::Handle) {
|
||||
self.shutdown_generation_in_background(self.generation(), runtime_handle);
|
||||
}
|
||||
|
||||
pub(crate) fn shutdown_generation_in_background(
|
||||
&self,
|
||||
generation: GuardianReviewSessionGeneration,
|
||||
runtime_handle: &tokio::runtime::Handle,
|
||||
) {
|
||||
let previous = self.state.compare_and_swap(
|
||||
&generation.0,
|
||||
Arc::new(Mutex::new(GuardianReviewSessionState::default())),
|
||||
);
|
||||
let did_swap = Arc::ptr_eq(&*previous, &generation.0);
|
||||
drop(previous);
|
||||
if !did_swap {
|
||||
return;
|
||||
}
|
||||
let state = generation.0;
|
||||
drop(runtime_handle.spawn(async move {
|
||||
let (review_session, ephemeral_reviews) =
|
||||
GuardianReviewSessionManager::take_review_sessions(&state).await;
|
||||
if let Some(review_session) = review_session {
|
||||
review_session.shutdown().await;
|
||||
}
|
||||
for review_session in ephemeral_reviews {
|
||||
review_session.shutdown().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "review session selection and trunk spawning must stay serialized"
|
||||
)]
|
||||
pub(super) async fn run_review(
|
||||
&self,
|
||||
generation: GuardianReviewSessionGeneration,
|
||||
params: GuardianReviewSessionParams,
|
||||
) -> (GuardianReviewSessionOutcome, GuardianReviewAnalyticsResult) {
|
||||
if params.review_activity.cancellation_token().is_cancelled() {
|
||||
return (
|
||||
GuardianReviewSessionOutcome::Aborted,
|
||||
GuardianReviewAnalyticsResult::without_session(),
|
||||
);
|
||||
}
|
||||
let state = generation.0;
|
||||
let deadline = tokio::time::Instant::now() + GUARDIAN_REVIEW_TIMEOUT;
|
||||
let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(¶ms.spawn_config);
|
||||
let mut stale_trunk_to_shutdown = None;
|
||||
@@ -319,11 +385,17 @@ impl GuardianReviewSessionManager {
|
||||
let trunk_candidate = match run_before_review_deadline(
|
||||
deadline,
|
||||
params.external_cancel.as_ref(),
|
||||
self.state.lock(),
|
||||
state.lock(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(mut state) => {
|
||||
if state.closed {
|
||||
return (
|
||||
GuardianReviewSessionOutcome::Aborted,
|
||||
GuardianReviewAnalyticsResult::without_session(),
|
||||
);
|
||||
}
|
||||
if let Some(trunk) = state.trunk.as_ref()
|
||||
&& trunk.reuse_key != next_reuse_key
|
||||
&& trunk.review_lock.try_acquire().is_ok()
|
||||
@@ -382,6 +454,7 @@ impl GuardianReviewSessionManager {
|
||||
|
||||
if trunk.reuse_key != next_reuse_key {
|
||||
return Box::pin(self.run_ephemeral_review(
|
||||
Arc::clone(&state),
|
||||
params,
|
||||
next_reuse_key,
|
||||
deadline,
|
||||
@@ -394,6 +467,7 @@ impl GuardianReviewSessionManager {
|
||||
Ok(trunk_guard) => trunk_guard,
|
||||
Err(_) => {
|
||||
return Box::pin(self.run_ephemeral_review(
|
||||
Arc::clone(&state),
|
||||
params,
|
||||
next_reuse_key,
|
||||
deadline,
|
||||
@@ -423,7 +497,7 @@ impl GuardianReviewSessionManager {
|
||||
if keep_review_session {
|
||||
(outcome, analytics_result)
|
||||
} else {
|
||||
if let Some(review_session) = self.remove_trunk_if_current(&trunk).await {
|
||||
if let Some(review_session) = Self::remove_trunk_if_current(&state, &trunk).await {
|
||||
review_session.shutdown_in_background();
|
||||
}
|
||||
(outcome, analytics_result)
|
||||
@@ -435,7 +509,8 @@ impl GuardianReviewSessionManager {
|
||||
let reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(
|
||||
codex.session.get_config().await.as_ref(),
|
||||
);
|
||||
self.state.lock().await.trunk = Some(Arc::new(GuardianReviewSession {
|
||||
let state = self.state();
|
||||
state.lock().await.trunk = Some(Arc::new(GuardianReviewSession {
|
||||
reuse_key,
|
||||
codex,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -453,7 +528,8 @@ impl GuardianReviewSessionManager {
|
||||
let reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(
|
||||
codex.session.get_config().await.as_ref(),
|
||||
);
|
||||
self.state
|
||||
let state = self.state();
|
||||
state
|
||||
.lock()
|
||||
.await
|
||||
.ephemeral_reviews
|
||||
@@ -472,7 +548,8 @@ impl GuardianReviewSessionManager {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn committed_fork_rollout_items_for_test(&self) -> Option<Vec<RolloutItem>> {
|
||||
let trunk = self.state.lock().await.trunk.clone()?;
|
||||
let manager_state = self.state();
|
||||
let trunk = manager_state.lock().await.trunk.clone()?;
|
||||
let state = trunk.state.lock().await;
|
||||
let snapshot = state.last_committed_fork_snapshot.as_ref()?;
|
||||
match &snapshot.initial_history {
|
||||
@@ -483,8 +560,8 @@ impl GuardianReviewSessionManager {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn send_trunk_event_raw_for_test(&self, event: Event) {
|
||||
let trunk = self
|
||||
.state
|
||||
let state = self.state();
|
||||
let trunk = state
|
||||
.lock()
|
||||
.await
|
||||
.trunk
|
||||
@@ -494,10 +571,10 @@ impl GuardianReviewSessionManager {
|
||||
}
|
||||
|
||||
async fn remove_trunk_if_current(
|
||||
&self,
|
||||
state: &Mutex<GuardianReviewSessionState>,
|
||||
trunk: &Arc<GuardianReviewSession>,
|
||||
) -> Option<Arc<GuardianReviewSession>> {
|
||||
let mut state = self.state.lock().await;
|
||||
let mut state = state.lock().await;
|
||||
if state
|
||||
.trunk
|
||||
.as_ref()
|
||||
@@ -509,19 +586,23 @@ impl GuardianReviewSessionManager {
|
||||
}
|
||||
}
|
||||
|
||||
async fn register_active_ephemeral(&self, review_session: Arc<GuardianReviewSession>) {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.ephemeral_reviews
|
||||
.push(review_session);
|
||||
async fn register_active_ephemeral(
|
||||
state: &Mutex<GuardianReviewSessionState>,
|
||||
review_session: Arc<GuardianReviewSession>,
|
||||
) -> bool {
|
||||
let mut state = state.lock().await;
|
||||
if state.closed {
|
||||
return false;
|
||||
}
|
||||
state.ephemeral_reviews.push(review_session);
|
||||
true
|
||||
}
|
||||
|
||||
async fn take_active_ephemeral(
|
||||
&self,
|
||||
state: &Mutex<GuardianReviewSessionState>,
|
||||
review_session: &Arc<GuardianReviewSession>,
|
||||
) -> Option<Arc<GuardianReviewSession>> {
|
||||
let mut state = self.state.lock().await;
|
||||
let mut state = state.lock().await;
|
||||
let ephemeral_review_index = state
|
||||
.ephemeral_reviews
|
||||
.iter()
|
||||
@@ -531,6 +612,7 @@ impl GuardianReviewSessionManager {
|
||||
|
||||
async fn run_ephemeral_review(
|
||||
&self,
|
||||
state: Arc<Mutex<GuardianReviewSessionState>>,
|
||||
params: GuardianReviewSessionParams,
|
||||
reuse_key: GuardianReviewSessionReuseKey,
|
||||
deadline: tokio::time::Instant,
|
||||
@@ -562,10 +644,15 @@ impl GuardianReviewSessionManager {
|
||||
}
|
||||
Err(outcome) => return (outcome, GuardianReviewAnalyticsResult::without_session()),
|
||||
};
|
||||
self.register_active_ephemeral(Arc::clone(&review_session))
|
||||
.await;
|
||||
if !Self::register_active_ephemeral(&state, Arc::clone(&review_session)).await {
|
||||
review_session.shutdown_in_background();
|
||||
return (
|
||||
GuardianReviewSessionOutcome::Aborted,
|
||||
GuardianReviewAnalyticsResult::without_session(),
|
||||
);
|
||||
}
|
||||
let mut cleanup =
|
||||
EphemeralReviewCleanup::new(Arc::clone(&self.state), Arc::clone(&review_session));
|
||||
EphemeralReviewCleanup::new(Arc::clone(&state), Arc::clone(&review_session));
|
||||
|
||||
let (outcome, _, analytics_result) = Box::pin(run_review_on_session(
|
||||
review_session.as_ref(),
|
||||
@@ -574,7 +661,7 @@ impl GuardianReviewSessionManager {
|
||||
deadline,
|
||||
))
|
||||
.await;
|
||||
if let Some(review_session) = self.take_active_ephemeral(&review_session).await {
|
||||
if let Some(review_session) = Self::take_active_ephemeral(&state, &review_session).await {
|
||||
cleanup.disarm();
|
||||
review_session.shutdown_in_background();
|
||||
}
|
||||
@@ -773,7 +860,7 @@ async fn run_review_on_session(
|
||||
};
|
||||
analytics_result.reviewed_action_truncated = reviewed_action_truncated;
|
||||
|
||||
let outcome = wait_for_guardian_review(
|
||||
let (mut outcome, mut keep_review_session, should_commit_review) = wait_for_guardian_review(
|
||||
review_session,
|
||||
child_turn_id.as_str(),
|
||||
deadline,
|
||||
@@ -781,10 +868,15 @@ async fn run_review_on_session(
|
||||
&mut analytics_result,
|
||||
)
|
||||
.await;
|
||||
if matches!(outcome.0, GuardianReviewSessionOutcome::Completed(_)) {
|
||||
if outcome.2
|
||||
&& let Some(total_token_usage) = review_session.codex.session.total_token_usage().await
|
||||
{
|
||||
if should_commit_review
|
||||
&& matches!(outcome, GuardianReviewSessionOutcome::Completed(_))
|
||||
&& !params.review_activity.try_commit()
|
||||
{
|
||||
outcome = GuardianReviewSessionOutcome::Aborted;
|
||||
keep_review_session = false;
|
||||
}
|
||||
if should_commit_review && matches!(outcome, GuardianReviewSessionOutcome::Completed(_)) {
|
||||
if let Some(total_token_usage) = review_session.codex.session.total_token_usage().await {
|
||||
analytics_result.token_usage = Some(token_usage_delta(
|
||||
&token_usage_at_review_start,
|
||||
&total_token_usage,
|
||||
@@ -794,7 +886,7 @@ async fn run_review_on_session(
|
||||
state.prior_review_count = state.prior_review_count.saturating_add(1);
|
||||
state.last_reviewed_transcript_cursor = Some(transcript_cursor);
|
||||
}
|
||||
(outcome.0, outcome.1, analytics_result)
|
||||
(outcome, keep_review_session, analytics_result)
|
||||
}
|
||||
|
||||
async fn append_guardian_followup_reminder(review_session: &GuardianReviewSession) {
|
||||
@@ -845,13 +937,12 @@ async fn wait_for_guardian_review(
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
} => {
|
||||
let keep_review_session = interrupt_and_drain_turn(
|
||||
let _ = interrupt_and_drain_turn(
|
||||
&review_session.codex,
|
||||
expected_turn_id,
|
||||
)
|
||||
.await
|
||||
.is_ok();
|
||||
return (GuardianReviewSessionOutcome::Aborted, keep_review_session, false);
|
||||
.await;
|
||||
return (GuardianReviewSessionOutcome::Aborted, false, false);
|
||||
}
|
||||
event = review_session.codex.next_event() => {
|
||||
match event {
|
||||
@@ -880,7 +971,7 @@ async fn wait_for_guardian_review(
|
||||
last_error_message = Some(error.message);
|
||||
}
|
||||
EventMsg::TurnAborted(_) => {
|
||||
return (GuardianReviewSessionOutcome::Aborted, true, false);
|
||||
return (GuardianReviewSessionOutcome::Aborted, false, false);
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
@@ -1129,9 +1220,14 @@ mod tests {
|
||||
)
|
||||
.expect("guardian config");
|
||||
|
||||
let parent_turn = Arc::new(turn);
|
||||
let review_activity = parent_turn
|
||||
.guardian_reviews
|
||||
.begin()
|
||||
.expect("guardian review activity");
|
||||
GuardianReviewSessionParams {
|
||||
parent_session: Arc::new(session),
|
||||
parent_turn: Arc::new(turn),
|
||||
parent_turn,
|
||||
spawn_config,
|
||||
request: GuardianApprovalRequest::Shell {
|
||||
id: "shell-1".to_string(),
|
||||
@@ -1148,6 +1244,7 @@ mod tests {
|
||||
reasoning_summary,
|
||||
personality,
|
||||
external_cancel: None,
|
||||
review_activity,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1183,6 +1280,52 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_shutdown_replaces_manager_state_without_waiting() {
|
||||
let manager = GuardianReviewSessionManager::default();
|
||||
let old_generation = manager.generation();
|
||||
let old_state = Arc::clone(&old_generation.0);
|
||||
let old_state_guard = old_state.lock().await;
|
||||
|
||||
manager.shutdown_generation_in_background(
|
||||
old_generation.clone(),
|
||||
&tokio::runtime::Handle::current(),
|
||||
);
|
||||
|
||||
let new_state = manager.state();
|
||||
assert!(!Arc::ptr_eq(&old_state, &new_state));
|
||||
manager
|
||||
.shutdown_generation_in_background(old_generation, &tokio::runtime::Handle::current());
|
||||
assert!(Arc::ptr_eq(&new_state, &manager.state()));
|
||||
drop(old_state_guard);
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if old_state.lock().await.closed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retired manager state should close");
|
||||
assert!(!new_state.lock().await.closed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_review_does_not_enter_manager_generation() {
|
||||
let manager = GuardianReviewSessionManager::default();
|
||||
let params = test_review_params().await;
|
||||
assert!(params.review_activity.cancel());
|
||||
|
||||
let (outcome, _) = manager.run_review(manager.generation(), params).await;
|
||||
|
||||
assert!(matches!(outcome, GuardianReviewSessionOutcome::Aborted));
|
||||
let state = manager.state();
|
||||
let state = state.lock().await;
|
||||
assert!(state.trunk.is_none());
|
||||
assert!(state.ephemeral_reviews.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guardian_prompt_cache_key_is_scoped_to_parent_thread() {
|
||||
let session_source =
|
||||
@@ -1473,6 +1616,51 @@ mod tests {
|
||||
assert!(keep_review_session);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn run_review_on_session_does_not_commit_cancelled_activity() {
|
||||
let (review_session, tx_event, rx_sub) = test_review_session().await;
|
||||
let mut params = test_review_params().await;
|
||||
params.review_activity = params
|
||||
.parent_turn
|
||||
.guardian_reviews
|
||||
.begin()
|
||||
.expect("guardian review activity");
|
||||
let review_activity = params.review_activity.clone();
|
||||
|
||||
let review = tokio::spawn(async move {
|
||||
let outcome = run_review_on_session(
|
||||
&review_session,
|
||||
¶ms,
|
||||
GuardianReviewSessionKind::TrunkReused,
|
||||
tokio::time::Instant::now() + Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
let state = review_session.state.lock().await;
|
||||
(
|
||||
outcome,
|
||||
state.prior_review_count,
|
||||
state.last_reviewed_transcript_cursor,
|
||||
)
|
||||
});
|
||||
let submission = rx_sub.recv().await.expect("guardian submission");
|
||||
assert!(review_activity.cancel());
|
||||
tx_event
|
||||
.send(turn_complete_event(
|
||||
submission.id.as_str(),
|
||||
Some("completed after cancellation"),
|
||||
Some(42),
|
||||
))
|
||||
.await
|
||||
.expect("queue submitted turn completion");
|
||||
|
||||
let ((outcome, keep_review_session, _), prior_review_count, transcript_cursor) =
|
||||
review.await.expect("review task should complete");
|
||||
assert!(matches!(outcome, GuardianReviewSessionOutcome::Aborted));
|
||||
assert!(!keep_review_session);
|
||||
assert_eq!(prior_review_count, 0);
|
||||
assert!(transcript_cursor.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_for_guardian_review_ignores_prior_turn_completion() {
|
||||
let (review_session, tx_event, _rx_sub) = test_review_session().await;
|
||||
@@ -1644,7 +1832,7 @@ mod tests {
|
||||
.await
|
||||
.expect("interrupt response task should complete");
|
||||
assert!(matches!(outcome, GuardianReviewSessionOutcome::Aborted));
|
||||
assert!(keep_review_session);
|
||||
assert!(!keep_review_session);
|
||||
assert!(!capture_token_usage);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::runtime::Handle;
|
||||
@@ -53,7 +55,9 @@ impl GuardianReviewTaskOwner {
|
||||
return None;
|
||||
}
|
||||
Some(GuardianReviewActivity {
|
||||
owner: Arc::clone(self),
|
||||
cancellation_token: self.cancellation_token.child_token(),
|
||||
committed: Arc::new(AtomicBool::new(false)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -97,7 +101,9 @@ impl GuardianReviewTaskOwner {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GuardianReviewActivity {
|
||||
owner: Arc<GuardianReviewTaskOwner>,
|
||||
cancellation_token: CancellationToken,
|
||||
committed: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl GuardianReviewActivity {
|
||||
@@ -105,8 +111,35 @@ impl GuardianReviewActivity {
|
||||
self.cancellation_token.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn cancel(&self) {
|
||||
pub(crate) fn cancel(&self) -> bool {
|
||||
if self.committed.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
let _state = self.owner.lock_state();
|
||||
if self.committed.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
self.cancellation_token.cancel();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn is_committed(&self) -> bool {
|
||||
self.committed.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn try_commit(&self) -> bool {
|
||||
if self.committed.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
let state = self.owner.lock_state();
|
||||
if self.committed.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
let can_commit = state.closed_at.is_none() && !self.cancellation_token.is_cancelled();
|
||||
if can_commit {
|
||||
self.committed.store(true, Ordering::Release);
|
||||
}
|
||||
can_commit
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -47,3 +47,25 @@ async fn forced_drain_aborts_review_task() {
|
||||
Err(oneshot::error::TryRecvError::Closed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commit_is_linearized_against_cancellation() {
|
||||
let committed_owner = Arc::new(GuardianReviewTaskOwner::default());
|
||||
let committed = committed_owner.begin().expect("review should start");
|
||||
assert!(committed.try_commit());
|
||||
let _drain = committed_owner.close();
|
||||
assert!(committed.try_commit());
|
||||
|
||||
let interrupted_owner = Arc::new(GuardianReviewTaskOwner::default());
|
||||
let interrupted = interrupted_owner
|
||||
.begin()
|
||||
.expect("review should start before interruption");
|
||||
let _drain = interrupted_owner.close();
|
||||
assert!(!interrupted.try_commit());
|
||||
assert!(interrupted_owner.begin().is_none());
|
||||
|
||||
let cancelled_owner = Arc::new(GuardianReviewTaskOwner::default());
|
||||
let cancelled = cancelled_owner.begin().expect("review should start");
|
||||
assert!(cancelled.cancel());
|
||||
assert!(!cancelled.try_commit());
|
||||
}
|
||||
|
||||
@@ -79,6 +79,37 @@ fn fixed_guardian_parent_session_id() -> ThreadId {
|
||||
.expect("fixed parent session id should be a valid UUID")
|
||||
}
|
||||
|
||||
async fn run_guardian_review_session_for_test(
|
||||
session: Arc<Session>,
|
||||
turn: Arc<TurnContext>,
|
||||
request: GuardianApprovalRequest,
|
||||
retry_reason: Option<String>,
|
||||
schema: serde_json::Value,
|
||||
external_cancel: Option<CancellationToken>,
|
||||
) -> (
|
||||
GuardianReviewOutcome,
|
||||
codex_analytics::GuardianReviewAnalyticsResult,
|
||||
) {
|
||||
let review_activity = turn
|
||||
.guardian_reviews
|
||||
.begin()
|
||||
.expect("guardian review activity");
|
||||
let review_generation = session.guardian_review_session.generation();
|
||||
super::review::run_guardian_review_session(
|
||||
session,
|
||||
turn,
|
||||
super::review::GuardianReviewRunContext {
|
||||
activity: review_activity,
|
||||
generation: review_generation,
|
||||
},
|
||||
request,
|
||||
retry_reason,
|
||||
schema,
|
||||
external_cancel,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guardian_rejection_circuit_breaker_interrupts_after_three_consecutive_denials() {
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
|
||||
@@ -552,7 +552,8 @@ impl Session {
|
||||
|
||||
async fn drain_guardian_reviews(&self, drain: crate::guardian::GuardianReviewDrain) {
|
||||
if drain.drain().await == crate::guardian::GuardianReviewDrainOutcome::Forced {
|
||||
self.guardian_review_session.shutdown().await;
|
||||
self.guardian_review_session
|
||||
.shutdown_in_background(&self.services.runtime_handle);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user