Fix turn interrupt reply ordering and shutdown handling (#62066)

## Why

A delayed terminal event from an earlier turn could acknowledge interrupts for a later turn. Interrupts could also remain pending after the thread listener shut down.

## What changed

- Associate pending `turn/interrupt` requests with their target turn and reply only when that turn completes or aborts.
- Track the current Core turn through stop hooks and resolve review delegate IDs to the owning turn when validating interrupts.
- Settle pending interrupts with an error on listener shutdown and reject further interrupts against the retired runtime.

## Testing

Add regression tests for delayed terminal events, queued turn starts, review delegates, and interrupt handling during thread revert and runtime replacement.

GitOrigin-RevId: 9695fc3f718baa6b792a14d54136bb87fc003100
This commit is contained in:
jif
2026-11-17 15:33:21 +00:00
committed by copyberry
parent 0cc88d0c74
commit 58627e6fea
9 changed files with 659 additions and 47 deletions
@@ -200,7 +200,7 @@ pub(crate) async fn apply_bespoke_event_handling(
.await;
}
EventMsg::TurnComplete(turn_complete_event) => {
respond_to_pending_interrupts(&thread_state, &outgoing).await;
respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await;
if finishes_latest_turn {
let turn_failed = thread_state
.lock()
@@ -1297,7 +1297,7 @@ pub(crate) async fn apply_bespoke_event_handling(
}
// If this is a TurnAborted, reply to any pending interrupt requests.
EventMsg::TurnAborted(turn_aborted_event) => {
respond_to_pending_interrupts(&thread_state, &outgoing).await;
respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await;
if finishes_latest_turn {
thread_watch_manager
@@ -1652,12 +1652,17 @@ async fn handle_turn_interrupted(
}
async fn respond_to_pending_interrupts(
event_turn_id: &str,
thread_state: &Arc<Mutex<ThreadState>>,
outgoing: &ThreadScopedOutgoingMessageSender,
) {
let pending = {
let mut state = thread_state.lock().await;
std::mem::take(&mut state.pending_interrupts)
state
.pending_interrupts
.extract_if(.., |(turn_id, _)| turn_id == event_turn_id)
.map(|(_, request_id)| request_id)
.collect::<Vec<_>>()
};
for request_id in pending {
@@ -756,6 +756,25 @@ async fn start_listener(
drop(turn_cost_worker);
if shutdown_complete {
// Close registration and take the queue under the same lock used by turn/interrupt.
// Detached finalizers can publish their terminal events after this listener retires.
let pending_interrupts = {
let mut state = thread_state.lock().await;
if state.listener_generation == listener_generation {
state.listener_shutdown = true;
std::mem::take(&mut state.pending_interrupts)
} else {
Vec::new()
}
};
for (_, request_id) in pending_interrupts {
outgoing_for_task
.send_error(
request_id,
internal_error("thread shut down before turn interruption completed"),
)
.await;
}
drained_tx.send_replace(true);
let relay_command = |command| {
handle_thread_listener_command(
@@ -610,3 +610,228 @@ async fn resume_preserves_recovery_turn_before_status_publication() -> Result<()
processor.skills_watcher.shutdown();
Ok(())
}
#[tokio::test]
async fn revert_settles_interrupts_before_replacing_a_runtime() -> Result<()> {
use super::super::turn_processor::TurnRequestProcessor;
use crate::outgoing_message::OutgoingError;
use codex_app_server_protocol::ThreadRevertParams;
use codex_app_server_protocol::TurnInterruptParams;
use codex_core::TurnInputRequest;
use codex_core::TurnInputSubmission;
use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::TurnLifecycleContributor;
use codex_extension_api::TurnStopInput;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::ThreadHistoryMode;
use tokio::sync::Notify;
#[derive(Default)]
struct StopGate {
entered: Notify,
release: Notify,
}
impl TurnLifecycleContributor for StopGate {
fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async {
self.entered.notify_one();
self.release.notified().await;
})
}
}
let gate = Arc::new(StopGate::default());
let mut extensions = ExtensionRegistryBuilder::new();
extensions.turn_lifecycle_contributor(gate.clone());
let server = responses::start_mock_server().await;
responses::mount_sse_once(&server, responses::sse_completed("first")).await;
let test = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_history_mode(ThreadHistoryMode::Paginated)
.with_config(|config| {
config.agents_enabled = false;
config
.features
.disable(codex_features::Feature::MultiAgentV2)
.unwrap();
})
.build_with_auto_env(&server)
.await?;
let thread_id = test.session_configured.thread_id;
let (processor, mut outgoing_rx) = test_processor(&test).await?;
// Keep transport flowing, retaining every reply and turn boundary in client-visible order.
let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
let collector = tokio::spawn(async move {
while let Some(envelope) = outgoing_rx.recv().await {
let OutgoingEnvelope::ToConnection { message, .. } = envelope else {
continue;
};
let relevant = match &message {
OutgoingMessage::Response(_) | OutgoingMessage::Error(_) => true,
OutgoingMessage::AppServerNotification(envelope) => matches!(
envelope.notification,
ServerNotification::TurnStarted(_) | ServerNotification::TurnCompleted(_)
),
OutgoingMessage::Request(_) => false,
};
if relevant && observed_tx.send(message).is_err() {
break;
}
}
});
let connection_id = ConnectionId(1);
processor
.connection_initialized(connection_id, ConnectionCapabilities::default())
.await;
processor
.try_attach_thread_listener(thread_id, test.codex.clone(), vec![connection_id])
.await;
let turn_processor = TurnRequestProcessor::new(
test.thread_manager.clone(),
processor.outgoing.clone(),
AnalyticsEventsClient::disabled(),
processor.config.clone(),
processor.config_manager.clone(),
processor.pending_thread_unloads.clone(),
processor.thread_state_manager.clone(),
processor.thread_watch_manager.clone(),
processor.skills_watcher.clone(),
/*turn_cost_worker*/ None,
);
let request_id = |id| ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(id),
};
let interrupt = async |id, turn_id: &str| {
turn_processor
.turn_interrupt(
&request_id(id),
TurnInterruptParams {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_owned(),
},
)
.await
};
let start = async |thread: &Arc<CodexThread>, text: &str| -> Result<String> {
let TurnInputSubmission::Started { turn_id, .. } = thread
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: text.to_owned(),
text_elements: vec![],
}]))
.await?
else {
anyhow::bail!("expected a new turn")
};
Ok(turn_id)
};
let first = start(&test.codex, "first").await?;
assert_eq!(
test.codex.multi_agent_version(),
Some(MultiAgentVersion::Disabled)
);
gate.entered.notified().await;
assert!(matches!(observed_rx.recv().await,
Some(OutgoingMessage::AppServerNotification(n))
if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == first)
));
assert!(
interrupt(/*id*/ 99, &first)
.await
.expect("interrupt accepted")
.is_none()
);
let state = processor.thread_state_manager.thread_state(thread_id).await;
assert_eq!(state.lock().await.pending_interrupts.len(), 1);
let mut drain = state
.lock()
.await
.shutdown_drain
.clone()
.expect("listener installed");
let revert = processor.thread_revert(
request_id(/*id*/ 100),
ThreadRevertParams {
thread_id: thread_id.to_string(),
before_turn_id: first.clone(),
},
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
);
tokio::pin!(revert);
// Stop polling revert at its shutdown barrier so late requests still load the retired runtime.
tokio::select! {
biased;
drained = drain.wait_for(|drained| *drained) => { drained?; }
result = &mut revert => { anyhow::bail!("revert passed the shutdown barrier: {result:?}"); }
}
assert!(Arc::ptr_eq(
&test.codex,
&test.thread_manager.get_thread(thread_id).await?
));
let Some(OutgoingMessage::Error(error)) = observed_rx.recv().await else {
anyhow::bail!("expected the pending interrupt's shutdown error");
};
assert_eq!(
error,
OutgoingError {
id: request_id(/*id*/ 99).request_id,
error: internal_error("thread shut down before turn interruption completed"),
}
);
assert!(state.lock().await.pending_interrupts.is_empty());
assert_eq!(
interrupt(/*id*/ 101, &first).await.unwrap_err(),
invalid_request("thread runtime is no longer accepting interrupts")
);
gate.release.notify_one();
wait_for_event(
&test.codex,
|msg| matches!(msg, EventMsg::TurnComplete(done) if done.turn_id == first),
)
.await;
// Late finalization changes Core's status back from Shutdown; the listener must stay closed.
assert!(!test.codex.has_shutdown_status());
assert_eq!(
interrupt(/*id*/ 102, &first).await.unwrap_err(),
invalid_request("thread runtime is no longer accepting interrupts")
);
revert.await.expect("revert replaces the retired runtime");
assert!(matches!(observed_rx.recv().await,
Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 100).request_id
));
let replacement = test.thread_manager.get_thread(thread_id).await?;
assert!(!Arc::ptr_eq(&test.codex, &replacement));
responses::mount_sse_once(&server, responses::sse_completed("second")).await;
let second = start(&replacement, "second").await?;
gate.entered.notified().await;
assert!(matches!(observed_rx.recv().await,
Some(OutgoingMessage::AppServerNotification(n))
if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == second)
));
assert!(
interrupt(/*id*/ 103, &second)
.await
.expect("replacement accepts interrupts")
.is_none()
);
gate.release.notify_one();
assert!(matches!(observed_rx.recv().await,
Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 103).request_id
));
assert!(matches!(observed_rx.recv().await,
Some(OutgoingMessage::AppServerNotification(n))
if matches!(&n.notification, ServerNotification::TurnCompleted(done) if done.turn.id == second)
));
assert!(state.lock().await.pending_interrupts.is_empty());
replacement.shutdown_and_wait().await?;
processor.thread_state_manager.clear_all_listeners().await;
processor.skills_watcher.shutdown();
collector.abort();
Ok(())
}
@@ -0,0 +1,278 @@
#![cfg(test)]
//! Checks interrupt replies against the order in which the client sees terminal events.
use super::*;
use crate::bespoke_event_handling::apply_bespoke_event_handling;
use crate::outgoing_message::OutgoingEnvelope;
use crate::outgoing_message::OutgoingMessage;
use app_test_support::create_mock_responses_server_repeating_assistant;
use codex_app_server_protocol::ThreadQueueAddParams;
use codex_app_server_protocol::ThreadQueueStartParams;
use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::NoopExtensionEventSink;
use codex_extension_api::TurnLifecycleContributor;
use codex_extension_api::TurnStartInput;
use codex_extension_api::TurnStopInput;
use codex_protocol::protocol::Event;
use codex_queue_extension::QueuedItemService;
use codex_thread_store::LocalQueueStore;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use serde_json::json;
use test_case::test_case;
use tokio::sync::Notify;
#[test]
fn late_completion_keeps_a_finished_review_delegate_ineligible_for_interrupt() {
let mut state = ThreadState::default();
let event = |kind, id| serde_json::from_value(json!({"type": kind, "turn_id": id})).unwrap();
state.track_current_turn_event("previous", &event("turn_started", "previous"));
state.track_current_turn_event("review", &event("turn_started", "delegate"));
assert_eq!(
interrupt_terminal_turn_id(&state, "delegate"),
Some("review")
);
for id in ["review", "previous"] {
state.track_current_turn_event(id, &event("turn_complete", id));
state.take_turn_summary(id);
}
assert_eq!(interrupt_terminal_turn_id(&state, "delegate"), None);
}
#[derive(Default)]
struct TerminalGates {
parent: std::sync::OnceLock<String>,
delegate_start: Notify,
run_delegate: Notify,
stopped: Notify,
finish: Notify,
}
impl TurnLifecycleContributor for TerminalGates {
fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
if self
.parent
.get()
.is_some_and(|id| id != input.thread_store.level_id())
{
self.delegate_start.notify_one();
self.run_delegate.notified().await;
}
})
}
fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async {
self.stopped.notify_one();
self.finish.notified().await;
})
}
}
#[test_case(false; "turn_start")]
#[test_case(true; "thread_queue_start")]
#[tokio::test]
async fn interrupt_reply_waits_for_the_requested_turn(queued: bool) -> anyhow::Result<()> {
core_test_support::skip_if_no_network!(Ok(()));
let gates = Arc::new(TerminalGates::default());
let mut registry = ExtensionRegistryBuilder::new();
registry.turn_lifecycle_contributor(gates.clone());
let server = create_mock_responses_server_repeating_assistant("Done").await;
let test = test_codex()
.with_extensions(Arc::new(registry.build()))
.build_with_auto_env(&server)
.await?;
let thread_id = test.session_configured.thread_id;
gates.parent.set(thread_id.to_string()).unwrap();
let connection_id = ConnectionId(1);
let analytics = AnalyticsEventsClient::disabled();
let (tx, mut rx) = tokio::sync::mpsc::channel(crate::CHANNEL_CAPACITY);
let outgoing = Arc::new(OutgoingMessageSender::new(tx, analytics.clone()));
let watcher = SkillsWatcher::new(
test.thread_manager.skills_service(),
&test.config.codex_home,
outgoing.clone(),
);
let processor = TurnRequestProcessor::new(
test.thread_manager.clone(),
outgoing.clone(),
analytics,
Arc::new(test.config.clone()),
ConfigManager::without_managed_config_for_tests(test.config.codex_home.to_path_buf()),
Default::default(),
ThreadStateManager::new(),
ThreadWatchManager::new(),
watcher.clone(),
/*turn_cost_worker*/ None,
);
let request_id = |id| ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(id),
};
let turn = TurnStartParams {
thread_id: thread_id.to_string(),
input: vec![V2UserInput::Text {
text: "first".into(),
text_elements: vec![],
}],
..Default::default()
};
let interrupt = async |id, turn_id: &str| {
processor
.turn_interrupt_inner(
&request_id(id),
TurnInterruptParams {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_owned(),
},
)
.await
};
let first = if queued {
let store = LocalQueueStore::new(test.codex.state_db().expect("state DB"));
let service = QueuedItemService::new(
Arc::new(store),
Arc::downgrade(&test.thread_manager),
Arc::new(NoopExtensionEventSink),
);
let queue = ThreadQueueRequestProcessor::new(
test.thread_manager.clone(),
test.thread_store.clone(),
outgoing.clone(),
processor.config_manager.clone(),
Some(Arc::new(service)),
);
let add = ThreadQueueAddParams {
thread_id: turn.thread_id.clone(),
input: turn.input,
client_user_message_id: "queued-first".into(),
responsesapi_client_metadata: None,
};
queue.add(add).await.expect("enqueue");
let params = ThreadQueueStartParams {
thread_id: turn.thread_id,
queued_submission_id: None,
};
queue
.start(&request_id(1), params)
.await
.expect("start queued turn")
.turn
.id
} else {
processor
.turn_start_inner(
request_id(1),
turn,
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
)
.await
.expect("start turn")
.turn
.id
};
gates.stopped.notified().await;
assert_eq!(
interrupt(0, "unknown").await.unwrap_err(),
invalid_request("no active turn to interrupt")
);
assert_eq!(interrupt(2, &first).await, Ok(None));
gates.finish.notify_one();
// Hold A's actual terminal event until B has an interrupt waiting for a reply.
let completed = wait_for_event(
&test.codex,
|msg| matches!(msg, EventMsg::TurnComplete(e) if e.turn_id == first && e.error.is_none()),
)
.await;
let params = ReviewStartParams {
thread_id: thread_id.to_string(),
target: ApiReviewTarget::UncommittedChanges,
delivery: None,
};
let review = processor
.review_start(&request_id(3), params)
.await
.expect("start review");
let Some(ClientResponsePayload::ReviewStart(review)) = review else {
panic!("expected review response")
};
let second = review.turn.id;
gates.delegate_start.notified().await;
assert_eq!(
interrupt(5, &first).await.unwrap_err(),
invalid_request("no active turn to interrupt")
);
// The review owns the slot, but Core has not announced its delegate yet.
assert_eq!(
interrupt(6, &second).await,
Err(invalid_request("no active turn to interrupt"))
);
gates.run_delegate.notify_one();
wait_for_event(&test.codex, |msg| matches!(msg, EventMsg::TurnStarted(_))).await;
// The delegate finishes its model work after the stale interrupt. Its parent is still running.
gates.stopped.notified().await;
assert_eq!(interrupt(4, &second).await, Ok(None));
gates.finish.notify_one();
let aborted = wait_for_event(
&test.codex,
|msg| matches!(msg, EventMsg::TurnAborted(e) if e.turn_id.as_deref() == Some(&second)),
)
.await;
let state = processor.thread_state_manager.thread_state(thread_id).await;
let outgoing = ThreadScopedOutgoingMessageSender::new(outgoing, vec![connection_id], thread_id);
let mut receive = || match rx.try_recv().expect("terminal event already processed") {
OutgoingEnvelope::ToConnection { message, .. } => message,
other => panic!("unexpected outgoing message {other:?}"),
};
for (turn_id, msg, reply, status) in [
(first.clone(), completed, 2, TurnStatus::Completed),
(second, aborted, 4, TurnStatus::Interrupted),
] {
state.lock().await.track_current_turn_event(&turn_id, &msg);
apply_bespoke_event_handling(
Event {
id: turn_id.clone(),
msg,
},
thread_id,
test.codex.clone(),
test.thread_manager.clone(),
outgoing.clone(),
state.clone(),
processor.thread_watch_manager.clone(),
)
.await;
assert!(
matches!(receive(), OutgoingMessage::Response(r) if r.id == RequestId::Integer(reply))
);
assert!(
matches!(receive(), OutgoingMessage::AppServerNotification(n)
if matches!(n.notification, ServerNotification::TurnCompleted(ref done) if done.turn.id == turn_id && done.turn.status == status))
);
}
// Late child activity opens a history turn after Core has finished both turns.
let activity = serde_json::from_value(json!({
"type": "sub_agent_activity", "kind": "completed", "event_id": "late-child",
"agent_thread_id": ThreadId::new(), "agent_path": "/root/worker"
}))?;
let history_id = {
let mut state = state.lock().await;
state.track_current_turn_event(&first, &activity);
state.active_turn_id().expect("history turn").to_owned()
};
assert_eq!(
interrupt(7, &history_id).await,
Err(invalid_request("no active turn to interrupt"))
);
assert!(state.lock().await.pending_interrupts.is_empty());
test.codex.shutdown_and_wait().await?;
watcher.shutdown();
Ok(())
}
@@ -1525,66 +1525,108 @@ impl TurnRequestProcessor {
}
}
#[expect(
clippy::await_holding_invalid_type,
reason = "register the interrupt before the listener can consume the validated turn's terminal event"
)]
async fn turn_interrupt_inner(
&self,
request_id: &ConnectionRequestId,
params: TurnInterruptParams,
) -> Result<Option<TurnInterruptResponse>, JSONRPCErrorError> {
let TurnInterruptParams { thread_id, turn_id } = params;
let is_startup_interrupt = turn_id.is_empty();
let (thread_uuid, thread) = self.load_thread(&thread_id).await?;
// Record turn interrupts so we can reply when TurnAborted arrives. Startup
// interrupts do not have a turn and are acknowledged after submission.
if !is_startup_interrupt {
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let is_running = matches!(thread.agent_status().await, AgentStatus::Running);
{
let mut thread_state = thread_state.lock().await;
if let Some(active_turn_id) = thread_state.active_turn_id() {
if active_turn_id != turn_id {
return Err(invalid_request(format!(
"expected active turn id {turn_id} but found {active_turn_id}"
)));
}
} else if thread_state.last_terminal_turn_id.as_deref() == Some(turn_id.as_str())
|| !is_running
{
return Err(invalid_request("no active turn to interrupt"));
}
thread_state.pending_interrupts.push(request_id.clone());
}
self.outgoing
.record_request_turn_id(request_id, &turn_id)
.await;
// Startup interrupts have no target and are acknowledged on submission.
if turn_id.is_empty() {
self.submit_core_op(request_id, thread.as_ref(), Op::Interrupt)
.await
.map_err(|err| internal_error(format!("failed to interrupt startup: {err}")))?;
return Ok(Some(TurnInterruptResponse {}));
}
// Reply when the target completes or aborts.
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let submit_interrupt = {
let mut thread_state = thread_state.lock().await;
// Serialize registration with listener shutdown. A handler may also have
// loaded the old runtime before a replacement listener was installed.
if thread_state.listener_shutdown
|| (thread_state.shutdown_drain.is_some()
&& !thread_state.listener_matches(&thread))
{
return Err(invalid_request(
"thread runtime is no longer accepting interrupts",
));
}
let listener_turn_id = thread_state.active_turn_id();
if let Some(active_turn_id) = listener_turn_id
&& active_turn_id != turn_id
{
return Err(invalid_request(format!(
"expected active turn id {turn_id} but found {active_turn_id}"
)));
}
let terminal_turn_id = interrupt_terminal_turn_id(&thread_state, &turn_id)
.ok_or_else(|| invalid_request("no active turn to interrupt"))?;
let submit_interrupt = match thread.current_turn_id().await {
// Core reserves the ID before it announces the turn.
Some(id)
if id == terminal_turn_id
&& (listener_turn_id.is_some()
|| matches!(thread.agent_status().await, AgentStatus::Running)) =>
{
true
}
// Core drops the slot before publishing TurnAborted.
None if thread_state
.pending_interrupts
.iter()
.any(|(id, _)| id == terminal_turn_id) =>
{
false
}
// A listener-visible turn may already be aborting. Only a review
// announced before task registration still needs an interrupt here.
None if listener_turn_id.is_some()
&& thread_state.turn_summaries.contains_key(terminal_turn_id) =>
{
listener_turn_id == Some(terminal_turn_id)
&& thread_state.last_started_turn_id.as_deref() != Some(terminal_turn_id)
}
Some(_) | None => return Err(invalid_request("no active turn to interrupt")),
};
let terminal_turn_id = terminal_turn_id.to_owned();
thread_state
.pending_interrupts
.push((terminal_turn_id, request_id.clone()));
submit_interrupt
};
self.outgoing
.record_request_turn_id(request_id, &turn_id)
.await;
if !submit_interrupt {
return Ok(None);
}
// Submit the interrupt. Turn interrupts respond upon TurnAborted; startup
// interrupts respond here because startup cancellation has no turn event.
match self
.submit_core_op(request_id, thread.as_ref(), Op::Interrupt)
.await
{
Ok(_) if is_startup_interrupt => Ok(Some(TurnInterruptResponse {})),
Ok(_) => Ok(None),
Err(err) => {
if !is_startup_interrupt {
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let mut thread_state = thread_state.lock().await;
thread_state
.pending_interrupts
.retain(|pending_request_id| pending_request_id != request_id);
}
let interrupt_target = if is_startup_interrupt {
"startup"
} else {
"turn"
let mut state = thread_state.lock().await;
let Some(index) = state
.pending_interrupts
.iter()
.position(|(_, id)| id == request_id)
else {
// The listener already settled this request.
return Ok(None);
};
Err(internal_error(format!(
"failed to interrupt {interrupt_target}: {err}"
)))
state.pending_interrupts.remove(index);
Err(internal_error(format!("failed to interrupt turn: {err}")))
}
}
}
@@ -1617,3 +1659,21 @@ impl TurnRequestProcessor {
.await
}
}
fn interrupt_terminal_turn_id<'a>(state: &'a ThreadState, turn_id: &'a str) -> Option<&'a str> {
let terminal_id = if state.forwarded_turn_id.as_deref() == Some(turn_id) {
// History can still name the delegate after the outer turn has finished.
let outer_id = state.last_started_turn_id.as_deref()?;
if !state.turn_summaries.contains_key(outer_id) {
return None;
}
outer_id
} else {
turn_id
};
(state.last_terminal_turn_id.as_deref() != Some(terminal_id)).then_some(terminal_id)
}
#[cfg(test)]
#[path = "turn_interrupt_tests.rs"]
mod turn_interrupt_tests;
+11 -1
View File
@@ -36,7 +36,7 @@ use tokio::sync::oneshot;
use tokio::sync::watch;
use tracing::error;
type PendingInterruptQueue = Vec<ConnectionRequestId>;
type PendingInterruptQueue = Vec<(String, ConnectionRequestId)>;
// The unload that inserts an entry owns its removal. Dropping the sender wakes deferred
// listener attachment only after that unload finishes cleanup.
@@ -124,6 +124,8 @@ pub(crate) struct ThreadState {
pub(crate) command_start_times: HashMap<String, HashMap<String, i64>>,
/// A terminal event can arrive after the listener has already started the next turn.
pub(crate) last_started_turn_id: Option<String>,
// A delegate may announce a different ID from the one on its terminal event.
pub(crate) forwarded_turn_id: Option<String>,
pub(crate) last_terminal_turn_id: Option<String>,
/// Lets an internal runtime replacement wait until the old listener has processed Core's
/// `ShutdownComplete` event before that listener is superseded.
@@ -131,6 +133,9 @@ pub(crate) struct ThreadState {
pub(crate) cancel_tx: Option<oneshot::Sender<()>>,
pub(crate) experimental_raw_events: bool,
pub(crate) listener_generation: u64,
/// The listener no longer consumes terminal events from this runtime. Unlike Core's
/// agent status, this stays closed when a detached finalizer completes after shutdown.
pub(crate) listener_shutdown: bool,
last_thread_settings: Option<ThreadSettings>,
listener_command_tx: Option<mpsc::UnboundedSender<ThreadListenerCommand>>,
current_turn_history: ThreadHistoryBuilder,
@@ -174,10 +179,12 @@ impl ThreadState {
if let Some(previous) = self.cancel_tx.replace(cancel_tx) {
let _ = previous.send(());
}
self.forwarded_turn_id = None;
self.turn_summaries.clear();
self.command_start_times.clear();
self.shutdown_drain = Some(shutdown_drain);
self.listener_generation = self.listener_generation.wrapping_add(1);
self.listener_shutdown = false;
self.last_thread_settings = Some(thread_settings_baseline);
self.history_mode = history_mode;
self.evict_superseded_agent_text =
@@ -197,6 +204,7 @@ impl ThreadState {
self.listener_command_tx = None;
self.current_turn_history.reset();
self.last_started_turn_id = None;
self.forwarded_turn_id = None;
self.turn_summaries.clear();
self.command_start_times.clear();
self.listener_thread = None;
@@ -280,6 +288,8 @@ impl ThreadState {
pub(crate) fn track_current_turn_event(&mut self, event_turn_id: &str, event: &EventMsg) {
if let EventMsg::TurnStarted(payload) = event {
self.note_turn_started(event_turn_id, payload.started_at);
self.forwarded_turn_id =
(payload.turn_id != event_turn_id).then(|| payload.turn_id.clone());
} else if let EventMsg::ItemStarted(payload) = event
&& matches!(payload.item, CoreTurnItem::EnteredReviewMode(_))
{
+11
View File
@@ -892,6 +892,17 @@ impl CodexThread {
self.session.active_turn.lock().await.is_some()
}
/// Returns the current turn ID, including while its stop hook runs.
pub async fn current_turn_id(&self) -> Option<String> {
self.session
.active_turn
.lock()
.await
.as_ref()?
.turn_id
.clone()
}
/// Returns the trusted root when the expected turn is currently active.
pub async fn active_turn_root(&self, expected_turn_id: &str) -> Option<String> {
let active = self.session.active_turn.lock().await;
+3
View File
@@ -36,6 +36,8 @@ use codex_protocol::protocol::TokenUsage;
/// Metadata about the currently running turn.
pub(crate) struct ActiveTurn {
/// The slot's owner, retained after its task detaches to emit the terminal event.
pub(crate) turn_id: Option<String>,
pub(crate) task: Option<RunningTask>,
pub(crate) sampling_interrupt: Option<SamplingInterrupt>,
/// Present when input admission is closed; cancelled when the final batch is recorded.
@@ -67,6 +69,7 @@ pub(crate) enum MailboxDeliveryPhase {
impl Default for ActiveTurn {
fn default() -> Self {
Self {
turn_id: None,
task: None,
sampling_interrupt: None,
finishing: None,
+1
View File
@@ -367,6 +367,7 @@ impl Session {
}
let turn = active.get_or_insert_with(ActiveTurn::default);
debug_assert!(turn.task.is_none());
turn.turn_id = Some(turn_context.sub_id.clone());
turn.finishing = None;
Arc::clone(&turn.turn_state)
};