mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Fix turn interrupt reply ordering and shutdown handling (#62066)
## Why A delayed terminal event from an earlier turn could acknowledge interrupts for a later turn. Interrupts could also remain pending after the thread listener shut down. ## What changed - Associate pending `turn/interrupt` requests with their target turn and reply only when that turn completes or aborts. - Track the current Core turn through stop hooks and resolve review delegate IDs to the owning turn when validating interrupts. - Settle pending interrupts with an error on listener shutdown and reject further interrupts against the retired runtime. ## Testing Add regression tests for delayed terminal events, queued turn starts, review delegates, and interrupt handling during thread revert and runtime replacement. GitOrigin-RevId: 9695fc3f718baa6b792a14d54136bb87fc003100
This commit is contained in:
@@ -200,7 +200,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
.await;
|
||||
}
|
||||
EventMsg::TurnComplete(turn_complete_event) => {
|
||||
respond_to_pending_interrupts(&thread_state, &outgoing).await;
|
||||
respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await;
|
||||
if finishes_latest_turn {
|
||||
let turn_failed = thread_state
|
||||
.lock()
|
||||
@@ -1297,7 +1297,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
}
|
||||
// If this is a TurnAborted, reply to any pending interrupt requests.
|
||||
EventMsg::TurnAborted(turn_aborted_event) => {
|
||||
respond_to_pending_interrupts(&thread_state, &outgoing).await;
|
||||
respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await;
|
||||
|
||||
if finishes_latest_turn {
|
||||
thread_watch_manager
|
||||
@@ -1652,12 +1652,17 @@ async fn handle_turn_interrupted(
|
||||
}
|
||||
|
||||
async fn respond_to_pending_interrupts(
|
||||
event_turn_id: &str,
|
||||
thread_state: &Arc<Mutex<ThreadState>>,
|
||||
outgoing: &ThreadScopedOutgoingMessageSender,
|
||||
) {
|
||||
let pending = {
|
||||
let mut state = thread_state.lock().await;
|
||||
std::mem::take(&mut state.pending_interrupts)
|
||||
state
|
||||
.pending_interrupts
|
||||
.extract_if(.., |(turn_id, _)| turn_id == event_turn_id)
|
||||
.map(|(_, request_id)| request_id)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for request_id in pending {
|
||||
|
||||
@@ -756,6 +756,25 @@ async fn start_listener(
|
||||
drop(turn_cost_worker);
|
||||
|
||||
if shutdown_complete {
|
||||
// Close registration and take the queue under the same lock used by turn/interrupt.
|
||||
// Detached finalizers can publish their terminal events after this listener retires.
|
||||
let pending_interrupts = {
|
||||
let mut state = thread_state.lock().await;
|
||||
if state.listener_generation == listener_generation {
|
||||
state.listener_shutdown = true;
|
||||
std::mem::take(&mut state.pending_interrupts)
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
for (_, request_id) in pending_interrupts {
|
||||
outgoing_for_task
|
||||
.send_error(
|
||||
request_id,
|
||||
internal_error("thread shut down before turn interruption completed"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drained_tx.send_replace(true);
|
||||
let relay_command = |command| {
|
||||
handle_thread_listener_command(
|
||||
|
||||
@@ -610,3 +610,228 @@ async fn resume_preserves_recovery_turn_before_status_publication() -> Result<()
|
||||
processor.skills_watcher.shutdown();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revert_settles_interrupts_before_replacing_a_runtime() -> Result<()> {
|
||||
use super::super::turn_processor::TurnRequestProcessor;
|
||||
use crate::outgoing_message::OutgoingError;
|
||||
use codex_app_server_protocol::ThreadRevertParams;
|
||||
use codex_app_server_protocol::TurnInterruptParams;
|
||||
use codex_core::TurnInputRequest;
|
||||
use codex_core::TurnInputSubmission;
|
||||
use codex_extension_api::ExtensionFuture;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::TurnLifecycleContributor;
|
||||
use codex_extension_api::TurnStopInput;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[derive(Default)]
|
||||
struct StopGate {
|
||||
entered: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
impl TurnLifecycleContributor for StopGate {
|
||||
fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async {
|
||||
self.entered.notify_one();
|
||||
self.release.notified().await;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let gate = Arc::new(StopGate::default());
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.turn_lifecycle_contributor(gate.clone());
|
||||
let server = responses::start_mock_server().await;
|
||||
responses::mount_sse_once(&server, responses::sse_completed("first")).await;
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.with_config(|config| {
|
||||
config.agents_enabled = false;
|
||||
config
|
||||
.features
|
||||
.disable(codex_features::Feature::MultiAgentV2)
|
||||
.unwrap();
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let thread_id = test.session_configured.thread_id;
|
||||
let (processor, mut outgoing_rx) = test_processor(&test).await?;
|
||||
// Keep transport flowing, retaining every reply and turn boundary in client-visible order.
|
||||
let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
|
||||
let collector = tokio::spawn(async move {
|
||||
while let Some(envelope) = outgoing_rx.recv().await {
|
||||
let OutgoingEnvelope::ToConnection { message, .. } = envelope else {
|
||||
continue;
|
||||
};
|
||||
let relevant = match &message {
|
||||
OutgoingMessage::Response(_) | OutgoingMessage::Error(_) => true,
|
||||
OutgoingMessage::AppServerNotification(envelope) => matches!(
|
||||
envelope.notification,
|
||||
ServerNotification::TurnStarted(_) | ServerNotification::TurnCompleted(_)
|
||||
),
|
||||
OutgoingMessage::Request(_) => false,
|
||||
};
|
||||
if relevant && observed_tx.send(message).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let connection_id = ConnectionId(1);
|
||||
processor
|
||||
.connection_initialized(connection_id, ConnectionCapabilities::default())
|
||||
.await;
|
||||
processor
|
||||
.try_attach_thread_listener(thread_id, test.codex.clone(), vec![connection_id])
|
||||
.await;
|
||||
let turn_processor = TurnRequestProcessor::new(
|
||||
test.thread_manager.clone(),
|
||||
processor.outgoing.clone(),
|
||||
AnalyticsEventsClient::disabled(),
|
||||
processor.config.clone(),
|
||||
processor.config_manager.clone(),
|
||||
processor.pending_thread_unloads.clone(),
|
||||
processor.thread_state_manager.clone(),
|
||||
processor.thread_watch_manager.clone(),
|
||||
processor.skills_watcher.clone(),
|
||||
/*turn_cost_worker*/ None,
|
||||
);
|
||||
let request_id = |id| ConnectionRequestId {
|
||||
connection_id,
|
||||
request_id: RequestId::Integer(id),
|
||||
};
|
||||
let interrupt = async |id, turn_id: &str| {
|
||||
turn_processor
|
||||
.turn_interrupt(
|
||||
&request_id(id),
|
||||
TurnInterruptParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: turn_id.to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
};
|
||||
let start = async |thread: &Arc<CodexThread>, text: &str| -> Result<String> {
|
||||
let TurnInputSubmission::Started { turn_id, .. } = thread
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: text.to_owned(),
|
||||
text_elements: vec![],
|
||||
}]))
|
||||
.await?
|
||||
else {
|
||||
anyhow::bail!("expected a new turn")
|
||||
};
|
||||
Ok(turn_id)
|
||||
};
|
||||
let first = start(&test.codex, "first").await?;
|
||||
assert_eq!(
|
||||
test.codex.multi_agent_version(),
|
||||
Some(MultiAgentVersion::Disabled)
|
||||
);
|
||||
gate.entered.notified().await;
|
||||
assert!(matches!(observed_rx.recv().await,
|
||||
Some(OutgoingMessage::AppServerNotification(n))
|
||||
if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == first)
|
||||
));
|
||||
assert!(
|
||||
interrupt(/*id*/ 99, &first)
|
||||
.await
|
||||
.expect("interrupt accepted")
|
||||
.is_none()
|
||||
);
|
||||
let state = processor.thread_state_manager.thread_state(thread_id).await;
|
||||
assert_eq!(state.lock().await.pending_interrupts.len(), 1);
|
||||
let mut drain = state
|
||||
.lock()
|
||||
.await
|
||||
.shutdown_drain
|
||||
.clone()
|
||||
.expect("listener installed");
|
||||
|
||||
let revert = processor.thread_revert(
|
||||
request_id(/*id*/ 100),
|
||||
ThreadRevertParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
before_turn_id: first.clone(),
|
||||
},
|
||||
/*app_server_client_name*/ None,
|
||||
/*app_server_client_version*/ None,
|
||||
);
|
||||
tokio::pin!(revert);
|
||||
// Stop polling revert at its shutdown barrier so late requests still load the retired runtime.
|
||||
tokio::select! {
|
||||
biased;
|
||||
drained = drain.wait_for(|drained| *drained) => { drained?; }
|
||||
result = &mut revert => { anyhow::bail!("revert passed the shutdown barrier: {result:?}"); }
|
||||
}
|
||||
assert!(Arc::ptr_eq(
|
||||
&test.codex,
|
||||
&test.thread_manager.get_thread(thread_id).await?
|
||||
));
|
||||
let Some(OutgoingMessage::Error(error)) = observed_rx.recv().await else {
|
||||
anyhow::bail!("expected the pending interrupt's shutdown error");
|
||||
};
|
||||
assert_eq!(
|
||||
error,
|
||||
OutgoingError {
|
||||
id: request_id(/*id*/ 99).request_id,
|
||||
error: internal_error("thread shut down before turn interruption completed"),
|
||||
}
|
||||
);
|
||||
assert!(state.lock().await.pending_interrupts.is_empty());
|
||||
assert_eq!(
|
||||
interrupt(/*id*/ 101, &first).await.unwrap_err(),
|
||||
invalid_request("thread runtime is no longer accepting interrupts")
|
||||
);
|
||||
|
||||
gate.release.notify_one();
|
||||
wait_for_event(
|
||||
&test.codex,
|
||||
|msg| matches!(msg, EventMsg::TurnComplete(done) if done.turn_id == first),
|
||||
)
|
||||
.await;
|
||||
// Late finalization changes Core's status back from Shutdown; the listener must stay closed.
|
||||
assert!(!test.codex.has_shutdown_status());
|
||||
assert_eq!(
|
||||
interrupt(/*id*/ 102, &first).await.unwrap_err(),
|
||||
invalid_request("thread runtime is no longer accepting interrupts")
|
||||
);
|
||||
revert.await.expect("revert replaces the retired runtime");
|
||||
assert!(matches!(observed_rx.recv().await,
|
||||
Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 100).request_id
|
||||
));
|
||||
let replacement = test.thread_manager.get_thread(thread_id).await?;
|
||||
assert!(!Arc::ptr_eq(&test.codex, &replacement));
|
||||
|
||||
responses::mount_sse_once(&server, responses::sse_completed("second")).await;
|
||||
let second = start(&replacement, "second").await?;
|
||||
gate.entered.notified().await;
|
||||
assert!(matches!(observed_rx.recv().await,
|
||||
Some(OutgoingMessage::AppServerNotification(n))
|
||||
if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == second)
|
||||
));
|
||||
assert!(
|
||||
interrupt(/*id*/ 103, &second)
|
||||
.await
|
||||
.expect("replacement accepts interrupts")
|
||||
.is_none()
|
||||
);
|
||||
gate.release.notify_one();
|
||||
assert!(matches!(observed_rx.recv().await,
|
||||
Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 103).request_id
|
||||
));
|
||||
assert!(matches!(observed_rx.recv().await,
|
||||
Some(OutgoingMessage::AppServerNotification(n))
|
||||
if matches!(&n.notification, ServerNotification::TurnCompleted(done) if done.turn.id == second)
|
||||
));
|
||||
assert!(state.lock().await.pending_interrupts.is_empty());
|
||||
replacement.shutdown_and_wait().await?;
|
||||
processor.thread_state_manager.clear_all_listeners().await;
|
||||
processor.skills_watcher.shutdown();
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
#![cfg(test)]
|
||||
|
||||
//! Checks interrupt replies against the order in which the client sees terminal events.
|
||||
|
||||
use super::*;
|
||||
use crate::bespoke_event_handling::apply_bespoke_event_handling;
|
||||
use crate::outgoing_message::OutgoingEnvelope;
|
||||
use crate::outgoing_message::OutgoingMessage;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use codex_app_server_protocol::ThreadQueueAddParams;
|
||||
use codex_app_server_protocol::ThreadQueueStartParams;
|
||||
use codex_extension_api::ExtensionFuture;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::NoopExtensionEventSink;
|
||||
use codex_extension_api::TurnLifecycleContributor;
|
||||
use codex_extension_api::TurnStartInput;
|
||||
use codex_extension_api::TurnStopInput;
|
||||
use codex_protocol::protocol::Event;
|
||||
use codex_queue_extension::QueuedItemService;
|
||||
use codex_thread_store::LocalQueueStore;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use test_case::test_case;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[test]
|
||||
fn late_completion_keeps_a_finished_review_delegate_ineligible_for_interrupt() {
|
||||
let mut state = ThreadState::default();
|
||||
let event = |kind, id| serde_json::from_value(json!({"type": kind, "turn_id": id})).unwrap();
|
||||
state.track_current_turn_event("previous", &event("turn_started", "previous"));
|
||||
state.track_current_turn_event("review", &event("turn_started", "delegate"));
|
||||
assert_eq!(
|
||||
interrupt_terminal_turn_id(&state, "delegate"),
|
||||
Some("review")
|
||||
);
|
||||
for id in ["review", "previous"] {
|
||||
state.track_current_turn_event(id, &event("turn_complete", id));
|
||||
state.take_turn_summary(id);
|
||||
}
|
||||
assert_eq!(interrupt_terminal_turn_id(&state, "delegate"), None);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TerminalGates {
|
||||
parent: std::sync::OnceLock<String>,
|
||||
delegate_start: Notify,
|
||||
run_delegate: Notify,
|
||||
stopped: Notify,
|
||||
finish: Notify,
|
||||
}
|
||||
|
||||
impl TurnLifecycleContributor for TerminalGates {
|
||||
fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
if self
|
||||
.parent
|
||||
.get()
|
||||
.is_some_and(|id| id != input.thread_store.level_id())
|
||||
{
|
||||
self.delegate_start.notify_one();
|
||||
self.run_delegate.notified().await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async {
|
||||
self.stopped.notify_one();
|
||||
self.finish.notified().await;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case(false; "turn_start")]
|
||||
#[test_case(true; "thread_queue_start")]
|
||||
#[tokio::test]
|
||||
async fn interrupt_reply_waits_for_the_requested_turn(queued: bool) -> anyhow::Result<()> {
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
let gates = Arc::new(TerminalGates::default());
|
||||
let mut registry = ExtensionRegistryBuilder::new();
|
||||
registry.turn_lifecycle_contributor(gates.clone());
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(registry.build()))
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let thread_id = test.session_configured.thread_id;
|
||||
gates.parent.set(thread_id.to_string()).unwrap();
|
||||
let connection_id = ConnectionId(1);
|
||||
let analytics = AnalyticsEventsClient::disabled();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(crate::CHANNEL_CAPACITY);
|
||||
let outgoing = Arc::new(OutgoingMessageSender::new(tx, analytics.clone()));
|
||||
let watcher = SkillsWatcher::new(
|
||||
test.thread_manager.skills_service(),
|
||||
&test.config.codex_home,
|
||||
outgoing.clone(),
|
||||
);
|
||||
let processor = TurnRequestProcessor::new(
|
||||
test.thread_manager.clone(),
|
||||
outgoing.clone(),
|
||||
analytics,
|
||||
Arc::new(test.config.clone()),
|
||||
ConfigManager::without_managed_config_for_tests(test.config.codex_home.to_path_buf()),
|
||||
Default::default(),
|
||||
ThreadStateManager::new(),
|
||||
ThreadWatchManager::new(),
|
||||
watcher.clone(),
|
||||
/*turn_cost_worker*/ None,
|
||||
);
|
||||
let request_id = |id| ConnectionRequestId {
|
||||
connection_id,
|
||||
request_id: RequestId::Integer(id),
|
||||
};
|
||||
let turn = TurnStartParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "first".into(),
|
||||
text_elements: vec![],
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let interrupt = async |id, turn_id: &str| {
|
||||
processor
|
||||
.turn_interrupt_inner(
|
||||
&request_id(id),
|
||||
TurnInterruptParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: turn_id.to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
};
|
||||
let first = if queued {
|
||||
let store = LocalQueueStore::new(test.codex.state_db().expect("state DB"));
|
||||
let service = QueuedItemService::new(
|
||||
Arc::new(store),
|
||||
Arc::downgrade(&test.thread_manager),
|
||||
Arc::new(NoopExtensionEventSink),
|
||||
);
|
||||
let queue = ThreadQueueRequestProcessor::new(
|
||||
test.thread_manager.clone(),
|
||||
test.thread_store.clone(),
|
||||
outgoing.clone(),
|
||||
processor.config_manager.clone(),
|
||||
Some(Arc::new(service)),
|
||||
);
|
||||
let add = ThreadQueueAddParams {
|
||||
thread_id: turn.thread_id.clone(),
|
||||
input: turn.input,
|
||||
client_user_message_id: "queued-first".into(),
|
||||
responsesapi_client_metadata: None,
|
||||
};
|
||||
queue.add(add).await.expect("enqueue");
|
||||
let params = ThreadQueueStartParams {
|
||||
thread_id: turn.thread_id,
|
||||
queued_submission_id: None,
|
||||
};
|
||||
queue
|
||||
.start(&request_id(1), params)
|
||||
.await
|
||||
.expect("start queued turn")
|
||||
.turn
|
||||
.id
|
||||
} else {
|
||||
processor
|
||||
.turn_start_inner(
|
||||
request_id(1),
|
||||
turn,
|
||||
/*app_server_client_name*/ None,
|
||||
/*app_server_client_version*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("start turn")
|
||||
.turn
|
||||
.id
|
||||
};
|
||||
gates.stopped.notified().await;
|
||||
assert_eq!(
|
||||
interrupt(0, "unknown").await.unwrap_err(),
|
||||
invalid_request("no active turn to interrupt")
|
||||
);
|
||||
assert_eq!(interrupt(2, &first).await, Ok(None));
|
||||
gates.finish.notify_one();
|
||||
// Hold A's actual terminal event until B has an interrupt waiting for a reply.
|
||||
let completed = wait_for_event(
|
||||
&test.codex,
|
||||
|msg| matches!(msg, EventMsg::TurnComplete(e) if e.turn_id == first && e.error.is_none()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let params = ReviewStartParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
target: ApiReviewTarget::UncommittedChanges,
|
||||
delivery: None,
|
||||
};
|
||||
let review = processor
|
||||
.review_start(&request_id(3), params)
|
||||
.await
|
||||
.expect("start review");
|
||||
let Some(ClientResponsePayload::ReviewStart(review)) = review else {
|
||||
panic!("expected review response")
|
||||
};
|
||||
let second = review.turn.id;
|
||||
gates.delegate_start.notified().await;
|
||||
assert_eq!(
|
||||
interrupt(5, &first).await.unwrap_err(),
|
||||
invalid_request("no active turn to interrupt")
|
||||
);
|
||||
// The review owns the slot, but Core has not announced its delegate yet.
|
||||
assert_eq!(
|
||||
interrupt(6, &second).await,
|
||||
Err(invalid_request("no active turn to interrupt"))
|
||||
);
|
||||
gates.run_delegate.notify_one();
|
||||
wait_for_event(&test.codex, |msg| matches!(msg, EventMsg::TurnStarted(_))).await;
|
||||
// The delegate finishes its model work after the stale interrupt. Its parent is still running.
|
||||
gates.stopped.notified().await;
|
||||
assert_eq!(interrupt(4, &second).await, Ok(None));
|
||||
gates.finish.notify_one();
|
||||
let aborted = wait_for_event(
|
||||
&test.codex,
|
||||
|msg| matches!(msg, EventMsg::TurnAborted(e) if e.turn_id.as_deref() == Some(&second)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let state = processor.thread_state_manager.thread_state(thread_id).await;
|
||||
let outgoing = ThreadScopedOutgoingMessageSender::new(outgoing, vec![connection_id], thread_id);
|
||||
let mut receive = || match rx.try_recv().expect("terminal event already processed") {
|
||||
OutgoingEnvelope::ToConnection { message, .. } => message,
|
||||
other => panic!("unexpected outgoing message {other:?}"),
|
||||
};
|
||||
for (turn_id, msg, reply, status) in [
|
||||
(first.clone(), completed, 2, TurnStatus::Completed),
|
||||
(second, aborted, 4, TurnStatus::Interrupted),
|
||||
] {
|
||||
state.lock().await.track_current_turn_event(&turn_id, &msg);
|
||||
apply_bespoke_event_handling(
|
||||
Event {
|
||||
id: turn_id.clone(),
|
||||
msg,
|
||||
},
|
||||
thread_id,
|
||||
test.codex.clone(),
|
||||
test.thread_manager.clone(),
|
||||
outgoing.clone(),
|
||||
state.clone(),
|
||||
processor.thread_watch_manager.clone(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(receive(), OutgoingMessage::Response(r) if r.id == RequestId::Integer(reply))
|
||||
);
|
||||
assert!(
|
||||
matches!(receive(), OutgoingMessage::AppServerNotification(n)
|
||||
if matches!(n.notification, ServerNotification::TurnCompleted(ref done) if done.turn.id == turn_id && done.turn.status == status))
|
||||
);
|
||||
}
|
||||
// Late child activity opens a history turn after Core has finished both turns.
|
||||
let activity = serde_json::from_value(json!({
|
||||
"type": "sub_agent_activity", "kind": "completed", "event_id": "late-child",
|
||||
"agent_thread_id": ThreadId::new(), "agent_path": "/root/worker"
|
||||
}))?;
|
||||
let history_id = {
|
||||
let mut state = state.lock().await;
|
||||
state.track_current_turn_event(&first, &activity);
|
||||
state.active_turn_id().expect("history turn").to_owned()
|
||||
};
|
||||
assert_eq!(
|
||||
interrupt(7, &history_id).await,
|
||||
Err(invalid_request("no active turn to interrupt"))
|
||||
);
|
||||
assert!(state.lock().await.pending_interrupts.is_empty());
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
watcher.shutdown();
|
||||
Ok(())
|
||||
}
|
||||
@@ -1525,66 +1525,108 @@ impl TurnRequestProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "register the interrupt before the listener can consume the validated turn's terminal event"
|
||||
)]
|
||||
async fn turn_interrupt_inner(
|
||||
&self,
|
||||
request_id: &ConnectionRequestId,
|
||||
params: TurnInterruptParams,
|
||||
) -> Result<Option<TurnInterruptResponse>, JSONRPCErrorError> {
|
||||
let TurnInterruptParams { thread_id, turn_id } = params;
|
||||
let is_startup_interrupt = turn_id.is_empty();
|
||||
|
||||
let (thread_uuid, thread) = self.load_thread(&thread_id).await?;
|
||||
|
||||
// Record turn interrupts so we can reply when TurnAborted arrives. Startup
|
||||
// interrupts do not have a turn and are acknowledged after submission.
|
||||
if !is_startup_interrupt {
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let is_running = matches!(thread.agent_status().await, AgentStatus::Running);
|
||||
{
|
||||
let mut thread_state = thread_state.lock().await;
|
||||
if let Some(active_turn_id) = thread_state.active_turn_id() {
|
||||
if active_turn_id != turn_id {
|
||||
return Err(invalid_request(format!(
|
||||
"expected active turn id {turn_id} but found {active_turn_id}"
|
||||
)));
|
||||
}
|
||||
} else if thread_state.last_terminal_turn_id.as_deref() == Some(turn_id.as_str())
|
||||
|| !is_running
|
||||
{
|
||||
return Err(invalid_request("no active turn to interrupt"));
|
||||
}
|
||||
thread_state.pending_interrupts.push(request_id.clone());
|
||||
}
|
||||
|
||||
self.outgoing
|
||||
.record_request_turn_id(request_id, &turn_id)
|
||||
.await;
|
||||
// Startup interrupts have no target and are acknowledged on submission.
|
||||
if turn_id.is_empty() {
|
||||
self.submit_core_op(request_id, thread.as_ref(), Op::Interrupt)
|
||||
.await
|
||||
.map_err(|err| internal_error(format!("failed to interrupt startup: {err}")))?;
|
||||
return Ok(Some(TurnInterruptResponse {}));
|
||||
}
|
||||
|
||||
// Reply when the target completes or aborts.
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let submit_interrupt = {
|
||||
let mut thread_state = thread_state.lock().await;
|
||||
// Serialize registration with listener shutdown. A handler may also have
|
||||
// loaded the old runtime before a replacement listener was installed.
|
||||
if thread_state.listener_shutdown
|
||||
|| (thread_state.shutdown_drain.is_some()
|
||||
&& !thread_state.listener_matches(&thread))
|
||||
{
|
||||
return Err(invalid_request(
|
||||
"thread runtime is no longer accepting interrupts",
|
||||
));
|
||||
}
|
||||
let listener_turn_id = thread_state.active_turn_id();
|
||||
if let Some(active_turn_id) = listener_turn_id
|
||||
&& active_turn_id != turn_id
|
||||
{
|
||||
return Err(invalid_request(format!(
|
||||
"expected active turn id {turn_id} but found {active_turn_id}"
|
||||
)));
|
||||
}
|
||||
let terminal_turn_id = interrupt_terminal_turn_id(&thread_state, &turn_id)
|
||||
.ok_or_else(|| invalid_request("no active turn to interrupt"))?;
|
||||
let submit_interrupt = match thread.current_turn_id().await {
|
||||
// Core reserves the ID before it announces the turn.
|
||||
Some(id)
|
||||
if id == terminal_turn_id
|
||||
&& (listener_turn_id.is_some()
|
||||
|| matches!(thread.agent_status().await, AgentStatus::Running)) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
// Core drops the slot before publishing TurnAborted.
|
||||
None if thread_state
|
||||
.pending_interrupts
|
||||
.iter()
|
||||
.any(|(id, _)| id == terminal_turn_id) =>
|
||||
{
|
||||
false
|
||||
}
|
||||
// A listener-visible turn may already be aborting. Only a review
|
||||
// announced before task registration still needs an interrupt here.
|
||||
None if listener_turn_id.is_some()
|
||||
&& thread_state.turn_summaries.contains_key(terminal_turn_id) =>
|
||||
{
|
||||
listener_turn_id == Some(terminal_turn_id)
|
||||
&& thread_state.last_started_turn_id.as_deref() != Some(terminal_turn_id)
|
||||
}
|
||||
Some(_) | None => return Err(invalid_request("no active turn to interrupt")),
|
||||
};
|
||||
let terminal_turn_id = terminal_turn_id.to_owned();
|
||||
thread_state
|
||||
.pending_interrupts
|
||||
.push((terminal_turn_id, request_id.clone()));
|
||||
submit_interrupt
|
||||
};
|
||||
|
||||
self.outgoing
|
||||
.record_request_turn_id(request_id, &turn_id)
|
||||
.await;
|
||||
if !submit_interrupt {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Submit the interrupt. Turn interrupts respond upon TurnAborted; startup
|
||||
// interrupts respond here because startup cancellation has no turn event.
|
||||
match self
|
||||
.submit_core_op(request_id, thread.as_ref(), Op::Interrupt)
|
||||
.await
|
||||
{
|
||||
Ok(_) if is_startup_interrupt => Ok(Some(TurnInterruptResponse {})),
|
||||
Ok(_) => Ok(None),
|
||||
Err(err) => {
|
||||
if !is_startup_interrupt {
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let mut thread_state = thread_state.lock().await;
|
||||
thread_state
|
||||
.pending_interrupts
|
||||
.retain(|pending_request_id| pending_request_id != request_id);
|
||||
}
|
||||
let interrupt_target = if is_startup_interrupt {
|
||||
"startup"
|
||||
} else {
|
||||
"turn"
|
||||
let mut state = thread_state.lock().await;
|
||||
let Some(index) = state
|
||||
.pending_interrupts
|
||||
.iter()
|
||||
.position(|(_, id)| id == request_id)
|
||||
else {
|
||||
// The listener already settled this request.
|
||||
return Ok(None);
|
||||
};
|
||||
Err(internal_error(format!(
|
||||
"failed to interrupt {interrupt_target}: {err}"
|
||||
)))
|
||||
state.pending_interrupts.remove(index);
|
||||
Err(internal_error(format!("failed to interrupt turn: {err}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1617,3 +1659,21 @@ impl TurnRequestProcessor {
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn interrupt_terminal_turn_id<'a>(state: &'a ThreadState, turn_id: &'a str) -> Option<&'a str> {
|
||||
let terminal_id = if state.forwarded_turn_id.as_deref() == Some(turn_id) {
|
||||
// History can still name the delegate after the outer turn has finished.
|
||||
let outer_id = state.last_started_turn_id.as_deref()?;
|
||||
if !state.turn_summaries.contains_key(outer_id) {
|
||||
return None;
|
||||
}
|
||||
outer_id
|
||||
} else {
|
||||
turn_id
|
||||
};
|
||||
(state.last_terminal_turn_id.as_deref() != Some(terminal_id)).then_some(terminal_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "turn_interrupt_tests.rs"]
|
||||
mod turn_interrupt_tests;
|
||||
|
||||
@@ -36,7 +36,7 @@ use tokio::sync::oneshot;
|
||||
use tokio::sync::watch;
|
||||
use tracing::error;
|
||||
|
||||
type PendingInterruptQueue = Vec<ConnectionRequestId>;
|
||||
type PendingInterruptQueue = Vec<(String, ConnectionRequestId)>;
|
||||
|
||||
// The unload that inserts an entry owns its removal. Dropping the sender wakes deferred
|
||||
// listener attachment only after that unload finishes cleanup.
|
||||
@@ -124,6 +124,8 @@ pub(crate) struct ThreadState {
|
||||
pub(crate) command_start_times: HashMap<String, HashMap<String, i64>>,
|
||||
/// A terminal event can arrive after the listener has already started the next turn.
|
||||
pub(crate) last_started_turn_id: Option<String>,
|
||||
// A delegate may announce a different ID from the one on its terminal event.
|
||||
pub(crate) forwarded_turn_id: Option<String>,
|
||||
pub(crate) last_terminal_turn_id: Option<String>,
|
||||
/// Lets an internal runtime replacement wait until the old listener has processed Core's
|
||||
/// `ShutdownComplete` event before that listener is superseded.
|
||||
@@ -131,6 +133,9 @@ pub(crate) struct ThreadState {
|
||||
pub(crate) cancel_tx: Option<oneshot::Sender<()>>,
|
||||
pub(crate) experimental_raw_events: bool,
|
||||
pub(crate) listener_generation: u64,
|
||||
/// The listener no longer consumes terminal events from this runtime. Unlike Core's
|
||||
/// agent status, this stays closed when a detached finalizer completes after shutdown.
|
||||
pub(crate) listener_shutdown: bool,
|
||||
last_thread_settings: Option<ThreadSettings>,
|
||||
listener_command_tx: Option<mpsc::UnboundedSender<ThreadListenerCommand>>,
|
||||
current_turn_history: ThreadHistoryBuilder,
|
||||
@@ -174,10 +179,12 @@ impl ThreadState {
|
||||
if let Some(previous) = self.cancel_tx.replace(cancel_tx) {
|
||||
let _ = previous.send(());
|
||||
}
|
||||
self.forwarded_turn_id = None;
|
||||
self.turn_summaries.clear();
|
||||
self.command_start_times.clear();
|
||||
self.shutdown_drain = Some(shutdown_drain);
|
||||
self.listener_generation = self.listener_generation.wrapping_add(1);
|
||||
self.listener_shutdown = false;
|
||||
self.last_thread_settings = Some(thread_settings_baseline);
|
||||
self.history_mode = history_mode;
|
||||
self.evict_superseded_agent_text =
|
||||
@@ -197,6 +204,7 @@ impl ThreadState {
|
||||
self.listener_command_tx = None;
|
||||
self.current_turn_history.reset();
|
||||
self.last_started_turn_id = None;
|
||||
self.forwarded_turn_id = None;
|
||||
self.turn_summaries.clear();
|
||||
self.command_start_times.clear();
|
||||
self.listener_thread = None;
|
||||
@@ -280,6 +288,8 @@ impl ThreadState {
|
||||
pub(crate) fn track_current_turn_event(&mut self, event_turn_id: &str, event: &EventMsg) {
|
||||
if let EventMsg::TurnStarted(payload) = event {
|
||||
self.note_turn_started(event_turn_id, payload.started_at);
|
||||
self.forwarded_turn_id =
|
||||
(payload.turn_id != event_turn_id).then(|| payload.turn_id.clone());
|
||||
} else if let EventMsg::ItemStarted(payload) = event
|
||||
&& matches!(payload.item, CoreTurnItem::EnteredReviewMode(_))
|
||||
{
|
||||
|
||||
@@ -892,6 +892,17 @@ impl CodexThread {
|
||||
self.session.active_turn.lock().await.is_some()
|
||||
}
|
||||
|
||||
/// Returns the current turn ID, including while its stop hook runs.
|
||||
pub async fn current_turn_id(&self) -> Option<String> {
|
||||
self.session
|
||||
.active_turn
|
||||
.lock()
|
||||
.await
|
||||
.as_ref()?
|
||||
.turn_id
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Returns the trusted root when the expected turn is currently active.
|
||||
pub async fn active_turn_root(&self, expected_turn_id: &str) -> Option<String> {
|
||||
let active = self.session.active_turn.lock().await;
|
||||
|
||||
@@ -36,6 +36,8 @@ use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
/// Metadata about the currently running turn.
|
||||
pub(crate) struct ActiveTurn {
|
||||
/// The slot's owner, retained after its task detaches to emit the terminal event.
|
||||
pub(crate) turn_id: Option<String>,
|
||||
pub(crate) task: Option<RunningTask>,
|
||||
pub(crate) sampling_interrupt: Option<SamplingInterrupt>,
|
||||
/// Present when input admission is closed; cancelled when the final batch is recorded.
|
||||
@@ -67,6 +69,7 @@ pub(crate) enum MailboxDeliveryPhase {
|
||||
impl Default for ActiveTurn {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
turn_id: None,
|
||||
task: None,
|
||||
sampling_interrupt: None,
|
||||
finishing: None,
|
||||
|
||||
@@ -367,6 +367,7 @@ impl Session {
|
||||
}
|
||||
let turn = active.get_or_insert_with(ActiveTurn::default);
|
||||
debug_assert!(turn.task.is_none());
|
||||
turn.turn_id = Some(turn_context.sub_id.clone());
|
||||
turn.finishing = None;
|
||||
Arc::clone(&turn.turn_state)
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user