diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index c1ed1a7544..e992ffa54c 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -200,7 +200,7 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::TurnComplete(turn_complete_event) => { - respond_to_pending_interrupts(&thread_state, &outgoing).await; + respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await; if finishes_latest_turn { let turn_failed = thread_state .lock() @@ -1297,7 +1297,7 @@ pub(crate) async fn apply_bespoke_event_handling( } // If this is a TurnAborted, reply to any pending interrupt requests. EventMsg::TurnAborted(turn_aborted_event) => { - respond_to_pending_interrupts(&thread_state, &outgoing).await; + respond_to_pending_interrupts(&event_turn_id, &thread_state, &outgoing).await; if finishes_latest_turn { thread_watch_manager @@ -1652,12 +1652,17 @@ async fn handle_turn_interrupted( } async fn respond_to_pending_interrupts( + event_turn_id: &str, thread_state: &Arc>, outgoing: &ThreadScopedOutgoingMessageSender, ) { let pending = { let mut state = thread_state.lock().await; - std::mem::take(&mut state.pending_interrupts) + state + .pending_interrupts + .extract_if(.., |(turn_id, _)| turn_id == event_turn_id) + .map(|(_, request_id)| request_id) + .collect::>() }; for request_id in pending { diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 7787313c63..f95d09f8cc 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -756,6 +756,25 @@ async fn start_listener( drop(turn_cost_worker); if shutdown_complete { + // Close registration and take the queue under the same lock used by turn/interrupt. + // Detached finalizers can publish their terminal events after this listener retires. + let pending_interrupts = { + let mut state = thread_state.lock().await; + if state.listener_generation == listener_generation { + state.listener_shutdown = true; + std::mem::take(&mut state.pending_interrupts) + } else { + Vec::new() + } + }; + for (_, request_id) in pending_interrupts { + outgoing_for_task + .send_error( + request_id, + internal_error("thread shut down before turn interruption completed"), + ) + .await; + } drained_tx.send_replace(true); let relay_command = |command| { handle_thread_listener_command( diff --git a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs index 001e82abee..a35db8e91f 100644 --- a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs @@ -610,3 +610,228 @@ async fn resume_preserves_recovery_turn_before_status_publication() -> Result<() processor.skills_watcher.shutdown(); Ok(()) } + +#[tokio::test] +async fn revert_settles_interrupts_before_replacing_a_runtime() -> Result<()> { + use super::super::turn_processor::TurnRequestProcessor; + use crate::outgoing_message::OutgoingError; + use codex_app_server_protocol::ThreadRevertParams; + use codex_app_server_protocol::TurnInterruptParams; + use codex_core::TurnInputRequest; + use codex_core::TurnInputSubmission; + use codex_extension_api::ExtensionFuture; + use codex_extension_api::ExtensionRegistryBuilder; + use codex_extension_api::TurnLifecycleContributor; + use codex_extension_api::TurnStopInput; + use codex_protocol::protocol::MultiAgentVersion; + use codex_protocol::protocol::ThreadHistoryMode; + use tokio::sync::Notify; + + #[derive(Default)] + struct StopGate { + entered: Notify, + release: Notify, + } + + impl TurnLifecycleContributor for StopGate { + fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async { + self.entered.notify_one(); + self.release.notified().await; + }) + } + } + + let gate = Arc::new(StopGate::default()); + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.turn_lifecycle_contributor(gate.clone()); + let server = responses::start_mock_server().await; + responses::mount_sse_once(&server, responses::sse_completed("first")).await; + let test = test_codex() + .with_extensions(Arc::new(extensions.build())) + .with_history_mode(ThreadHistoryMode::Paginated) + .with_config(|config| { + config.agents_enabled = false; + config + .features + .disable(codex_features::Feature::MultiAgentV2) + .unwrap(); + }) + .build_with_auto_env(&server) + .await?; + let thread_id = test.session_configured.thread_id; + let (processor, mut outgoing_rx) = test_processor(&test).await?; + // Keep transport flowing, retaining every reply and turn boundary in client-visible order. + let (observed_tx, mut observed_rx) = mpsc::unbounded_channel(); + let collector = tokio::spawn(async move { + while let Some(envelope) = outgoing_rx.recv().await { + let OutgoingEnvelope::ToConnection { message, .. } = envelope else { + continue; + }; + let relevant = match &message { + OutgoingMessage::Response(_) | OutgoingMessage::Error(_) => true, + OutgoingMessage::AppServerNotification(envelope) => matches!( + envelope.notification, + ServerNotification::TurnStarted(_) | ServerNotification::TurnCompleted(_) + ), + OutgoingMessage::Request(_) => false, + }; + if relevant && observed_tx.send(message).is_err() { + break; + } + } + }); + let connection_id = ConnectionId(1); + processor + .connection_initialized(connection_id, ConnectionCapabilities::default()) + .await; + processor + .try_attach_thread_listener(thread_id, test.codex.clone(), vec![connection_id]) + .await; + let turn_processor = TurnRequestProcessor::new( + test.thread_manager.clone(), + processor.outgoing.clone(), + AnalyticsEventsClient::disabled(), + processor.config.clone(), + processor.config_manager.clone(), + processor.pending_thread_unloads.clone(), + processor.thread_state_manager.clone(), + processor.thread_watch_manager.clone(), + processor.skills_watcher.clone(), + /*turn_cost_worker*/ None, + ); + let request_id = |id| ConnectionRequestId { + connection_id, + request_id: RequestId::Integer(id), + }; + let interrupt = async |id, turn_id: &str| { + turn_processor + .turn_interrupt( + &request_id(id), + TurnInterruptParams { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_owned(), + }, + ) + .await + }; + let start = async |thread: &Arc, text: &str| -> Result { + let TurnInputSubmission::Started { turn_id, .. } = thread + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: text.to_owned(), + text_elements: vec![], + }])) + .await? + else { + anyhow::bail!("expected a new turn") + }; + Ok(turn_id) + }; + let first = start(&test.codex, "first").await?; + assert_eq!( + test.codex.multi_agent_version(), + Some(MultiAgentVersion::Disabled) + ); + gate.entered.notified().await; + assert!(matches!(observed_rx.recv().await, + Some(OutgoingMessage::AppServerNotification(n)) + if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == first) + )); + assert!( + interrupt(/*id*/ 99, &first) + .await + .expect("interrupt accepted") + .is_none() + ); + let state = processor.thread_state_manager.thread_state(thread_id).await; + assert_eq!(state.lock().await.pending_interrupts.len(), 1); + let mut drain = state + .lock() + .await + .shutdown_drain + .clone() + .expect("listener installed"); + + let revert = processor.thread_revert( + request_id(/*id*/ 100), + ThreadRevertParams { + thread_id: thread_id.to_string(), + before_turn_id: first.clone(), + }, + /*app_server_client_name*/ None, + /*app_server_client_version*/ None, + ); + tokio::pin!(revert); + // Stop polling revert at its shutdown barrier so late requests still load the retired runtime. + tokio::select! { + biased; + drained = drain.wait_for(|drained| *drained) => { drained?; } + result = &mut revert => { anyhow::bail!("revert passed the shutdown barrier: {result:?}"); } + } + assert!(Arc::ptr_eq( + &test.codex, + &test.thread_manager.get_thread(thread_id).await? + )); + let Some(OutgoingMessage::Error(error)) = observed_rx.recv().await else { + anyhow::bail!("expected the pending interrupt's shutdown error"); + }; + assert_eq!( + error, + OutgoingError { + id: request_id(/*id*/ 99).request_id, + error: internal_error("thread shut down before turn interruption completed"), + } + ); + assert!(state.lock().await.pending_interrupts.is_empty()); + assert_eq!( + interrupt(/*id*/ 101, &first).await.unwrap_err(), + invalid_request("thread runtime is no longer accepting interrupts") + ); + + gate.release.notify_one(); + wait_for_event( + &test.codex, + |msg| matches!(msg, EventMsg::TurnComplete(done) if done.turn_id == first), + ) + .await; + // Late finalization changes Core's status back from Shutdown; the listener must stay closed. + assert!(!test.codex.has_shutdown_status()); + assert_eq!( + interrupt(/*id*/ 102, &first).await.unwrap_err(), + invalid_request("thread runtime is no longer accepting interrupts") + ); + revert.await.expect("revert replaces the retired runtime"); + assert!(matches!(observed_rx.recv().await, + Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 100).request_id + )); + let replacement = test.thread_manager.get_thread(thread_id).await?; + assert!(!Arc::ptr_eq(&test.codex, &replacement)); + + responses::mount_sse_once(&server, responses::sse_completed("second")).await; + let second = start(&replacement, "second").await?; + gate.entered.notified().await; + assert!(matches!(observed_rx.recv().await, + Some(OutgoingMessage::AppServerNotification(n)) + if matches!(&n.notification, ServerNotification::TurnStarted(started) if started.turn.id == second) + )); + assert!( + interrupt(/*id*/ 103, &second) + .await + .expect("replacement accepts interrupts") + .is_none() + ); + gate.release.notify_one(); + assert!(matches!(observed_rx.recv().await, + Some(OutgoingMessage::Response(response)) if response.id == request_id(/*id*/ 103).request_id + )); + assert!(matches!(observed_rx.recv().await, + Some(OutgoingMessage::AppServerNotification(n)) + if matches!(&n.notification, ServerNotification::TurnCompleted(done) if done.turn.id == second) + )); + assert!(state.lock().await.pending_interrupts.is_empty()); + replacement.shutdown_and_wait().await?; + processor.thread_state_manager.clear_all_listeners().await; + processor.skills_watcher.shutdown(); + collector.abort(); + Ok(()) +} diff --git a/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs b/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs new file mode 100644 index 0000000000..47649b189a --- /dev/null +++ b/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs @@ -0,0 +1,278 @@ +#![cfg(test)] + +//! Checks interrupt replies against the order in which the client sees terminal events. + +use super::*; +use crate::bespoke_event_handling::apply_bespoke_event_handling; +use crate::outgoing_message::OutgoingEnvelope; +use crate::outgoing_message::OutgoingMessage; +use app_test_support::create_mock_responses_server_repeating_assistant; +use codex_app_server_protocol::ThreadQueueAddParams; +use codex_app_server_protocol::ThreadQueueStartParams; +use codex_extension_api::ExtensionFuture; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::NoopExtensionEventSink; +use codex_extension_api::TurnLifecycleContributor; +use codex_extension_api::TurnStartInput; +use codex_extension_api::TurnStopInput; +use codex_protocol::protocol::Event; +use codex_queue_extension::QueuedItemService; +use codex_thread_store::LocalQueueStore; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; +use serde_json::json; +use test_case::test_case; +use tokio::sync::Notify; + +#[test] +fn late_completion_keeps_a_finished_review_delegate_ineligible_for_interrupt() { + let mut state = ThreadState::default(); + let event = |kind, id| serde_json::from_value(json!({"type": kind, "turn_id": id})).unwrap(); + state.track_current_turn_event("previous", &event("turn_started", "previous")); + state.track_current_turn_event("review", &event("turn_started", "delegate")); + assert_eq!( + interrupt_terminal_turn_id(&state, "delegate"), + Some("review") + ); + for id in ["review", "previous"] { + state.track_current_turn_event(id, &event("turn_complete", id)); + state.take_turn_summary(id); + } + assert_eq!(interrupt_terminal_turn_id(&state, "delegate"), None); +} + +#[derive(Default)] +struct TerminalGates { + parent: std::sync::OnceLock, + delegate_start: Notify, + run_delegate: Notify, + stopped: Notify, + finish: Notify, +} + +impl TurnLifecycleContributor for TerminalGates { + fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if self + .parent + .get() + .is_some_and(|id| id != input.thread_store.level_id()) + { + self.delegate_start.notify_one(); + self.run_delegate.notified().await; + } + }) + } + + fn on_turn_stop<'a>(&'a self, _: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async { + self.stopped.notify_one(); + self.finish.notified().await; + }) + } +} + +#[test_case(false; "turn_start")] +#[test_case(true; "thread_queue_start")] +#[tokio::test] +async fn interrupt_reply_waits_for_the_requested_turn(queued: bool) -> anyhow::Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + let gates = Arc::new(TerminalGates::default()); + let mut registry = ExtensionRegistryBuilder::new(); + registry.turn_lifecycle_contributor(gates.clone()); + let server = create_mock_responses_server_repeating_assistant("Done").await; + let test = test_codex() + .with_extensions(Arc::new(registry.build())) + .build_with_auto_env(&server) + .await?; + let thread_id = test.session_configured.thread_id; + gates.parent.set(thread_id.to_string()).unwrap(); + let connection_id = ConnectionId(1); + let analytics = AnalyticsEventsClient::disabled(); + let (tx, mut rx) = tokio::sync::mpsc::channel(crate::CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx, analytics.clone())); + let watcher = SkillsWatcher::new( + test.thread_manager.skills_service(), + &test.config.codex_home, + outgoing.clone(), + ); + let processor = TurnRequestProcessor::new( + test.thread_manager.clone(), + outgoing.clone(), + analytics, + Arc::new(test.config.clone()), + ConfigManager::without_managed_config_for_tests(test.config.codex_home.to_path_buf()), + Default::default(), + ThreadStateManager::new(), + ThreadWatchManager::new(), + watcher.clone(), + /*turn_cost_worker*/ None, + ); + let request_id = |id| ConnectionRequestId { + connection_id, + request_id: RequestId::Integer(id), + }; + let turn = TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![V2UserInput::Text { + text: "first".into(), + text_elements: vec![], + }], + ..Default::default() + }; + let interrupt = async |id, turn_id: &str| { + processor + .turn_interrupt_inner( + &request_id(id), + TurnInterruptParams { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_owned(), + }, + ) + .await + }; + let first = if queued { + let store = LocalQueueStore::new(test.codex.state_db().expect("state DB")); + let service = QueuedItemService::new( + Arc::new(store), + Arc::downgrade(&test.thread_manager), + Arc::new(NoopExtensionEventSink), + ); + let queue = ThreadQueueRequestProcessor::new( + test.thread_manager.clone(), + test.thread_store.clone(), + outgoing.clone(), + processor.config_manager.clone(), + Some(Arc::new(service)), + ); + let add = ThreadQueueAddParams { + thread_id: turn.thread_id.clone(), + input: turn.input, + client_user_message_id: "queued-first".into(), + responsesapi_client_metadata: None, + }; + queue.add(add).await.expect("enqueue"); + let params = ThreadQueueStartParams { + thread_id: turn.thread_id, + queued_submission_id: None, + }; + queue + .start(&request_id(1), params) + .await + .expect("start queued turn") + .turn + .id + } else { + processor + .turn_start_inner( + request_id(1), + turn, + /*app_server_client_name*/ None, + /*app_server_client_version*/ None, + ) + .await + .expect("start turn") + .turn + .id + }; + gates.stopped.notified().await; + assert_eq!( + interrupt(0, "unknown").await.unwrap_err(), + invalid_request("no active turn to interrupt") + ); + assert_eq!(interrupt(2, &first).await, Ok(None)); + gates.finish.notify_one(); + // Hold A's actual terminal event until B has an interrupt waiting for a reply. + let completed = wait_for_event( + &test.codex, + |msg| matches!(msg, EventMsg::TurnComplete(e) if e.turn_id == first && e.error.is_none()), + ) + .await; + + let params = ReviewStartParams { + thread_id: thread_id.to_string(), + target: ApiReviewTarget::UncommittedChanges, + delivery: None, + }; + let review = processor + .review_start(&request_id(3), params) + .await + .expect("start review"); + let Some(ClientResponsePayload::ReviewStart(review)) = review else { + panic!("expected review response") + }; + let second = review.turn.id; + gates.delegate_start.notified().await; + assert_eq!( + interrupt(5, &first).await.unwrap_err(), + invalid_request("no active turn to interrupt") + ); + // The review owns the slot, but Core has not announced its delegate yet. + assert_eq!( + interrupt(6, &second).await, + Err(invalid_request("no active turn to interrupt")) + ); + gates.run_delegate.notify_one(); + wait_for_event(&test.codex, |msg| matches!(msg, EventMsg::TurnStarted(_))).await; + // The delegate finishes its model work after the stale interrupt. Its parent is still running. + gates.stopped.notified().await; + assert_eq!(interrupt(4, &second).await, Ok(None)); + gates.finish.notify_one(); + let aborted = wait_for_event( + &test.codex, + |msg| matches!(msg, EventMsg::TurnAborted(e) if e.turn_id.as_deref() == Some(&second)), + ) + .await; + + let state = processor.thread_state_manager.thread_state(thread_id).await; + let outgoing = ThreadScopedOutgoingMessageSender::new(outgoing, vec![connection_id], thread_id); + let mut receive = || match rx.try_recv().expect("terminal event already processed") { + OutgoingEnvelope::ToConnection { message, .. } => message, + other => panic!("unexpected outgoing message {other:?}"), + }; + for (turn_id, msg, reply, status) in [ + (first.clone(), completed, 2, TurnStatus::Completed), + (second, aborted, 4, TurnStatus::Interrupted), + ] { + state.lock().await.track_current_turn_event(&turn_id, &msg); + apply_bespoke_event_handling( + Event { + id: turn_id.clone(), + msg, + }, + thread_id, + test.codex.clone(), + test.thread_manager.clone(), + outgoing.clone(), + state.clone(), + processor.thread_watch_manager.clone(), + ) + .await; + assert!( + matches!(receive(), OutgoingMessage::Response(r) if r.id == RequestId::Integer(reply)) + ); + assert!( + matches!(receive(), OutgoingMessage::AppServerNotification(n) + if matches!(n.notification, ServerNotification::TurnCompleted(ref done) if done.turn.id == turn_id && done.turn.status == status)) + ); + } + // Late child activity opens a history turn after Core has finished both turns. + let activity = serde_json::from_value(json!({ + "type": "sub_agent_activity", "kind": "completed", "event_id": "late-child", + "agent_thread_id": ThreadId::new(), "agent_path": "/root/worker" + }))?; + let history_id = { + let mut state = state.lock().await; + state.track_current_turn_event(&first, &activity); + state.active_turn_id().expect("history turn").to_owned() + }; + assert_eq!( + interrupt(7, &history_id).await, + Err(invalid_request("no active turn to interrupt")) + ); + assert!(state.lock().await.pending_interrupts.is_empty()); + test.codex.shutdown_and_wait().await?; + watcher.shutdown(); + Ok(()) +} diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index f4f50639fd..cacecd27ac 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -1525,66 +1525,108 @@ impl TurnRequestProcessor { } } + #[expect( + clippy::await_holding_invalid_type, + reason = "register the interrupt before the listener can consume the validated turn's terminal event" + )] async fn turn_interrupt_inner( &self, request_id: &ConnectionRequestId, params: TurnInterruptParams, ) -> Result, JSONRPCErrorError> { let TurnInterruptParams { thread_id, turn_id } = params; - let is_startup_interrupt = turn_id.is_empty(); - let (thread_uuid, thread) = self.load_thread(&thread_id).await?; - // Record turn interrupts so we can reply when TurnAborted arrives. Startup - // interrupts do not have a turn and are acknowledged after submission. - if !is_startup_interrupt { - let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; - let is_running = matches!(thread.agent_status().await, AgentStatus::Running); - { - let mut thread_state = thread_state.lock().await; - if let Some(active_turn_id) = thread_state.active_turn_id() { - if active_turn_id != turn_id { - return Err(invalid_request(format!( - "expected active turn id {turn_id} but found {active_turn_id}" - ))); - } - } else if thread_state.last_terminal_turn_id.as_deref() == Some(turn_id.as_str()) - || !is_running - { - return Err(invalid_request("no active turn to interrupt")); - } - thread_state.pending_interrupts.push(request_id.clone()); - } - - self.outgoing - .record_request_turn_id(request_id, &turn_id) - .await; + // Startup interrupts have no target and are acknowledged on submission. + if turn_id.is_empty() { + self.submit_core_op(request_id, thread.as_ref(), Op::Interrupt) + .await + .map_err(|err| internal_error(format!("failed to interrupt startup: {err}")))?; + return Ok(Some(TurnInterruptResponse {})); + } + + // Reply when the target completes or aborts. + let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; + let submit_interrupt = { + let mut thread_state = thread_state.lock().await; + // Serialize registration with listener shutdown. A handler may also have + // loaded the old runtime before a replacement listener was installed. + if thread_state.listener_shutdown + || (thread_state.shutdown_drain.is_some() + && !thread_state.listener_matches(&thread)) + { + return Err(invalid_request( + "thread runtime is no longer accepting interrupts", + )); + } + let listener_turn_id = thread_state.active_turn_id(); + if let Some(active_turn_id) = listener_turn_id + && active_turn_id != turn_id + { + return Err(invalid_request(format!( + "expected active turn id {turn_id} but found {active_turn_id}" + ))); + } + let terminal_turn_id = interrupt_terminal_turn_id(&thread_state, &turn_id) + .ok_or_else(|| invalid_request("no active turn to interrupt"))?; + let submit_interrupt = match thread.current_turn_id().await { + // Core reserves the ID before it announces the turn. + Some(id) + if id == terminal_turn_id + && (listener_turn_id.is_some() + || matches!(thread.agent_status().await, AgentStatus::Running)) => + { + true + } + // Core drops the slot before publishing TurnAborted. + None if thread_state + .pending_interrupts + .iter() + .any(|(id, _)| id == terminal_turn_id) => + { + false + } + // A listener-visible turn may already be aborting. Only a review + // announced before task registration still needs an interrupt here. + None if listener_turn_id.is_some() + && thread_state.turn_summaries.contains_key(terminal_turn_id) => + { + listener_turn_id == Some(terminal_turn_id) + && thread_state.last_started_turn_id.as_deref() != Some(terminal_turn_id) + } + Some(_) | None => return Err(invalid_request("no active turn to interrupt")), + }; + let terminal_turn_id = terminal_turn_id.to_owned(); + thread_state + .pending_interrupts + .push((terminal_turn_id, request_id.clone())); + submit_interrupt + }; + + self.outgoing + .record_request_turn_id(request_id, &turn_id) + .await; + if !submit_interrupt { + return Ok(None); } - // Submit the interrupt. Turn interrupts respond upon TurnAborted; startup - // interrupts respond here because startup cancellation has no turn event. match self .submit_core_op(request_id, thread.as_ref(), Op::Interrupt) .await { - Ok(_) if is_startup_interrupt => Ok(Some(TurnInterruptResponse {})), Ok(_) => Ok(None), Err(err) => { - if !is_startup_interrupt { - let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; - let mut thread_state = thread_state.lock().await; - thread_state - .pending_interrupts - .retain(|pending_request_id| pending_request_id != request_id); - } - let interrupt_target = if is_startup_interrupt { - "startup" - } else { - "turn" + let mut state = thread_state.lock().await; + let Some(index) = state + .pending_interrupts + .iter() + .position(|(_, id)| id == request_id) + else { + // The listener already settled this request. + return Ok(None); }; - Err(internal_error(format!( - "failed to interrupt {interrupt_target}: {err}" - ))) + state.pending_interrupts.remove(index); + Err(internal_error(format!("failed to interrupt turn: {err}"))) } } } @@ -1617,3 +1659,21 @@ impl TurnRequestProcessor { .await } } + +fn interrupt_terminal_turn_id<'a>(state: &'a ThreadState, turn_id: &'a str) -> Option<&'a str> { + let terminal_id = if state.forwarded_turn_id.as_deref() == Some(turn_id) { + // History can still name the delegate after the outer turn has finished. + let outer_id = state.last_started_turn_id.as_deref()?; + if !state.turn_summaries.contains_key(outer_id) { + return None; + } + outer_id + } else { + turn_id + }; + (state.last_terminal_turn_id.as_deref() != Some(terminal_id)).then_some(terminal_id) +} + +#[cfg(test)] +#[path = "turn_interrupt_tests.rs"] +mod turn_interrupt_tests; diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index a5f5abf80a..42e46cab00 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -36,7 +36,7 @@ use tokio::sync::oneshot; use tokio::sync::watch; use tracing::error; -type PendingInterruptQueue = Vec; +type PendingInterruptQueue = Vec<(String, ConnectionRequestId)>; // The unload that inserts an entry owns its removal. Dropping the sender wakes deferred // listener attachment only after that unload finishes cleanup. @@ -124,6 +124,8 @@ pub(crate) struct ThreadState { pub(crate) command_start_times: HashMap>, /// A terminal event can arrive after the listener has already started the next turn. pub(crate) last_started_turn_id: Option, + // A delegate may announce a different ID from the one on its terminal event. + pub(crate) forwarded_turn_id: Option, pub(crate) last_terminal_turn_id: Option, /// Lets an internal runtime replacement wait until the old listener has processed Core's /// `ShutdownComplete` event before that listener is superseded. @@ -131,6 +133,9 @@ pub(crate) struct ThreadState { pub(crate) cancel_tx: Option>, pub(crate) experimental_raw_events: bool, pub(crate) listener_generation: u64, + /// The listener no longer consumes terminal events from this runtime. Unlike Core's + /// agent status, this stays closed when a detached finalizer completes after shutdown. + pub(crate) listener_shutdown: bool, last_thread_settings: Option, listener_command_tx: Option>, current_turn_history: ThreadHistoryBuilder, @@ -174,10 +179,12 @@ impl ThreadState { if let Some(previous) = self.cancel_tx.replace(cancel_tx) { let _ = previous.send(()); } + self.forwarded_turn_id = None; self.turn_summaries.clear(); self.command_start_times.clear(); self.shutdown_drain = Some(shutdown_drain); self.listener_generation = self.listener_generation.wrapping_add(1); + self.listener_shutdown = false; self.last_thread_settings = Some(thread_settings_baseline); self.history_mode = history_mode; self.evict_superseded_agent_text = @@ -197,6 +204,7 @@ impl ThreadState { self.listener_command_tx = None; self.current_turn_history.reset(); self.last_started_turn_id = None; + self.forwarded_turn_id = None; self.turn_summaries.clear(); self.command_start_times.clear(); self.listener_thread = None; @@ -280,6 +288,8 @@ impl ThreadState { pub(crate) fn track_current_turn_event(&mut self, event_turn_id: &str, event: &EventMsg) { if let EventMsg::TurnStarted(payload) = event { self.note_turn_started(event_turn_id, payload.started_at); + self.forwarded_turn_id = + (payload.turn_id != event_turn_id).then(|| payload.turn_id.clone()); } else if let EventMsg::ItemStarted(payload) = event && matches!(payload.item, CoreTurnItem::EnteredReviewMode(_)) { diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 67287994a4..1d443d3a4d 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -892,6 +892,17 @@ impl CodexThread { self.session.active_turn.lock().await.is_some() } + /// Returns the current turn ID, including while its stop hook runs. + pub async fn current_turn_id(&self) -> Option { + self.session + .active_turn + .lock() + .await + .as_ref()? + .turn_id + .clone() + } + /// Returns the trusted root when the expected turn is currently active. pub async fn active_turn_root(&self, expected_turn_id: &str) -> Option { let active = self.session.active_turn.lock().await; diff --git a/codex-rs/core/src/state/turn.rs b/codex-rs/core/src/state/turn.rs index d54dbff5a3..dc7d916b25 100644 --- a/codex-rs/core/src/state/turn.rs +++ b/codex-rs/core/src/state/turn.rs @@ -36,6 +36,8 @@ use codex_protocol::protocol::TokenUsage; /// Metadata about the currently running turn. pub(crate) struct ActiveTurn { + /// The slot's owner, retained after its task detaches to emit the terminal event. + pub(crate) turn_id: Option, pub(crate) task: Option, pub(crate) sampling_interrupt: Option, /// Present when input admission is closed; cancelled when the final batch is recorded. @@ -67,6 +69,7 @@ pub(crate) enum MailboxDeliveryPhase { impl Default for ActiveTurn { fn default() -> Self { Self { + turn_id: None, task: None, sampling_interrupt: None, finishing: None, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index af1f14d657..81d0d24f94 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -367,6 +367,7 @@ impl Session { } let turn = active.get_or_insert_with(ActiveTurn::default); debug_assert!(turn.task.is_none()); + turn.turn_id = Some(turn_context.sub_id.clone()); turn.finishing = None; Arc::clone(&turn.turn_state) };