diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 73037682ec..4666ca9c1b 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -255,10 +255,8 @@ async fn run_compact_task_inner( let result = result.and_then(|_| sess.check_rollout_budget()); if let Err(err) = &result && !matches!(phase, CompactionPhase::PostTurn) - && !matches!( - err.details(), - CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted - ) + && !matches!(err.details(), CodexErrorDetails::Interrupted) + && !crate::tasks::ends_turn_as_interrupted(err) { sess.track_turn_codex_error(turn_context.as_ref(), err); // Pre-turn failures are reported after preserving the incoming prompt. @@ -573,10 +571,8 @@ pub(crate) fn compaction_status_from_result(result: &CodexResult) -> Compa match result { Ok(_) => CompactionStatus::Completed, Err(err) - if matches!( - err.details(), - CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted - ) => + if matches!(err.details(), CodexErrorDetails::Interrupted) + || crate::tasks::ends_turn_as_interrupted(err) => { CompactionStatus::Interrupted } diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 0c68011664..637743c6a6 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -268,7 +268,7 @@ async fn run_remote_compact_task_inner( match result { Ok(installed) => Ok(installed), Err(err) - if matches!(err.details(), CodexErrorDetails::TurnAborted) + if crate::tasks::ends_turn_as_interrupted(&err) || matches!(phase, CompactionPhase::PostTurn) => { Err(err) diff --git a/codex-rs/core/src/context/turn_aborted.rs b/codex-rs/core/src/context/turn_aborted.rs index f0bd958060..254c69593f 100644 --- a/codex-rs/core/src/context/turn_aborted.rs +++ b/codex-rs/core/src/context/turn_aborted.rs @@ -10,7 +10,7 @@ pub(crate) struct TurnAborted { impl TurnAborted { pub(crate) const INTERRUPTED_GUIDANCE: &'static str = "The user interrupted the previous turn on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed."; - pub(crate) const INTERRUPTED_DEVELOPER_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed."; + pub(crate) const INTERRUPTED_NEUTRAL_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed."; pub(crate) fn new(guidance: impl Into) -> Self { Self { diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 8350baba82..7496e7c894 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -537,9 +537,8 @@ async fn handle_approved_mcp_tool_call( let mut admission_error = None; let admit = || { let reason = sess - .rollout_budget() - .tools_stopped() - .then(|| "rollout budget exhausted".to_owned()); + .tool_start_denial() + .map_err(|error| McpAdmissionError(FunctionCallError::Fatal(error.to_string())))?; if let Some(reason) = reason { return Err(McpAdmissionError(FunctionCallError::RespondToModel(format!( "aborted before MCP handoff: {reason}" diff --git a/codex-rs/core/src/session/rollout_budget.rs b/codex-rs/core/src/session/rollout_budget.rs index 12bbc11bbb..43bd80b6a6 100644 --- a/codex-rs/core/src/session/rollout_budget.rs +++ b/codex-rs/core/src/session/rollout_budget.rs @@ -46,6 +46,14 @@ impl Session { &self.services.local_agent_runtime.rollout_budget } + /// Return the diagnostic for a refused tool or propagate an accounting error. + pub(crate) fn tool_start_denial(&self) -> CodexResult> { + Ok(self + .rollout_budget() + .tools_stopped() + .then(|| "rollout budget exhausted".into())) + } + pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> { if self.rollout_budget().tools_stopped() { Err(CodexErr::SessionBudgetExceeded) diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index ee6a16237c..1cfd2a535f 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -277,7 +277,7 @@ pub(crate) async fn run_turn( PersistContext::Standard, ) .await; - if matches!(err.details(), CodexErrorDetails::TurnAborted) { + if crate::tasks::ends_turn_as_interrupted(&err) { return Err(err); } if matches!(err.details(), CodexErrorDetails::ToolCollision(_)) { @@ -345,7 +345,7 @@ pub(crate) async fn run_turn( .await { Ok(step_context) => step_context, - Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => { + Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => { let _ = run_hooks_and_record_inputs( &sess, &turn_context, @@ -793,7 +793,7 @@ pub(crate) async fn run_turn( } Ok(false) => {} Err(err) => { - if matches!(err.details(), CodexErrorDetails::TurnAborted) { + if crate::tasks::ends_turn_as_interrupted(&err) { return Err(err); } let error = err.to_codex_protocol_error(); @@ -897,10 +897,9 @@ pub(crate) async fn run_turn( ) .await { - if matches!( - err.details(), - CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted - ) { + if matches!(err.details(), CodexErrorDetails::Interrupted) + || crate::tasks::ends_turn_as_interrupted(&err) + { return Err(err); } let error = err.to_codex_protocol_error(); @@ -951,7 +950,7 @@ pub(crate) async fn run_turn( can_drain_pending_input = false; continue; } - Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => { + Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => { return Err(err); } Err(codex_error) diff --git a/codex-rs/core/src/tasks/compact.rs b/codex-rs/core/src/tasks/compact.rs index 6014a2d7f7..35b249841f 100644 --- a/codex-rs/core/src/tasks/compact.rs +++ b/codex-rs/core/src/tasks/compact.rs @@ -9,7 +9,6 @@ use crate::session::turn_context::TurnContext; use crate::state::TaskKind; use codex_features::Feature; use codex_model_provider::RemoteCompactionSupport; -use codex_protocol::error::CodexErrorDetails; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::user_input::UserInput; use tokio_util::sync::CancellationToken; @@ -95,7 +94,7 @@ impl SessionTask for CompactTask { } }; if let Err(err) = result { - if matches!(err.details(), CodexErrorDetails::TurnAborted) { + if super::ends_turn_as_interrupted(&err) { return Err(err); } let error = err.to_codex_protocol_error(); diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 44f195ef57..c3c1d20b4c 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -53,6 +53,7 @@ use codex_otel::TURN_TOOL_CALL_METRIC; use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC; use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::MultiAgentVersion; @@ -67,6 +68,7 @@ use codex_thread_store::PersistContext; pub(crate) use admission::HostTurnStarted; use codex_features::Feature; +use codex_protocol::error::CodexErr; use codex_protocol::error::CodexErrorDetails; use codex_protocol::error::Result as CodexResult; pub(crate) use compact::CompactTask; @@ -82,6 +84,11 @@ static ACTIVE_TURNS: Gauge = Gauge::new("core.turns.active"); pub(crate) type SessionTaskResult = CodexResult>; +/// Whether task completion should publish an interrupted turn. +pub(crate) fn ends_turn_as_interrupted(error: &CodexErr) -> bool { + matches!(error.details(), CodexErrorDetails::TurnAborted) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InterruptedTurnHistoryMarker { Disabled, @@ -109,15 +116,19 @@ impl InterruptedTurnHistoryMarker { /// interrupted fork snapshots. pub(crate) fn interrupted_turn_history_marker( marker: InterruptedTurnHistoryMarker, + cause: Option<&CodexErrorInfo>, ) -> Option { - match marker { - InterruptedTurnHistoryMarker::Disabled => None, - InterruptedTurnHistoryMarker::ContextualUser => Some(ContextualUserFragment::into( - crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE), - )), - InterruptedTurnHistoryMarker::Developer => { + match (marker, cause.is_some_and(is_automatic_rollout_stop)) { + (InterruptedTurnHistoryMarker::Disabled, _) => None, + (InterruptedTurnHistoryMarker::ContextualUser, false) => { + Some(ContextualUserFragment::into( + crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE), + )) + } + (InterruptedTurnHistoryMarker::ContextualUser, true) + | (InterruptedTurnHistoryMarker::Developer, _) => { let marker = crate::context::TurnAborted::new( - crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE, + crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE, ); let (_, content) = marker.render_fragment().into_parts(); Some(RenderedFragment::new("developer", content).into()) @@ -125,6 +136,10 @@ pub(crate) fn interrupted_turn_history_marker( } } +fn is_automatic_rollout_stop(cause: &CodexErrorInfo) -> bool { + matches!(cause, CodexErrorInfo::SessionBudgetExceeded) +} + fn emit_turn_network_proxy_metric( session_telemetry: &SessionTelemetry, network_proxy_active: bool, @@ -667,25 +682,7 @@ impl Session { reason: TurnAbortReason, error: Option, ) -> bool { - let active_turn = { - let mut active = self.active_turn.lock().await; - if active - .as_ref() - .and_then(|active_turn| active_turn.task.as_ref()) - .is_some_and(|task| task.turn_context.sub_id == turn_id) - { - if matches!( - reason, - TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited - ) { - self.mark_interrupted(); - } - active.take() - } else { - None - } - }; - let Some(active_turn) = active_turn else { + let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else { return false; }; @@ -693,6 +690,24 @@ impl Session { true } + async fn take_named_active_turn( + &self, + turn_id: &str, + reason: &TurnAbortReason, + ) -> Option { + let mut active = self.active_turn.lock().await; + if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id { + return None; + } + if matches!( + reason, + TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited + ) { + self.mark_interrupted(); + } + active.take() + } + pub(crate) async fn finish_turn_abort( self: &Arc, mut active_turn: ActiveTurn, @@ -723,10 +738,11 @@ impl Session { turn_context: Arc, task_result: SessionTaskResult, ) -> Option { - let (regular_output, abort_reason, task_error) = match task_result { + let (regular_output, abort, task_error) = match task_result { Ok(output) => (output, None, None), - Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => { - (None, Some(TurnAbortReason::Interrupted), None) + Err(err) if ends_turn_as_interrupted(&err) => { + let error: Option = None; + (None, Some((TurnAbortReason::Interrupted, error)), None) } Err(err) => { warn!(%err, "session task returned an unexpected error"); @@ -799,6 +815,24 @@ impl Session { task.abort(Arc::clone(self), Arc::clone(&turn_context)) .await; } + let stop_cause = { + let terminal_error = turn_context.terminal_error.lock().await; + abort + .as_ref() + .and_then(|(_, error)| error.as_ref()) + .and_then(|error| error.codex_error_info.clone()) + .or_else(|| task_error.as_ref().map(CodexErr::to_codex_protocol_error)) + .or_else(|| { + terminal_error + .as_ref() + .and_then(|error| error.codex_error_info.clone()) + }) + .filter(is_automatic_rollout_stop) + }; + if let Some(cause) = &stop_cause { + self.record_interrupted_turn_history(&turn_context, Some(cause)) + .await; + } drop(finish_recording); // A retry on the first error must start a new turn with the denied turn's input in history. for err in task_error.into_iter().chain(record_result.err()) { @@ -932,16 +966,16 @@ impl Session { profile, }); let idle_cause = if matches!( - abort_reason.as_ref(), + abort.as_ref().map(|(reason, _)| reason), Some(TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited) ) { ThreadIdleCause::Interrupted - } else if abort_reason.is_none() && turn_context.terminal_error.lock().await.is_some() { + } else if abort.is_none() && turn_context.terminal_error.lock().await.is_some() { ThreadIdleCause::Failed } else { ThreadIdleCause::Completed }; - let event = if let Some(reason) = abort_reason { + let event = if let Some((reason, error)) = abort { if reason == TurnAbortReason::Interrupted { run_turn_interrupt_hooks(self, &turn_context, &turn_state).await; } @@ -951,7 +985,7 @@ impl Session { root_turn_id: Some(turn_context.root_turn_id()), turn_id: Some(turn_context.sub_id.clone()), reason, - error: None, + error, started_at, completed_at, duration_ms, @@ -1021,10 +1055,9 @@ impl Session { turn.task.is_some() || (self.enabled(Feature::WakeOnAgentMessages) && turn.finishing.is_none()) }) { - // Message wakes reserve work before registering a task. With the - // feature enabled, stopping a reservation must block later mail too. - // Finishing turns must retain their status until completion is - // persisted, so capacity eviction cannot close their rollout writer. + // A message wake can reserve work before registering a task. Mark + // it interrupted too, but leave a finishing turn's status intact + // until its terminal event is persisted. self.mark_interrupted(); } active.take() @@ -1048,6 +1081,32 @@ impl Session { .await } + async fn record_interrupted_turn_history( + &self, + turn_context: &TurnContext, + cause: Option<&CodexErrorInfo>, + ) { + let Some(marker) = interrupted_turn_history_marker( + InterruptedTurnHistoryMarker::from_config_and_version( + turn_context.config.as_ref(), + turn_context.multi_agent_version, + ), + cause, + ) else { + return; + }; + self.record_conversation_items( + turn_context, + turn_context.model_info(), + std::slice::from_ref(&marker), + ) + .await; + // Clients may re-read the rollout as soon as they receive the terminal event. + if let Err(err) = self.flush_rollout().await { + warn!("failed to flush interrupted-turn marker before terminal event: {err}"); + } + } + async fn handle_task_abort( self: &Arc, task: RunningTask, @@ -1117,28 +1176,14 @@ impl Session { .abort(Arc::clone(self), Arc::clone(&task.turn_context)) .await; - if reason == TurnAbortReason::Interrupted - && let Some(marker) = interrupted_turn_history_marker( - InterruptedTurnHistoryMarker::from_config_and_version( - task.turn_context.config.as_ref(), - task.turn_context.multi_agent_version, - ), - ) - { - self.record_conversation_items( - task.turn_context.as_ref(), - task.turn_context.model_info(), - std::slice::from_ref(&marker), + if reason == TurnAbortReason::Interrupted { + self.record_interrupted_turn_history( + &task.turn_context, + error + .as_ref() + .and_then(|error| error.codex_error_info.as_ref()), ) .await; - // Ensure the marker is durably visible before emitting TurnAborted: some clients - // synchronously re-read the rollout on receipt of the abort event. - if let Err(err) = self.flush_rollout().await { - warn!("failed to flush interrupted-turn marker before emitting TurnAborted: {err}"); - } - } - - if reason == TurnAbortReason::Interrupted { run_turn_interrupt_hooks(self, &task.turn_context, turn_state).await; } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 8c2fd45869..35e5c84d7a 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -3315,14 +3315,18 @@ fn append_interrupted_boundary( match history { InitialHistory::New | InitialHistory::Cleared => { let mut history = Vec::new(); - if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) { + if let Some(marker) = + interrupted_turn_history_marker(interrupted_marker, /*cause*/ None) + { history.push(RolloutItem::ResponseItem(marker.into())); } history.push(aborted_event); InitialHistory::Forked(history) } InitialHistory::Forked(mut history) => { - if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) { + if let Some(marker) = + interrupted_turn_history_marker(interrupted_marker, /*cause*/ None) + { history.push(RolloutItem::ResponseItem(marker.into())); } history.push(aborted_event); @@ -3330,7 +3334,9 @@ fn append_interrupted_boundary( } InitialHistory::Resumed(resumed) => { let mut history = Arc::unwrap_or_clone(resumed.history); - if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) { + if let Some(marker) = + interrupted_turn_history_marker(interrupted_marker, /*cause*/ None) + { history.push(RolloutItem::ResponseItem(marker.into())); } history.push(aborted_event); diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 2d7090db02..a038e1e641 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -1141,12 +1141,15 @@ fn assistant_msg(text: &str) -> ResponseItem { } fn contextual_user_interrupted_marker() -> ResponseItem { - interrupted_turn_history_marker(InterruptedTurnHistoryMarker::ContextualUser) - .expect("contextual-user interrupted marker should be enabled") + interrupted_turn_history_marker( + InterruptedTurnHistoryMarker::ContextualUser, + /*cause*/ None, + ) + .expect("contextual-user interrupted marker should be enabled") } fn developer_interrupted_marker() -> ResponseItem { - interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer) + interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer, /*cause*/ None) .expect("developer interrupted marker should be enabled") } @@ -3846,7 +3849,7 @@ fn multi_agent_v2_interrupted_marker_uses_developer_input_message() { content: vec![ContentItem::InputText { text: format!( "\n{}\n", - crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE + crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE ), }], phase: None, diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 38dfbc1a0e..13dcba3fd8 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -338,7 +338,11 @@ impl ToolCallRuntime { .or_cancel_tool(&invocation_cancellation_token) .await?; // A peer may have exhausted the account while this call waited for its slot. - if session.rollout_budget().tools_stopped() { + if session + .tool_start_denial() + .map_err(|error| FunctionCallError::Fatal(error.to_string()))? + .is_some() + { tracing::Span::current().record("aborted", true); record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id); return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch)); diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index ac8288c7e4..6b78419ca7 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -933,26 +933,32 @@ impl ToolRegistry { } // Environment resolution, pre-tool hooks and Guardian may all have waited. - let invocation_guard = if invocation.session.rollout_budget().tools_stopped() { - None - } else { - let guard = reservation.activate(&invocation); + let mut start_denial = invocation.session.tool_start_denial(); + let mut invocation_guard = None; + if matches!(start_denial, Ok(None)) { + invocation_guard = reservation.activate(&invocation); if tool.mcp_server_name().is_none() { notify_tool_start(&invocation, /*mcp_tool*/ None) .or_cancel_tool(&cancellation_token) .await?; } - guard + // The rollout can stop while the start observer waits. + start_denial = invocation.session.tool_start_denial(); + } + let reason = match start_denial { + Ok(reason) => reason, + Err(error) => { + let err = FunctionCallError::Fatal(error.to_string()); + return Err(fail_before_execution(dispatch_trace, &invocation, err).await); + } }; - // A start observer can also wait. This is the last check before polling the - // handler; a stop after admission does not cancel an already-running tool. - if invocation.session.rollout_budget().tools_stopped() { + if let Some(reason) = reason { drop(invocation_guard); let result = AnyToolResult { call_id: invocation.call_id.clone(), payload: invocation.payload.clone(), result: Box::new(AbortedToolOutput { - message: "aborted before handler start: rollout budget exhausted".into(), + message: format!("aborted before handler start: {reason}"), }), post_tool_use_payload: None, }; diff --git a/codex-rs/core/tests/suite/rollout_budget.rs b/codex-rs/core/tests/suite/rollout_budget.rs index 455fab0851..b84cb8a1b3 100644 --- a/codex-rs/core/tests/suite/rollout_budget.rs +++ b/codex-rs/core/tests/suite/rollout_budget.rs @@ -14,8 +14,11 @@ use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; use codex_features::Feature; +use codex_history::RolloutItem; use codex_model_provider_info::built_in_model_providers; use codex_protocol::error::CodexErr; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -514,6 +517,32 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( .await; } + if owner == AgentRuntimeOwnership::Controller && enforce_limit { + let history = test.codex.load_history(/*include_archived*/ false).await?; + let warning_roles = history + .items + .iter() + .filter_map(|item| { + let RolloutItem::ResponseItem(item) = item else { + return None; + }; + let ResponseItem::Message { role, content, .. } = &item.item else { + return None; + }; + content + .iter() + .any(|part| { + matches!(part, + ContentItem::InputText { text } + if text.contains("") + ) + }) + .then_some(role.as_str()) + }) + .collect::>(); + assert_eq!(warning_roles, ["developer", "developer"]); + } + if owner == AgentRuntimeOwnership::Host { let requests = responses.requests(); assert_eq!(requests.len(), charges.len());