Record neutral interruption markers for rollout budget stops (#61581)

## Why

Turns stopped by an exhausted rollout budget need a history marker without attributing the interruption to the user.

## What changed

- Record and flush a `<turn_aborted>` marker before the terminal event for `SessionBudgetExceeded`, respecting the interrupt-message setting.
- Use neutral guidance in a `developer` message for automatic budget stops.
- Centralize tool-start budget checks across parallel dispatch, tool handlers, and MCP handoff, retaining the check after tool-start observers finish waiting.
- Share interruption classification and history recording across turn and compaction paths.

## Testing

Extend the rollout-budget integration test to assert that both the budget-crossing turn and a later denied turn persist `developer` interruption markers.

GitOrigin-RevId: 2a49ac9824a4e95e4799f2d0f638f8c46889e8e8
This commit is contained in:
tongzhou wang
2026-11-15 00:02:01 +00:00
committed by copyberry
parent c8c3f70893
commit 2e61e8ec4e
13 changed files with 191 additions and 97 deletions
+4 -8
View File
@@ -255,10 +255,8 @@ async fn run_compact_task_inner(
let result = result.and_then(|_| sess.check_rollout_budget());
if let Err(err) = &result
&& !matches!(phase, CompactionPhase::PostTurn)
&& !matches!(
err.details(),
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
)
&& !matches!(err.details(), CodexErrorDetails::Interrupted)
&& !crate::tasks::ends_turn_as_interrupted(err)
{
sess.track_turn_codex_error(turn_context.as_ref(), err);
// Pre-turn failures are reported after preserving the incoming prompt.
@@ -573,10 +571,8 @@ pub(crate) fn compaction_status_from_result<T>(result: &CodexResult<T>) -> Compa
match result {
Ok(_) => CompactionStatus::Completed,
Err(err)
if matches!(
err.details(),
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
) =>
if matches!(err.details(), CodexErrorDetails::Interrupted)
|| crate::tasks::ends_turn_as_interrupted(err) =>
{
CompactionStatus::Interrupted
}
+1 -1
View File
@@ -268,7 +268,7 @@ async fn run_remote_compact_task_inner(
match result {
Ok(installed) => Ok(installed),
Err(err)
if matches!(err.details(), CodexErrorDetails::TurnAborted)
if crate::tasks::ends_turn_as_interrupted(&err)
|| matches!(phase, CompactionPhase::PostTurn) =>
{
Err(err)
+1 -1
View File
@@ -10,7 +10,7 @@ pub(crate) struct TurnAborted {
impl TurnAborted {
pub(crate) const INTERRUPTED_GUIDANCE: &'static str = "The user interrupted the previous turn on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
pub(crate) const INTERRUPTED_DEVELOPER_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
pub(crate) const INTERRUPTED_NEUTRAL_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
pub(crate) fn new(guidance: impl Into<String>) -> Self {
Self {
+2 -3
View File
@@ -537,9 +537,8 @@ async fn handle_approved_mcp_tool_call(
let mut admission_error = None;
let admit = || {
let reason = sess
.rollout_budget()
.tools_stopped()
.then(|| "rollout budget exhausted".to_owned());
.tool_start_denial()
.map_err(|error| McpAdmissionError(FunctionCallError::Fatal(error.to_string())))?;
if let Some(reason) = reason {
return Err(McpAdmissionError(FunctionCallError::RespondToModel(format!(
"aborted before MCP handoff: {reason}"
@@ -46,6 +46,14 @@ impl Session {
&self.services.local_agent_runtime.rollout_budget
}
/// Return the diagnostic for a refused tool or propagate an accounting error.
pub(crate) fn tool_start_denial(&self) -> CodexResult<Option<String>> {
Ok(self
.rollout_budget()
.tools_stopped()
.then(|| "rollout budget exhausted".into()))
}
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
if self.rollout_budget().tools_stopped() {
Err(CodexErr::SessionBudgetExceeded)
+7 -8
View File
@@ -277,7 +277,7 @@ pub(crate) async fn run_turn(
PersistContext::Standard,
)
.await;
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
if crate::tasks::ends_turn_as_interrupted(&err) {
return Err(err);
}
if matches!(err.details(), CodexErrorDetails::ToolCollision(_)) {
@@ -345,7 +345,7 @@ pub(crate) async fn run_turn(
.await
{
Ok(step_context) => step_context,
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => {
let _ = run_hooks_and_record_inputs(
&sess,
&turn_context,
@@ -793,7 +793,7 @@ pub(crate) async fn run_turn(
}
Ok(false) => {}
Err(err) => {
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
if crate::tasks::ends_turn_as_interrupted(&err) {
return Err(err);
}
let error = err.to_codex_protocol_error();
@@ -897,10 +897,9 @@ pub(crate) async fn run_turn(
)
.await
{
if matches!(
err.details(),
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
) {
if matches!(err.details(), CodexErrorDetails::Interrupted)
|| crate::tasks::ends_turn_as_interrupted(&err)
{
return Err(err);
}
let error = err.to_codex_protocol_error();
@@ -951,7 +950,7 @@ pub(crate) async fn run_turn(
can_drain_pending_input = false;
continue;
}
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => {
return Err(err);
}
Err(codex_error)
+1 -2
View File
@@ -9,7 +9,6 @@ use crate::session::turn_context::TurnContext;
use crate::state::TaskKind;
use codex_features::Feature;
use codex_model_provider::RemoteCompactionSupport;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::user_input::UserInput;
use tokio_util::sync::CancellationToken;
@@ -95,7 +94,7 @@ impl SessionTask for CompactTask {
}
};
if let Err(err) = result {
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
if super::ends_turn_as_interrupted(&err) {
return Err(err);
}
let error = err.to_codex_protocol_error();
+102 -57
View File
@@ -53,6 +53,7 @@ use codex_otel::TURN_TOOL_CALL_METRIC;
use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC;
use codex_otel::auth_storage::AuthStorageOriginator;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::MultiAgentVersion;
@@ -67,6 +68,7 @@ use codex_thread_store::PersistContext;
pub(crate) use admission::HostTurnStarted;
use codex_features::Feature;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result as CodexResult;
pub(crate) use compact::CompactTask;
@@ -82,6 +84,11 @@ static ACTIVE_TURNS: Gauge = Gauge::new("core.turns.active");
pub(crate) type SessionTaskResult = CodexResult<Option<regular::RegularTaskOutput>>;
/// Whether task completion should publish an interrupted turn.
pub(crate) fn ends_turn_as_interrupted(error: &CodexErr) -> bool {
matches!(error.details(), CodexErrorDetails::TurnAborted)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum InterruptedTurnHistoryMarker {
Disabled,
@@ -109,15 +116,19 @@ impl InterruptedTurnHistoryMarker {
/// interrupted fork snapshots.
pub(crate) fn interrupted_turn_history_marker(
marker: InterruptedTurnHistoryMarker,
cause: Option<&CodexErrorInfo>,
) -> Option<ResponseItem> {
match marker {
InterruptedTurnHistoryMarker::Disabled => None,
InterruptedTurnHistoryMarker::ContextualUser => Some(ContextualUserFragment::into(
crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE),
)),
InterruptedTurnHistoryMarker::Developer => {
match (marker, cause.is_some_and(is_automatic_rollout_stop)) {
(InterruptedTurnHistoryMarker::Disabled, _) => None,
(InterruptedTurnHistoryMarker::ContextualUser, false) => {
Some(ContextualUserFragment::into(
crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE),
))
}
(InterruptedTurnHistoryMarker::ContextualUser, true)
| (InterruptedTurnHistoryMarker::Developer, _) => {
let marker = crate::context::TurnAborted::new(
crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE,
crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE,
);
let (_, content) = marker.render_fragment().into_parts();
Some(RenderedFragment::new("developer", content).into())
@@ -125,6 +136,10 @@ pub(crate) fn interrupted_turn_history_marker(
}
}
fn is_automatic_rollout_stop(cause: &CodexErrorInfo) -> bool {
matches!(cause, CodexErrorInfo::SessionBudgetExceeded)
}
fn emit_turn_network_proxy_metric(
session_telemetry: &SessionTelemetry,
network_proxy_active: bool,
@@ -667,25 +682,7 @@ impl Session {
reason: TurnAbortReason,
error: Option<ErrorEvent>,
) -> bool {
let active_turn = {
let mut active = self.active_turn.lock().await;
if active
.as_ref()
.and_then(|active_turn| active_turn.task.as_ref())
.is_some_and(|task| task.turn_context.sub_id == turn_id)
{
if matches!(
reason,
TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited
) {
self.mark_interrupted();
}
active.take()
} else {
None
}
};
let Some(active_turn) = active_turn else {
let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else {
return false;
};
@@ -693,6 +690,24 @@ impl Session {
true
}
async fn take_named_active_turn(
&self,
turn_id: &str,
reason: &TurnAbortReason,
) -> Option<ActiveTurn> {
let mut active = self.active_turn.lock().await;
if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id {
return None;
}
if matches!(
reason,
TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited
) {
self.mark_interrupted();
}
active.take()
}
pub(crate) async fn finish_turn_abort(
self: &Arc<Self>,
mut active_turn: ActiveTurn,
@@ -723,10 +738,11 @@ impl Session {
turn_context: Arc<TurnContext>,
task_result: SessionTaskResult,
) -> Option<McpStartupRequirements> {
let (regular_output, abort_reason, task_error) = match task_result {
let (regular_output, abort, task_error) = match task_result {
Ok(output) => (output, None, None),
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
(None, Some(TurnAbortReason::Interrupted), None)
Err(err) if ends_turn_as_interrupted(&err) => {
let error: Option<ErrorEvent> = None;
(None, Some((TurnAbortReason::Interrupted, error)), None)
}
Err(err) => {
warn!(%err, "session task returned an unexpected error");
@@ -799,6 +815,24 @@ impl Session {
task.abort(Arc::clone(self), Arc::clone(&turn_context))
.await;
}
let stop_cause = {
let terminal_error = turn_context.terminal_error.lock().await;
abort
.as_ref()
.and_then(|(_, error)| error.as_ref())
.and_then(|error| error.codex_error_info.clone())
.or_else(|| task_error.as_ref().map(CodexErr::to_codex_protocol_error))
.or_else(|| {
terminal_error
.as_ref()
.and_then(|error| error.codex_error_info.clone())
})
.filter(is_automatic_rollout_stop)
};
if let Some(cause) = &stop_cause {
self.record_interrupted_turn_history(&turn_context, Some(cause))
.await;
}
drop(finish_recording);
// A retry on the first error must start a new turn with the denied turn's input in history.
for err in task_error.into_iter().chain(record_result.err()) {
@@ -932,16 +966,16 @@ impl Session {
profile,
});
let idle_cause = if matches!(
abort_reason.as_ref(),
abort.as_ref().map(|(reason, _)| reason),
Some(TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited)
) {
ThreadIdleCause::Interrupted
} else if abort_reason.is_none() && turn_context.terminal_error.lock().await.is_some() {
} else if abort.is_none() && turn_context.terminal_error.lock().await.is_some() {
ThreadIdleCause::Failed
} else {
ThreadIdleCause::Completed
};
let event = if let Some(reason) = abort_reason {
let event = if let Some((reason, error)) = abort {
if reason == TurnAbortReason::Interrupted {
run_turn_interrupt_hooks(self, &turn_context, &turn_state).await;
}
@@ -951,7 +985,7 @@ impl Session {
root_turn_id: Some(turn_context.root_turn_id()),
turn_id: Some(turn_context.sub_id.clone()),
reason,
error: None,
error,
started_at,
completed_at,
duration_ms,
@@ -1021,10 +1055,9 @@ impl Session {
turn.task.is_some()
|| (self.enabled(Feature::WakeOnAgentMessages) && turn.finishing.is_none())
}) {
// Message wakes reserve work before registering a task. With the
// feature enabled, stopping a reservation must block later mail too.
// Finishing turns must retain their status until completion is
// persisted, so capacity eviction cannot close their rollout writer.
// A message wake can reserve work before registering a task. Mark
// it interrupted too, but leave a finishing turn's status intact
// until its terminal event is persisted.
self.mark_interrupted();
}
active.take()
@@ -1048,6 +1081,32 @@ impl Session {
.await
}
async fn record_interrupted_turn_history(
&self,
turn_context: &TurnContext,
cause: Option<&CodexErrorInfo>,
) {
let Some(marker) = interrupted_turn_history_marker(
InterruptedTurnHistoryMarker::from_config_and_version(
turn_context.config.as_ref(),
turn_context.multi_agent_version,
),
cause,
) else {
return;
};
self.record_conversation_items(
turn_context,
turn_context.model_info(),
std::slice::from_ref(&marker),
)
.await;
// Clients may re-read the rollout as soon as they receive the terminal event.
if let Err(err) = self.flush_rollout().await {
warn!("failed to flush interrupted-turn marker before terminal event: {err}");
}
}
async fn handle_task_abort(
self: &Arc<Self>,
task: RunningTask,
@@ -1117,28 +1176,14 @@ impl Session {
.abort(Arc::clone(self), Arc::clone(&task.turn_context))
.await;
if reason == TurnAbortReason::Interrupted
&& let Some(marker) = interrupted_turn_history_marker(
InterruptedTurnHistoryMarker::from_config_and_version(
task.turn_context.config.as_ref(),
task.turn_context.multi_agent_version,
),
)
{
self.record_conversation_items(
task.turn_context.as_ref(),
task.turn_context.model_info(),
std::slice::from_ref(&marker),
if reason == TurnAbortReason::Interrupted {
self.record_interrupted_turn_history(
&task.turn_context,
error
.as_ref()
.and_then(|error| error.codex_error_info.as_ref()),
)
.await;
// Ensure the marker is durably visible before emitting TurnAborted: some clients
// synchronously re-read the rollout on receipt of the abort event.
if let Err(err) = self.flush_rollout().await {
warn!("failed to flush interrupted-turn marker before emitting TurnAborted: {err}");
}
}
if reason == TurnAbortReason::Interrupted {
run_turn_interrupt_hooks(self, &task.turn_context, turn_state).await;
}
+9 -3
View File
@@ -3315,14 +3315,18 @@ fn append_interrupted_boundary(
match history {
InitialHistory::New | InitialHistory::Cleared => {
let mut history = Vec::new();
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
if let Some(marker) =
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
{
history.push(RolloutItem::ResponseItem(marker.into()));
}
history.push(aborted_event);
InitialHistory::Forked(history)
}
InitialHistory::Forked(mut history) => {
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
if let Some(marker) =
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
{
history.push(RolloutItem::ResponseItem(marker.into()));
}
history.push(aborted_event);
@@ -3330,7 +3334,9 @@ fn append_interrupted_boundary(
}
InitialHistory::Resumed(resumed) => {
let mut history = Arc::unwrap_or_clone(resumed.history);
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
if let Some(marker) =
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
{
history.push(RolloutItem::ResponseItem(marker.into()));
}
history.push(aborted_event);
+7 -4
View File
@@ -1141,12 +1141,15 @@ fn assistant_msg(text: &str) -> ResponseItem {
}
fn contextual_user_interrupted_marker() -> ResponseItem {
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::ContextualUser)
.expect("contextual-user interrupted marker should be enabled")
interrupted_turn_history_marker(
InterruptedTurnHistoryMarker::ContextualUser,
/*cause*/ None,
)
.expect("contextual-user interrupted marker should be enabled")
}
fn developer_interrupted_marker() -> ResponseItem {
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer)
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer, /*cause*/ None)
.expect("developer interrupted marker should be enabled")
}
@@ -3846,7 +3849,7 @@ fn multi_agent_v2_interrupted_marker_uses_developer_input_message() {
content: vec![ContentItem::InputText {
text: format!(
"<turn_aborted>\n{}\n</turn_aborted>",
crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE
crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE
),
}],
phase: None,
+5 -1
View File
@@ -338,7 +338,11 @@ impl ToolCallRuntime {
.or_cancel_tool(&invocation_cancellation_token)
.await?;
// A peer may have exhausted the account while this call waited for its slot.
if session.rollout_budget().tools_stopped() {
if session
.tool_start_denial()
.map_err(|error| FunctionCallError::Fatal(error.to_string()))?
.is_some()
{
tracing::Span::current().record("aborted", true);
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch));
+15 -9
View File
@@ -933,26 +933,32 @@ impl ToolRegistry {
}
// Environment resolution, pre-tool hooks and Guardian may all have waited.
let invocation_guard = if invocation.session.rollout_budget().tools_stopped() {
None
} else {
let guard = reservation.activate(&invocation);
let mut start_denial = invocation.session.tool_start_denial();
let mut invocation_guard = None;
if matches!(start_denial, Ok(None)) {
invocation_guard = reservation.activate(&invocation);
if tool.mcp_server_name().is_none() {
notify_tool_start(&invocation, /*mcp_tool*/ None)
.or_cancel_tool(&cancellation_token)
.await?;
}
guard
// The rollout can stop while the start observer waits.
start_denial = invocation.session.tool_start_denial();
}
let reason = match start_denial {
Ok(reason) => reason,
Err(error) => {
let err = FunctionCallError::Fatal(error.to_string());
return Err(fail_before_execution(dispatch_trace, &invocation, err).await);
}
};
// A start observer can also wait. This is the last check before polling the
// handler; a stop after admission does not cancel an already-running tool.
if invocation.session.rollout_budget().tools_stopped() {
if let Some(reason) = reason {
drop(invocation_guard);
let result = AnyToolResult {
call_id: invocation.call_id.clone(),
payload: invocation.payload.clone(),
result: Box::new(AbortedToolOutput {
message: "aborted before handler start: rollout budget exhausted".into(),
message: format!("aborted before handler start: {reason}"),
}),
post_tool_use_payload: None,
};
@@ -14,8 +14,11 @@ use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ThreadLifecycleContributor;
use codex_extension_api::ThreadStartInput;
use codex_features::Feature;
use codex_history::RolloutItem;
use codex_model_provider_info::built_in_model_providers;
use codex_protocol::error::CodexErr;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
@@ -514,6 +517,32 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
.await;
}
if owner == AgentRuntimeOwnership::Controller && enforce_limit {
let history = test.codex.load_history(/*include_archived*/ false).await?;
let warning_roles = history
.items
.iter()
.filter_map(|item| {
let RolloutItem::ResponseItem(item) = item else {
return None;
};
let ResponseItem::Message { role, content, .. } = &item.item else {
return None;
};
content
.iter()
.any(|part| {
matches!(part,
ContentItem::InputText { text }
if text.contains("<turn_aborted>")
)
})
.then_some(role.as_str())
})
.collect::<Vec<_>>();
assert_eq!(warning_roles, ["developer", "developer"]);
}
if owner == AgentRuntimeOwnership::Host {
let requests = responses.requests();
assert_eq!(requests.len(), charges.len());