mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Record neutral interruption markers for rollout budget stops (#61581)
## Why Turns stopped by an exhausted rollout budget need a history marker without attributing the interruption to the user. ## What changed - Record and flush a `<turn_aborted>` marker before the terminal event for `SessionBudgetExceeded`, respecting the interrupt-message setting. - Use neutral guidance in a `developer` message for automatic budget stops. - Centralize tool-start budget checks across parallel dispatch, tool handlers, and MCP handoff, retaining the check after tool-start observers finish waiting. - Share interruption classification and history recording across turn and compaction paths. ## Testing Extend the rollout-budget integration test to assert that both the budget-crossing turn and a later denied turn persist `developer` interruption markers. GitOrigin-RevId: 2a49ac9824a4e95e4799f2d0f638f8c46889e8e8
This commit is contained in:
@@ -255,10 +255,8 @@ async fn run_compact_task_inner(
|
||||
let result = result.and_then(|_| sess.check_rollout_budget());
|
||||
if let Err(err) = &result
|
||||
&& !matches!(phase, CompactionPhase::PostTurn)
|
||||
&& !matches!(
|
||||
err.details(),
|
||||
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
|
||||
)
|
||||
&& !matches!(err.details(), CodexErrorDetails::Interrupted)
|
||||
&& !crate::tasks::ends_turn_as_interrupted(err)
|
||||
{
|
||||
sess.track_turn_codex_error(turn_context.as_ref(), err);
|
||||
// Pre-turn failures are reported after preserving the incoming prompt.
|
||||
@@ -573,10 +571,8 @@ pub(crate) fn compaction_status_from_result<T>(result: &CodexResult<T>) -> Compa
|
||||
match result {
|
||||
Ok(_) => CompactionStatus::Completed,
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.details(),
|
||||
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
|
||||
) =>
|
||||
if matches!(err.details(), CodexErrorDetails::Interrupted)
|
||||
|| crate::tasks::ends_turn_as_interrupted(err) =>
|
||||
{
|
||||
CompactionStatus::Interrupted
|
||||
}
|
||||
|
||||
@@ -268,7 +268,7 @@ async fn run_remote_compact_task_inner(
|
||||
match result {
|
||||
Ok(installed) => Ok(installed),
|
||||
Err(err)
|
||||
if matches!(err.details(), CodexErrorDetails::TurnAborted)
|
||||
if crate::tasks::ends_turn_as_interrupted(&err)
|
||||
|| matches!(phase, CompactionPhase::PostTurn) =>
|
||||
{
|
||||
Err(err)
|
||||
|
||||
@@ -10,7 +10,7 @@ pub(crate) struct TurnAborted {
|
||||
|
||||
impl TurnAborted {
|
||||
pub(crate) const INTERRUPTED_GUIDANCE: &'static str = "The user interrupted the previous turn on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
|
||||
pub(crate) const INTERRUPTED_DEVELOPER_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
|
||||
pub(crate) const INTERRUPTED_NEUTRAL_GUIDANCE: &'static str = "The previous turn was interrupted on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed.";
|
||||
|
||||
pub(crate) fn new(guidance: impl Into<String>) -> Self {
|
||||
Self {
|
||||
|
||||
@@ -537,9 +537,8 @@ async fn handle_approved_mcp_tool_call(
|
||||
let mut admission_error = None;
|
||||
let admit = || {
|
||||
let reason = sess
|
||||
.rollout_budget()
|
||||
.tools_stopped()
|
||||
.then(|| "rollout budget exhausted".to_owned());
|
||||
.tool_start_denial()
|
||||
.map_err(|error| McpAdmissionError(FunctionCallError::Fatal(error.to_string())))?;
|
||||
if let Some(reason) = reason {
|
||||
return Err(McpAdmissionError(FunctionCallError::RespondToModel(format!(
|
||||
"aborted before MCP handoff: {reason}"
|
||||
|
||||
@@ -46,6 +46,14 @@ impl Session {
|
||||
&self.services.local_agent_runtime.rollout_budget
|
||||
}
|
||||
|
||||
/// Return the diagnostic for a refused tool or propagate an accounting error.
|
||||
pub(crate) fn tool_start_denial(&self) -> CodexResult<Option<String>> {
|
||||
Ok(self
|
||||
.rollout_budget()
|
||||
.tools_stopped()
|
||||
.then(|| "rollout budget exhausted".into()))
|
||||
}
|
||||
|
||||
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
|
||||
if self.rollout_budget().tools_stopped() {
|
||||
Err(CodexErr::SessionBudgetExceeded)
|
||||
|
||||
@@ -277,7 +277,7 @@ pub(crate) async fn run_turn(
|
||||
PersistContext::Standard,
|
||||
)
|
||||
.await;
|
||||
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
|
||||
if crate::tasks::ends_turn_as_interrupted(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
if matches!(err.details(), CodexErrorDetails::ToolCollision(_)) {
|
||||
@@ -345,7 +345,7 @@ pub(crate) async fn run_turn(
|
||||
.await
|
||||
{
|
||||
Ok(step_context) => step_context,
|
||||
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
|
||||
Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => {
|
||||
let _ = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&turn_context,
|
||||
@@ -793,7 +793,7 @@ pub(crate) async fn run_turn(
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => {
|
||||
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
|
||||
if crate::tasks::ends_turn_as_interrupted(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
@@ -897,10 +897,9 @@ pub(crate) async fn run_turn(
|
||||
)
|
||||
.await
|
||||
{
|
||||
if matches!(
|
||||
err.details(),
|
||||
CodexErrorDetails::Interrupted | CodexErrorDetails::TurnAborted
|
||||
) {
|
||||
if matches!(err.details(), CodexErrorDetails::Interrupted)
|
||||
|| crate::tasks::ends_turn_as_interrupted(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
@@ -951,7 +950,7 @@ pub(crate) async fn run_turn(
|
||||
can_drain_pending_input = false;
|
||||
continue;
|
||||
}
|
||||
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
|
||||
Err(err) if crate::tasks::ends_turn_as_interrupted(&err) => {
|
||||
return Err(err);
|
||||
}
|
||||
Err(codex_error)
|
||||
|
||||
@@ -9,7 +9,6 @@ use crate::session::turn_context::TurnContext;
|
||||
use crate::state::TaskKind;
|
||||
use codex_features::Feature;
|
||||
use codex_model_provider::RemoteCompactionSupport;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -95,7 +94,7 @@ impl SessionTask for CompactTask {
|
||||
}
|
||||
};
|
||||
if let Err(err) = result {
|
||||
if matches!(err.details(), CodexErrorDetails::TurnAborted) {
|
||||
if super::ends_turn_as_interrupted(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
|
||||
+102
-57
@@ -53,6 +53,7 @@ use codex_otel::TURN_TOOL_CALL_METRIC;
|
||||
use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC;
|
||||
use codex_otel::auth_storage::AuthStorageOriginator;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::ErrorEvent;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
@@ -67,6 +68,7 @@ use codex_thread_store::PersistContext;
|
||||
|
||||
pub(crate) use admission::HostTurnStarted;
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
pub(crate) use compact::CompactTask;
|
||||
@@ -82,6 +84,11 @@ static ACTIVE_TURNS: Gauge = Gauge::new("core.turns.active");
|
||||
|
||||
pub(crate) type SessionTaskResult = CodexResult<Option<regular::RegularTaskOutput>>;
|
||||
|
||||
/// Whether task completion should publish an interrupted turn.
|
||||
pub(crate) fn ends_turn_as_interrupted(error: &CodexErr) -> bool {
|
||||
matches!(error.details(), CodexErrorDetails::TurnAborted)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum InterruptedTurnHistoryMarker {
|
||||
Disabled,
|
||||
@@ -109,15 +116,19 @@ impl InterruptedTurnHistoryMarker {
|
||||
/// interrupted fork snapshots.
|
||||
pub(crate) fn interrupted_turn_history_marker(
|
||||
marker: InterruptedTurnHistoryMarker,
|
||||
cause: Option<&CodexErrorInfo>,
|
||||
) -> Option<ResponseItem> {
|
||||
match marker {
|
||||
InterruptedTurnHistoryMarker::Disabled => None,
|
||||
InterruptedTurnHistoryMarker::ContextualUser => Some(ContextualUserFragment::into(
|
||||
crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE),
|
||||
)),
|
||||
InterruptedTurnHistoryMarker::Developer => {
|
||||
match (marker, cause.is_some_and(is_automatic_rollout_stop)) {
|
||||
(InterruptedTurnHistoryMarker::Disabled, _) => None,
|
||||
(InterruptedTurnHistoryMarker::ContextualUser, false) => {
|
||||
Some(ContextualUserFragment::into(
|
||||
crate::context::TurnAborted::new(crate::context::TurnAborted::INTERRUPTED_GUIDANCE),
|
||||
))
|
||||
}
|
||||
(InterruptedTurnHistoryMarker::ContextualUser, true)
|
||||
| (InterruptedTurnHistoryMarker::Developer, _) => {
|
||||
let marker = crate::context::TurnAborted::new(
|
||||
crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE,
|
||||
crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE,
|
||||
);
|
||||
let (_, content) = marker.render_fragment().into_parts();
|
||||
Some(RenderedFragment::new("developer", content).into())
|
||||
@@ -125,6 +136,10 @@ pub(crate) fn interrupted_turn_history_marker(
|
||||
}
|
||||
}
|
||||
|
||||
fn is_automatic_rollout_stop(cause: &CodexErrorInfo) -> bool {
|
||||
matches!(cause, CodexErrorInfo::SessionBudgetExceeded)
|
||||
}
|
||||
|
||||
fn emit_turn_network_proxy_metric(
|
||||
session_telemetry: &SessionTelemetry,
|
||||
network_proxy_active: bool,
|
||||
@@ -667,25 +682,7 @@ impl Session {
|
||||
reason: TurnAbortReason,
|
||||
error: Option<ErrorEvent>,
|
||||
) -> bool {
|
||||
let active_turn = {
|
||||
let mut active = self.active_turn.lock().await;
|
||||
if active
|
||||
.as_ref()
|
||||
.and_then(|active_turn| active_turn.task.as_ref())
|
||||
.is_some_and(|task| task.turn_context.sub_id == turn_id)
|
||||
{
|
||||
if matches!(
|
||||
reason,
|
||||
TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited
|
||||
) {
|
||||
self.mark_interrupted();
|
||||
}
|
||||
active.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let Some(active_turn) = active_turn else {
|
||||
let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -693,6 +690,24 @@ impl Session {
|
||||
true
|
||||
}
|
||||
|
||||
async fn take_named_active_turn(
|
||||
&self,
|
||||
turn_id: &str,
|
||||
reason: &TurnAbortReason,
|
||||
) -> Option<ActiveTurn> {
|
||||
let mut active = self.active_turn.lock().await;
|
||||
if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id {
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
reason,
|
||||
TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited
|
||||
) {
|
||||
self.mark_interrupted();
|
||||
}
|
||||
active.take()
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_turn_abort(
|
||||
self: &Arc<Self>,
|
||||
mut active_turn: ActiveTurn,
|
||||
@@ -723,10 +738,11 @@ impl Session {
|
||||
turn_context: Arc<TurnContext>,
|
||||
task_result: SessionTaskResult,
|
||||
) -> Option<McpStartupRequirements> {
|
||||
let (regular_output, abort_reason, task_error) = match task_result {
|
||||
let (regular_output, abort, task_error) = match task_result {
|
||||
Ok(output) => (output, None, None),
|
||||
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
|
||||
(None, Some(TurnAbortReason::Interrupted), None)
|
||||
Err(err) if ends_turn_as_interrupted(&err) => {
|
||||
let error: Option<ErrorEvent> = None;
|
||||
(None, Some((TurnAbortReason::Interrupted, error)), None)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(%err, "session task returned an unexpected error");
|
||||
@@ -799,6 +815,24 @@ impl Session {
|
||||
task.abort(Arc::clone(self), Arc::clone(&turn_context))
|
||||
.await;
|
||||
}
|
||||
let stop_cause = {
|
||||
let terminal_error = turn_context.terminal_error.lock().await;
|
||||
abort
|
||||
.as_ref()
|
||||
.and_then(|(_, error)| error.as_ref())
|
||||
.and_then(|error| error.codex_error_info.clone())
|
||||
.or_else(|| task_error.as_ref().map(CodexErr::to_codex_protocol_error))
|
||||
.or_else(|| {
|
||||
terminal_error
|
||||
.as_ref()
|
||||
.and_then(|error| error.codex_error_info.clone())
|
||||
})
|
||||
.filter(is_automatic_rollout_stop)
|
||||
};
|
||||
if let Some(cause) = &stop_cause {
|
||||
self.record_interrupted_turn_history(&turn_context, Some(cause))
|
||||
.await;
|
||||
}
|
||||
drop(finish_recording);
|
||||
// A retry on the first error must start a new turn with the denied turn's input in history.
|
||||
for err in task_error.into_iter().chain(record_result.err()) {
|
||||
@@ -932,16 +966,16 @@ impl Session {
|
||||
profile,
|
||||
});
|
||||
let idle_cause = if matches!(
|
||||
abort_reason.as_ref(),
|
||||
abort.as_ref().map(|(reason, _)| reason),
|
||||
Some(TurnAbortReason::Interrupted | TurnAbortReason::BudgetLimited)
|
||||
) {
|
||||
ThreadIdleCause::Interrupted
|
||||
} else if abort_reason.is_none() && turn_context.terminal_error.lock().await.is_some() {
|
||||
} else if abort.is_none() && turn_context.terminal_error.lock().await.is_some() {
|
||||
ThreadIdleCause::Failed
|
||||
} else {
|
||||
ThreadIdleCause::Completed
|
||||
};
|
||||
let event = if let Some(reason) = abort_reason {
|
||||
let event = if let Some((reason, error)) = abort {
|
||||
if reason == TurnAbortReason::Interrupted {
|
||||
run_turn_interrupt_hooks(self, &turn_context, &turn_state).await;
|
||||
}
|
||||
@@ -951,7 +985,7 @@ impl Session {
|
||||
root_turn_id: Some(turn_context.root_turn_id()),
|
||||
turn_id: Some(turn_context.sub_id.clone()),
|
||||
reason,
|
||||
error: None,
|
||||
error,
|
||||
started_at,
|
||||
completed_at,
|
||||
duration_ms,
|
||||
@@ -1021,10 +1055,9 @@ impl Session {
|
||||
turn.task.is_some()
|
||||
|| (self.enabled(Feature::WakeOnAgentMessages) && turn.finishing.is_none())
|
||||
}) {
|
||||
// Message wakes reserve work before registering a task. With the
|
||||
// feature enabled, stopping a reservation must block later mail too.
|
||||
// Finishing turns must retain their status until completion is
|
||||
// persisted, so capacity eviction cannot close their rollout writer.
|
||||
// A message wake can reserve work before registering a task. Mark
|
||||
// it interrupted too, but leave a finishing turn's status intact
|
||||
// until its terminal event is persisted.
|
||||
self.mark_interrupted();
|
||||
}
|
||||
active.take()
|
||||
@@ -1048,6 +1081,32 @@ impl Session {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn record_interrupted_turn_history(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
cause: Option<&CodexErrorInfo>,
|
||||
) {
|
||||
let Some(marker) = interrupted_turn_history_marker(
|
||||
InterruptedTurnHistoryMarker::from_config_and_version(
|
||||
turn_context.config.as_ref(),
|
||||
turn_context.multi_agent_version,
|
||||
),
|
||||
cause,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
self.record_conversation_items(
|
||||
turn_context,
|
||||
turn_context.model_info(),
|
||||
std::slice::from_ref(&marker),
|
||||
)
|
||||
.await;
|
||||
// Clients may re-read the rollout as soon as they receive the terminal event.
|
||||
if let Err(err) = self.flush_rollout().await {
|
||||
warn!("failed to flush interrupted-turn marker before terminal event: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_task_abort(
|
||||
self: &Arc<Self>,
|
||||
task: RunningTask,
|
||||
@@ -1117,28 +1176,14 @@ impl Session {
|
||||
.abort(Arc::clone(self), Arc::clone(&task.turn_context))
|
||||
.await;
|
||||
|
||||
if reason == TurnAbortReason::Interrupted
|
||||
&& let Some(marker) = interrupted_turn_history_marker(
|
||||
InterruptedTurnHistoryMarker::from_config_and_version(
|
||||
task.turn_context.config.as_ref(),
|
||||
task.turn_context.multi_agent_version,
|
||||
),
|
||||
)
|
||||
{
|
||||
self.record_conversation_items(
|
||||
task.turn_context.as_ref(),
|
||||
task.turn_context.model_info(),
|
||||
std::slice::from_ref(&marker),
|
||||
if reason == TurnAbortReason::Interrupted {
|
||||
self.record_interrupted_turn_history(
|
||||
&task.turn_context,
|
||||
error
|
||||
.as_ref()
|
||||
.and_then(|error| error.codex_error_info.as_ref()),
|
||||
)
|
||||
.await;
|
||||
// Ensure the marker is durably visible before emitting TurnAborted: some clients
|
||||
// synchronously re-read the rollout on receipt of the abort event.
|
||||
if let Err(err) = self.flush_rollout().await {
|
||||
warn!("failed to flush interrupted-turn marker before emitting TurnAborted: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
if reason == TurnAbortReason::Interrupted {
|
||||
run_turn_interrupt_hooks(self, &task.turn_context, turn_state).await;
|
||||
}
|
||||
|
||||
|
||||
@@ -3315,14 +3315,18 @@ fn append_interrupted_boundary(
|
||||
match history {
|
||||
InitialHistory::New | InitialHistory::Cleared => {
|
||||
let mut history = Vec::new();
|
||||
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
|
||||
if let Some(marker) =
|
||||
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
|
||||
{
|
||||
history.push(RolloutItem::ResponseItem(marker.into()));
|
||||
}
|
||||
history.push(aborted_event);
|
||||
InitialHistory::Forked(history)
|
||||
}
|
||||
InitialHistory::Forked(mut history) => {
|
||||
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
|
||||
if let Some(marker) =
|
||||
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
|
||||
{
|
||||
history.push(RolloutItem::ResponseItem(marker.into()));
|
||||
}
|
||||
history.push(aborted_event);
|
||||
@@ -3330,7 +3334,9 @@ fn append_interrupted_boundary(
|
||||
}
|
||||
InitialHistory::Resumed(resumed) => {
|
||||
let mut history = Arc::unwrap_or_clone(resumed.history);
|
||||
if let Some(marker) = interrupted_turn_history_marker(interrupted_marker) {
|
||||
if let Some(marker) =
|
||||
interrupted_turn_history_marker(interrupted_marker, /*cause*/ None)
|
||||
{
|
||||
history.push(RolloutItem::ResponseItem(marker.into()));
|
||||
}
|
||||
history.push(aborted_event);
|
||||
|
||||
@@ -1141,12 +1141,15 @@ fn assistant_msg(text: &str) -> ResponseItem {
|
||||
}
|
||||
|
||||
fn contextual_user_interrupted_marker() -> ResponseItem {
|
||||
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::ContextualUser)
|
||||
.expect("contextual-user interrupted marker should be enabled")
|
||||
interrupted_turn_history_marker(
|
||||
InterruptedTurnHistoryMarker::ContextualUser,
|
||||
/*cause*/ None,
|
||||
)
|
||||
.expect("contextual-user interrupted marker should be enabled")
|
||||
}
|
||||
|
||||
fn developer_interrupted_marker() -> ResponseItem {
|
||||
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer)
|
||||
interrupted_turn_history_marker(InterruptedTurnHistoryMarker::Developer, /*cause*/ None)
|
||||
.expect("developer interrupted marker should be enabled")
|
||||
}
|
||||
|
||||
@@ -3846,7 +3849,7 @@ fn multi_agent_v2_interrupted_marker_uses_developer_input_message() {
|
||||
content: vec![ContentItem::InputText {
|
||||
text: format!(
|
||||
"<turn_aborted>\n{}\n</turn_aborted>",
|
||||
crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE
|
||||
crate::context::TurnAborted::INTERRUPTED_NEUTRAL_GUIDANCE
|
||||
),
|
||||
}],
|
||||
phase: None,
|
||||
|
||||
@@ -338,7 +338,11 @@ impl ToolCallRuntime {
|
||||
.or_cancel_tool(&invocation_cancellation_token)
|
||||
.await?;
|
||||
// A peer may have exhausted the account while this call waited for its slot.
|
||||
if session.rollout_budget().tools_stopped() {
|
||||
if session
|
||||
.tool_start_denial()
|
||||
.map_err(|error| FunctionCallError::Fatal(error.to_string()))?
|
||||
.is_some()
|
||||
{
|
||||
tracing::Span::current().record("aborted", true);
|
||||
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
|
||||
return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch));
|
||||
|
||||
@@ -933,26 +933,32 @@ impl ToolRegistry {
|
||||
}
|
||||
|
||||
// Environment resolution, pre-tool hooks and Guardian may all have waited.
|
||||
let invocation_guard = if invocation.session.rollout_budget().tools_stopped() {
|
||||
None
|
||||
} else {
|
||||
let guard = reservation.activate(&invocation);
|
||||
let mut start_denial = invocation.session.tool_start_denial();
|
||||
let mut invocation_guard = None;
|
||||
if matches!(start_denial, Ok(None)) {
|
||||
invocation_guard = reservation.activate(&invocation);
|
||||
if tool.mcp_server_name().is_none() {
|
||||
notify_tool_start(&invocation, /*mcp_tool*/ None)
|
||||
.or_cancel_tool(&cancellation_token)
|
||||
.await?;
|
||||
}
|
||||
guard
|
||||
// The rollout can stop while the start observer waits.
|
||||
start_denial = invocation.session.tool_start_denial();
|
||||
}
|
||||
let reason = match start_denial {
|
||||
Ok(reason) => reason,
|
||||
Err(error) => {
|
||||
let err = FunctionCallError::Fatal(error.to_string());
|
||||
return Err(fail_before_execution(dispatch_trace, &invocation, err).await);
|
||||
}
|
||||
};
|
||||
// A start observer can also wait. This is the last check before polling the
|
||||
// handler; a stop after admission does not cancel an already-running tool.
|
||||
if invocation.session.rollout_budget().tools_stopped() {
|
||||
if let Some(reason) = reason {
|
||||
drop(invocation_guard);
|
||||
let result = AnyToolResult {
|
||||
call_id: invocation.call_id.clone(),
|
||||
payload: invocation.payload.clone(),
|
||||
result: Box::new(AbortedToolOutput {
|
||||
message: "aborted before handler start: rollout budget exhausted".into(),
|
||||
message: format!("aborted before handler start: {reason}"),
|
||||
}),
|
||||
post_tool_use_payload: None,
|
||||
};
|
||||
|
||||
@@ -14,8 +14,11 @@ use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::ThreadLifecycleContributor;
|
||||
use codex_extension_api::ThreadStartInput;
|
||||
use codex_features::Feature;
|
||||
use codex_history::RolloutItem;
|
||||
use codex_model_provider_info::built_in_model_providers;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
@@ -514,6 +517,32 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
.await;
|
||||
}
|
||||
|
||||
if owner == AgentRuntimeOwnership::Controller && enforce_limit {
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
let warning_roles = history
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| {
|
||||
let RolloutItem::ResponseItem(item) = item else {
|
||||
return None;
|
||||
};
|
||||
let ResponseItem::Message { role, content, .. } = &item.item else {
|
||||
return None;
|
||||
};
|
||||
content
|
||||
.iter()
|
||||
.any(|part| {
|
||||
matches!(part,
|
||||
ContentItem::InputText { text }
|
||||
if text.contains("<turn_aborted>")
|
||||
)
|
||||
})
|
||||
.then_some(role.as_str())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(warning_roles, ["developer", "developer"]);
|
||||
}
|
||||
|
||||
if owner == AgentRuntimeOwnership::Host {
|
||||
let requests = responses.requests();
|
||||
assert_eq!(requests.len(), charges.len());
|
||||
|
||||
Reference in New Issue
Block a user