Gate new rollout work on shared budget exhaustion (#61582)

## Why

A budget can be exhausted while another thread waits to start work. New requests, tools, and automatic turns must observe that stop without discarding an already-consumed response whose cost exhausted the budget.

## What changed

- Share a work gate across the rollout tree while preserving native and host accounting ownership.
- Recheck the budget before HTTP and WebSocket requests, hook execution, tool dispatch, and compaction. Prevent automatic wakeups and late input from restarting stopped work, leaving pending input queued.
- Record response costs without rejecting the consumed response solely because its bill exhausts the budget, and stop new work when enforced accounting overflows.
- Expose a thread-registration receipt that remains committed after unload.

## Testing

Extend rollout-budget coverage to verify that accepted response text remains in history after exhaustion and that waiting tools observe the shared stop while completed tools retain their results.

GitOrigin-RevId: e583d657c6f29e374d374cdb014bf23fd6c28614
This commit is contained in:
tongzhou wang
2026-11-15 00:03:23 +00:00
committed by copyberry
parent 2e61e8ec4e
commit ce82e38b16
26 changed files with 333 additions and 108 deletions
+6
View File
@@ -39,6 +39,12 @@ const _: Option<&dyn AgentControl> = None;
/// remote ownership, retries and recovery are separate backend work.
/// Boxed Send futures allow callers to use `Arc<dyn AgentControl>`.
pub trait AgentControl: Send + Sync {
/// Expose the account only when this controller actually owns native billing.
/// External controllers keep their own ledger and need not provide a numeric read.
fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
None
}
fn identity(&self) -> SessionId;
/// Resolve an ID or a name relative to the caller's captured source, without loading.
+4
View File
@@ -33,6 +33,10 @@ use futures::future::BoxFuture;
use std::collections::HashSet;
impl AgentControl for LocalAgentControl {
fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
Some(std::sync::Arc::clone(&self.runtime.rollout_budget))
}
fn identity(&self) -> SessionId {
self.session_id()
}
+2 -9
View File
@@ -4,7 +4,6 @@
use super::LocalAgentControl;
use crate::rollout_budget::RolloutBudgetReminder;
use codex_protocol::ThreadId;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
@@ -14,14 +13,8 @@ impl LocalAgentControl {
usage: Option<&TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> CodexResult<()> {
if self
.runtime
.rollout_budget
.record_usage(usage, hook_charge)?
{
return Err(CodexErr::SessionBudgetExceeded);
}
Ok(())
// The caller keeps a consumed response even when its bill stops new work.
self.runtime.rollout_budget.record_usage(usage, hook_charge)
}
pub(crate) fn pending_budget_reminder(
+16
View File
@@ -312,6 +312,7 @@ pub struct ModelClient {
http_client_factory: HttpClientFactory,
restored_history: bool,
request_contributors: Vec<Arc<dyn codex_extension_api::ModelRequestContributor>>,
budget_policy: Option<crate::rollout_budget::ThreadBudgetPolicy>,
executed_tool_calls: Option<ExecutedToolCalls>,
// Resolved once when the session is created, like other session feature flags.
api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms,
@@ -660,6 +661,7 @@ impl ModelClient {
http_client_factory,
restored_history: false,
request_contributors,
budget_policy: None,
executed_tool_calls: None,
api_key_cyber_access_programs:
cyber_access_program::ApiKeyCyberAccessPrograms::UnsupportedProvider,
@@ -672,6 +674,14 @@ impl ModelClient {
self
}
pub(crate) fn with_budget_policy(
mut self,
policy: crate::rollout_budget::ThreadBudgetPolicy,
) -> Self {
self.budget_policy = Some(policy);
self
}
pub(crate) fn reasoning_effort_override_enabled(&self, model_info: &ModelInfo) -> bool {
self.state.reasoning_effort_override_enabled
&& self.state.provider.info().is_openai()
@@ -2120,6 +2130,9 @@ impl ResponsesHttpExecutor<'_> {
client_setup.api_auth,
)
.with_telemetry(Some(request_telemetry), Some(sse_telemetry));
if let Some(policy) = &self.client.budget_policy {
policy.check_start()?;
}
let stream_result = client
.stream_request_with_observer(request, options, |wire| {
inference_trace_attempt.record_started(
@@ -2474,6 +2487,9 @@ impl ModelClientSession {
),
],
);
if !warmup && let Some(policy) = &self.client.budget_policy {
policy.check_start()?;
}
let stream_result = websocket_connection
.stream_request_with_observer(
ws_request,
+4 -2
View File
@@ -571,8 +571,10 @@ pub(crate) fn compaction_status_from_result<T>(result: &CodexResult<T>) -> Compa
match result {
Ok(_) => CompactionStatus::Completed,
Err(err)
if matches!(err.details(), CodexErrorDetails::Interrupted)
|| crate::tasks::ends_turn_as_interrupted(err) =>
if matches!(
err.details(),
CodexErrorDetails::Interrupted | CodexErrorDetails::SessionBudgetExceeded
) || crate::tasks::ends_turn_as_interrupted(err) =>
{
CompactionStatus::Interrupted
}
+2 -2
View File
@@ -1482,13 +1482,13 @@ mod tests {
assert!(
matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted))
);
assert!(!sess.rollout_budget().tools_stopped());
assert!(!sess.work_budget().is_exhausted());
assert!(accounting.reports.lock().unwrap().is_empty());
return;
}
assert!(matches!(result, Err(ref err)
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_))));
assert!(sess.rollout_budget().tools_stopped());
assert!(sess.work_budget().is_exhausted());
assert_eq!(
*accounting.reports.lock().unwrap(),
vec![AgentUsageReport {
+1 -3
View File
@@ -251,9 +251,7 @@ pub(crate) async fn finalize(
.pending_budget_reminder(session.thread_id(), &session.current_window_id().await)
.await
{
let reminder = ContextualUserFragment::into(crate::context::RolloutBudgetContext {
remaining_units: reminder.remaining_units,
});
let reminder = session.budget_reminder_item(&reminder);
existing = existing.saturating_add(
usize::try_from(estimate_item_token_count(&reminder)).unwrap_or(usize::MAX),
);
+3 -4
View File
@@ -54,10 +54,6 @@ impl SamplingRequest {
owner_cancellation: &CancellationToken,
send: impl Future<Output = Result<OpenedModelResponse>>,
) -> Result<SamplingStream> {
let send = async {
session.check_rollout_budget()?;
send.await
};
let sampling_turn = match self.kind {
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
@@ -83,6 +79,9 @@ impl SamplingRequest {
agent_id: subagent.as_ref().map(|context| context.agent_id.clone()),
agent_type: subagent.map(|context| context.agent_type),
};
// Hook input preparation may wait. Do not start a new hook pair after a stop;
// the HTTP/WS send independently rechecks after a pre-hook has finished.
session.check_rollout_budget()?;
let mut attempt = post_mode.map(|mode| SamplingAttempt {
owner_cancellation: owner_cancellation.clone(),
post: Some(PostHooks {
+2
View File
@@ -85,6 +85,7 @@ pub use agent::types::ResolvedMultiAgentV2UsageHints;
pub use agent::types::SpawnAgentForkMode;
pub use agent::types::SpawnAgentOptions;
pub use agent::types::SpawnProgramSource;
pub use rollout_budget::RolloutBudget;
pub use rollout_budget::RolloutBudgetReminder;
mod agent_communication;
pub use agent_communication::PendingAgentMessage;
@@ -202,6 +203,7 @@ pub use thread_manager::StartupModels;
pub use thread_manager::SuspendedAgentTree;
pub use thread_manager::ThreadManager;
pub use thread_manager::ThreadRegistration;
pub use thread_manager::ThreadRegistrationReceipt;
pub use thread_manager::ThreadShutdownReport;
pub use thread_manager::build_models_manager;
pub use thread_manager::build_startup_models;
+84 -35
View File
@@ -1,3 +1,5 @@
mod gate;
use crate::config::RolloutBudgetConfig;
use crate::config::rollout_budget_threshold_reached;
use codex_protocol::BudgetUnitsSum;
@@ -5,12 +7,12 @@ use codex_protocol::ThreadId;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
pub use gate::RolloutWorkGate;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::MutexGuard;
use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
/// Budget reminder returned by a controller and acknowledged after history insertion.
pub struct RolloutBudgetReminder {
@@ -19,14 +21,47 @@ pub struct RolloutBudgetReminder {
pub reminder_index: i64,
}
/// Shared accounting and reminder state for one root-thread session tree.
/// Native spending and reminders, plus the stop published by the tree's selected
/// accounting owner (native or host). In-flight work can still report its bill.
#[derive(Default)]
pub(crate) struct RolloutBudget {
pub struct RolloutBudget {
state: OnceLock<Mutex<RolloutBudgetState>>,
/// An explicit stop gates new tools across the tree, even with per-thread host accountants.
stop_tools: AtomicBool,
gate: Arc<RolloutWorkGate>,
}
/// A native account owns this thread's bills. A host-owned thread only shares its work gate.
pub(crate) enum ThreadBudgetAccount {
Native(Arc<RolloutBudget>),
Shared(Arc<RolloutBudget>),
}
/// Every thread consults the shared work gate before starting new work.
#[derive(Clone, Debug, Default)]
pub(crate) struct ThreadBudgetPolicy {
rollout: Arc<RolloutBudget>,
}
impl ThreadBudgetPolicy {
pub(crate) fn new(account: ThreadBudgetAccount) -> CodexResult<Self> {
let (ThreadBudgetAccount::Native(rollout) | ThreadBudgetAccount::Shared(rollout)) = account;
Ok(Self { rollout })
}
pub(crate) fn check_start(&self) -> CodexResult<()> {
self.rollout.gate.check_start()
}
}
impl std::fmt::Debug for RolloutBudget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RolloutBudget")
.field("state", &self.state)
.field("stopped", &self.is_exhausted())
.finish_non_exhaustive()
}
}
#[derive(Debug)]
struct RolloutBudgetState {
config: RolloutBudgetConfig,
posted: BudgetUnitsSum,
@@ -34,6 +69,7 @@ struct RolloutBudgetState {
deliveries: HashMap<ThreadId, ThreadBudgetDelivery>,
}
#[derive(Debug)]
struct ThreadBudgetDelivery {
window_id: String,
reminder_index: i64,
@@ -50,25 +86,22 @@ impl RolloutBudget {
});
}
/// Records usage and returns whether an enforced budget is exhausted.
/// Post the response cost and enforce the allowance under the same lock.
pub(crate) fn record_usage(
&self,
usage: Option<&TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> CodexResult<bool> {
) -> CodexResult<()> {
let Some(mut state) = self.lock() else {
return Ok(false);
return Ok(());
};
if !hook_charge.units.is_finite() || hook_charge.units < 0.0 {
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
return Err(CodexErr::Fatal("invalid additional sampling charge".into()));
}
if hook_charge.overflowed {
// All allowances are finite. No future nonnegative receipt can make
// this total representable; the remaining allowance is known to be zero.
if hook_charge.overflowed && state.posted.total().is_finite() {
// Stop new work before validating the rest of an overflowing receipt.
state.posted.add(f64::INFINITY);
if state.config.enforce_limit {
self.stop_new_tools();
}
self.enforce_posted_limit(&state)?;
}
let provider_units = if let Some(units) =
usage.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
@@ -88,31 +121,31 @@ impl RolloutBudget {
0.0
};
if state.posted.total().is_infinite() {
return Ok(state.config.enforce_limit);
// Later valid receipts cannot restore precision. The original
// overflow was reported; it must not veto every advisory turn.
return Ok(());
}
// Keep the full bill even if enforcing the new allowance fails.
state.posted.add(provider_units);
state.posted.add(hook_charge.units);
// Enforce the allowance before reporting an overflowing sum.
self.enforce_posted_limit(&state)?;
if !state.posted.total().is_finite() {
if state.config.enforce_limit {
self.stop_new_tools();
}
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
return Err(CodexErr::Fatal(
"rollout sampling cost exceeds f64 range".into(),
));
}
// Equality is allowed, including one rounding step (0.1 + 0.2 versus 0.3).
let exhausted = state.config.enforce_limit
&& state.posted.total() > state.config.limit_budget.next_up();
if exhausted {
self.stop_new_tools();
}
Ok(exhausted)
Ok(())
}
pub(crate) fn stop_new_tools(&self) {
self.stop_tools.store(true, Ordering::Release);
}
pub(crate) fn tools_stopped(&self) -> bool {
self.stop_tools.load(Ordering::Acquire)
fn enforce_posted_limit(&self, state: &RolloutBudgetState) -> CodexResult<()> {
let posted = state.posted.total();
if state.config.enforce_limit
&& (!posted.is_finite() || posted > state.config.limit_budget.next_up())
{
self.stop_new_work();
}
Ok(())
}
pub(crate) fn pending_reminder(
@@ -121,7 +154,7 @@ impl RolloutBudget {
window_id: &str,
) -> Option<RolloutBudgetReminder> {
let state = self.lock()?;
let remaining_units = (state.config.limit_budget - state.posted.total()).max(0.0);
let remaining_units = self.remaining_units(&state).ok()?;
let reminder_index = state
.config
.reminder_at_remaining_tokens
@@ -167,4 +200,20 @@ impl RolloutBudget {
.unwrap_or_else(std::sync::PoisonError::into_inner)
})
}
pub(crate) fn stop_new_work(&self) {
self.gate.stop_budget();
}
pub(crate) fn is_exhausted(&self) -> bool {
self.gate.is_stopped()
}
fn remaining_units(&self, state: &RolloutBudgetState) -> CodexResult<f64> {
if state.posted.total().is_infinite() {
return Ok(0.0);
}
let used = state.posted.total();
Ok((state.config.limit_budget - used).max(0.0))
}
}
+31
View File
@@ -0,0 +1,31 @@
//! The first stop closes every work-start boundary in this rollout tree.
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result;
use std::sync::OnceLock;
/// Controls new work across the rollout tree.
#[derive(Debug, Default)]
pub struct RolloutWorkGate(OnceLock<Stop>);
#[derive(Debug)]
enum Stop {
Budget,
}
impl RolloutWorkGate {
pub(super) fn stop_budget(&self) {
let _ = self.0.set(Stop::Budget);
}
pub(super) fn is_stopped(&self) -> bool {
self.0.get().is_some()
}
pub(super) fn check_start(&self) -> Result<()> {
match self.0.get() {
None => Ok(()),
Some(Stop::Budget) => Err(CodexErr::SessionBudgetExceeded),
}
}
}
@@ -66,6 +66,7 @@ pub(super) async fn maybe_start(
return;
}
if !enabled(turn_context)
|| sess.work_budget().is_exhausted()
|| token_status.token_limit_reached
|| (matches!(phase, CompactionPhase::PostTurn)
&& token_status.turn_end_compaction_threshold_reached)
+34 -18
View File
@@ -5,7 +5,6 @@ use crate::agent::host::AgentUsageReport;
use crate::agent::host::UsageAccountingDecision;
use crate::client_common::SamplingUsage;
use crate::context::ContextualUserFragment;
use crate::rollout_budget::RolloutBudget;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result as CodexResult;
@@ -27,9 +26,7 @@ pub(super) async fn maybe_record_reminder(
else {
return;
};
let response_item = ContextualUserFragment::into(crate::context::RolloutBudgetContext {
remaining_units: reminder.remaining_units,
});
let response_item = sess.budget_reminder_item(&reminder);
sess.record_conversation_items(
turn_context,
turn_context.model_info(),
@@ -42,23 +39,41 @@ pub(super) async fn maybe_record_reminder(
}
impl Session {
pub(crate) fn rollout_budget(&self) -> &RolloutBudget {
pub(crate) fn budget_reminder_item(
&self,
reminder: &crate::RolloutBudgetReminder,
) -> codex_protocol::models::ResponseItem {
ContextualUserFragment::into(crate::context::RolloutBudgetContext {
remaining_units: reminder.remaining_units,
})
}
pub(crate) fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
match &self.agent_runtime {
AgentRuntimeServices::Controller => self.services.agent_control.native_rollout_budget(),
AgentRuntimeServices::Host(_) => None,
}
}
/// All threads consult the same work gate, even when a host owns one thread's bill.
pub(crate) fn work_budget(&self) -> &crate::RolloutBudget {
&self.services.local_agent_runtime.rollout_budget
}
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
self.services.budget_policy.check_start()
}
/// Return the diagnostic for a refused tool or propagate an accounting error.
pub(crate) fn tool_start_denial(&self) -> CodexResult<Option<String>> {
Ok(self
.rollout_budget()
.tools_stopped()
.then(|| "rollout budget exhausted".into()))
}
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
if self.rollout_budget().tools_stopped() {
Err(CodexErr::SessionBudgetExceeded)
} else {
Ok(())
match self.check_rollout_budget() {
Ok(()) => Ok(None),
Err(error) => match error.details() {
CodexErrorDetails::SessionBudgetExceeded => {
Ok(Some("rollout budget exhausted".into()))
}
_ => Err(error),
},
}
}
@@ -98,10 +113,11 @@ impl Session {
{
Ok(()) => false,
// A legacy controller can report a posted bill and a stop through this error.
// A native account has already chosen its own deadline action.
Err(error)
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
{
true
self.native_rollout_budget().is_none()
}
Err(error) => return Err(CodexErr::UsageAccountingFailed(error.to_string())),
},
@@ -127,7 +143,7 @@ impl Session {
}
};
if stop_new_work {
self.rollout_budget().stop_new_tools();
self.work_budget().stop_new_work();
}
Ok(())
}
+20
View File
@@ -1807,6 +1807,24 @@ impl Session {
// This placeholder is used only for teardown, before the session is published.
Err(err) => (AgentRuntimeServices::Controller, Some(err)),
};
let mut local_agent_runtime = local_agent_runtime;
let mut account = crate::rollout_budget::ThreadBudgetAccount::Shared(Arc::clone(
&local_agent_runtime.rollout_budget,
));
if startup_error.is_none() && matches!(&agent_runtime, AgentRuntimeServices::Controller) {
if let Some(native) = agent_control.native_rollout_budget() {
local_agent_runtime.rollout_budget = Arc::clone(&native);
account = crate::rollout_budget::ThreadBudgetAccount::Native(native);
}
}
let budget_policy = crate::rollout_budget::ThreadBudgetPolicy::new(
account,
)
.unwrap_or_else(|err| {
startup_error = Some(err);
// Teardown needs a session. Its empty policy is never published.
Default::default()
});
state.shared_service_tier_pending = parent_thread_id.is_none()
&& match &agent_runtime {
AgentRuntimeServices::Controller => has_provided_agent_control,
@@ -1854,6 +1872,7 @@ impl Session {
workspace_routing.as_ref().clone(),
extensions.model_request_contributors().to_vec(),
)
.with_budget_policy(budget_policy.clone())
.with_executed_tool_calls(executed_tool_calls.clone())
.with_restored_history(matches!(
&initial_history,
@@ -1946,6 +1965,7 @@ impl Session {
mcp_thread_init,
client_mcp_extensions,
local_agent_runtime,
budget_policy,
agent_control,
network_proxy: arc_swap::ArcSwapOption::from(network_proxy.map(Arc::new)),
network_proxy_audit_metadata,
+12
View File
@@ -6898,6 +6898,12 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
mcp_thread_init: codex_extension_api::ExtensionDataInit::default(),
client_mcp_extensions: ClientMcpExtensions::default(),
local_agent_runtime: agent_control.runtime.clone(),
budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new(
crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone(
&agent_control.runtime.rollout_budget,
)),
)
.expect("test budget policy"),
agent_control: Arc::new(agent_control),
network_proxy: arc_swap::ArcSwapOption::from(None),
network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(),
@@ -9249,6 +9255,12 @@ where
mcp_thread_init: codex_extension_api::ExtensionDataInit::default(),
client_mcp_extensions: ClientMcpExtensions::default(),
local_agent_runtime: agent_control.runtime.clone(),
budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new(
crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone(
&agent_control.runtime.rollout_budget,
)),
)
.expect("test budget policy"),
agent_control: Arc::new(agent_control),
network_proxy: arc_swap::ArcSwapOption::from(None),
network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(),
+7 -2
View File
@@ -541,7 +541,7 @@ pub(crate) async fn run_turn(
// Note that pending_input would be something like a message the user
// submitted through the UI while the model was running. Though the UI
// may support this, the model might not.
let pending_input = if can_drain_pending_input {
let pending_input = if can_drain_pending_input && !sess.work_budget().is_exhausted() {
sess.input_queue
.get_pending_input(&sess.active_turn)
.await
@@ -654,7 +654,8 @@ pub(crate) async fn run_turn(
.await
}
.await;
// Consume the whole response before deciding whether the turn may continue.
// Completed can precede Guardian's accepted-text replay. Preserve the
// entire consumed response before deciding whether the turn may continue.
let sampling_request_result = sampling_request_result.and_then(|output| {
sess.check_rollout_budget()?;
Ok(output)
@@ -1694,6 +1695,7 @@ async fn run_auto_compact(
candidate: Option<Box<BackgroundCompactionCandidate>>,
) -> CodexResult<bool> {
let turn_context = &step_context.turn;
sess.check_rollout_budget()?;
if matches!(phase, CompactionPhase::PreTurn)
&& crate::guardian::is_basic_session_source(&turn_context.session_source)
&& !crate::guardian::should_compact_guardian_input(sess)?
@@ -1712,6 +1714,9 @@ async fn run_auto_compact(
turn.id = %turn_context.sub_id,
);
if turn_context.config.features.enabled(Feature::TokenBudget) {
// Unlike sampling compaction, this reset has no transport admission check.
// World-state preparation above may have waited while a peer posted a bill.
sess.check_rollout_budget()?;
// Compaction is the reset request, so force a new context window
// instead of consuming a pending `new_context` tool request.
crate::compact_token_budget::run_inline_auto_compact_task(
+11
View File
@@ -609,6 +609,11 @@ async fn start_if_idle(
reason: NotSubmittedReason::NotIdle,
});
}
// Idle observers can retry a failed turn. Check before reserving so a permanent
// stop cannot create a loop of new turns that never reach the provider.
if kind != TurnStartKind::User {
session.check_rollout_budget()?;
}
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
Arc::clone(&active_turn.turn_state)
};
@@ -843,6 +848,12 @@ impl Session {
{
return None;
}
// Triggered mail and durable sleep bypass start_if_idle. Leave their mail
// pending rather than consuming it to start a turn that the work gate will refuse.
if let Err(error) = self.check_rollout_budget() {
tracing::debug!(%error, "rollout budget stopped a pending-work wake");
return None;
}
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
let previous_options = if has_trigger_turn {
Default::default()
@@ -275,11 +275,11 @@ async fn mcp_budget_stop_at_handoff(phase: McpPause) -> anyhow::Result<()> {
extensions.tool_lifecycle_contributor(observer.clone());
session.services.extensions = Arc::new(extensions.build());
session
.rollout_budget()
.work_budget()
.configure(crate::config::RolloutBudgetConfig {
enforce_limit: true,
limit_budget: 1.0,
reminder_at_context_start: true,
reminder_at_context_start: false,
reminder_at_remaining_tokens: Vec::new(),
sampling_token_weight: 1.0,
prefill_token_weight: 1.0,
+1
View File
@@ -89,6 +89,7 @@ pub(crate) struct SessionServices {
pub(crate) mcp_thread_init: ExtensionDataInit,
pub(crate) agent_control: Arc<dyn AgentControl>,
pub(crate) local_agent_runtime: LocalAgentRuntime,
pub(crate) budget_policy: crate::rollout_budget::ThreadBudgetPolicy,
pub(crate) network_proxy: ArcSwapOption<StartedNetworkProxy>,
pub(crate) network_proxy_audit_metadata: NetworkProxyAuditMetadata,
pub(crate) managed_network_requirements_configured: bool,
+3
View File
@@ -41,6 +41,7 @@ impl SessionTask for CompactTask {
);
// Preparation errors must reach the task runner, which reports them to the client.
session.emit_turn_started(&ctx, TaskKind::Compact).await;
session.check_rollout_budget()?;
let step_context = session
.capture_step_context(Arc::clone(&ctx), &cancellation_token)
.await?;
@@ -50,6 +51,8 @@ impl SessionTask for CompactTask {
.await?,
);
if ctx.config.features.enabled(Feature::TokenBudget) {
// Preparing the context may wait; this reset never sends a model request.
session.check_rollout_budget()?;
crate::compact_token_budget::run_manual_compact_task(
session,
step_context,
+8 -4
View File
@@ -682,7 +682,8 @@ impl Session {
reason: TurnAbortReason,
error: Option<ErrorEvent>,
) -> bool {
let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else {
let active_turn = self.take_named_active_turn(turn_id, &reason).await;
let Some(active_turn) = active_turn else {
return false;
};
@@ -696,7 +697,8 @@ impl Session {
reason: &TurnAbortReason,
) -> Option<ActiveTurn> {
let mut active = self.active_turn.lock().await;
if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id {
let turn = &active.as_ref()?.task.as_ref()?.turn_context;
if turn.sub_id != turn_id {
return None;
}
if matches!(
@@ -749,8 +751,10 @@ impl Session {
(None, None, Some(err))
}
};
let can_continue =
regular_output.is_some() && turn_context.terminal_error.lock().await.is_none();
// A late steer cannot reopen the stopped rollout after the turn loop has finished.
let can_continue = regular_output.is_some()
&& turn_context.terminal_error.lock().await.is_none()
&& !self.work_budget().is_exhausted();
let finishing = CancellationToken::new();
let finish_recording = finishing.clone().drop_guard();
let (turn_state, unadmitted_task, mut pending_input, outcome_handoff) = {
+1
View File
@@ -163,6 +163,7 @@ impl RegularTask {
// Terminal errors are already reported. Let task completion preserve pending
// input instead of restarting the failed turn for that same input.
if ctx.terminal_error.lock().await.is_some()
|| sess.work_budget().is_exhausted()
|| !sess.input_queue.has_pending_input(&sess.active_turn).await
{
return Ok(Some(RegularTaskOutput {
+16 -1
View File
@@ -207,6 +207,16 @@ pub enum ThreadRegistration {
Replaced,
}
/// A manager-started runtime's receipt stays committed across unload. A runtime
/// started without a manager has no receipt in its thread extension store.
pub struct ThreadRegistrationReceipt(Arc<AtomicBool>);
impl ThreadRegistrationReceipt {
pub fn was_registered(&self) -> bool {
self.0.load(Ordering::Acquire)
}
}
// TODO(ccunningham): Add an explicit non-interrupting live-turn snapshot once
// core can represent sampling boundaries directly instead of relying on
// whichever items happened to be persisted mid-turn.
@@ -2903,6 +2913,8 @@ impl ThreadManagerState {
&session_source,
) => manager,
};
let registered = Arc::new(AtomicBool::new(false));
thread_extension_init.insert(ThreadRegistrationReceipt(Arc::clone(&registered)));
let spawn_result = Session::spawn(SessionSpawnArgs {
attached_thread,
startup,
@@ -2997,7 +3009,7 @@ impl ThreadManagerState {
}
}
let finalize_result = self
.finalize_thread_spawn(session, io, tracked_session_source)
.finalize_thread_spawn(session, io, tracked_session_source, &registered)
.await;
let (new_thread, tier_acceptance) = match finalize_result {
Ok(registered) => registered,
@@ -3040,6 +3052,7 @@ impl ThreadManagerState {
session: Arc<Session>,
io: SessionIo,
session_source: SessionSource,
registered: &AtomicBool,
) -> CodexResult<(NewThread, Option<ServiceTierAcceptance>)> {
let thread_id = session.thread_id();
let event = io.next_event().await?;
@@ -3080,6 +3093,8 @@ impl ThreadManagerState {
session_configured.rollout_path.clone(),
session_source,
));
// All registration checks passed; publish the receipt with this insertion.
registered.store(true, Ordering::Release);
e.insert(thread.clone());
let tier_acceptance = pending_service_tier.and_then(|(mut state, update)| {
state.service_tier_acceptance =
+27 -13
View File
@@ -309,6 +309,13 @@ impl ToolCallRuntime {
let mut dispatch_handle = AbortOnDropHandle::new(tokio::spawn(
async move {
let _message_admission = message_admission?;
// A response replayed after settlement can already have exhausted the budget.
// Record its denial without waiting for a handler that will never start.
if session.work_budget().is_exhausted() {
tracing::Span::current().record("aborted", true);
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch));
}
// Readiness and admission remain cancellable even when the handler must finish.
let (guard, admission_started) = async {
if let Some(tool_runtime) = tool_runtime
@@ -1034,17 +1041,20 @@ mod tests {
}
}
#[test_case::test_case(false; "posted cost checked after the slot opens")]
#[tokio::test]
async fn completed_and_budget_stopped_tools_report_their_own_lifecycle() -> anyhow::Result<()> {
async fn completed_and_budget_stopped_tools_report_their_own_lifecycle(
weighted: bool,
) -> anyhow::Result<()> {
let (mut session, turn_context) = crate::session::tests::make_session_and_context().await;
session
.rollout_budget()
.work_budget()
.configure(crate::config::RolloutBudgetConfig {
enforce_limit: true,
limit_budget: 1.0,
reminder_at_context_start: true,
reminder_at_context_start: weighted,
reminder_at_remaining_tokens: Vec::new(),
sampling_token_weight: 1.0,
sampling_token_weight: if weighted { f64::MAX } else { 1.0 },
prefill_token_weight: 1.0,
});
let lifecycle = Arc::new(PausedToolLifecycle::default());
@@ -1136,25 +1146,33 @@ mod tests {
output_tokens: 2,
..Default::default()
};
session
let bill = session
.record_rollout_budget_usage("peer-turn", &usage.into())
.await?;
.await;
bill?;
assert!(session.work_budget().is_exhausted());
cancellation_token.cancel();
// Let the caller observe cancellation while the already-finished handler is held.
assert!(futures::poll!(response.as_mut()).is_pending());
// Both waiting handlers must see the posted stop themselves.
lifecycle.release_start.notify_one();
let held_reply = tokio::time::timeout(Duration::from_secs(1), held).await??;
let expected_denial = "rollout budget exhausted";
assert_eq!(
held_reply.item,
ResponseInputItem::FunctionCallOutput {
call_id: "held-at-start".into(),
output: FunctionCallOutputPayload::from_text(
"aborted before handler start: rollout budget exhausted".into(),
),
output: FunctionCallOutputPayload::from_text(format!(
"aborted before handler start: {expected_denial}"
),),
}
.into()
);
lifecycle.release_finish.notify_one();
assert_eq!(
ResponseItemEnvelope::new(expected_abort.into()),
tokio::time::timeout(Duration::from_secs(1), queued).await??
);
let response = tokio::time::timeout(Duration::from_secs(1), response).await??;
let expected_response = ResponseInputItem::FunctionCallOutput {
@@ -1168,10 +1186,6 @@ mod tests {
ResponseItemEnvelope::new(expected_response.into()),
response
);
assert_eq!(
ResponseItemEnvelope::new(expected_abort.into()),
tokio::time::timeout(Duration::from_secs(1), queued).await??
);
assert_eq!(parallel_handler.calls.load(Ordering::Relaxed), 1);
assert_eq!(queued_handler.calls.load(Ordering::Relaxed), 0);
+3 -3
View File
@@ -6205,8 +6205,8 @@ enum ToolDispatchCase {
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
#[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")]
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered output waits for completion without hooks")]
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered text and tools wait for completion")]
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming text and tools do not wait for synchronous pricing")]
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "the host receives an overflow lower bound and keeps its policy")]
@@ -6324,7 +6324,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
.with_pre_build_hook(move |home| {
if buffered {
fs::write(home.join("config.toml"), "[responses]\nprocess_mode = \"after_complete_response\"\n")
.expect("configure response processing after completion");
.expect("configure buffered response processing");
}
if let Some(hook_async) = hook_async {
let post = format!(
+32 -10
View File
@@ -440,6 +440,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
const ACCEPTED_REVIEW: &str = r#"{"outcome":"allow"}"#;
// Enforced: decimal equality is allowed; the next material charge exceeds it.
// Advisory: a real overspend still permits a later request with a zero-balance reminder.
let (limit_budget, reminder_at_remaining_tokens, charges): (f64, Vec<i64>, &[f64]) =
@@ -462,7 +463,12 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
let id = format!("budget-{turn}");
let mut completed = ev_completed_with_tokens(&id, /*total_tokens*/ 1);
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(units);
sse(vec![ev_response_created(&id), completed])
let mut events = vec![ev_response_created(&id)];
if turn == crossing_turn {
events.push(ev_assistant_message("accepted-review", ACCEPTED_REVIEW));
}
events.push(completed);
sse(events)
})
.collect();
let responses = mount_sse_sequence(&server, sequence).await;
@@ -483,6 +489,10 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
.build_with_auto_env(&server)
.await?;
let late = Arc::new(HostAccounting::default());
// This real Guardian policy replays accepted text only after Completed.
test.codex
.thread_extension_data()
.insert(codex_guardian_reviewer::DenialResample::default());
if owner == AgentRuntimeOwnership::Host {
// Neither the owner nor the accounting provider can change after startup.
let data = test.codex.thread_extension_data();
@@ -502,7 +512,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_))
})
.await;
if turn < crossing_turn || owner == AgentRuntimeOwnership::Host || !enforce_limit {
if turn < 2 || owner == AgentRuntimeOwnership::Host || !enforce_limit {
assert!(matches!(event, EventMsg::TurnComplete(done) if done.error.is_none()));
continue;
}
@@ -517,8 +527,14 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
.await;
}
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
assert!(history.items.iter().any(|item| matches!(item, codex_history::RolloutItem::ResponseItem(envelope)
if matches!(&envelope.item, codex_protocol::models::ResponseItem::Message { role, content, .. }
if role == "assistant" && content.iter().any(|part| matches!(part,
codex_protocol::models::ContentItem::OutputText { text, .. } if text == ACCEPTED_REVIEW))))),
"a shared stop must not discard Guardian's already-selected final text");
if owner == AgentRuntimeOwnership::Controller && enforce_limit {
let history = test.codex.load_history(/*include_archived*/ false).await?;
let warning_roles = history
.items
.iter()
@@ -545,7 +561,6 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
if owner == AgentRuntimeOwnership::Host {
let requests = responses.requests();
assert_eq!(requests.len(), charges.len());
for request in &requests {
assert!(rollout_budget_texts(request).is_empty());
}
@@ -770,14 +785,21 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
extensions.thread_lifecycle_contributor(accounting.clone());
let test = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_config(|config| {
config
.features
.enable(Feature::InstantInterrupt)
.expect("enable instant interrupt");
.with_config(move |config| {
for feature in [
Feature::InstantInterrupt,
Feature::DefaultModeRequestUserInput,
] {
config
.features
.enable(feature)
.expect("enable test feature");
}
config.model_provider.stream_max_retries = Some(2);
config.model_provider.request_max_retries = Some(0);
config.rollout_budget = None;
// This allowance would admit the response. The selected host can still stop it.
config.rollout_budget =
matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget);
})
.build_with_auto_env(&server)
.await?;