mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Gate new rollout work on shared budget exhaustion (#61582)
## Why A budget can be exhausted while another thread waits to start work. New requests, tools, and automatic turns must observe that stop without discarding an already-consumed response whose cost exhausted the budget. ## What changed - Share a work gate across the rollout tree while preserving native and host accounting ownership. - Recheck the budget before HTTP and WebSocket requests, hook execution, tool dispatch, and compaction. Prevent automatic wakeups and late input from restarting stopped work, leaving pending input queued. - Record response costs without rejecting the consumed response solely because its bill exhausts the budget, and stop new work when enforced accounting overflows. - Expose a thread-registration receipt that remains committed after unload. ## Testing Extend rollout-budget coverage to verify that accepted response text remains in history after exhaustion and that waiting tools observe the shared stop while completed tools retain their results. GitOrigin-RevId: e583d657c6f29e374d374cdb014bf23fd6c28614
This commit is contained in:
@@ -39,6 +39,12 @@ const _: Option<&dyn AgentControl> = None;
|
||||
/// remote ownership, retries and recovery are separate backend work.
|
||||
/// Boxed Send futures allow callers to use `Arc<dyn AgentControl>`.
|
||||
pub trait AgentControl: Send + Sync {
|
||||
/// Expose the account only when this controller actually owns native billing.
|
||||
/// External controllers keep their own ledger and need not provide a numeric read.
|
||||
fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
|
||||
None
|
||||
}
|
||||
|
||||
fn identity(&self) -> SessionId;
|
||||
|
||||
/// Resolve an ID or a name relative to the caller's captured source, without loading.
|
||||
|
||||
@@ -33,6 +33,10 @@ use futures::future::BoxFuture;
|
||||
use std::collections::HashSet;
|
||||
|
||||
impl AgentControl for LocalAgentControl {
|
||||
fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
|
||||
Some(std::sync::Arc::clone(&self.runtime.rollout_budget))
|
||||
}
|
||||
|
||||
fn identity(&self) -> SessionId {
|
||||
self.session_id()
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
use super::LocalAgentControl;
|
||||
use crate::rollout_budget::RolloutBudgetReminder;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
@@ -14,14 +13,8 @@ impl LocalAgentControl {
|
||||
usage: Option<&TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
) -> CodexResult<()> {
|
||||
if self
|
||||
.runtime
|
||||
.rollout_budget
|
||||
.record_usage(usage, hook_charge)?
|
||||
{
|
||||
return Err(CodexErr::SessionBudgetExceeded);
|
||||
}
|
||||
Ok(())
|
||||
// The caller keeps a consumed response even when its bill stops new work.
|
||||
self.runtime.rollout_budget.record_usage(usage, hook_charge)
|
||||
}
|
||||
|
||||
pub(crate) fn pending_budget_reminder(
|
||||
|
||||
@@ -312,6 +312,7 @@ pub struct ModelClient {
|
||||
http_client_factory: HttpClientFactory,
|
||||
restored_history: bool,
|
||||
request_contributors: Vec<Arc<dyn codex_extension_api::ModelRequestContributor>>,
|
||||
budget_policy: Option<crate::rollout_budget::ThreadBudgetPolicy>,
|
||||
executed_tool_calls: Option<ExecutedToolCalls>,
|
||||
// Resolved once when the session is created, like other session feature flags.
|
||||
api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms,
|
||||
@@ -660,6 +661,7 @@ impl ModelClient {
|
||||
http_client_factory,
|
||||
restored_history: false,
|
||||
request_contributors,
|
||||
budget_policy: None,
|
||||
executed_tool_calls: None,
|
||||
api_key_cyber_access_programs:
|
||||
cyber_access_program::ApiKeyCyberAccessPrograms::UnsupportedProvider,
|
||||
@@ -672,6 +674,14 @@ impl ModelClient {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_budget_policy(
|
||||
mut self,
|
||||
policy: crate::rollout_budget::ThreadBudgetPolicy,
|
||||
) -> Self {
|
||||
self.budget_policy = Some(policy);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn reasoning_effort_override_enabled(&self, model_info: &ModelInfo) -> bool {
|
||||
self.state.reasoning_effort_override_enabled
|
||||
&& self.state.provider.info().is_openai()
|
||||
@@ -2120,6 +2130,9 @@ impl ResponsesHttpExecutor<'_> {
|
||||
client_setup.api_auth,
|
||||
)
|
||||
.with_telemetry(Some(request_telemetry), Some(sse_telemetry));
|
||||
if let Some(policy) = &self.client.budget_policy {
|
||||
policy.check_start()?;
|
||||
}
|
||||
let stream_result = client
|
||||
.stream_request_with_observer(request, options, |wire| {
|
||||
inference_trace_attempt.record_started(
|
||||
@@ -2474,6 +2487,9 @@ impl ModelClientSession {
|
||||
),
|
||||
],
|
||||
);
|
||||
if !warmup && let Some(policy) = &self.client.budget_policy {
|
||||
policy.check_start()?;
|
||||
}
|
||||
let stream_result = websocket_connection
|
||||
.stream_request_with_observer(
|
||||
ws_request,
|
||||
|
||||
@@ -571,8 +571,10 @@ pub(crate) fn compaction_status_from_result<T>(result: &CodexResult<T>) -> Compa
|
||||
match result {
|
||||
Ok(_) => CompactionStatus::Completed,
|
||||
Err(err)
|
||||
if matches!(err.details(), CodexErrorDetails::Interrupted)
|
||||
|| crate::tasks::ends_turn_as_interrupted(err) =>
|
||||
if matches!(
|
||||
err.details(),
|
||||
CodexErrorDetails::Interrupted | CodexErrorDetails::SessionBudgetExceeded
|
||||
) || crate::tasks::ends_turn_as_interrupted(err) =>
|
||||
{
|
||||
CompactionStatus::Interrupted
|
||||
}
|
||||
|
||||
@@ -1482,13 +1482,13 @@ mod tests {
|
||||
assert!(
|
||||
matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted))
|
||||
);
|
||||
assert!(!sess.rollout_budget().tools_stopped());
|
||||
assert!(!sess.work_budget().is_exhausted());
|
||||
assert!(accounting.reports.lock().unwrap().is_empty());
|
||||
return;
|
||||
}
|
||||
assert!(matches!(result, Err(ref err)
|
||||
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_))));
|
||||
assert!(sess.rollout_budget().tools_stopped());
|
||||
assert!(sess.work_budget().is_exhausted());
|
||||
assert_eq!(
|
||||
*accounting.reports.lock().unwrap(),
|
||||
vec![AgentUsageReport {
|
||||
|
||||
@@ -251,9 +251,7 @@ pub(crate) async fn finalize(
|
||||
.pending_budget_reminder(session.thread_id(), &session.current_window_id().await)
|
||||
.await
|
||||
{
|
||||
let reminder = ContextualUserFragment::into(crate::context::RolloutBudgetContext {
|
||||
remaining_units: reminder.remaining_units,
|
||||
});
|
||||
let reminder = session.budget_reminder_item(&reminder);
|
||||
existing = existing.saturating_add(
|
||||
usize::try_from(estimate_item_token_count(&reminder)).unwrap_or(usize::MAX),
|
||||
);
|
||||
|
||||
@@ -54,10 +54,6 @@ impl SamplingRequest {
|
||||
owner_cancellation: &CancellationToken,
|
||||
send: impl Future<Output = Result<OpenedModelResponse>>,
|
||||
) -> Result<SamplingStream> {
|
||||
let send = async {
|
||||
session.check_rollout_budget()?;
|
||||
send.await
|
||||
};
|
||||
let sampling_turn = match self.kind {
|
||||
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
|
||||
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
|
||||
@@ -83,6 +79,9 @@ impl SamplingRequest {
|
||||
agent_id: subagent.as_ref().map(|context| context.agent_id.clone()),
|
||||
agent_type: subagent.map(|context| context.agent_type),
|
||||
};
|
||||
// Hook input preparation may wait. Do not start a new hook pair after a stop;
|
||||
// the HTTP/WS send independently rechecks after a pre-hook has finished.
|
||||
session.check_rollout_budget()?;
|
||||
let mut attempt = post_mode.map(|mode| SamplingAttempt {
|
||||
owner_cancellation: owner_cancellation.clone(),
|
||||
post: Some(PostHooks {
|
||||
|
||||
@@ -85,6 +85,7 @@ pub use agent::types::ResolvedMultiAgentV2UsageHints;
|
||||
pub use agent::types::SpawnAgentForkMode;
|
||||
pub use agent::types::SpawnAgentOptions;
|
||||
pub use agent::types::SpawnProgramSource;
|
||||
pub use rollout_budget::RolloutBudget;
|
||||
pub use rollout_budget::RolloutBudgetReminder;
|
||||
mod agent_communication;
|
||||
pub use agent_communication::PendingAgentMessage;
|
||||
@@ -202,6 +203,7 @@ pub use thread_manager::StartupModels;
|
||||
pub use thread_manager::SuspendedAgentTree;
|
||||
pub use thread_manager::ThreadManager;
|
||||
pub use thread_manager::ThreadRegistration;
|
||||
pub use thread_manager::ThreadRegistrationReceipt;
|
||||
pub use thread_manager::ThreadShutdownReport;
|
||||
pub use thread_manager::build_models_manager;
|
||||
pub use thread_manager::build_startup_models;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod gate;
|
||||
|
||||
use crate::config::RolloutBudgetConfig;
|
||||
use crate::config::rollout_budget_threshold_reached;
|
||||
use codex_protocol::BudgetUnitsSum;
|
||||
@@ -5,12 +7,12 @@ use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
pub use gate::RolloutWorkGate;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::MutexGuard;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
/// Budget reminder returned by a controller and acknowledged after history insertion.
|
||||
pub struct RolloutBudgetReminder {
|
||||
@@ -19,14 +21,47 @@ pub struct RolloutBudgetReminder {
|
||||
pub reminder_index: i64,
|
||||
}
|
||||
|
||||
/// Shared accounting and reminder state for one root-thread session tree.
|
||||
/// Native spending and reminders, plus the stop published by the tree's selected
|
||||
/// accounting owner (native or host). In-flight work can still report its bill.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RolloutBudget {
|
||||
pub struct RolloutBudget {
|
||||
state: OnceLock<Mutex<RolloutBudgetState>>,
|
||||
/// An explicit stop gates new tools across the tree, even with per-thread host accountants.
|
||||
stop_tools: AtomicBool,
|
||||
gate: Arc<RolloutWorkGate>,
|
||||
}
|
||||
|
||||
/// A native account owns this thread's bills. A host-owned thread only shares its work gate.
|
||||
pub(crate) enum ThreadBudgetAccount {
|
||||
Native(Arc<RolloutBudget>),
|
||||
Shared(Arc<RolloutBudget>),
|
||||
}
|
||||
|
||||
/// Every thread consults the shared work gate before starting new work.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct ThreadBudgetPolicy {
|
||||
rollout: Arc<RolloutBudget>,
|
||||
}
|
||||
|
||||
impl ThreadBudgetPolicy {
|
||||
pub(crate) fn new(account: ThreadBudgetAccount) -> CodexResult<Self> {
|
||||
let (ThreadBudgetAccount::Native(rollout) | ThreadBudgetAccount::Shared(rollout)) = account;
|
||||
Ok(Self { rollout })
|
||||
}
|
||||
|
||||
pub(crate) fn check_start(&self) -> CodexResult<()> {
|
||||
self.rollout.gate.check_start()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for RolloutBudget {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RolloutBudget")
|
||||
.field("state", &self.state)
|
||||
.field("stopped", &self.is_exhausted())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RolloutBudgetState {
|
||||
config: RolloutBudgetConfig,
|
||||
posted: BudgetUnitsSum,
|
||||
@@ -34,6 +69,7 @@ struct RolloutBudgetState {
|
||||
deliveries: HashMap<ThreadId, ThreadBudgetDelivery>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ThreadBudgetDelivery {
|
||||
window_id: String,
|
||||
reminder_index: i64,
|
||||
@@ -50,25 +86,22 @@ impl RolloutBudget {
|
||||
});
|
||||
}
|
||||
|
||||
/// Records usage and returns whether an enforced budget is exhausted.
|
||||
/// Post the response cost and enforce the allowance under the same lock.
|
||||
pub(crate) fn record_usage(
|
||||
&self,
|
||||
usage: Option<&TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
) -> CodexResult<bool> {
|
||||
) -> CodexResult<()> {
|
||||
let Some(mut state) = self.lock() else {
|
||||
return Ok(false);
|
||||
return Ok(());
|
||||
};
|
||||
if !hook_charge.units.is_finite() || hook_charge.units < 0.0 {
|
||||
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
|
||||
return Err(CodexErr::Fatal("invalid additional sampling charge".into()));
|
||||
}
|
||||
if hook_charge.overflowed {
|
||||
// All allowances are finite. No future nonnegative receipt can make
|
||||
// this total representable; the remaining allowance is known to be zero.
|
||||
if hook_charge.overflowed && state.posted.total().is_finite() {
|
||||
// Stop new work before validating the rest of an overflowing receipt.
|
||||
state.posted.add(f64::INFINITY);
|
||||
if state.config.enforce_limit {
|
||||
self.stop_new_tools();
|
||||
}
|
||||
self.enforce_posted_limit(&state)?;
|
||||
}
|
||||
let provider_units = if let Some(units) =
|
||||
usage.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
|
||||
@@ -88,31 +121,31 @@ impl RolloutBudget {
|
||||
0.0
|
||||
};
|
||||
if state.posted.total().is_infinite() {
|
||||
return Ok(state.config.enforce_limit);
|
||||
// Later valid receipts cannot restore precision. The original
|
||||
// overflow was reported; it must not veto every advisory turn.
|
||||
return Ok(());
|
||||
}
|
||||
// Keep the full bill even if enforcing the new allowance fails.
|
||||
state.posted.add(provider_units);
|
||||
state.posted.add(hook_charge.units);
|
||||
// Enforce the allowance before reporting an overflowing sum.
|
||||
self.enforce_posted_limit(&state)?;
|
||||
if !state.posted.total().is_finite() {
|
||||
if state.config.enforce_limit {
|
||||
self.stop_new_tools();
|
||||
}
|
||||
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
|
||||
return Err(CodexErr::Fatal(
|
||||
"rollout sampling cost exceeds f64 range".into(),
|
||||
));
|
||||
}
|
||||
// Equality is allowed, including one rounding step (0.1 + 0.2 versus 0.3).
|
||||
let exhausted = state.config.enforce_limit
|
||||
&& state.posted.total() > state.config.limit_budget.next_up();
|
||||
if exhausted {
|
||||
self.stop_new_tools();
|
||||
}
|
||||
Ok(exhausted)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn stop_new_tools(&self) {
|
||||
self.stop_tools.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn tools_stopped(&self) -> bool {
|
||||
self.stop_tools.load(Ordering::Acquire)
|
||||
fn enforce_posted_limit(&self, state: &RolloutBudgetState) -> CodexResult<()> {
|
||||
let posted = state.posted.total();
|
||||
if state.config.enforce_limit
|
||||
&& (!posted.is_finite() || posted > state.config.limit_budget.next_up())
|
||||
{
|
||||
self.stop_new_work();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn pending_reminder(
|
||||
@@ -121,7 +154,7 @@ impl RolloutBudget {
|
||||
window_id: &str,
|
||||
) -> Option<RolloutBudgetReminder> {
|
||||
let state = self.lock()?;
|
||||
let remaining_units = (state.config.limit_budget - state.posted.total()).max(0.0);
|
||||
let remaining_units = self.remaining_units(&state).ok()?;
|
||||
let reminder_index = state
|
||||
.config
|
||||
.reminder_at_remaining_tokens
|
||||
@@ -167,4 +200,20 @@ impl RolloutBudget {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn stop_new_work(&self) {
|
||||
self.gate.stop_budget();
|
||||
}
|
||||
|
||||
pub(crate) fn is_exhausted(&self) -> bool {
|
||||
self.gate.is_stopped()
|
||||
}
|
||||
|
||||
fn remaining_units(&self, state: &RolloutBudgetState) -> CodexResult<f64> {
|
||||
if state.posted.total().is_infinite() {
|
||||
return Ok(0.0);
|
||||
}
|
||||
let used = state.posted.total();
|
||||
Ok((state.config.limit_budget - used).max(0.0))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
//! The first stop closes every work-start boundary in this rollout tree.
|
||||
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Controls new work across the rollout tree.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct RolloutWorkGate(OnceLock<Stop>);
|
||||
|
||||
#[derive(Debug)]
|
||||
enum Stop {
|
||||
Budget,
|
||||
}
|
||||
|
||||
impl RolloutWorkGate {
|
||||
pub(super) fn stop_budget(&self) {
|
||||
let _ = self.0.set(Stop::Budget);
|
||||
}
|
||||
|
||||
pub(super) fn is_stopped(&self) -> bool {
|
||||
self.0.get().is_some()
|
||||
}
|
||||
|
||||
pub(super) fn check_start(&self) -> Result<()> {
|
||||
match self.0.get() {
|
||||
None => Ok(()),
|
||||
Some(Stop::Budget) => Err(CodexErr::SessionBudgetExceeded),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,7 @@ pub(super) async fn maybe_start(
|
||||
return;
|
||||
}
|
||||
if !enabled(turn_context)
|
||||
|| sess.work_budget().is_exhausted()
|
||||
|| token_status.token_limit_reached
|
||||
|| (matches!(phase, CompactionPhase::PostTurn)
|
||||
&& token_status.turn_end_compaction_threshold_reached)
|
||||
|
||||
@@ -5,7 +5,6 @@ use crate::agent::host::AgentUsageReport;
|
||||
use crate::agent::host::UsageAccountingDecision;
|
||||
use crate::client_common::SamplingUsage;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::rollout_budget::RolloutBudget;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
@@ -27,9 +26,7 @@ pub(super) async fn maybe_record_reminder(
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let response_item = ContextualUserFragment::into(crate::context::RolloutBudgetContext {
|
||||
remaining_units: reminder.remaining_units,
|
||||
});
|
||||
let response_item = sess.budget_reminder_item(&reminder);
|
||||
sess.record_conversation_items(
|
||||
turn_context,
|
||||
turn_context.model_info(),
|
||||
@@ -42,23 +39,41 @@ pub(super) async fn maybe_record_reminder(
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) fn rollout_budget(&self) -> &RolloutBudget {
|
||||
pub(crate) fn budget_reminder_item(
|
||||
&self,
|
||||
reminder: &crate::RolloutBudgetReminder,
|
||||
) -> codex_protocol::models::ResponseItem {
|
||||
ContextualUserFragment::into(crate::context::RolloutBudgetContext {
|
||||
remaining_units: reminder.remaining_units,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn native_rollout_budget(&self) -> Option<std::sync::Arc<crate::RolloutBudget>> {
|
||||
match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => self.services.agent_control.native_rollout_budget(),
|
||||
AgentRuntimeServices::Host(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// All threads consult the same work gate, even when a host owns one thread's bill.
|
||||
pub(crate) fn work_budget(&self) -> &crate::RolloutBudget {
|
||||
&self.services.local_agent_runtime.rollout_budget
|
||||
}
|
||||
|
||||
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
|
||||
self.services.budget_policy.check_start()
|
||||
}
|
||||
|
||||
/// Return the diagnostic for a refused tool or propagate an accounting error.
|
||||
pub(crate) fn tool_start_denial(&self) -> CodexResult<Option<String>> {
|
||||
Ok(self
|
||||
.rollout_budget()
|
||||
.tools_stopped()
|
||||
.then(|| "rollout budget exhausted".into()))
|
||||
}
|
||||
|
||||
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
|
||||
if self.rollout_budget().tools_stopped() {
|
||||
Err(CodexErr::SessionBudgetExceeded)
|
||||
} else {
|
||||
Ok(())
|
||||
match self.check_rollout_budget() {
|
||||
Ok(()) => Ok(None),
|
||||
Err(error) => match error.details() {
|
||||
CodexErrorDetails::SessionBudgetExceeded => {
|
||||
Ok(Some("rollout budget exhausted".into()))
|
||||
}
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,10 +113,11 @@ impl Session {
|
||||
{
|
||||
Ok(()) => false,
|
||||
// A legacy controller can report a posted bill and a stop through this error.
|
||||
// A native account has already chosen its own deadline action.
|
||||
Err(error)
|
||||
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
|
||||
{
|
||||
true
|
||||
self.native_rollout_budget().is_none()
|
||||
}
|
||||
Err(error) => return Err(CodexErr::UsageAccountingFailed(error.to_string())),
|
||||
},
|
||||
@@ -127,7 +143,7 @@ impl Session {
|
||||
}
|
||||
};
|
||||
if stop_new_work {
|
||||
self.rollout_budget().stop_new_tools();
|
||||
self.work_budget().stop_new_work();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1807,6 +1807,24 @@ impl Session {
|
||||
// This placeholder is used only for teardown, before the session is published.
|
||||
Err(err) => (AgentRuntimeServices::Controller, Some(err)),
|
||||
};
|
||||
let mut local_agent_runtime = local_agent_runtime;
|
||||
let mut account = crate::rollout_budget::ThreadBudgetAccount::Shared(Arc::clone(
|
||||
&local_agent_runtime.rollout_budget,
|
||||
));
|
||||
if startup_error.is_none() && matches!(&agent_runtime, AgentRuntimeServices::Controller) {
|
||||
if let Some(native) = agent_control.native_rollout_budget() {
|
||||
local_agent_runtime.rollout_budget = Arc::clone(&native);
|
||||
account = crate::rollout_budget::ThreadBudgetAccount::Native(native);
|
||||
}
|
||||
}
|
||||
let budget_policy = crate::rollout_budget::ThreadBudgetPolicy::new(
|
||||
account,
|
||||
)
|
||||
.unwrap_or_else(|err| {
|
||||
startup_error = Some(err);
|
||||
// Teardown needs a session. Its empty policy is never published.
|
||||
Default::default()
|
||||
});
|
||||
state.shared_service_tier_pending = parent_thread_id.is_none()
|
||||
&& match &agent_runtime {
|
||||
AgentRuntimeServices::Controller => has_provided_agent_control,
|
||||
@@ -1854,6 +1872,7 @@ impl Session {
|
||||
workspace_routing.as_ref().clone(),
|
||||
extensions.model_request_contributors().to_vec(),
|
||||
)
|
||||
.with_budget_policy(budget_policy.clone())
|
||||
.with_executed_tool_calls(executed_tool_calls.clone())
|
||||
.with_restored_history(matches!(
|
||||
&initial_history,
|
||||
@@ -1946,6 +1965,7 @@ impl Session {
|
||||
mcp_thread_init,
|
||||
client_mcp_extensions,
|
||||
local_agent_runtime,
|
||||
budget_policy,
|
||||
agent_control,
|
||||
network_proxy: arc_swap::ArcSwapOption::from(network_proxy.map(Arc::new)),
|
||||
network_proxy_audit_metadata,
|
||||
|
||||
@@ -6898,6 +6898,12 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
|
||||
mcp_thread_init: codex_extension_api::ExtensionDataInit::default(),
|
||||
client_mcp_extensions: ClientMcpExtensions::default(),
|
||||
local_agent_runtime: agent_control.runtime.clone(),
|
||||
budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new(
|
||||
crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone(
|
||||
&agent_control.runtime.rollout_budget,
|
||||
)),
|
||||
)
|
||||
.expect("test budget policy"),
|
||||
agent_control: Arc::new(agent_control),
|
||||
network_proxy: arc_swap::ArcSwapOption::from(None),
|
||||
network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(),
|
||||
@@ -9249,6 +9255,12 @@ where
|
||||
mcp_thread_init: codex_extension_api::ExtensionDataInit::default(),
|
||||
client_mcp_extensions: ClientMcpExtensions::default(),
|
||||
local_agent_runtime: agent_control.runtime.clone(),
|
||||
budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new(
|
||||
crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone(
|
||||
&agent_control.runtime.rollout_budget,
|
||||
)),
|
||||
)
|
||||
.expect("test budget policy"),
|
||||
agent_control: Arc::new(agent_control),
|
||||
network_proxy: arc_swap::ArcSwapOption::from(None),
|
||||
network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(),
|
||||
|
||||
@@ -541,7 +541,7 @@ pub(crate) async fn run_turn(
|
||||
// Note that pending_input would be something like a message the user
|
||||
// submitted through the UI while the model was running. Though the UI
|
||||
// may support this, the model might not.
|
||||
let pending_input = if can_drain_pending_input {
|
||||
let pending_input = if can_drain_pending_input && !sess.work_budget().is_exhausted() {
|
||||
sess.input_queue
|
||||
.get_pending_input(&sess.active_turn)
|
||||
.await
|
||||
@@ -654,7 +654,8 @@ pub(crate) async fn run_turn(
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
// Consume the whole response before deciding whether the turn may continue.
|
||||
// Completed can precede Guardian's accepted-text replay. Preserve the
|
||||
// entire consumed response before deciding whether the turn may continue.
|
||||
let sampling_request_result = sampling_request_result.and_then(|output| {
|
||||
sess.check_rollout_budget()?;
|
||||
Ok(output)
|
||||
@@ -1694,6 +1695,7 @@ async fn run_auto_compact(
|
||||
candidate: Option<Box<BackgroundCompactionCandidate>>,
|
||||
) -> CodexResult<bool> {
|
||||
let turn_context = &step_context.turn;
|
||||
sess.check_rollout_budget()?;
|
||||
if matches!(phase, CompactionPhase::PreTurn)
|
||||
&& crate::guardian::is_basic_session_source(&turn_context.session_source)
|
||||
&& !crate::guardian::should_compact_guardian_input(sess)?
|
||||
@@ -1712,6 +1714,9 @@ async fn run_auto_compact(
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
if turn_context.config.features.enabled(Feature::TokenBudget) {
|
||||
// Unlike sampling compaction, this reset has no transport admission check.
|
||||
// World-state preparation above may have waited while a peer posted a bill.
|
||||
sess.check_rollout_budget()?;
|
||||
// Compaction is the reset request, so force a new context window
|
||||
// instead of consuming a pending `new_context` tool request.
|
||||
crate::compact_token_budget::run_inline_auto_compact_task(
|
||||
|
||||
@@ -609,6 +609,11 @@ async fn start_if_idle(
|
||||
reason: NotSubmittedReason::NotIdle,
|
||||
});
|
||||
}
|
||||
// Idle observers can retry a failed turn. Check before reserving so a permanent
|
||||
// stop cannot create a loop of new turns that never reach the provider.
|
||||
if kind != TurnStartKind::User {
|
||||
session.check_rollout_budget()?;
|
||||
}
|
||||
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
|
||||
Arc::clone(&active_turn.turn_state)
|
||||
};
|
||||
@@ -843,6 +848,12 @@ impl Session {
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// Triggered mail and durable sleep bypass start_if_idle. Leave their mail
|
||||
// pending rather than consuming it to start a turn that the work gate will refuse.
|
||||
if let Err(error) = self.check_rollout_budget() {
|
||||
tracing::debug!(%error, "rollout budget stopped a pending-work wake");
|
||||
return None;
|
||||
}
|
||||
let active_turn = active_turn.get_or_insert_with(ActiveTurn::default);
|
||||
let previous_options = if has_trigger_turn {
|
||||
Default::default()
|
||||
|
||||
@@ -275,11 +275,11 @@ async fn mcp_budget_stop_at_handoff(phase: McpPause) -> anyhow::Result<()> {
|
||||
extensions.tool_lifecycle_contributor(observer.clone());
|
||||
session.services.extensions = Arc::new(extensions.build());
|
||||
session
|
||||
.rollout_budget()
|
||||
.work_budget()
|
||||
.configure(crate::config::RolloutBudgetConfig {
|
||||
enforce_limit: true,
|
||||
limit_budget: 1.0,
|
||||
reminder_at_context_start: true,
|
||||
reminder_at_context_start: false,
|
||||
reminder_at_remaining_tokens: Vec::new(),
|
||||
sampling_token_weight: 1.0,
|
||||
prefill_token_weight: 1.0,
|
||||
|
||||
@@ -89,6 +89,7 @@ pub(crate) struct SessionServices {
|
||||
pub(crate) mcp_thread_init: ExtensionDataInit,
|
||||
pub(crate) agent_control: Arc<dyn AgentControl>,
|
||||
pub(crate) local_agent_runtime: LocalAgentRuntime,
|
||||
pub(crate) budget_policy: crate::rollout_budget::ThreadBudgetPolicy,
|
||||
pub(crate) network_proxy: ArcSwapOption<StartedNetworkProxy>,
|
||||
pub(crate) network_proxy_audit_metadata: NetworkProxyAuditMetadata,
|
||||
pub(crate) managed_network_requirements_configured: bool,
|
||||
|
||||
@@ -41,6 +41,7 @@ impl SessionTask for CompactTask {
|
||||
);
|
||||
// Preparation errors must reach the task runner, which reports them to the client.
|
||||
session.emit_turn_started(&ctx, TaskKind::Compact).await;
|
||||
session.check_rollout_budget()?;
|
||||
let step_context = session
|
||||
.capture_step_context(Arc::clone(&ctx), &cancellation_token)
|
||||
.await?;
|
||||
@@ -50,6 +51,8 @@ impl SessionTask for CompactTask {
|
||||
.await?,
|
||||
);
|
||||
if ctx.config.features.enabled(Feature::TokenBudget) {
|
||||
// Preparing the context may wait; this reset never sends a model request.
|
||||
session.check_rollout_budget()?;
|
||||
crate::compact_token_budget::run_manual_compact_task(
|
||||
session,
|
||||
step_context,
|
||||
|
||||
@@ -682,7 +682,8 @@ impl Session {
|
||||
reason: TurnAbortReason,
|
||||
error: Option<ErrorEvent>,
|
||||
) -> bool {
|
||||
let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else {
|
||||
let active_turn = self.take_named_active_turn(turn_id, &reason).await;
|
||||
let Some(active_turn) = active_turn else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -696,7 +697,8 @@ impl Session {
|
||||
reason: &TurnAbortReason,
|
||||
) -> Option<ActiveTurn> {
|
||||
let mut active = self.active_turn.lock().await;
|
||||
if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id {
|
||||
let turn = &active.as_ref()?.task.as_ref()?.turn_context;
|
||||
if turn.sub_id != turn_id {
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
@@ -749,8 +751,10 @@ impl Session {
|
||||
(None, None, Some(err))
|
||||
}
|
||||
};
|
||||
let can_continue =
|
||||
regular_output.is_some() && turn_context.terminal_error.lock().await.is_none();
|
||||
// A late steer cannot reopen the stopped rollout after the turn loop has finished.
|
||||
let can_continue = regular_output.is_some()
|
||||
&& turn_context.terminal_error.lock().await.is_none()
|
||||
&& !self.work_budget().is_exhausted();
|
||||
let finishing = CancellationToken::new();
|
||||
let finish_recording = finishing.clone().drop_guard();
|
||||
let (turn_state, unadmitted_task, mut pending_input, outcome_handoff) = {
|
||||
|
||||
@@ -163,6 +163,7 @@ impl RegularTask {
|
||||
// Terminal errors are already reported. Let task completion preserve pending
|
||||
// input instead of restarting the failed turn for that same input.
|
||||
if ctx.terminal_error.lock().await.is_some()
|
||||
|| sess.work_budget().is_exhausted()
|
||||
|| !sess.input_queue.has_pending_input(&sess.active_turn).await
|
||||
{
|
||||
return Ok(Some(RegularTaskOutput {
|
||||
|
||||
@@ -207,6 +207,16 @@ pub enum ThreadRegistration {
|
||||
Replaced,
|
||||
}
|
||||
|
||||
/// A manager-started runtime's receipt stays committed across unload. A runtime
|
||||
/// started without a manager has no receipt in its thread extension store.
|
||||
pub struct ThreadRegistrationReceipt(Arc<AtomicBool>);
|
||||
|
||||
impl ThreadRegistrationReceipt {
|
||||
pub fn was_registered(&self) -> bool {
|
||||
self.0.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(ccunningham): Add an explicit non-interrupting live-turn snapshot once
|
||||
// core can represent sampling boundaries directly instead of relying on
|
||||
// whichever items happened to be persisted mid-turn.
|
||||
@@ -2903,6 +2913,8 @@ impl ThreadManagerState {
|
||||
&session_source,
|
||||
) => manager,
|
||||
};
|
||||
let registered = Arc::new(AtomicBool::new(false));
|
||||
thread_extension_init.insert(ThreadRegistrationReceipt(Arc::clone(®istered)));
|
||||
let spawn_result = Session::spawn(SessionSpawnArgs {
|
||||
attached_thread,
|
||||
startup,
|
||||
@@ -2997,7 +3009,7 @@ impl ThreadManagerState {
|
||||
}
|
||||
}
|
||||
let finalize_result = self
|
||||
.finalize_thread_spawn(session, io, tracked_session_source)
|
||||
.finalize_thread_spawn(session, io, tracked_session_source, ®istered)
|
||||
.await;
|
||||
let (new_thread, tier_acceptance) = match finalize_result {
|
||||
Ok(registered) => registered,
|
||||
@@ -3040,6 +3052,7 @@ impl ThreadManagerState {
|
||||
session: Arc<Session>,
|
||||
io: SessionIo,
|
||||
session_source: SessionSource,
|
||||
registered: &AtomicBool,
|
||||
) -> CodexResult<(NewThread, Option<ServiceTierAcceptance>)> {
|
||||
let thread_id = session.thread_id();
|
||||
let event = io.next_event().await?;
|
||||
@@ -3080,6 +3093,8 @@ impl ThreadManagerState {
|
||||
session_configured.rollout_path.clone(),
|
||||
session_source,
|
||||
));
|
||||
// All registration checks passed; publish the receipt with this insertion.
|
||||
registered.store(true, Ordering::Release);
|
||||
e.insert(thread.clone());
|
||||
let tier_acceptance = pending_service_tier.and_then(|(mut state, update)| {
|
||||
state.service_tier_acceptance =
|
||||
|
||||
@@ -309,6 +309,13 @@ impl ToolCallRuntime {
|
||||
let mut dispatch_handle = AbortOnDropHandle::new(tokio::spawn(
|
||||
async move {
|
||||
let _message_admission = message_admission?;
|
||||
// A response replayed after settlement can already have exhausted the budget.
|
||||
// Record its denial without waiting for a handler that will never start.
|
||||
if session.work_budget().is_exhausted() {
|
||||
tracing::Span::current().record("aborted", true);
|
||||
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
|
||||
return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch));
|
||||
}
|
||||
// Readiness and admission remain cancellable even when the handler must finish.
|
||||
let (guard, admission_started) = async {
|
||||
if let Some(tool_runtime) = tool_runtime
|
||||
@@ -1034,17 +1041,20 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case::test_case(false; "posted cost checked after the slot opens")]
|
||||
#[tokio::test]
|
||||
async fn completed_and_budget_stopped_tools_report_their_own_lifecycle() -> anyhow::Result<()> {
|
||||
async fn completed_and_budget_stopped_tools_report_their_own_lifecycle(
|
||||
weighted: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let (mut session, turn_context) = crate::session::tests::make_session_and_context().await;
|
||||
session
|
||||
.rollout_budget()
|
||||
.work_budget()
|
||||
.configure(crate::config::RolloutBudgetConfig {
|
||||
enforce_limit: true,
|
||||
limit_budget: 1.0,
|
||||
reminder_at_context_start: true,
|
||||
reminder_at_context_start: weighted,
|
||||
reminder_at_remaining_tokens: Vec::new(),
|
||||
sampling_token_weight: 1.0,
|
||||
sampling_token_weight: if weighted { f64::MAX } else { 1.0 },
|
||||
prefill_token_weight: 1.0,
|
||||
});
|
||||
let lifecycle = Arc::new(PausedToolLifecycle::default());
|
||||
@@ -1136,25 +1146,33 @@ mod tests {
|
||||
output_tokens: 2,
|
||||
..Default::default()
|
||||
};
|
||||
session
|
||||
let bill = session
|
||||
.record_rollout_budget_usage("peer-turn", &usage.into())
|
||||
.await?;
|
||||
.await;
|
||||
bill?;
|
||||
assert!(session.work_budget().is_exhausted());
|
||||
cancellation_token.cancel();
|
||||
// Let the caller observe cancellation while the already-finished handler is held.
|
||||
assert!(futures::poll!(response.as_mut()).is_pending());
|
||||
// Both waiting handlers must see the posted stop themselves.
|
||||
lifecycle.release_start.notify_one();
|
||||
let held_reply = tokio::time::timeout(Duration::from_secs(1), held).await??;
|
||||
let expected_denial = "rollout budget exhausted";
|
||||
assert_eq!(
|
||||
held_reply.item,
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: "held-at-start".into(),
|
||||
output: FunctionCallOutputPayload::from_text(
|
||||
"aborted before handler start: rollout budget exhausted".into(),
|
||||
),
|
||||
output: FunctionCallOutputPayload::from_text(format!(
|
||||
"aborted before handler start: {expected_denial}"
|
||||
),),
|
||||
}
|
||||
.into()
|
||||
);
|
||||
lifecycle.release_finish.notify_one();
|
||||
assert_eq!(
|
||||
ResponseItemEnvelope::new(expected_abort.into()),
|
||||
tokio::time::timeout(Duration::from_secs(1), queued).await??
|
||||
);
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_secs(1), response).await??;
|
||||
let expected_response = ResponseInputItem::FunctionCallOutput {
|
||||
@@ -1168,10 +1186,6 @@ mod tests {
|
||||
ResponseItemEnvelope::new(expected_response.into()),
|
||||
response
|
||||
);
|
||||
assert_eq!(
|
||||
ResponseItemEnvelope::new(expected_abort.into()),
|
||||
tokio::time::timeout(Duration::from_secs(1), queued).await??
|
||||
);
|
||||
assert_eq!(parallel_handler.calls.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(queued_handler.calls.load(Ordering::Relaxed), 0);
|
||||
|
||||
|
||||
@@ -6205,8 +6205,8 @@ enum ToolDispatchCase {
|
||||
|
||||
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
|
||||
#[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")]
|
||||
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered output waits for completion without hooks")]
|
||||
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
|
||||
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered text and tools wait for completion")]
|
||||
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming text and tools do not wait for synchronous pricing")]
|
||||
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
|
||||
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
|
||||
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "the host receives an overflow lower bound and keeps its policy")]
|
||||
@@ -6324,7 +6324,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
.with_pre_build_hook(move |home| {
|
||||
if buffered {
|
||||
fs::write(home.join("config.toml"), "[responses]\nprocess_mode = \"after_complete_response\"\n")
|
||||
.expect("configure response processing after completion");
|
||||
.expect("configure buffered response processing");
|
||||
}
|
||||
if let Some(hook_async) = hook_async {
|
||||
let post = format!(
|
||||
|
||||
@@ -440,6 +440,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
const ACCEPTED_REVIEW: &str = r#"{"outcome":"allow"}"#;
|
||||
// Enforced: decimal equality is allowed; the next material charge exceeds it.
|
||||
// Advisory: a real overspend still permits a later request with a zero-balance reminder.
|
||||
let (limit_budget, reminder_at_remaining_tokens, charges): (f64, Vec<i64>, &[f64]) =
|
||||
@@ -462,7 +463,12 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
let id = format!("budget-{turn}");
|
||||
let mut completed = ev_completed_with_tokens(&id, /*total_tokens*/ 1);
|
||||
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(units);
|
||||
sse(vec![ev_response_created(&id), completed])
|
||||
let mut events = vec![ev_response_created(&id)];
|
||||
if turn == crossing_turn {
|
||||
events.push(ev_assistant_message("accepted-review", ACCEPTED_REVIEW));
|
||||
}
|
||||
events.push(completed);
|
||||
sse(events)
|
||||
})
|
||||
.collect();
|
||||
let responses = mount_sse_sequence(&server, sequence).await;
|
||||
@@ -483,6 +489,10 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let late = Arc::new(HostAccounting::default());
|
||||
// This real Guardian policy replays accepted text only after Completed.
|
||||
test.codex
|
||||
.thread_extension_data()
|
||||
.insert(codex_guardian_reviewer::DenialResample::default());
|
||||
if owner == AgentRuntimeOwnership::Host {
|
||||
// Neither the owner nor the accounting provider can change after startup.
|
||||
let data = test.codex.thread_extension_data();
|
||||
@@ -502,7 +512,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
if turn < crossing_turn || owner == AgentRuntimeOwnership::Host || !enforce_limit {
|
||||
if turn < 2 || owner == AgentRuntimeOwnership::Host || !enforce_limit {
|
||||
assert!(matches!(event, EventMsg::TurnComplete(done) if done.error.is_none()));
|
||||
continue;
|
||||
}
|
||||
@@ -517,8 +527,14 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
.await;
|
||||
}
|
||||
|
||||
test.codex.ensure_rollout_materialized().await;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
assert!(history.items.iter().any(|item| matches!(item, codex_history::RolloutItem::ResponseItem(envelope)
|
||||
if matches!(&envelope.item, codex_protocol::models::ResponseItem::Message { role, content, .. }
|
||||
if role == "assistant" && content.iter().any(|part| matches!(part,
|
||||
codex_protocol::models::ContentItem::OutputText { text, .. } if text == ACCEPTED_REVIEW))))),
|
||||
"a shared stop must not discard Guardian's already-selected final text");
|
||||
if owner == AgentRuntimeOwnership::Controller && enforce_limit {
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
let warning_roles = history
|
||||
.items
|
||||
.iter()
|
||||
@@ -545,7 +561,6 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
|
||||
if owner == AgentRuntimeOwnership::Host {
|
||||
let requests = responses.requests();
|
||||
assert_eq!(requests.len(), charges.len());
|
||||
for request in &requests {
|
||||
assert!(rollout_budget_texts(request).is_empty());
|
||||
}
|
||||
@@ -770,14 +785,21 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::InstantInterrupt)
|
||||
.expect("enable instant interrupt");
|
||||
.with_config(move |config| {
|
||||
for feature in [
|
||||
Feature::InstantInterrupt,
|
||||
Feature::DefaultModeRequestUserInput,
|
||||
] {
|
||||
config
|
||||
.features
|
||||
.enable(feature)
|
||||
.expect("enable test feature");
|
||||
}
|
||||
config.model_provider.stream_max_retries = Some(2);
|
||||
config.model_provider.request_max_retries = Some(0);
|
||||
config.rollout_budget = None;
|
||||
// This allowance would admit the response. The selected host can still stop it.
|
||||
config.rollout_budget =
|
||||
matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
|
||||
Reference in New Issue
Block a user