mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Reject overflowing rollout budget charges before posting (#62559)
## Why An overflowing hook total could send a partial charge to the host, and an overflow in native accounting could leave the posted balance infinite. ## What changed Reject overflowing hook totals before reporting usage. Check both the combined provider and hook charge and the cumulative rollout balance before posting, preserving the previous balance on overflow. Native budgets with limit enforcement enabled still stop new work on overflow. ## Testing Update the hook overflow integration case to verify that the response's tool is blocked and the host receives no usage report. GitOrigin-RevId: 2ac80a6aa1d857924a8807df7bb70c6b7c107cd2
This commit is contained in:
@@ -130,11 +130,8 @@ pub trait AgentControl: Send + Sync {
|
||||
/// Account for one inference response, including compaction. Each call records usage;
|
||||
/// callers report it once. `SessionBudgetExceeded` means the usage was recorded and
|
||||
/// the shared budget is now exhausted.
|
||||
fn record_usage(
|
||||
&self,
|
||||
usage: Option<TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
) -> BoxFuture<'_, Result<()>>;
|
||||
fn record_usage(&self, usage: Option<TokenUsage>, hook_units: f64)
|
||||
-> BoxFuture<'_, Result<()>>;
|
||||
|
||||
/// Report the terminal result to the parent and completion activity to the task
|
||||
/// initiator. Local delivery remains best effort and uses the reporting runtime's
|
||||
|
||||
@@ -290,9 +290,9 @@ impl AgentControl for LocalAgentControl {
|
||||
fn record_usage(
|
||||
&self,
|
||||
usage: Option<TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
hook_units: f64,
|
||||
) -> BoxFuture<'_, Result<()>> {
|
||||
Box::pin(async move { self.record_rollout_budget_usage(usage.as_ref(), hook_charge) })
|
||||
Box::pin(async move { self.record_rollout_budget_usage(usage.as_ref(), hook_units) })
|
||||
}
|
||||
|
||||
fn turn_finished<'a>(
|
||||
|
||||
@@ -11,10 +11,10 @@ impl LocalAgentControl {
|
||||
pub(crate) fn record_rollout_budget_usage(
|
||||
&self,
|
||||
usage: Option<&TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
hook_units: f64,
|
||||
) -> CodexResult<()> {
|
||||
// The caller keeps a consumed response even when its bill stops new work.
|
||||
self.runtime.rollout_budget.record_usage(usage, hook_charge)
|
||||
self.runtime.rollout_budget.record_usage(usage, hook_units)
|
||||
}
|
||||
|
||||
pub(crate) fn pending_budget_reminder(
|
||||
|
||||
@@ -15,8 +15,6 @@ pub struct AgentUsageReport {
|
||||
pub usage: Option<TokenUsage>,
|
||||
/// Synchronous post-hook contributions, independent of the provider's usage.
|
||||
pub additional_budget_units: f64,
|
||||
/// If f64 summation overflowed, this is the largest individual hook price, not the total.
|
||||
pub additional_budget_units_overflowed: bool,
|
||||
}
|
||||
|
||||
/// Each decision means the host recorded the reported spending.
|
||||
|
||||
@@ -1469,10 +1469,7 @@ mod tests {
|
||||
))),
|
||||
hook_output: crate::client_common::SamplingCharge {
|
||||
hooks: codex_hooks::ModelRequestHookOutcome {
|
||||
charge: codex_hooks::HookCharge {
|
||||
units: 3.0,
|
||||
overflowed: false,
|
||||
},
|
||||
charge: codex_hooks::HookCharge::Units(3.0),
|
||||
invalid_price: Some("negative sibling price".into()),
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1520,7 +1517,6 @@ mod tests {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
usage: Some(usage),
|
||||
additional_budget_units: 3.0,
|
||||
additional_budget_units_overflowed: false,
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -1560,10 +1556,7 @@ mod tests {
|
||||
}),
|
||||
hook_output: crate::client_common::SamplingCharge {
|
||||
hooks: codex_hooks::ModelRequestHookOutcome {
|
||||
charge: codex_hooks::HookCharge {
|
||||
units: 42.0,
|
||||
overflowed: false,
|
||||
},
|
||||
charge: codex_hooks::HookCharge::Units(42.0),
|
||||
invalid_price: bad_sibling.then(|| "negative sibling price".into()),
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1601,7 +1594,6 @@ mod tests {
|
||||
turn_id: turn.sub_id.clone(),
|
||||
usage: provider_usage,
|
||||
additional_budget_units: 42.0,
|
||||
additional_budget_units_overflowed: false,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -95,23 +95,22 @@ impl RolloutBudget {
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn config(&self) -> Option<RolloutBudgetConfig> {
|
||||
self.lock().map(|state| state.config.clone())
|
||||
}
|
||||
|
||||
/// Post the response cost and enforce the allowance under the same lock.
|
||||
pub(crate) fn record_usage(
|
||||
&self,
|
||||
usage: Option<&TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
hook_units: f64,
|
||||
) -> CodexResult<()> {
|
||||
let Some(mut state) = self.lock() else {
|
||||
return Ok(());
|
||||
};
|
||||
if !hook_charge.units.is_finite() || hook_charge.units < 0.0 {
|
||||
if !hook_units.is_finite() || hook_units < 0.0 {
|
||||
return Err(CodexErr::Fatal("invalid additional sampling charge".into()));
|
||||
}
|
||||
if hook_charge.overflowed && state.posted.total().is_finite() {
|
||||
// Stop new work before validating the rest of an overflowing receipt.
|
||||
state.posted.add(f64::INFINITY);
|
||||
self.enforce_posted_limit(&state)?;
|
||||
}
|
||||
let provider_units = if let Some(units) =
|
||||
usage.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
|
||||
{
|
||||
@@ -129,29 +128,38 @@ impl RolloutBudget {
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
if state.posted.total().is_infinite() {
|
||||
// Later valid receipts cannot restore precision. The original
|
||||
// overflow was reported; it must not veto every advisory turn.
|
||||
return Ok(());
|
||||
let response_charge_amount = provider_units + hook_units;
|
||||
if !response_charge_amount.is_finite() {
|
||||
return self.reject_overflow(&state, "responseChargeAmount");
|
||||
}
|
||||
// Keep the full bill even if enforcing the new allowance fails.
|
||||
state.posted.add(provider_units);
|
||||
state.posted.add(hook_charge.units);
|
||||
// Enforce the allowance before reporting an overflowing sum.
|
||||
self.enforce_posted_limit(&state)?;
|
||||
if !state.posted.total().is_finite() {
|
||||
return Err(CodexErr::Fatal(
|
||||
"rollout sampling cost exceeds f64 range".into(),
|
||||
));
|
||||
// Stage the bill so a failed check cannot change the previously posted balance.
|
||||
let Some(posted) = state
|
||||
.posted
|
||||
.checked_add(provider_units)
|
||||
.and_then(|sum| sum.checked_add(hook_units))
|
||||
else {
|
||||
return self.reject_overflow(&state, "totalBudgetUsed");
|
||||
};
|
||||
state.posted = posted;
|
||||
let enforcement = self.enforce_posted_limit(&state);
|
||||
enforcement
|
||||
}
|
||||
|
||||
fn reject_overflow(&self, state: &RolloutBudgetState, field: &str) -> CodexResult<()> {
|
||||
if state.config.enforce_limit {
|
||||
self.stop_on_overflow();
|
||||
}
|
||||
Ok(())
|
||||
Err(CodexErr::Fatal(format!("{field} must be finite")))
|
||||
}
|
||||
|
||||
pub(crate) fn stop_on_overflow(&self) {
|
||||
// Decline new work immediately and interrupt active turns on clocks that support it.
|
||||
self.stop_new_work();
|
||||
}
|
||||
|
||||
fn enforce_posted_limit(&self, state: &RolloutBudgetState) -> CodexResult<()> {
|
||||
let posted = state.posted.total();
|
||||
if state.config.enforce_limit
|
||||
&& (!posted.is_finite() || posted > state.config.limit_budget.next_up())
|
||||
{
|
||||
if state.config.enforce_limit && posted > state.config.limit_budget.next_up() {
|
||||
self.stop_new_work();
|
||||
}
|
||||
Ok(())
|
||||
@@ -223,9 +231,6 @@ impl RolloutBudget {
|
||||
}
|
||||
|
||||
fn remaining_units(&self, state: &RolloutBudgetState) -> CodexResult<f64> {
|
||||
if state.posted.total().is_infinite() {
|
||||
return Ok(0.0);
|
||||
}
|
||||
let used = state.posted.total();
|
||||
Ok((state.config.limit_budget - used).max(0.0))
|
||||
}
|
||||
|
||||
@@ -151,14 +151,27 @@ impl Session {
|
||||
usage: Option<&TokenUsage>,
|
||||
hook_charge: codex_hooks::HookCharge,
|
||||
) -> CodexResult<()> {
|
||||
if usage.is_none() && hook_charge == codex_hooks::HookCharge::default() {
|
||||
let hook_units = match hook_charge {
|
||||
codex_hooks::HookCharge::Units(units) => units,
|
||||
codex_hooks::HookCharge::Overflow => {
|
||||
if let Some(account) = self.native_rollout_budget()
|
||||
&& account.config().is_some_and(|config| config.enforce_limit)
|
||||
{
|
||||
account.stop_on_overflow();
|
||||
}
|
||||
return Err(CodexErr::UsageAccountingFailed(
|
||||
"responseChargeAmount: total hook cost exceeds f64 range".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
if usage.is_none() && hook_units == 0.0 {
|
||||
return Ok(());
|
||||
}
|
||||
let stop_new_work = match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => match self
|
||||
.services
|
||||
.agent_control
|
||||
.record_usage(usage.cloned(), hook_charge)
|
||||
.record_usage(usage.cloned(), hook_units)
|
||||
.await
|
||||
{
|
||||
Ok(()) => false,
|
||||
@@ -180,8 +193,7 @@ impl Session {
|
||||
thread_id: self.thread_id(),
|
||||
turn_id: turn_id.to_owned(),
|
||||
usage: usage.cloned(),
|
||||
additional_budget_units: hook_charge.units,
|
||||
additional_budget_units_overflowed: hook_charge.overflowed,
|
||||
additional_budget_units: hook_units,
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
|
||||
@@ -1985,7 +1985,7 @@ async fn spawn_internal_session_preserves_parent_lineage_without_forking_history
|
||||
output_tokens: 25,
|
||||
..Default::default()
|
||||
}),
|
||||
codex_hooks::HookCharge::default(),
|
||||
/*hook_units*/ 0.0,
|
||||
)
|
||||
.await
|
||||
.expect("record reviewer usage");
|
||||
|
||||
@@ -276,7 +276,7 @@ impl AgentControl for TestAgentControl {
|
||||
fn record_usage(
|
||||
&self,
|
||||
_usage: Option<TokenUsage>,
|
||||
_hook_charge: codex_core::HookCharge,
|
||||
_hook_units: f64,
|
||||
) -> BoxFuture<'_, CodexResult<()>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
@@ -6217,7 +6217,7 @@ enum ToolDispatchCase {
|
||||
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming text and tools do not wait for synchronous pricing")]
|
||||
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
|
||||
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
|
||||
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "the host receives an overflow lower bound and keeps its policy")]
|
||||
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "an overflowing hook charge never reaches the host")]
|
||||
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Controller); "a native overflow closes even the largest allowance")]
|
||||
#[tokio::test]
|
||||
async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> Result<()> {
|
||||
@@ -6520,19 +6520,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
);
|
||||
assert!(!marker.exists());
|
||||
if owner == AgentRuntimeOwnership::Host {
|
||||
let reports = accounting.reports.lock().expect("usage reports");
|
||||
let [bill] = reports.as_slice() else {
|
||||
anyhow::bail!("expected one partial bill: {reports:?}");
|
||||
};
|
||||
assert_eq!(bill.additional_budget_units, 1e308);
|
||||
assert!(bill.additional_budget_units_overflowed);
|
||||
assert_eq!(
|
||||
bill.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
|
||||
.and_then(serde_json::Number::as_f64),
|
||||
Some(1.0)
|
||||
);
|
||||
assert!(accounting.reports.lock().expect("usage reports").is_empty());
|
||||
}
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
|
||||
@@ -969,7 +969,6 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
..Default::default()
|
||||
}),
|
||||
additional_budget_units: 0.0,
|
||||
additional_budget_units_overflowed: false,
|
||||
}]
|
||||
);
|
||||
Ok(())
|
||||
|
||||
@@ -164,12 +164,17 @@ pub(crate) fn preview(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// One response's successful hook prices. On overflow, units is a finite
|
||||
/// subtotal of real contributions, not the price of the entire response.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq)]
|
||||
pub struct HookCharge {
|
||||
pub units: f64,
|
||||
pub overflowed: bool,
|
||||
/// Sum of successful hook prices. An overflowing sum is rejected before posting.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum HookCharge {
|
||||
Units(f64),
|
||||
Overflow,
|
||||
}
|
||||
|
||||
impl Default for HookCharge {
|
||||
fn default() -> Self {
|
||||
Self::Units(0.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// A failed observer does not erase prices reported by successful siblings.
|
||||
@@ -245,30 +250,23 @@ pub(crate) async fn run(
|
||||
)
|
||||
.await;
|
||||
let mut outcome = ModelRequestHookOutcome::default();
|
||||
let mut reported_units = BudgetUnitsSum::default();
|
||||
let mut largest_price = 0.0_f64;
|
||||
let mut reported_units = Some(BudgetUnitsSum::default());
|
||||
let mut events = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
if billable && let Some(price) = result.data {
|
||||
match price {
|
||||
Ok(units) => {
|
||||
largest_price = largest_price.max(units);
|
||||
reported_units.add(units);
|
||||
reported_units = reported_units.and_then(|sum| sum.checked_add(units));
|
||||
}
|
||||
Err(error) => outcome.invalid_price = Some(error),
|
||||
}
|
||||
}
|
||||
events.push(result.completed);
|
||||
}
|
||||
outcome.charge.units = reported_units.total();
|
||||
if !outcome.charge.units.is_finite() {
|
||||
// An individual price is a real lower bound, even if earlier f64 additions rounded up.
|
||||
outcome.charge = HookCharge {
|
||||
units: largest_price,
|
||||
overflowed: true,
|
||||
};
|
||||
outcome.invalid_price = Some("total hook cost exceeds f64 range".into());
|
||||
}
|
||||
outcome.charge = match reported_units {
|
||||
Some(sum) => HookCharge::Units(sum.total()),
|
||||
None => HookCharge::Overflow,
|
||||
};
|
||||
(events, outcome)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,23 +2,21 @@
|
||||
//! used for one response's hook contributions and for the rollout's posted bills.
|
||||
|
||||
/// Accumulate validated, nonnegative `f64` charges without dropping every small
|
||||
/// contribution after a large one. An unrepresentable total stays infinite.
|
||||
#[derive(Debug, Default)]
|
||||
/// contribution after a large one.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct BudgetUnitsSum {
|
||||
total: f64,
|
||||
remainder: f64,
|
||||
}
|
||||
|
||||
impl BudgetUnitsSum {
|
||||
pub fn add(&mut self, units: f64) {
|
||||
debug_assert!(units >= 0.0);
|
||||
if units == 0.0 || self.total.is_infinite() {
|
||||
return;
|
||||
/// Compute the next balance. Invalid inputs and arithmetic overflow leave this sum intact.
|
||||
pub fn checked_add(&self, units: f64) -> Option<Self> {
|
||||
if !units.is_finite() || units < 0.0 {
|
||||
return None;
|
||||
}
|
||||
if units.is_infinite() {
|
||||
self.total = f64::INFINITY;
|
||||
self.remainder = 0.0;
|
||||
return;
|
||||
if units == 0.0 {
|
||||
return Some(self.clone());
|
||||
}
|
||||
// Near f64::MAX the first addition can overflow before a negative
|
||||
// remainder brings it back into range. Halving both inputs avoids that.
|
||||
@@ -40,12 +38,9 @@ impl BudgetUnitsSum {
|
||||
// A nonnegative bill cannot lower posted spending. Keep any unused
|
||||
// correction for later receipts rather than reopening the budget.
|
||||
let next = (sum + remainder).max(previous);
|
||||
self.total = next / scale;
|
||||
self.remainder = if self.total.is_finite() {
|
||||
((sum - next) + remainder) / scale
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let total = next / scale;
|
||||
let remainder = ((sum - next) + remainder) / scale;
|
||||
(total.is_finite() && remainder.is_finite()).then_some(Self { total, remainder })
|
||||
}
|
||||
|
||||
pub fn total(&self) -> f64 {
|
||||
|
||||
Reference in New Issue
Block a user