Support turn-scoped exhaustion in host usage accounting (#61958)

## What changed

- Add `UsageAccountingDecision::ExhaustedForTurn` so hosts can fail the current response with `SessionBudgetExceeded` while leaving later turns and peer threads eligible.
- Preserve completed response output and token usage before reporting exhaustion, and track posted charges to avoid accounting twice.
- When steering or mailbox preemption stops a stream, wait for accounting of any completion already observed before starting a follow-up request, and propagate budget errors.

## Testing

Add coverage for preserving output and usage on turn exhaustion, successfully starting a later turn, and reporting turn exhaustion during steering.

GitOrigin-RevId: e673bf30775f27a057fdc39d8429b74ff22121db
This commit is contained in:
rhan-oai
2026-11-17 03:28:51 +00:00
committed by copyberry
parent e9a2a9e115
commit 44834c58fa
7 changed files with 190 additions and 51 deletions
+4 -1
View File
@@ -19,12 +19,15 @@ pub struct AgentUsageReport {
pub additional_budget_units_overflowed: bool,
}
/// Both decisions mean the host recorded the reported spending.
/// Each decision means the host recorded the reported spending.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum UsageAccountingDecision {
Continue,
/// Do not start more tool handlers in this local rollout tree. Already-running work continues.
Exhausted,
/// Fail the reporting response with `SessionBudgetExceeded` after recording its output and usage.
/// Background compaction fails only its job; later turns and peer threads remain eligible.
ExhaustedForTurn,
}
/// Replaces this runtime's `AgentControl::record_usage` and native rollout-budget reminders.
+1
View File
@@ -2920,6 +2920,7 @@ where
}
loop {
let event = tokio::select! {
biased;
_ = consumer_dropped.cancelled() => {
inference_trace_attempt.record_cancelled(
STREAM_DROPPED_REASON,
+14 -4
View File
@@ -1,11 +1,13 @@
//! Settles completed responses and releases output according to the configured processing mode.
use crate::client_common::ChargePosting;
use crate::client_common::ResponseEvent;
use crate::client_common::SamplingEvent;
use crate::hook_runtime::sampling::SamplingAttempt;
use codex_api::ApiError;
use codex_hooks::ModelRequestHookEvent;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
@@ -125,15 +127,23 @@ impl ResponseDelivery {
if let Some((session, turn)) = &self.settlement
&& let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event
{
let usage = crate::client_common::SamplingUsage {
let mut usage = crate::client_common::SamplingUsage {
provider: token_usage.clone(),
charge: std::mem::take(&mut terminal.hook_output),
};
session
usage.charge.posting = match session
.record_rollout_budget_usage(&turn.sub_id, &usage)
.await?;
.await
{
Ok(()) => ChargePosting::Posted,
Err(error)
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
{
ChargePosting::PostedTurnExhausted
}
Err(error) => return Err(error),
};
terminal.hook_output = usage.charge;
terminal.hook_output.posted = true;
}
if terminal.event.is_ok() {
for item in std::mem::take(&mut self.held) {
+11 -2
View File
@@ -121,11 +121,20 @@ fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelIn
}
}
/// Carries the hook price and whether the response producer has posted the bill.
/// Whether the producer has posted a response's bill and deferred a turn error.
#[derive(Clone, Copy, Debug, Default)]
pub(crate) enum ChargePosting {
#[default]
Unposted,
Posted,
PostedTurnExhausted,
}
/// Carries the hook price and the response producer's billing state.
#[derive(Debug, Default)]
pub(crate) struct SamplingCharge {
pub(crate) hooks: codex_hooks::ModelRequestHookOutcome,
pub(crate) posted: bool,
pub(crate) posting: ChargePosting,
}
/// Core consumes a terminal event and its hook output together. Other readers
+25 -5
View File
@@ -3,6 +3,7 @@ use super::turn_context::TurnContext;
use crate::agent::host::AgentRuntimeServices;
use crate::agent::host::AgentUsageReport;
use crate::agent::host::UsageAccountingDecision;
use crate::client_common::ChargePosting;
use crate::client_common::SamplingUsage;
use crate::context::ContextualUserFragment;
use codex_protocol::error::CodexErr;
@@ -118,9 +119,22 @@ impl Session {
turn_id: &str,
usage: &SamplingUsage,
) -> CodexResult<()> {
if !usage.charge.posted {
self.record_numeric_usage(turn_id, usage.provider.as_ref(), usage.charge.hooks.charge)
.await?;
let result = match usage.charge.posting {
ChargePosting::Unposted => {
self.record_numeric_usage(
turn_id,
usage.provider.as_ref(),
usage.charge.hooks.charge,
)
.await
}
ChargePosting::Posted => Ok(()),
ChargePosting::PostedTurnExhausted => Err(CodexErr::SessionBudgetExceeded),
};
if let Err(error) = &result
&& !matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded)
{
return result;
}
if let Some(message) = &usage.charge.hooks.invalid_price {
// A bad price does not erase successful siblings' contributions.
@@ -128,7 +142,7 @@ impl Session {
"Invalid post-hook budget units: {message}"
)));
}
Ok(())
result
}
async fn record_numeric_usage(
@@ -175,7 +189,13 @@ impl Session {
"Host usage accounting failed: {error}"
))
})?;
decision == UsageAccountingDecision::Exhausted
match decision {
UsageAccountingDecision::Continue => false,
UsageAccountingDecision::Exhausted => true,
UsageAccountingDecision::ExhaustedForTurn => {
return Err(CodexErr::SessionBudgetExceeded);
}
}
}
};
if stop_new_work {
+37 -35
View File
@@ -3177,20 +3177,17 @@ async fn try_run_sampling_request(
preempt = CancellationToken::new();
continue;
}
if stream.settlement_started.load(Ordering::Acquire) {
wait_for_bill_only = streaming && stream.interrupt.is_none();
} else {
let Some(interrupt) = stream.interrupt.take() else {
// TODO: Reconcile any response item already being presented to the client.
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: false,
last_agent_message,
});
};
if step_context.settings.model_info.use_responses_lite {
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
}
if stream.interrupt.is_none() {
// Stop upstream polling, but keep receiving until any completion already
// observed by the mapper has finished accounting.
wait_for_bill_only =
streaming || !stream.settlement_started.load(Ordering::Acquire);
stream.consumer_dropped.cancel();
} else if !stream.settlement_started.load(Ordering::Acquire)
&& let Some(interrupt) = stream.interrupt.take()
&& step_context.settings.model_info.use_responses_lite
{
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
}
// A steer cannot start another request before the accounting owner returns its decision.
preempt = CancellationToken::new();
@@ -3202,6 +3199,13 @@ async fn try_run_sampling_request(
let event = SamplingStream::accept_for_turn(event, &cancellation_token);
let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else {
if stream.consumer_dropped.is_cancelled() {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: *draining_for_mailbox,
last_agent_message,
});
}
break Err(CodexErr::Stream(
"stream closed before response.completed".into(),
));
@@ -3237,14 +3241,7 @@ async fn try_run_sampling_request(
// A steer can cut off unread items on an HTTP stream while the owner posts its bill.
// Consume the terminal accounting decision before opening the follow-up request.
if wait_for_bill_only {
if matches!(event, ResponseEvent::Completed { .. }) {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: *draining_for_mailbox,
last_agent_message,
});
}
if wait_for_bill_only && !matches!(event, ResponseEvent::Completed { .. }) {
continue;
}
@@ -3435,21 +3432,19 @@ async fn try_run_sampling_request(
conversation.id = %sess.thread_id,
turn.id = %turn_context.sub_id,
);
if stream.settlement_started.load(Ordering::Acquire) {
wait_for_bill_only = streaming
&& !(step_context.settings.model_info.use_responses_lite
&& stream.interrupt.is_some());
} else if step_context.settings.model_info.use_responses_lite
&& let Some(interrupt) = stream.interrupt.take()
if step_context.settings.model_info.use_responses_lite
&& stream.interrupt.is_some()
{
// Completion can already be buffered when the interrupt receiver closes.
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
if !stream.settlement_started.load(Ordering::Acquire)
&& let Some(interrupt) = stream.interrupt.take()
{
// Completion can already be buffered when the interrupt receiver closes.
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
}
} else {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: true,
last_agent_message,
});
wait_for_bill_only =
streaming || !stream.settlement_started.load(Ordering::Acquire);
stream.consumer_dropped.cancel();
}
// Keep the response ID and terminal decision. Later tools receive a denial.
*draining_for_mailbox = true;
@@ -3668,6 +3663,13 @@ async fn try_run_sampling_request(
if let Err(err) = budget_result {
break Err(err);
}
if wait_for_bill_only {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: *draining_for_mailbox,
last_agent_message,
});
}
if resampling.retry_pending {
response_completed = false;
response_id_for_assessment = None;
+98 -4
View File
@@ -40,6 +40,7 @@ use core_test_support::skip_if_no_network;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use core_test_support::wait_for_event_match;
use core_test_support::wait_for_successful_turn;
use futures::future::BoxFuture;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -58,6 +59,7 @@ enum HostAccountingBehavior {
#[default]
Continue,
Exhausted,
ExhaustedForTurnOnce,
Fail,
}
@@ -75,7 +77,11 @@ impl AgentUsageAccounting for HostAccounting {
report: AgentUsageReport,
) -> BoxFuture<'_, codex_protocol::error::Result<UsageAccountingDecision>> {
Box::pin(async move {
self.reports.lock().expect("usage reports").push(report);
let report_count = {
let mut reports = self.reports.lock().expect("usage reports");
reports.push(report);
reports.len()
};
if let Some(pause) = &self.pause_report {
let (resume, resumed) = oneshot::channel();
let _ = pause.send(resume);
@@ -84,6 +90,11 @@ impl AgentUsageAccounting for HostAccounting {
match self.behavior {
HostAccountingBehavior::Continue => Ok(UsageAccountingDecision::Continue),
HostAccountingBehavior::Exhausted => Ok(UsageAccountingDecision::Exhausted),
HostAccountingBehavior::ExhaustedForTurnOnce => Ok(if report_count == 1 {
UsageAccountingDecision::ExhaustedForTurn
} else {
UsageAccountingDecision::Continue
}),
HostAccountingBehavior::Fail => {
Err(CodexErr::Stream("accounting unavailable".into()))
}
@@ -284,6 +295,84 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn host_turn_exhaustion_posts_response_state_and_allows_a_later_turn() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("exhausted"),
ev_assistant_message("exhausted-message", "accepted output"),
ev_completed_with_tokens("exhausted", /*total_tokens*/ 11),
]),
sse(vec![
ev_response_created("fresh"),
ev_assistant_message("fresh-message", "fresh output"),
ev_completed_with_tokens("fresh", /*total_tokens*/ 7),
]),
],
)
.await;
let accounting = Arc::new(HostAccounting {
behavior: HostAccountingBehavior::ExhaustedForTurnOnce,
..Default::default()
});
let mut extensions = ExtensionRegistryBuilder::new();
extensions.thread_lifecycle_contributor(accounting.clone());
let test = test_codex()
.with_config(|config| {
config.response_process_mode =
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
})
.with_extensions(Arc::new(extensions.build()))
.build_with_auto_env(&server)
.await?;
let input = |text: &str| {
TurnInputRequest::user_input(vec![UserInput::Text {
text: text.into(),
text_elements: Vec::new(),
}])
};
test.codex
.start_or_steer_turn(input("spend the current turn allowance"))
.await?;
let mut saw_output = false;
let mut saw_usage = false;
let error = wait_for_event(&test.codex, |event| {
saw_output |= matches!(event, EventMsg::AgentMessage(message) if message.message == "accepted output");
saw_usage |= matches!(event, EventMsg::TokenCount(count)
if count.info.as_ref().is_some_and(|info| info.total_token_usage.total_tokens == 11));
matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_))
})
.await;
assert_eq!((saw_output, saw_usage), (true, true));
assert!(matches!(error, EventMsg::Error(error)
if error.codex_error_info == Some(CodexErrorInfo::SessionBudgetExceeded)));
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
test.codex
.start_or_steer_turn(input("use a fresh turn allowance"))
.await?;
wait_for_successful_turn(&test.codex).await;
let reports = accounting.reports.lock().expect("usage reports");
assert_eq!(
reports
.iter()
.map(|report| report.usage.as_ref().map(|usage| usage.total_tokens))
.collect::<Vec<_>>(),
vec![Some(11), Some(7)],
);
assert_ne!(reports[0].turn_id, reports[1].turn_id);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(false ; "local accounting")]
#[test_case(true ; "host accounting")]
@@ -745,6 +834,7 @@ async fn restates_the_current_remainder_after_compaction() -> Result<()> {
#[tokio::test]
#[test_case(HostAccountingBehavior::Exhausted, false ; "host exhausts budget")]
#[test_case(HostAccountingBehavior::ExhaustedForTurnOnce, false ; "host exhausts steered turn")]
#[test_case(HostAccountingBehavior::Fail, false ; "accounting failure never retries inference")]
#[test_case(HostAccountingBehavior::Fail, true ; "incomplete response keeps original error")]
async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
@@ -752,6 +842,10 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
incomplete: bool,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let exhausts_budget = matches!(
behavior,
HostAccountingBehavior::Exhausted | HostAccountingBehavior::ExhaustedForTurnOnce
);
let server = start_mock_server().await;
let mut response = json!({
"usage": {
@@ -798,8 +892,7 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
config.model_provider.stream_max_retries = Some(2);
config.model_provider.request_max_retries = Some(0);
// This allowance would admit the response. The selected host can still stop it.
config.rollout_budget =
matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget);
config.rollout_budget = exhausts_budget.then(rollout_budget);
})
.build_with_auto_env(&server)
.await?;
@@ -848,11 +941,12 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
error.message,
"Incomplete response returned, reason: max_output_tokens"
);
} else if matches!(behavior, HostAccountingBehavior::Exhausted) {
} else if exhausts_budget {
assert_eq!(
error.codex_error_info,
Some(CodexErrorInfo::SessionBudgetExceeded)
);
assert_eq!(completed.error.as_ref(), Some(error));
} else {
assert!(error.message.contains("Host usage accounting failed"));
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));