mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Support turn-scoped exhaustion in host usage accounting (#61958)
## What changed - Add `UsageAccountingDecision::ExhaustedForTurn` so hosts can fail the current response with `SessionBudgetExceeded` while leaving later turns and peer threads eligible. - Preserve completed response output and token usage before reporting exhaustion, and track posted charges to avoid accounting twice. - When steering or mailbox preemption stops a stream, wait for accounting of any completion already observed before starting a follow-up request, and propagate budget errors. ## Testing Add coverage for preserving output and usage on turn exhaustion, successfully starting a later turn, and reporting turn exhaustion during steering. GitOrigin-RevId: e673bf30775f27a057fdc39d8429b74ff22121db
This commit is contained in:
@@ -19,12 +19,15 @@ pub struct AgentUsageReport {
|
||||
pub additional_budget_units_overflowed: bool,
|
||||
}
|
||||
|
||||
/// Both decisions mean the host recorded the reported spending.
|
||||
/// Each decision means the host recorded the reported spending.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum UsageAccountingDecision {
|
||||
Continue,
|
||||
/// Do not start more tool handlers in this local rollout tree. Already-running work continues.
|
||||
Exhausted,
|
||||
/// Fail the reporting response with `SessionBudgetExceeded` after recording its output and usage.
|
||||
/// Background compaction fails only its job; later turns and peer threads remain eligible.
|
||||
ExhaustedForTurn,
|
||||
}
|
||||
|
||||
/// Replaces this runtime's `AgentControl::record_usage` and native rollout-budget reminders.
|
||||
|
||||
@@ -2920,6 +2920,7 @@ where
|
||||
}
|
||||
loop {
|
||||
let event = tokio::select! {
|
||||
biased;
|
||||
_ = consumer_dropped.cancelled() => {
|
||||
inference_trace_attempt.record_cancelled(
|
||||
STREAM_DROPPED_REASON,
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
//! Settles completed responses and releases output according to the configured processing mode.
|
||||
|
||||
use crate::client_common::ChargePosting;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::hook_runtime::sampling::SamplingAttempt;
|
||||
use codex_api::ApiError;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
@@ -125,15 +127,23 @@ impl ResponseDelivery {
|
||||
if let Some((session, turn)) = &self.settlement
|
||||
&& let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event
|
||||
{
|
||||
let usage = crate::client_common::SamplingUsage {
|
||||
let mut usage = crate::client_common::SamplingUsage {
|
||||
provider: token_usage.clone(),
|
||||
charge: std::mem::take(&mut terminal.hook_output),
|
||||
};
|
||||
session
|
||||
usage.charge.posting = match session
|
||||
.record_rollout_budget_usage(&turn.sub_id, &usage)
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
Ok(()) => ChargePosting::Posted,
|
||||
Err(error)
|
||||
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
|
||||
{
|
||||
ChargePosting::PostedTurnExhausted
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
terminal.hook_output = usage.charge;
|
||||
terminal.hook_output.posted = true;
|
||||
}
|
||||
if terminal.event.is_ok() {
|
||||
for item in std::mem::take(&mut self.held) {
|
||||
|
||||
@@ -121,11 +121,20 @@ fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelIn
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries the hook price and whether the response producer has posted the bill.
|
||||
/// Whether the producer has posted a response's bill and deferred a turn error.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub(crate) enum ChargePosting {
|
||||
#[default]
|
||||
Unposted,
|
||||
Posted,
|
||||
PostedTurnExhausted,
|
||||
}
|
||||
|
||||
/// Carries the hook price and the response producer's billing state.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct SamplingCharge {
|
||||
pub(crate) hooks: codex_hooks::ModelRequestHookOutcome,
|
||||
pub(crate) posted: bool,
|
||||
pub(crate) posting: ChargePosting,
|
||||
}
|
||||
|
||||
/// Core consumes a terminal event and its hook output together. Other readers
|
||||
|
||||
@@ -3,6 +3,7 @@ use super::turn_context::TurnContext;
|
||||
use crate::agent::host::AgentRuntimeServices;
|
||||
use crate::agent::host::AgentUsageReport;
|
||||
use crate::agent::host::UsageAccountingDecision;
|
||||
use crate::client_common::ChargePosting;
|
||||
use crate::client_common::SamplingUsage;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use codex_protocol::error::CodexErr;
|
||||
@@ -118,9 +119,22 @@ impl Session {
|
||||
turn_id: &str,
|
||||
usage: &SamplingUsage,
|
||||
) -> CodexResult<()> {
|
||||
if !usage.charge.posted {
|
||||
self.record_numeric_usage(turn_id, usage.provider.as_ref(), usage.charge.hooks.charge)
|
||||
.await?;
|
||||
let result = match usage.charge.posting {
|
||||
ChargePosting::Unposted => {
|
||||
self.record_numeric_usage(
|
||||
turn_id,
|
||||
usage.provider.as_ref(),
|
||||
usage.charge.hooks.charge,
|
||||
)
|
||||
.await
|
||||
}
|
||||
ChargePosting::Posted => Ok(()),
|
||||
ChargePosting::PostedTurnExhausted => Err(CodexErr::SessionBudgetExceeded),
|
||||
};
|
||||
if let Err(error) = &result
|
||||
&& !matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
if let Some(message) = &usage.charge.hooks.invalid_price {
|
||||
// A bad price does not erase successful siblings' contributions.
|
||||
@@ -128,7 +142,7 @@ impl Session {
|
||||
"Invalid post-hook budget units: {message}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
result
|
||||
}
|
||||
|
||||
async fn record_numeric_usage(
|
||||
@@ -175,7 +189,13 @@ impl Session {
|
||||
"Host usage accounting failed: {error}"
|
||||
))
|
||||
})?;
|
||||
decision == UsageAccountingDecision::Exhausted
|
||||
match decision {
|
||||
UsageAccountingDecision::Continue => false,
|
||||
UsageAccountingDecision::Exhausted => true,
|
||||
UsageAccountingDecision::ExhaustedForTurn => {
|
||||
return Err(CodexErr::SessionBudgetExceeded);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if stop_new_work {
|
||||
|
||||
@@ -3177,20 +3177,17 @@ async fn try_run_sampling_request(
|
||||
preempt = CancellationToken::new();
|
||||
continue;
|
||||
}
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming && stream.interrupt.is_none();
|
||||
} else {
|
||||
let Some(interrupt) = stream.interrupt.take() else {
|
||||
// TODO: Reconcile any response item already being presented to the client.
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: false,
|
||||
last_agent_message,
|
||||
});
|
||||
};
|
||||
if step_context.settings.model_info.use_responses_lite {
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
}
|
||||
if stream.interrupt.is_none() {
|
||||
// Stop upstream polling, but keep receiving until any completion already
|
||||
// observed by the mapper has finished accounting.
|
||||
wait_for_bill_only =
|
||||
streaming || !stream.settlement_started.load(Ordering::Acquire);
|
||||
stream.consumer_dropped.cancel();
|
||||
} else if !stream.settlement_started.load(Ordering::Acquire)
|
||||
&& let Some(interrupt) = stream.interrupt.take()
|
||||
&& step_context.settings.model_info.use_responses_lite
|
||||
{
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
}
|
||||
// A steer cannot start another request before the accounting owner returns its decision.
|
||||
preempt = CancellationToken::new();
|
||||
@@ -3202,6 +3199,13 @@ async fn try_run_sampling_request(
|
||||
|
||||
let event = SamplingStream::accept_for_turn(event, &cancellation_token);
|
||||
let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else {
|
||||
if stream.consumer_dropped.is_cancelled() {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: *draining_for_mailbox,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
break Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
));
|
||||
@@ -3237,14 +3241,7 @@ async fn try_run_sampling_request(
|
||||
|
||||
// A steer can cut off unread items on an HTTP stream while the owner posts its bill.
|
||||
// Consume the terminal accounting decision before opening the follow-up request.
|
||||
if wait_for_bill_only {
|
||||
if matches!(event, ResponseEvent::Completed { .. }) {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: *draining_for_mailbox,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
if wait_for_bill_only && !matches!(event, ResponseEvent::Completed { .. }) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -3435,21 +3432,19 @@ async fn try_run_sampling_request(
|
||||
conversation.id = %sess.thread_id,
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming
|
||||
&& !(step_context.settings.model_info.use_responses_lite
|
||||
&& stream.interrupt.is_some());
|
||||
} else if step_context.settings.model_info.use_responses_lite
|
||||
&& let Some(interrupt) = stream.interrupt.take()
|
||||
if step_context.settings.model_info.use_responses_lite
|
||||
&& stream.interrupt.is_some()
|
||||
{
|
||||
// Completion can already be buffered when the interrupt receiver closes.
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
if !stream.settlement_started.load(Ordering::Acquire)
|
||||
&& let Some(interrupt) = stream.interrupt.take()
|
||||
{
|
||||
// Completion can already be buffered when the interrupt receiver closes.
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
}
|
||||
} else {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: true,
|
||||
last_agent_message,
|
||||
});
|
||||
wait_for_bill_only =
|
||||
streaming || !stream.settlement_started.load(Ordering::Acquire);
|
||||
stream.consumer_dropped.cancel();
|
||||
}
|
||||
// Keep the response ID and terminal decision. Later tools receive a denial.
|
||||
*draining_for_mailbox = true;
|
||||
@@ -3668,6 +3663,13 @@ async fn try_run_sampling_request(
|
||||
if let Err(err) = budget_result {
|
||||
break Err(err);
|
||||
}
|
||||
if wait_for_bill_only {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: *draining_for_mailbox,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
if resampling.retry_pending {
|
||||
response_completed = false;
|
||||
response_id_for_assessment = None;
|
||||
|
||||
@@ -40,6 +40,7 @@ use core_test_support::skip_if_no_network;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_match;
|
||||
use core_test_support::wait_for_successful_turn;
|
||||
use futures::future::BoxFuture;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
@@ -58,6 +59,7 @@ enum HostAccountingBehavior {
|
||||
#[default]
|
||||
Continue,
|
||||
Exhausted,
|
||||
ExhaustedForTurnOnce,
|
||||
Fail,
|
||||
}
|
||||
|
||||
@@ -75,7 +77,11 @@ impl AgentUsageAccounting for HostAccounting {
|
||||
report: AgentUsageReport,
|
||||
) -> BoxFuture<'_, codex_protocol::error::Result<UsageAccountingDecision>> {
|
||||
Box::pin(async move {
|
||||
self.reports.lock().expect("usage reports").push(report);
|
||||
let report_count = {
|
||||
let mut reports = self.reports.lock().expect("usage reports");
|
||||
reports.push(report);
|
||||
reports.len()
|
||||
};
|
||||
if let Some(pause) = &self.pause_report {
|
||||
let (resume, resumed) = oneshot::channel();
|
||||
let _ = pause.send(resume);
|
||||
@@ -84,6 +90,11 @@ impl AgentUsageAccounting for HostAccounting {
|
||||
match self.behavior {
|
||||
HostAccountingBehavior::Continue => Ok(UsageAccountingDecision::Continue),
|
||||
HostAccountingBehavior::Exhausted => Ok(UsageAccountingDecision::Exhausted),
|
||||
HostAccountingBehavior::ExhaustedForTurnOnce => Ok(if report_count == 1 {
|
||||
UsageAccountingDecision::ExhaustedForTurn
|
||||
} else {
|
||||
UsageAccountingDecision::Continue
|
||||
}),
|
||||
HostAccountingBehavior::Fail => {
|
||||
Err(CodexErr::Stream("accounting unavailable".into()))
|
||||
}
|
||||
@@ -284,6 +295,84 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn host_turn_exhaustion_posts_response_state_and_allows_a_later_turn() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_response_created("exhausted"),
|
||||
ev_assistant_message("exhausted-message", "accepted output"),
|
||||
ev_completed_with_tokens("exhausted", /*total_tokens*/ 11),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("fresh"),
|
||||
ev_assistant_message("fresh-message", "fresh output"),
|
||||
ev_completed_with_tokens("fresh", /*total_tokens*/ 7),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let accounting = Arc::new(HostAccounting {
|
||||
behavior: HostAccountingBehavior::ExhaustedForTurnOnce,
|
||||
..Default::default()
|
||||
});
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.response_process_mode =
|
||||
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
|
||||
})
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
let input = |text: &str| {
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: text.into(),
|
||||
text_elements: Vec::new(),
|
||||
}])
|
||||
};
|
||||
test.codex
|
||||
.start_or_steer_turn(input("spend the current turn allowance"))
|
||||
.await?;
|
||||
let mut saw_output = false;
|
||||
let mut saw_usage = false;
|
||||
let error = wait_for_event(&test.codex, |event| {
|
||||
saw_output |= matches!(event, EventMsg::AgentMessage(message) if message.message == "accepted output");
|
||||
saw_usage |= matches!(event, EventMsg::TokenCount(count)
|
||||
if count.info.as_ref().is_some_and(|info| info.total_token_usage.total_tokens == 11));
|
||||
matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
assert_eq!((saw_output, saw_usage), (true, true));
|
||||
assert!(matches!(error, EventMsg::Error(error)
|
||||
if error.codex_error_info == Some(CodexErrorInfo::SessionBudgetExceeded)));
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
test.codex
|
||||
.start_or_steer_turn(input("use a fresh turn allowance"))
|
||||
.await?;
|
||||
wait_for_successful_turn(&test.codex).await;
|
||||
let reports = accounting.reports.lock().expect("usage reports");
|
||||
assert_eq!(
|
||||
reports
|
||||
.iter()
|
||||
.map(|report| report.usage.as_ref().map(|usage| usage.total_tokens))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![Some(11), Some(7)],
|
||||
);
|
||||
assert_ne!(reports[0].turn_id, reports[1].turn_id);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[test_case(false ; "local accounting")]
|
||||
#[test_case(true ; "host accounting")]
|
||||
@@ -745,6 +834,7 @@ async fn restates_the_current_remainder_after_compaction() -> Result<()> {
|
||||
|
||||
#[tokio::test]
|
||||
#[test_case(HostAccountingBehavior::Exhausted, false ; "host exhausts budget")]
|
||||
#[test_case(HostAccountingBehavior::ExhaustedForTurnOnce, false ; "host exhausts steered turn")]
|
||||
#[test_case(HostAccountingBehavior::Fail, false ; "accounting failure never retries inference")]
|
||||
#[test_case(HostAccountingBehavior::Fail, true ; "incomplete response keeps original error")]
|
||||
async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
@@ -752,6 +842,10 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
incomplete: bool,
|
||||
) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let exhausts_budget = matches!(
|
||||
behavior,
|
||||
HostAccountingBehavior::Exhausted | HostAccountingBehavior::ExhaustedForTurnOnce
|
||||
);
|
||||
let server = start_mock_server().await;
|
||||
let mut response = json!({
|
||||
"usage": {
|
||||
@@ -798,8 +892,7 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
config.model_provider.stream_max_retries = Some(2);
|
||||
config.model_provider.request_max_retries = Some(0);
|
||||
// This allowance would admit the response. The selected host can still stop it.
|
||||
config.rollout_budget =
|
||||
matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget);
|
||||
config.rollout_budget = exhausts_budget.then(rollout_budget);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
@@ -848,11 +941,12 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
|
||||
error.message,
|
||||
"Incomplete response returned, reason: max_output_tokens"
|
||||
);
|
||||
} else if matches!(behavior, HostAccountingBehavior::Exhausted) {
|
||||
} else if exhausts_budget {
|
||||
assert_eq!(
|
||||
error.codex_error_info,
|
||||
Some(CodexErrorInfo::SessionBudgetExceeded)
|
||||
);
|
||||
assert_eq!(completed.error.as_ref(), Some(error));
|
||||
} else {
|
||||
assert!(error.message.contains("Host usage accounting failed"));
|
||||
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
|
||||
|
||||
Reference in New Issue
Block a user