From 44834c58fa99d0a04f222e18817b99b430f7c0e8 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 17 Nov 2026 03:02:21 +0000 Subject: [PATCH] Support turn-scoped exhaustion in host usage accounting (#61958) ## What changed - Add `UsageAccountingDecision::ExhaustedForTurn` so hosts can fail the current response with `SessionBudgetExceeded` while leaving later turns and peer threads eligible. - Preserve completed response output and token usage before reporting exhaustion, and track posted charges to avoid accounting twice. - When steering or mailbox preemption stops a stream, wait for accounting of any completion already observed before starting a follow-up request, and propagate budget errors. ## Testing Add coverage for preserving output and usage on turn exhaustion, successfully starting a later turn, and reporting turn exhaustion during steering. GitOrigin-RevId: e673bf30775f27a057fdc39d8429b74ff22121db --- codex-rs/core/src/agent/host/usage.rs | 5 +- codex-rs/core/src/client.rs | 1 + codex-rs/core/src/client/response_delivery.rs | 18 +++- codex-rs/core/src/client_common.rs | 13 ++- codex-rs/core/src/session/rollout_budget.rs | 30 +++++- codex-rs/core/src/session/turn.rs | 72 +++++++------ codex-rs/core/tests/suite/rollout_budget.rs | 102 +++++++++++++++++- 7 files changed, 190 insertions(+), 51 deletions(-) diff --git a/codex-rs/core/src/agent/host/usage.rs b/codex-rs/core/src/agent/host/usage.rs index 65e7959c3f..0b9785fb48 100644 --- a/codex-rs/core/src/agent/host/usage.rs +++ b/codex-rs/core/src/agent/host/usage.rs @@ -19,12 +19,15 @@ pub struct AgentUsageReport { pub additional_budget_units_overflowed: bool, } -/// Both decisions mean the host recorded the reported spending. +/// Each decision means the host recorded the reported spending. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum UsageAccountingDecision { Continue, /// Do not start more tool handlers in this local rollout tree. Already-running work continues. Exhausted, + /// Fail the reporting response with `SessionBudgetExceeded` after recording its output and usage. + /// Background compaction fails only its job; later turns and peer threads remain eligible. + ExhaustedForTurn, } /// Replaces this runtime's `AgentControl::record_usage` and native rollout-budget reminders. diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 4492151727..e75e592f58 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -2920,6 +2920,7 @@ where } loop { let event = tokio::select! { + biased; _ = consumer_dropped.cancelled() => { inference_trace_attempt.record_cancelled( STREAM_DROPPED_REASON, diff --git a/codex-rs/core/src/client/response_delivery.rs b/codex-rs/core/src/client/response_delivery.rs index 888cdcfad5..65c6567c0b 100644 --- a/codex-rs/core/src/client/response_delivery.rs +++ b/codex-rs/core/src/client/response_delivery.rs @@ -1,11 +1,13 @@ //! Settles completed responses and releases output according to the configured processing mode. +use crate::client_common::ChargePosting; use crate::client_common::ResponseEvent; use crate::client_common::SamplingEvent; use crate::hook_runtime::sampling::SamplingAttempt; use codex_api::ApiError; use codex_hooks::ModelRequestHookEvent; use codex_protocol::error::CodexErr; +use codex_protocol::error::CodexErrorDetails; use codex_protocol::error::Result; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -125,15 +127,23 @@ impl ResponseDelivery { if let Some((session, turn)) = &self.settlement && let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event { - let usage = crate::client_common::SamplingUsage { + let mut usage = crate::client_common::SamplingUsage { provider: token_usage.clone(), charge: std::mem::take(&mut terminal.hook_output), }; - session + usage.charge.posting = match session .record_rollout_budget_usage(&turn.sub_id, &usage) - .await?; + .await + { + Ok(()) => ChargePosting::Posted, + Err(error) + if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) => + { + ChargePosting::PostedTurnExhausted + } + Err(error) => return Err(error), + }; terminal.hook_output = usage.charge; - terminal.hook_output.posted = true; } if terminal.event.is_ok() { for item in std::mem::take(&mut self.held) { diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 1a42418c81..47d806d010 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -121,11 +121,20 @@ fn normalize_image_detail(detail: &mut Option, model_info: &ModelIn } } -/// Carries the hook price and whether the response producer has posted the bill. +/// Whether the producer has posted a response's bill and deferred a turn error. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) enum ChargePosting { + #[default] + Unposted, + Posted, + PostedTurnExhausted, +} + +/// Carries the hook price and the response producer's billing state. #[derive(Debug, Default)] pub(crate) struct SamplingCharge { pub(crate) hooks: codex_hooks::ModelRequestHookOutcome, - pub(crate) posted: bool, + pub(crate) posting: ChargePosting, } /// Core consumes a terminal event and its hook output together. Other readers diff --git a/codex-rs/core/src/session/rollout_budget.rs b/codex-rs/core/src/session/rollout_budget.rs index 88b5ac0b95..caf8f989b8 100644 --- a/codex-rs/core/src/session/rollout_budget.rs +++ b/codex-rs/core/src/session/rollout_budget.rs @@ -3,6 +3,7 @@ use super::turn_context::TurnContext; use crate::agent::host::AgentRuntimeServices; use crate::agent::host::AgentUsageReport; use crate::agent::host::UsageAccountingDecision; +use crate::client_common::ChargePosting; use crate::client_common::SamplingUsage; use crate::context::ContextualUserFragment; use codex_protocol::error::CodexErr; @@ -118,9 +119,22 @@ impl Session { turn_id: &str, usage: &SamplingUsage, ) -> CodexResult<()> { - if !usage.charge.posted { - self.record_numeric_usage(turn_id, usage.provider.as_ref(), usage.charge.hooks.charge) - .await?; + let result = match usage.charge.posting { + ChargePosting::Unposted => { + self.record_numeric_usage( + turn_id, + usage.provider.as_ref(), + usage.charge.hooks.charge, + ) + .await + } + ChargePosting::Posted => Ok(()), + ChargePosting::PostedTurnExhausted => Err(CodexErr::SessionBudgetExceeded), + }; + if let Err(error) = &result + && !matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) + { + return result; } if let Some(message) = &usage.charge.hooks.invalid_price { // A bad price does not erase successful siblings' contributions. @@ -128,7 +142,7 @@ impl Session { "Invalid post-hook budget units: {message}" ))); } - Ok(()) + result } async fn record_numeric_usage( @@ -175,7 +189,13 @@ impl Session { "Host usage accounting failed: {error}" )) })?; - decision == UsageAccountingDecision::Exhausted + match decision { + UsageAccountingDecision::Continue => false, + UsageAccountingDecision::Exhausted => true, + UsageAccountingDecision::ExhaustedForTurn => { + return Err(CodexErr::SessionBudgetExceeded); + } + } } }; if stop_new_work { diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 04e3aab6c0..6b9e6f5cf2 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -3177,20 +3177,17 @@ async fn try_run_sampling_request( preempt = CancellationToken::new(); continue; } - if stream.settlement_started.load(Ordering::Acquire) { - wait_for_bill_only = streaming && stream.interrupt.is_none(); - } else { - let Some(interrupt) = stream.interrupt.take() else { - // TODO: Reconcile any response item already being presented to the client. - break Ok(SamplingRequestResult { - needs_follow_up: true, - mailbox_preempted: false, - last_agent_message, - }); - }; - if step_context.settings.model_info.use_responses_lite { - let _ = interrupt.send(codex_api::ResponseInterrupt::Drain); - } + if stream.interrupt.is_none() { + // Stop upstream polling, but keep receiving until any completion already + // observed by the mapper has finished accounting. + wait_for_bill_only = + streaming || !stream.settlement_started.load(Ordering::Acquire); + stream.consumer_dropped.cancel(); + } else if !stream.settlement_started.load(Ordering::Acquire) + && let Some(interrupt) = stream.interrupt.take() + && step_context.settings.model_info.use_responses_lite + { + let _ = interrupt.send(codex_api::ResponseInterrupt::Drain); } // A steer cannot start another request before the accounting owner returns its decision. preempt = CancellationToken::new(); @@ -3202,6 +3199,13 @@ async fn try_run_sampling_request( let event = SamplingStream::accept_for_turn(event, &cancellation_token); let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else { + if stream.consumer_dropped.is_cancelled() { + break Ok(SamplingRequestResult { + needs_follow_up: true, + mailbox_preempted: *draining_for_mailbox, + last_agent_message, + }); + } break Err(CodexErr::Stream( "stream closed before response.completed".into(), )); @@ -3237,14 +3241,7 @@ async fn try_run_sampling_request( // A steer can cut off unread items on an HTTP stream while the owner posts its bill. // Consume the terminal accounting decision before opening the follow-up request. - if wait_for_bill_only { - if matches!(event, ResponseEvent::Completed { .. }) { - break Ok(SamplingRequestResult { - needs_follow_up: true, - mailbox_preempted: *draining_for_mailbox, - last_agent_message, - }); - } + if wait_for_bill_only && !matches!(event, ResponseEvent::Completed { .. }) { continue; } @@ -3435,21 +3432,19 @@ async fn try_run_sampling_request( conversation.id = %sess.thread_id, turn.id = %turn_context.sub_id, ); - if stream.settlement_started.load(Ordering::Acquire) { - wait_for_bill_only = streaming - && !(step_context.settings.model_info.use_responses_lite - && stream.interrupt.is_some()); - } else if step_context.settings.model_info.use_responses_lite - && let Some(interrupt) = stream.interrupt.take() + if step_context.settings.model_info.use_responses_lite + && stream.interrupt.is_some() { - // Completion can already be buffered when the interrupt receiver closes. - let _ = interrupt.send(codex_api::ResponseInterrupt::Drain); + if !stream.settlement_started.load(Ordering::Acquire) + && let Some(interrupt) = stream.interrupt.take() + { + // Completion can already be buffered when the interrupt receiver closes. + let _ = interrupt.send(codex_api::ResponseInterrupt::Drain); + } } else { - break Ok(SamplingRequestResult { - needs_follow_up: true, - mailbox_preempted: true, - last_agent_message, - }); + wait_for_bill_only = + streaming || !stream.settlement_started.load(Ordering::Acquire); + stream.consumer_dropped.cancel(); } // Keep the response ID and terminal decision. Later tools receive a denial. *draining_for_mailbox = true; @@ -3668,6 +3663,13 @@ async fn try_run_sampling_request( if let Err(err) = budget_result { break Err(err); } + if wait_for_bill_only { + break Ok(SamplingRequestResult { + needs_follow_up: true, + mailbox_preempted: *draining_for_mailbox, + last_agent_message, + }); + } if resampling.retry_pending { response_completed = false; response_id_for_assessment = None; diff --git a/codex-rs/core/tests/suite/rollout_budget.rs b/codex-rs/core/tests/suite/rollout_budget.rs index 6bc386af2c..36aa3d6736 100644 --- a/codex-rs/core/tests/suite/rollout_budget.rs +++ b/codex-rs/core/tests/suite/rollout_budget.rs @@ -40,6 +40,7 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; +use core_test_support::wait_for_successful_turn; use futures::future::BoxFuture; use pretty_assertions::assert_eq; use serde_json::json; @@ -58,6 +59,7 @@ enum HostAccountingBehavior { #[default] Continue, Exhausted, + ExhaustedForTurnOnce, Fail, } @@ -75,7 +77,11 @@ impl AgentUsageAccounting for HostAccounting { report: AgentUsageReport, ) -> BoxFuture<'_, codex_protocol::error::Result> { Box::pin(async move { - self.reports.lock().expect("usage reports").push(report); + let report_count = { + let mut reports = self.reports.lock().expect("usage reports"); + reports.push(report); + reports.len() + }; if let Some(pause) = &self.pause_report { let (resume, resumed) = oneshot::channel(); let _ = pause.send(resume); @@ -84,6 +90,11 @@ impl AgentUsageAccounting for HostAccounting { match self.behavior { HostAccountingBehavior::Continue => Ok(UsageAccountingDecision::Continue), HostAccountingBehavior::Exhausted => Ok(UsageAccountingDecision::Exhausted), + HostAccountingBehavior::ExhaustedForTurnOnce => Ok(if report_count == 1 { + UsageAccountingDecision::ExhaustedForTurn + } else { + UsageAccountingDecision::Continue + }), HostAccountingBehavior::Fail => { Err(CodexErr::Stream("accounting unavailable".into())) } @@ -284,6 +295,84 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<() Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn host_turn_exhaustion_posts_response_state_and_allows_a_later_turn() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("exhausted"), + ev_assistant_message("exhausted-message", "accepted output"), + ev_completed_with_tokens("exhausted", /*total_tokens*/ 11), + ]), + sse(vec![ + ev_response_created("fresh"), + ev_assistant_message("fresh-message", "fresh output"), + ev_completed_with_tokens("fresh", /*total_tokens*/ 7), + ]), + ], + ) + .await; + let accounting = Arc::new(HostAccounting { + behavior: HostAccountingBehavior::ExhaustedForTurnOnce, + ..Default::default() + }); + let mut extensions = ExtensionRegistryBuilder::new(); + extensions.thread_lifecycle_contributor(accounting.clone()); + let test = test_codex() + .with_config(|config| { + config.response_process_mode = + codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse; + }) + .with_extensions(Arc::new(extensions.build())) + .build_with_auto_env(&server) + .await?; + let input = |text: &str| { + TurnInputRequest::user_input(vec![UserInput::Text { + text: text.into(), + text_elements: Vec::new(), + }]) + }; + test.codex + .start_or_steer_turn(input("spend the current turn allowance")) + .await?; + let mut saw_output = false; + let mut saw_usage = false; + let error = wait_for_event(&test.codex, |event| { + saw_output |= matches!(event, EventMsg::AgentMessage(message) if message.message == "accepted output"); + saw_usage |= matches!(event, EventMsg::TokenCount(count) + if count.info.as_ref().is_some_and(|info| info.total_token_usage.total_tokens == 11)); + matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_)) + }) + .await; + assert_eq!((saw_output, saw_usage), (true, true)); + assert!(matches!(error, EventMsg::Error(error) + if error.codex_error_info == Some(CodexErrorInfo::SessionBudgetExceeded))); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + test.codex + .start_or_steer_turn(input("use a fresh turn allowance")) + .await?; + wait_for_successful_turn(&test.codex).await; + let reports = accounting.reports.lock().expect("usage reports"); + assert_eq!( + reports + .iter() + .map(|report| report.usage.as_ref().map(|usage| usage.total_tokens)) + .collect::>(), + vec![Some(11), Some(7)], + ); + assert_ne!(reports[0].turn_id, reports[1].turn_id); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[test_case(false ; "local accounting")] #[test_case(true ; "host accounting")] @@ -745,6 +834,7 @@ async fn restates_the_current_remainder_after_compaction() -> Result<()> { #[tokio::test] #[test_case(HostAccountingBehavior::Exhausted, false ; "host exhausts budget")] +#[test_case(HostAccountingBehavior::ExhaustedForTurnOnce, false ; "host exhausts steered turn")] #[test_case(HostAccountingBehavior::Fail, false ; "accounting failure never retries inference")] #[test_case(HostAccountingBehavior::Fail, true ; "incomplete response keeps original error")] async fn host_accounting_reports_raw_usage_and_preserves_error_semantics( @@ -752,6 +842,10 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics( incomplete: bool, ) -> Result<()> { skip_if_no_network!(Ok(())); + let exhausts_budget = matches!( + behavior, + HostAccountingBehavior::Exhausted | HostAccountingBehavior::ExhaustedForTurnOnce + ); let server = start_mock_server().await; let mut response = json!({ "usage": { @@ -798,8 +892,7 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics( config.model_provider.stream_max_retries = Some(2); config.model_provider.request_max_retries = Some(0); // This allowance would admit the response. The selected host can still stop it. - config.rollout_budget = - matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget); + config.rollout_budget = exhausts_budget.then(rollout_budget); }) .build_with_auto_env(&server) .await?; @@ -848,11 +941,12 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics( error.message, "Incomplete response returned, reason: max_output_tokens" ); - } else if matches!(behavior, HostAccountingBehavior::Exhausted) { + } else if exhausts_budget { assert_eq!( error.codex_error_info, Some(CodexErrorInfo::SessionBudgetExceeded) ); + assert_eq!(completed.error.as_ref(), Some(error)); } else { assert!(error.message.contains("Host usage accounting failed")); assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));