Add response processing after completion and usage accounting (#61577)

## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
tongzhou wang
2026-11-14 23:50:12 +00:00
committed by copyberry
parent 81b995a6c9
commit 92c1a4dc2e
32 changed files with 780 additions and 259 deletions
@@ -106,11 +106,12 @@ async fn cancelled_turn_skips_launch_but_preserves_existing_work(trigger: Compac
]));
sess.update_token_usage_info(
&turn,
Some(&TokenUsage {
TokenUsage {
input_tokens: 60_000,
total_tokens: 60_000,
..Default::default()
}),
}
.into(),
/*includes_reasoning*/ false,
)
.await
+1 -1
View File
@@ -540,7 +540,7 @@ impl InputQueue {
}
impl TurnInputQueue {
fn has_user_input(&self) -> bool {
pub(crate) fn has_user_input(&self) -> bool {
self.items
.iter()
.any(|input| matches!(input, TurnInput::UserInput { .. }))
+27 -21
View File
@@ -5654,6 +5654,7 @@ impl Session {
turn_context: &TurnContext,
settings: &ResolvedStepSettings,
incomplete: &ResponseIncompleteError,
charge: crate::client_common::SamplingCharge,
) {
if let Some(response_id) = &incomplete.response_id {
self.record_response_token_usage(
@@ -5668,7 +5669,10 @@ impl Session {
.record_token_usage_info(
turn_context,
settings,
incomplete.token_usage.as_ref(),
crate::client_common::SamplingUsage {
provider: incomplete.token_usage.clone(),
charge,
},
TokenUsageContext::SpendingOnly,
)
.await;
@@ -5677,14 +5681,14 @@ impl Session {
pub(crate) async fn update_token_usage_info(
&self,
turn_context: &TurnContext,
token_usage: Option<&TokenUsage>,
usage: crate::client_common::SamplingUsage,
includes_reasoning: bool,
) -> CodexResult<()> {
let result = self
.record_token_usage_info(
turn_context,
&turn_context.initial_settings,
token_usage,
usage,
TokenUsageContext::Sampling {
includes_reasoning,
input_item_count: 0,
@@ -5743,10 +5747,10 @@ impl Session {
&self,
turn_context: &TurnContext,
settings: &ResolvedStepSettings,
token_usage: Option<&TokenUsage>,
usage: crate::client_common::SamplingUsage,
context: TokenUsageContext,
) -> CodexResult<()> {
if let Some(token_usage) = token_usage {
let token_info = if let Some(token_usage) = usage.provider.as_ref() {
let token_info = {
let mut state = self.state.lock().await;
match context {
@@ -5799,24 +5803,26 @@ impl Session {
token_usage,
);
}
let budget_result = self
.record_rollout_budget_usage(&turn_context.sub_id, token_usage)
.await;
if let Some(token_info) = token_info.as_ref() {
for contributor in self.services.extensions.token_usage_contributors() {
contributor
.on_token_usage(
&self.services.session_extension_data,
&self.services.thread_extension_data,
turn_context.extension_data.as_ref(),
token_info,
)
.await;
}
token_info
} else {
None
};
let budget_result = self
.record_rollout_budget_usage(&turn_context.sub_id, &usage)
.await;
if let Some(token_info) = token_info.as_ref() {
for contributor in self.services.extensions.token_usage_contributors() {
contributor
.on_token_usage(
&self.services.session_extension_data,
&self.services.thread_extension_data,
turn_context.extension_data.as_ref(),
token_info,
)
.await;
}
budget_result?;
}
Ok(())
budget_result
}
pub(crate) async fn recompute_token_usage(&self, turn_context: &TurnContext) {
+41 -13
View File
@@ -4,9 +4,10 @@ use crate::agent::host::AgentRuntimeServices;
use crate::agent::host::AgentUsageReport;
use crate::agent::host::UsageAccountingDecision;
use crate::context::ContextualUserFragment;
use crate::rollout_budget::RolloutBudget;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
pub(super) async fn maybe_record_reminder(
sess: &Session,
@@ -39,18 +40,44 @@ pub(super) async fn maybe_record_reminder(
}
impl Session {
pub(crate) fn rollout_budget(&self) -> &RolloutBudget {
&self.services.local_agent_runtime.rollout_budget
}
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
if self.rollout_budget().tools_stopped() {
Err(CodexErr::SessionBudgetExceeded)
} else {
Ok(())
}
}
pub(crate) async fn record_rollout_budget_usage(
&self,
turn_id: &str,
usage: &TokenUsage,
bill: &crate::client_common::SamplingUsage,
) -> CodexResult<()> {
match &self.agent_runtime {
AgentRuntimeServices::Controller => {
self.services
.agent_control
.record_usage(usage.clone())
.await
}
if bill.charge.posted {
return Ok(());
}
let Some(usage) = bill.provider.as_ref() else {
return Ok(());
};
let stop_new_work = match &self.agent_runtime {
AgentRuntimeServices::Controller => match self
.services
.agent_control
.record_usage(usage.clone())
.await
{
Ok(()) => false,
Err(error)
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
{
true
}
Err(error) => return Err(error),
},
AgentRuntimeServices::Host(services) => {
let Some(accounting) = &services.usage_accounting else {
return Ok(());
@@ -65,11 +92,12 @@ impl Session {
.map_err(|error| {
CodexErr::Fatal(format!("Host usage accounting failed: {error}"))
})?;
match decision {
UsageAccountingDecision::Continue => Ok(()),
UsageAccountingDecision::Exhausted => Err(CodexErr::SessionBudgetExceeded),
}
decision == UsageAccountingDecision::Exhausted
}
};
if stop_new_work {
self.rollout_budget().stop_new_tools();
}
Ok(())
}
}
+2 -3
View File
@@ -3398,7 +3398,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
.record_token_usage_info(
&turn_context,
&turn_context.initial_settings,
Some(&first_usage),
first_usage.clone().into(),
crate::session::TokenUsageContext::Sampling {
includes_reasoning: true,
input_item_count: 0,
@@ -3410,7 +3410,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
.record_token_usage_info(
&turn_context,
&turn_context.initial_settings,
Some(&second_usage),
second_usage.clone().into(),
context,
)
.await
@@ -13173,7 +13173,6 @@ async fn tool_calls_reopen_mailbox_delivery_for_current_turn() {
step_context: StepContext::for_test(Arc::clone(&tc)),
turn_store: Arc::new(codex_extension_api::ExtensionData::new(tc.sub_id.clone())),
tool_runtime: test_tool_runtime(Arc::clone(&sess), Arc::clone(&tc)).await,
defer_tool_dispatch: false,
cancellation_token: CancellationToken::new(),
};
+69 -38
View File
@@ -10,7 +10,7 @@ use crate::client::ModelClientSession;
use crate::client::SamplingOptions;
use crate::client_common::Prompt;
use crate::client_common::ResponseEvent;
use crate::client_common::ResponseStream;
use crate::client_common::SamplingStream;
use crate::compact::AutoCompactionDisabled;
use crate::compact::run_inline_auto_compact_task;
use crate::compact_remote_v2::BackgroundCompactionCandidate;
@@ -654,6 +654,11 @@ pub(crate) async fn run_turn(
.await
}
.await;
// Consume the whole response before deciding whether the turn may continue.
let sampling_request_result = sampling_request_result.and_then(|output| {
sess.check_rollout_budget()?;
Ok(output)
});
match sampling_request_result {
Ok((sampling_request_output, notify_messages)) => {
guardian_budget_compacted = false;
@@ -2827,7 +2832,7 @@ async fn drain_in_flight(
in_flight: &mut FuturesOrdered<InFlightFuture<'static>>,
sess: Arc<Session>,
step_context: &StepContext,
) -> CodexResult<()> {
) {
let turn_context = &step_context.turn;
while let Some(res) = in_flight.next().await {
match res {
@@ -2850,7 +2855,6 @@ async fn drain_in_flight(
}
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
@@ -2977,7 +2981,10 @@ async fn try_run_sampling_request(
if cancellation_token.is_cancelled() && !can_interrupt {
return Err(CodexErr::TurnAborted);
}
let mut in_flight: FuturesOrdered<InFlightFuture<'static>> = FuturesOrdered::new();
let streaming = turn_context.config.response_process_mode
== codex_config::config_toml::ResponseProcessMode::Streaming;
let mut wait_for_bill_only = false;
let mut in_flight = FuturesOrdered::new();
let mut needs_follow_up = false;
let mut last_agent_message: Option<String> = None;
let mut active_item: Option<TurnItem> = None;
@@ -3053,31 +3060,36 @@ async fn try_run_sampling_request(
preempt = CancellationToken::new();
continue;
}
if let Some(interrupt) = stream.interrupt.take() {
if stream.settlement_started.load(Ordering::Acquire) {
wait_for_bill_only = streaming && stream.interrupt.is_none();
} else {
let Some(interrupt) = stream.interrupt.take() else {
// TODO: Reconcile any response item already being presented to the client.
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: false,
last_agent_message,
});
};
if step_context.settings.model_info.use_responses_lite {
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
}
// Drain the response normally before reusing its connection and history.
preempt = CancellationToken::new();
needs_follow_up = true;
continue;
}
// TODO: Reconcile any response item already being presented to the client.
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: false,
last_agent_message,
});
// A steer cannot start another request before the accounting owner returns its decision.
preempt = CancellationToken::new();
needs_follow_up = true;
continue;
}
Err(_) => break Err(CodexErr::TurnAborted),
};
let event = ResponseStream::accept_for_turn(event, &cancellation_token);
let event = match event.unwrap_or_else(|| {
Err(CodexErr::Stream(
let event = SamplingStream::accept_for_turn(event, &cancellation_token);
let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else {
break Err(CodexErr::Stream(
"stream closed before response.completed".into(),
))
}) {
));
};
let event = match event {
Ok(event) => event,
Err(err) => {
if let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details() {
@@ -3085,6 +3097,7 @@ async fn try_run_sampling_request(
&turn_context,
&step_context.settings,
incomplete,
hook_output,
)
.await;
should_emit_token_count |= incomplete.token_usage.is_some();
@@ -3105,6 +3118,19 @@ async fn try_run_sampling_request(
}
};
// A steer can cut off unread items on an HTTP stream while the owner posts its bill.
// Consume the terminal accounting decision before opening the follow-up request.
if wait_for_bill_only {
if matches!(event, ResponseEvent::Completed { .. }) {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: *draining_for_mailbox,
last_agent_message,
});
}
continue;
}
let Some(event) = resampling.filter(event, &step_context.tool_router) else {
continue;
};
@@ -3215,7 +3241,6 @@ async fn try_run_sampling_request(
step_context: Arc::clone(&step_context),
turn_store: Arc::clone(&turn_store),
tool_runtime,
defer_tool_dispatch: *draining_for_mailbox,
cancellation_token: cancellation_token.child_token(),
};
if *draining_for_mailbox {
@@ -3263,8 +3288,8 @@ async fn try_run_sampling_request(
Ok(output_result) => output_result,
Err(err) => break Err(err),
};
if let Some(tool_future) = output_result.tool_future {
in_flight.push_back(tool_future);
if let Some(tool) = output_result.tool_future {
in_flight.push_back(tool);
}
if let Some(agent_message) = output_result.last_agent_message {
last_agent_message = Some(agent_message);
@@ -3291,23 +3316,26 @@ async fn try_run_sampling_request(
conversation.id = %sess.thread_id,
turn.id = %turn_context.sub_id,
);
if step_context.settings.model_info.use_responses_lite
if stream.settlement_started.load(Ordering::Acquire) {
wait_for_bill_only = streaming
&& !(step_context.settings.model_info.use_responses_lite
&& stream.interrupt.is_some());
} else if step_context.settings.model_info.use_responses_lite
&& let Some(interrupt) = stream.interrupt.take()
{
// A closed receiver can mean completion is already buffered. Drain it
// too; a transport failure below still takes the mailbox follow-up.
// Completion can already be buffered when the interrupt receiver closes.
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
// Retain completed items and the response ID for incremental continuation.
// Further item boundaries must not drop the stream while it drains.
*draining_for_mailbox = true;
needs_follow_up = true;
continue;
} else {
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: true,
last_agent_message,
});
}
break Ok(SamplingRequestResult {
needs_follow_up: true,
mailbox_preempted: true,
last_agent_message,
});
// Keep the response ID and terminal decision. Later tools receive a denial.
*draining_for_mailbox = true;
needs_follow_up = true;
continue;
}
}
ResponseEvent::OutputItemAdded(mut item) => {
@@ -3502,7 +3530,10 @@ async fn try_run_sampling_request(
.record_token_usage_info(
&turn_context,
&step_context.settings,
token_usage.as_ref(),
crate::client_common::SamplingUsage {
provider: token_usage,
charge: hook_output,
},
if resampling.retry_pending {
super::TokenUsageContext::SpendingOnly
} else {
@@ -3715,7 +3746,7 @@ async fn try_run_sampling_request(
conversation.id = %sess.thread_id,
turn.id = %turn_context.sub_id,
);
drain_in_flight(&mut in_flight, sess.clone(), &step_context).await?;
drain_in_flight(&mut in_flight, Arc::clone(&sess), &step_context).await;
}
if should_emit_token_count {