Add response processing after completion and usage accounting (#61577)

## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
tongzhou wang
2026-11-14 23:50:12 +00:00
committed by copyberry
parent 81b995a6c9
commit 92c1a4dc2e
32 changed files with 780 additions and 259 deletions
+10 -5
View File
@@ -8,7 +8,7 @@ use super::run_model_request_hooks;
use super::thread_spawn_subagent_hook_context;
use super::tool_hook_cwd;
use crate::client::OpenedModelResponse;
use crate::client_common::ResponseStream;
use crate::client_common::SamplingStream;
use crate::session::session::Session;
use crate::session::step_context::StepContext;
use codex_hooks::ModelRequestHookEvent;
@@ -40,7 +40,7 @@ impl SamplingRequest {
model: &str,
owner_cancellation: &CancellationToken,
send: impl Future<Output = Result<OpenedModelResponse>> + Send,
) -> impl Future<Output = Result<ResponseStream>> + Send {
) -> impl Future<Output = Result<SamplingStream>> + Send {
// Keep the transport and hook state out of the turn's cancellation and retry futures.
Box::pin(self.open_attempt(session, step, model, owner_cancellation, send))
}
@@ -52,16 +52,21 @@ impl SamplingRequest {
model: &str,
owner_cancellation: &CancellationToken,
send: impl Future<Output = Result<OpenedModelResponse>>,
) -> Result<ResponseStream> {
) -> Result<SamplingStream> {
let send = async {
session.check_rollout_budget()?;
send.await
};
let sampling_turn = match self.kind {
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
};
let settlement = Some((Arc::clone(session), Arc::clone(&step.turn)));
let hooks = hooks_for_model_request(session, step);
let pre_mode = hooks.execution_mode_for_event(HookEventName::PreModelRequest);
let post_mode = hooks.execution_mode_for_event(HookEventName::PostModelRequest);
if pre_mode.is_none() && post_mode.is_none() {
return send.await?.deliver(&mut None, sampling_turn);
return send.await?.deliver(&mut None, sampling_turn, settlement);
}
let subagent = thread_spawn_subagent_hook_context(session, &step.turn);
let input = ModelRequestHookInput {
@@ -92,7 +97,7 @@ impl SamplingRequest {
}
let stream = send
.await
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn));
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn, settlement));
if let Err(error) = &stream
&& let Some(mut attempt) = attempt
&& let Some(hooks) = attempt.start_post_hooks(ModelRequestHookEvent::failed(error))