Add response processing after completion and usage accounting (#61577)

## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
tongzhou wang
2026-11-14 23:50:12 +00:00
committed by copyberry
parent 81b995a6c9
commit 92c1a4dc2e
32 changed files with 780 additions and 259 deletions
+27 -10
View File
@@ -129,6 +129,7 @@ use crate::attestation::X_OAI_ATTESTATION_HEADER;
use crate::client_common::Prompt;
use crate::client_common::ResponseEvent;
use crate::client_common::ResponseStream;
use crate::client_common::SamplingStream;
use crate::context::BaseInstructionsFragment;
use crate::context::ContextualUserFragment;
use crate::cyber_access_program;
@@ -2614,7 +2615,7 @@ impl ModelClientSession {
.await
{
Ok(WebsocketStreamOutcome::Stream(opened)) => {
let mut stream = (*opened).into_stream()?;
let mut stream = (*opened).into_raw_stream()?;
// Wait for the v2 warmup request to complete before sending the first turn request.
while let Some(event) = stream.next().await {
match event {
@@ -2670,7 +2671,7 @@ impl ModelClientSession {
inference_trace,
)
.await?
.into_stream()
.into_raw_stream()
}
pub(crate) async fn open_model_response(
@@ -2814,16 +2815,20 @@ pub(crate) struct OpenedModelResponse {
}
impl OpenedModelResponse {
pub(crate) fn into_stream(self) -> Result<ResponseStream> {
self.deliver(&mut None, /*sampling_turn*/ None)
fn into_raw_stream(self) -> Result<ResponseStream> {
self.deliver(
&mut None, /*sampling_turn*/ None, /*settlement*/ None,
)
.map(ResponseStream)
}
pub(crate) fn deliver(
mut self,
attempt: &mut Option<crate::hook_runtime::sampling::SamplingAttempt>,
sampling_turn: Option<Arc<TurnContext>>,
) -> Result<ResponseStream> {
let delivery = ResponseDelivery::new(attempt);
settlement: Option<(Arc<crate::session::session::Session>, Arc<TurnContext>)>,
) -> Result<SamplingStream> {
let delivery = ResponseDelivery::new(attempt, settlement);
let routing_affinity = self.stream.routing_affinity.take();
let request_id = self.stream.upstream_request_id.take();
let interrupt = self.stream.interrupt.take();
@@ -2865,17 +2870,17 @@ fn map_response_events<S>(
client: &ModelClientState,
mut last_response_tx: Option<oneshot::Sender<LastResponse>>,
mut delivery: ResponseDelivery,
) -> ResponseStream
) -> SamplingStream
where
S: futures::Stream<Item = std::result::Result<ResponseEvent, ApiError>>
+ Unpin
+ Send
+ 'static,
{
let (tx_event, rx_event) =
mpsc::channel::<Result<ResponseEvent>>(RESPONSE_STREAM_CHANNEL_CAPACITY);
let (tx_event, rx_event) = mpsc::channel(RESPONSE_STREAM_CHANNEL_CAPACITY);
let consumer_dropped = CancellationToken::new();
let consumer_dropped_for_stream = consumer_dropped.clone();
let settlement_started = Arc::clone(&delivery.settlement_started);
let provider = Arc::clone(&client.provider);
let item_id_generator = Arc::clone(&client.item_id_generator);
@@ -2906,6 +2911,14 @@ where
let Some(event) = event else {
break;
};
let overflow = delivery.would_overflow(&event);
let event = if overflow {
Err(ApiError::Stream(
"sampling response exceeded event buffer limit".to_owned(),
))
} else {
event
};
let event = match event {
Ok(ResponseEvent::OutputItemDone(mut item)) => {
// Assign IDs after interceptors, which may replace provider items.
@@ -3020,6 +3033,9 @@ where
}
return;
}
if overflow {
return;
}
}
inference_trace_attempt.record_failed(
"stream closed before response.completed",
@@ -3029,11 +3045,12 @@ where
delivery.finish_eof(&tx_event).await;
});
ResponseStream {
SamplingStream {
routing_affinity: None,
rx_event,
interrupt: None,
consumer_dropped: consumer_dropped_for_stream,
settlement_started,
}
}