Add response processing after completion and usage accounting (#61577)

## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
tongzhou wang
2026-11-14 23:50:12 +00:00
committed by copyberry
parent 81b995a6c9
commit 92c1a4dc2e
32 changed files with 780 additions and 259 deletions
+94 -23
View File
@@ -1,34 +1,73 @@
//! Synchronous post-hooks finish before the turn receives response completion.
//! Settles completed responses and releases output according to the configured processing mode.
use crate::client_common::ResponseEvent;
use crate::client_common::SamplingEvent;
use crate::hook_runtime::sampling::SamplingAttempt;
use codex_api::ApiError;
use codex_hooks::ModelRequestHookEvent;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use tokio::sync::mpsc;
const MAX_BUFFERED_EVENTS: usize = 65_535;
#[derive(Default)]
pub(super) struct ResponseDelivery {
attempt: Option<SamplingAttempt>,
held: Vec<ResponseEvent>,
pub(super) settlement_started: Arc<AtomicBool>,
settlement: Option<(
Arc<crate::session::session::Session>,
Arc<crate::session::turn_context::TurnContext>,
)>,
}
impl ResponseDelivery {
pub(super) fn new(attempt: &mut Option<SamplingAttempt>) -> Self {
pub(super) fn new(
attempt: &mut Option<SamplingAttempt>,
settlement: Option<(
Arc<crate::session::session::Session>,
Arc<crate::session::turn_context::TurnContext>,
)>,
) -> Self {
Self {
attempt: attempt.take(),
held: Vec::new(),
settlement_started: Default::default(),
settlement,
}
}
pub(super) fn would_overflow(
&self,
event: &std::result::Result<ResponseEvent, ApiError>,
) -> bool {
self.held.len() >= MAX_BUFFERED_EVENTS && event.as_ref().is_ok_and(|item| self.holds(item))
}
fn holds(&self, item: &ResponseEvent) -> bool {
!matches!(
item,
ResponseEvent::Completed { .. } | ResponseEvent::SafetyBuffering(_)
) && self.settlement.as_ref().is_some_and(|(_, turn)| {
turn.config.response_process_mode
== codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse
})
}
/// Return false when the turn has stopped receiving this attempt's output.
pub(super) async fn send_or_hold(
&mut self,
event: Result<ResponseEvent>,
tx: &mpsc::Sender<Result<ResponseEvent>>,
tx: &mpsc::Sender<SamplingEvent>,
) -> bool {
let Some(attempt) = self.attempt.as_mut() else {
return tx.send(event).await.is_ok();
};
let passthrough = self.attempt.is_none() && self.settlement.is_none();
if passthrough {
return tx.send(event.into()).await.is_ok();
}
let report = match &event {
Ok(ResponseEvent::Completed {
response_id,
@@ -36,40 +75,72 @@ impl ResponseDelivery {
..
}) => ModelRequestHookEvent::completed(response_id, token_usage.as_ref()),
Err(error) => ModelRequestHookEvent::failed(error),
Ok(_) => return tx.send(event).await.is_ok(),
Ok(item) if self.holds(item) => {
if let Ok(item) = event {
self.held.push(item);
}
return true;
}
Ok(_) => return tx.send(event.into()).await.is_ok(),
};
let owner = attempt.owner_cancellation.clone();
if let Some(hooks) = attempt.start_post_hooks(report) {
let hooks = self.attempt.as_mut().and_then(|attempt| {
let owner = attempt.owner_cancellation.clone();
attempt.start_post_hooks(report).map(|hooks| (owner, hooks))
});
if let Some((owner, hooks)) = hooks {
tokio::select! {
biased;
() = tx.closed() => return false,
() = owner.cancelled() => return false,
() = hooks => {}
}
// The owner can interrupt before the reader is scheduled to close its channel.
if owner.is_cancelled() {
return false;
}
}
self.attempt = None;
let released = Self::release(event, tx).await;
let terminal = SamplingEvent {
event,
hook_output: Default::default(),
};
let released = self.release(terminal, tx).await;
match released {
Ok(delivered) => delivered,
Err(error) => tx.send(Err(codex_api::map_api_error(error))).await.is_ok(),
Err(error) => tx.send(Err(error).into()).await.is_ok(),
}
}
async fn release(
event: Result<ResponseEvent>,
tx: &mpsc::Sender<Result<ResponseEvent>>,
) -> std::result::Result<bool, ApiError> {
if tx.is_closed() {
return Ok(false);
&mut self,
mut terminal: SamplingEvent,
tx: &mpsc::Sender<SamplingEvent>,
) -> Result<bool> {
if self.settlement.is_some()
&& matches!(terminal.event, Ok(ResponseEvent::Completed { .. }))
{
self.settlement_started.store(true, Ordering::Release);
}
Ok(tx.send(event).await.is_ok())
if let Some((session, turn)) = &self.settlement
&& let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event
{
let usage = crate::client_common::SamplingUsage {
provider: token_usage.clone(),
charge: std::mem::take(&mut terminal.hook_output),
};
session
.record_rollout_budget_usage(&turn.sub_id, &usage)
.await?;
terminal.hook_output = usage.charge;
terminal.hook_output.posted = true;
}
if terminal.event.is_ok() {
for item in std::mem::take(&mut self.held) {
if tx.send(Ok(item).into()).await.is_err() {
return Ok(false);
}
}
}
Ok(tx.send(terminal).await.is_ok())
}
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<Result<ResponseEvent>>) {
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<SamplingEvent>) {
let Some(mut attempt) = self.attempt.take() else {
return;
};
@@ -81,7 +152,7 @@ impl ResponseDelivery {
biased;
() = tx.closed() => {}
() = owner.cancelled() => {}
() = hooks => {}
_ = hooks => {}
}
}
}