mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Add response processing after completion and usage accounting (#61577)
## Why Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget. ## What changed - Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately. - Track whether response usage has already been charged to avoid duplicate accounting, including during compaction. - Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation. - Bound buffered responses to 65,535 events and reject overflow. ## Testing Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch. GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
@@ -1,34 +1,73 @@
|
||||
//! Synchronous post-hooks finish before the turn receives response completion.
|
||||
//! Settles completed responses and releases output according to the configured processing mode.
|
||||
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::hook_runtime::sampling::SamplingAttempt;
|
||||
use codex_api::ApiError;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const MAX_BUFFERED_EVENTS: usize = 65_535;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ResponseDelivery {
|
||||
attempt: Option<SamplingAttempt>,
|
||||
held: Vec<ResponseEvent>,
|
||||
pub(super) settlement_started: Arc<AtomicBool>,
|
||||
settlement: Option<(
|
||||
Arc<crate::session::session::Session>,
|
||||
Arc<crate::session::turn_context::TurnContext>,
|
||||
)>,
|
||||
}
|
||||
|
||||
impl ResponseDelivery {
|
||||
pub(super) fn new(attempt: &mut Option<SamplingAttempt>) -> Self {
|
||||
pub(super) fn new(
|
||||
attempt: &mut Option<SamplingAttempt>,
|
||||
settlement: Option<(
|
||||
Arc<crate::session::session::Session>,
|
||||
Arc<crate::session::turn_context::TurnContext>,
|
||||
)>,
|
||||
) -> Self {
|
||||
Self {
|
||||
attempt: attempt.take(),
|
||||
held: Vec::new(),
|
||||
settlement_started: Default::default(),
|
||||
settlement,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn would_overflow(
|
||||
&self,
|
||||
event: &std::result::Result<ResponseEvent, ApiError>,
|
||||
) -> bool {
|
||||
self.held.len() >= MAX_BUFFERED_EVENTS && event.as_ref().is_ok_and(|item| self.holds(item))
|
||||
}
|
||||
|
||||
fn holds(&self, item: &ResponseEvent) -> bool {
|
||||
!matches!(
|
||||
item,
|
||||
ResponseEvent::Completed { .. } | ResponseEvent::SafetyBuffering(_)
|
||||
) && self.settlement.as_ref().is_some_and(|(_, turn)| {
|
||||
turn.config.response_process_mode
|
||||
== codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse
|
||||
})
|
||||
}
|
||||
|
||||
/// Return false when the turn has stopped receiving this attempt's output.
|
||||
pub(super) async fn send_or_hold(
|
||||
&mut self,
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
tx: &mpsc::Sender<SamplingEvent>,
|
||||
) -> bool {
|
||||
let Some(attempt) = self.attempt.as_mut() else {
|
||||
return tx.send(event).await.is_ok();
|
||||
};
|
||||
let passthrough = self.attempt.is_none() && self.settlement.is_none();
|
||||
if passthrough {
|
||||
return tx.send(event.into()).await.is_ok();
|
||||
}
|
||||
let report = match &event {
|
||||
Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
@@ -36,40 +75,72 @@ impl ResponseDelivery {
|
||||
..
|
||||
}) => ModelRequestHookEvent::completed(response_id, token_usage.as_ref()),
|
||||
Err(error) => ModelRequestHookEvent::failed(error),
|
||||
Ok(_) => return tx.send(event).await.is_ok(),
|
||||
Ok(item) if self.holds(item) => {
|
||||
if let Ok(item) = event {
|
||||
self.held.push(item);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Ok(_) => return tx.send(event.into()).await.is_ok(),
|
||||
};
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
if let Some(hooks) = attempt.start_post_hooks(report) {
|
||||
let hooks = self.attempt.as_mut().and_then(|attempt| {
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
attempt.start_post_hooks(report).map(|hooks| (owner, hooks))
|
||||
});
|
||||
if let Some((owner, hooks)) = hooks {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = tx.closed() => return false,
|
||||
() = owner.cancelled() => return false,
|
||||
() = hooks => {}
|
||||
}
|
||||
// The owner can interrupt before the reader is scheduled to close its channel.
|
||||
if owner.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
self.attempt = None;
|
||||
let released = Self::release(event, tx).await;
|
||||
let terminal = SamplingEvent {
|
||||
event,
|
||||
hook_output: Default::default(),
|
||||
};
|
||||
let released = self.release(terminal, tx).await;
|
||||
match released {
|
||||
Ok(delivered) => delivered,
|
||||
Err(error) => tx.send(Err(codex_api::map_api_error(error))).await.is_ok(),
|
||||
Err(error) => tx.send(Err(error).into()).await.is_ok(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
) -> std::result::Result<bool, ApiError> {
|
||||
if tx.is_closed() {
|
||||
return Ok(false);
|
||||
&mut self,
|
||||
mut terminal: SamplingEvent,
|
||||
tx: &mpsc::Sender<SamplingEvent>,
|
||||
) -> Result<bool> {
|
||||
if self.settlement.is_some()
|
||||
&& matches!(terminal.event, Ok(ResponseEvent::Completed { .. }))
|
||||
{
|
||||
self.settlement_started.store(true, Ordering::Release);
|
||||
}
|
||||
Ok(tx.send(event).await.is_ok())
|
||||
if let Some((session, turn)) = &self.settlement
|
||||
&& let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event
|
||||
{
|
||||
let usage = crate::client_common::SamplingUsage {
|
||||
provider: token_usage.clone(),
|
||||
charge: std::mem::take(&mut terminal.hook_output),
|
||||
};
|
||||
session
|
||||
.record_rollout_budget_usage(&turn.sub_id, &usage)
|
||||
.await?;
|
||||
terminal.hook_output = usage.charge;
|
||||
terminal.hook_output.posted = true;
|
||||
}
|
||||
if terminal.event.is_ok() {
|
||||
for item in std::mem::take(&mut self.held) {
|
||||
if tx.send(Ok(item).into()).await.is_err() {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(tx.send(terminal).await.is_ok())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<Result<ResponseEvent>>) {
|
||||
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<SamplingEvent>) {
|
||||
let Some(mut attempt) = self.attempt.take() else {
|
||||
return;
|
||||
};
|
||||
@@ -81,7 +152,7 @@ impl ResponseDelivery {
|
||||
biased;
|
||||
() = tx.closed() => {}
|
||||
() = owner.cancelled() => {}
|
||||
() = hooks => {}
|
||||
_ = hooks => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user