mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add response processing after completion and usage accounting (#61577)
## Why Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget. ## What changed - Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately. - Track whether response usage has already been charged to avoid duplicate accounting, including during compaction. - Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation. - Bound buffered responses to 65,535 events and reject overflow. ## Testing Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch. GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
+27
-10
@@ -129,6 +129,7 @@ use crate::attestation::X_OAI_ATTESTATION_HEADER;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::context::BaseInstructionsFragment;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::cyber_access_program;
|
||||
@@ -2614,7 +2615,7 @@ impl ModelClientSession {
|
||||
.await
|
||||
{
|
||||
Ok(WebsocketStreamOutcome::Stream(opened)) => {
|
||||
let mut stream = (*opened).into_stream()?;
|
||||
let mut stream = (*opened).into_raw_stream()?;
|
||||
// Wait for the v2 warmup request to complete before sending the first turn request.
|
||||
while let Some(event) = stream.next().await {
|
||||
match event {
|
||||
@@ -2670,7 +2671,7 @@ impl ModelClientSession {
|
||||
inference_trace,
|
||||
)
|
||||
.await?
|
||||
.into_stream()
|
||||
.into_raw_stream()
|
||||
}
|
||||
|
||||
pub(crate) async fn open_model_response(
|
||||
@@ -2814,16 +2815,20 @@ pub(crate) struct OpenedModelResponse {
|
||||
}
|
||||
|
||||
impl OpenedModelResponse {
|
||||
pub(crate) fn into_stream(self) -> Result<ResponseStream> {
|
||||
self.deliver(&mut None, /*sampling_turn*/ None)
|
||||
fn into_raw_stream(self) -> Result<ResponseStream> {
|
||||
self.deliver(
|
||||
&mut None, /*sampling_turn*/ None, /*settlement*/ None,
|
||||
)
|
||||
.map(ResponseStream)
|
||||
}
|
||||
|
||||
pub(crate) fn deliver(
|
||||
mut self,
|
||||
attempt: &mut Option<crate::hook_runtime::sampling::SamplingAttempt>,
|
||||
sampling_turn: Option<Arc<TurnContext>>,
|
||||
) -> Result<ResponseStream> {
|
||||
let delivery = ResponseDelivery::new(attempt);
|
||||
settlement: Option<(Arc<crate::session::session::Session>, Arc<TurnContext>)>,
|
||||
) -> Result<SamplingStream> {
|
||||
let delivery = ResponseDelivery::new(attempt, settlement);
|
||||
let routing_affinity = self.stream.routing_affinity.take();
|
||||
let request_id = self.stream.upstream_request_id.take();
|
||||
let interrupt = self.stream.interrupt.take();
|
||||
@@ -2865,17 +2870,17 @@ fn map_response_events<S>(
|
||||
client: &ModelClientState,
|
||||
mut last_response_tx: Option<oneshot::Sender<LastResponse>>,
|
||||
mut delivery: ResponseDelivery,
|
||||
) -> ResponseStream
|
||||
) -> SamplingStream
|
||||
where
|
||||
S: futures::Stream<Item = std::result::Result<ResponseEvent, ApiError>>
|
||||
+ Unpin
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let (tx_event, rx_event) =
|
||||
mpsc::channel::<Result<ResponseEvent>>(RESPONSE_STREAM_CHANNEL_CAPACITY);
|
||||
let (tx_event, rx_event) = mpsc::channel(RESPONSE_STREAM_CHANNEL_CAPACITY);
|
||||
let consumer_dropped = CancellationToken::new();
|
||||
let consumer_dropped_for_stream = consumer_dropped.clone();
|
||||
let settlement_started = Arc::clone(&delivery.settlement_started);
|
||||
let provider = Arc::clone(&client.provider);
|
||||
let item_id_generator = Arc::clone(&client.item_id_generator);
|
||||
|
||||
@@ -2906,6 +2911,14 @@ where
|
||||
let Some(event) = event else {
|
||||
break;
|
||||
};
|
||||
let overflow = delivery.would_overflow(&event);
|
||||
let event = if overflow {
|
||||
Err(ApiError::Stream(
|
||||
"sampling response exceeded event buffer limit".to_owned(),
|
||||
))
|
||||
} else {
|
||||
event
|
||||
};
|
||||
let event = match event {
|
||||
Ok(ResponseEvent::OutputItemDone(mut item)) => {
|
||||
// Assign IDs after interceptors, which may replace provider items.
|
||||
@@ -3020,6 +3033,9 @@ where
|
||||
}
|
||||
return;
|
||||
}
|
||||
if overflow {
|
||||
return;
|
||||
}
|
||||
}
|
||||
inference_trace_attempt.record_failed(
|
||||
"stream closed before response.completed",
|
||||
@@ -3029,11 +3045,12 @@ where
|
||||
delivery.finish_eof(&tx_event).await;
|
||||
});
|
||||
|
||||
ResponseStream {
|
||||
SamplingStream {
|
||||
routing_affinity: None,
|
||||
rx_event,
|
||||
interrupt: None,
|
||||
consumer_dropped: consumer_dropped_for_stream,
|
||||
settlement_started,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user