mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Add response processing after completion and usage accounting (#61577)
## Why Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget. ## What changed - Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately. - Track whether response usage has already been charged to avoid duplicate accounting, including during compaction. - Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation. - Bound buffered responses to 65,535 events and reject overflow. ## Testing Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch. GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
@@ -106,11 +106,12 @@ async fn cancelled_turn_skips_launch_but_preserves_existing_work(trigger: Compac
|
||||
]));
|
||||
sess.update_token_usage_info(
|
||||
&turn,
|
||||
Some(&TokenUsage {
|
||||
TokenUsage {
|
||||
input_tokens: 60_000,
|
||||
total_tokens: 60_000,
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
.into(),
|
||||
/*includes_reasoning*/ false,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -540,7 +540,7 @@ impl InputQueue {
|
||||
}
|
||||
|
||||
impl TurnInputQueue {
|
||||
fn has_user_input(&self) -> bool {
|
||||
pub(crate) fn has_user_input(&self) -> bool {
|
||||
self.items
|
||||
.iter()
|
||||
.any(|input| matches!(input, TurnInput::UserInput { .. }))
|
||||
|
||||
@@ -5654,6 +5654,7 @@ impl Session {
|
||||
turn_context: &TurnContext,
|
||||
settings: &ResolvedStepSettings,
|
||||
incomplete: &ResponseIncompleteError,
|
||||
charge: crate::client_common::SamplingCharge,
|
||||
) {
|
||||
if let Some(response_id) = &incomplete.response_id {
|
||||
self.record_response_token_usage(
|
||||
@@ -5668,7 +5669,10 @@ impl Session {
|
||||
.record_token_usage_info(
|
||||
turn_context,
|
||||
settings,
|
||||
incomplete.token_usage.as_ref(),
|
||||
crate::client_common::SamplingUsage {
|
||||
provider: incomplete.token_usage.clone(),
|
||||
charge,
|
||||
},
|
||||
TokenUsageContext::SpendingOnly,
|
||||
)
|
||||
.await;
|
||||
@@ -5677,14 +5681,14 @@ impl Session {
|
||||
pub(crate) async fn update_token_usage_info(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
usage: crate::client_common::SamplingUsage,
|
||||
includes_reasoning: bool,
|
||||
) -> CodexResult<()> {
|
||||
let result = self
|
||||
.record_token_usage_info(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
token_usage,
|
||||
usage,
|
||||
TokenUsageContext::Sampling {
|
||||
includes_reasoning,
|
||||
input_item_count: 0,
|
||||
@@ -5743,10 +5747,10 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
settings: &ResolvedStepSettings,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
usage: crate::client_common::SamplingUsage,
|
||||
context: TokenUsageContext,
|
||||
) -> CodexResult<()> {
|
||||
if let Some(token_usage) = token_usage {
|
||||
let token_info = if let Some(token_usage) = usage.provider.as_ref() {
|
||||
let token_info = {
|
||||
let mut state = self.state.lock().await;
|
||||
match context {
|
||||
@@ -5799,24 +5803,26 @@ impl Session {
|
||||
token_usage,
|
||||
);
|
||||
}
|
||||
let budget_result = self
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, token_usage)
|
||||
.await;
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
contributor
|
||||
.on_token_usage(
|
||||
&self.services.session_extension_data,
|
||||
&self.services.thread_extension_data,
|
||||
turn_context.extension_data.as_ref(),
|
||||
token_info,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
token_info
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let budget_result = self
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, &usage)
|
||||
.await;
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
contributor
|
||||
.on_token_usage(
|
||||
&self.services.session_extension_data,
|
||||
&self.services.thread_extension_data,
|
||||
turn_context.extension_data.as_ref(),
|
||||
token_info,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
budget_result?;
|
||||
}
|
||||
Ok(())
|
||||
budget_result
|
||||
}
|
||||
|
||||
pub(crate) async fn recompute_token_usage(&self, turn_context: &TurnContext) {
|
||||
|
||||
@@ -4,9 +4,10 @@ use crate::agent::host::AgentRuntimeServices;
|
||||
use crate::agent::host::AgentUsageReport;
|
||||
use crate::agent::host::UsageAccountingDecision;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::rollout_budget::RolloutBudget;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
pub(super) async fn maybe_record_reminder(
|
||||
sess: &Session,
|
||||
@@ -39,18 +40,44 @@ pub(super) async fn maybe_record_reminder(
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) fn rollout_budget(&self) -> &RolloutBudget {
|
||||
&self.services.local_agent_runtime.rollout_budget
|
||||
}
|
||||
|
||||
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
|
||||
if self.rollout_budget().tools_stopped() {
|
||||
Err(CodexErr::SessionBudgetExceeded)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn record_rollout_budget_usage(
|
||||
&self,
|
||||
turn_id: &str,
|
||||
usage: &TokenUsage,
|
||||
bill: &crate::client_common::SamplingUsage,
|
||||
) -> CodexResult<()> {
|
||||
match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => {
|
||||
self.services
|
||||
.agent_control
|
||||
.record_usage(usage.clone())
|
||||
.await
|
||||
}
|
||||
if bill.charge.posted {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(usage) = bill.provider.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let stop_new_work = match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => match self
|
||||
.services
|
||||
.agent_control
|
||||
.record_usage(usage.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => false,
|
||||
Err(error)
|
||||
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
},
|
||||
AgentRuntimeServices::Host(services) => {
|
||||
let Some(accounting) = &services.usage_accounting else {
|
||||
return Ok(());
|
||||
@@ -65,11 +92,12 @@ impl Session {
|
||||
.map_err(|error| {
|
||||
CodexErr::Fatal(format!("Host usage accounting failed: {error}"))
|
||||
})?;
|
||||
match decision {
|
||||
UsageAccountingDecision::Continue => Ok(()),
|
||||
UsageAccountingDecision::Exhausted => Err(CodexErr::SessionBudgetExceeded),
|
||||
}
|
||||
decision == UsageAccountingDecision::Exhausted
|
||||
}
|
||||
};
|
||||
if stop_new_work {
|
||||
self.rollout_budget().stop_new_tools();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3398,7 +3398,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&first_usage),
|
||||
first_usage.clone().into(),
|
||||
crate::session::TokenUsageContext::Sampling {
|
||||
includes_reasoning: true,
|
||||
input_item_count: 0,
|
||||
@@ -3410,7 +3410,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&second_usage),
|
||||
second_usage.clone().into(),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
@@ -13173,7 +13173,6 @@ async fn tool_calls_reopen_mailbox_delivery_for_current_turn() {
|
||||
step_context: StepContext::for_test(Arc::clone(&tc)),
|
||||
turn_store: Arc::new(codex_extension_api::ExtensionData::new(tc.sub_id.clone())),
|
||||
tool_runtime: test_tool_runtime(Arc::clone(&sess), Arc::clone(&tc)).await,
|
||||
defer_tool_dispatch: false,
|
||||
cancellation_token: CancellationToken::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::client::ModelClientSession;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::compact::AutoCompactionDisabled;
|
||||
use crate::compact::run_inline_auto_compact_task;
|
||||
use crate::compact_remote_v2::BackgroundCompactionCandidate;
|
||||
@@ -654,6 +654,11 @@ pub(crate) async fn run_turn(
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
// Consume the whole response before deciding whether the turn may continue.
|
||||
let sampling_request_result = sampling_request_result.and_then(|output| {
|
||||
sess.check_rollout_budget()?;
|
||||
Ok(output)
|
||||
});
|
||||
match sampling_request_result {
|
||||
Ok((sampling_request_output, notify_messages)) => {
|
||||
guardian_budget_compacted = false;
|
||||
@@ -2827,7 +2832,7 @@ async fn drain_in_flight(
|
||||
in_flight: &mut FuturesOrdered<InFlightFuture<'static>>,
|
||||
sess: Arc<Session>,
|
||||
step_context: &StepContext,
|
||||
) -> CodexResult<()> {
|
||||
) {
|
||||
let turn_context = &step_context.turn;
|
||||
while let Some(res) = in_flight.next().await {
|
||||
match res {
|
||||
@@ -2850,7 +2855,6 @@ async fn drain_in_flight(
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2977,7 +2981,10 @@ async fn try_run_sampling_request(
|
||||
if cancellation_token.is_cancelled() && !can_interrupt {
|
||||
return Err(CodexErr::TurnAborted);
|
||||
}
|
||||
let mut in_flight: FuturesOrdered<InFlightFuture<'static>> = FuturesOrdered::new();
|
||||
let streaming = turn_context.config.response_process_mode
|
||||
== codex_config::config_toml::ResponseProcessMode::Streaming;
|
||||
let mut wait_for_bill_only = false;
|
||||
let mut in_flight = FuturesOrdered::new();
|
||||
let mut needs_follow_up = false;
|
||||
let mut last_agent_message: Option<String> = None;
|
||||
let mut active_item: Option<TurnItem> = None;
|
||||
@@ -3053,31 +3060,36 @@ async fn try_run_sampling_request(
|
||||
preempt = CancellationToken::new();
|
||||
continue;
|
||||
}
|
||||
if let Some(interrupt) = stream.interrupt.take() {
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming && stream.interrupt.is_none();
|
||||
} else {
|
||||
let Some(interrupt) = stream.interrupt.take() else {
|
||||
// TODO: Reconcile any response item already being presented to the client.
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: false,
|
||||
last_agent_message,
|
||||
});
|
||||
};
|
||||
if step_context.settings.model_info.use_responses_lite {
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
}
|
||||
// Drain the response normally before reusing its connection and history.
|
||||
preempt = CancellationToken::new();
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
// TODO: Reconcile any response item already being presented to the client.
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: false,
|
||||
last_agent_message,
|
||||
});
|
||||
// A steer cannot start another request before the accounting owner returns its decision.
|
||||
preempt = CancellationToken::new();
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
Err(_) => break Err(CodexErr::TurnAborted),
|
||||
};
|
||||
|
||||
let event = ResponseStream::accept_for_turn(event, &cancellation_token);
|
||||
let event = match event.unwrap_or_else(|| {
|
||||
Err(CodexErr::Stream(
|
||||
let event = SamplingStream::accept_for_turn(event, &cancellation_token);
|
||||
let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else {
|
||||
break Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
))
|
||||
}) {
|
||||
));
|
||||
};
|
||||
let event = match event {
|
||||
Ok(event) => event,
|
||||
Err(err) => {
|
||||
if let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details() {
|
||||
@@ -3085,6 +3097,7 @@ async fn try_run_sampling_request(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
incomplete,
|
||||
hook_output,
|
||||
)
|
||||
.await;
|
||||
should_emit_token_count |= incomplete.token_usage.is_some();
|
||||
@@ -3105,6 +3118,19 @@ async fn try_run_sampling_request(
|
||||
}
|
||||
};
|
||||
|
||||
// A steer can cut off unread items on an HTTP stream while the owner posts its bill.
|
||||
// Consume the terminal accounting decision before opening the follow-up request.
|
||||
if wait_for_bill_only {
|
||||
if matches!(event, ResponseEvent::Completed { .. }) {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: *draining_for_mailbox,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(event) = resampling.filter(event, &step_context.tool_router) else {
|
||||
continue;
|
||||
};
|
||||
@@ -3215,7 +3241,6 @@ async fn try_run_sampling_request(
|
||||
step_context: Arc::clone(&step_context),
|
||||
turn_store: Arc::clone(&turn_store),
|
||||
tool_runtime,
|
||||
defer_tool_dispatch: *draining_for_mailbox,
|
||||
cancellation_token: cancellation_token.child_token(),
|
||||
};
|
||||
if *draining_for_mailbox {
|
||||
@@ -3263,8 +3288,8 @@ async fn try_run_sampling_request(
|
||||
Ok(output_result) => output_result,
|
||||
Err(err) => break Err(err),
|
||||
};
|
||||
if let Some(tool_future) = output_result.tool_future {
|
||||
in_flight.push_back(tool_future);
|
||||
if let Some(tool) = output_result.tool_future {
|
||||
in_flight.push_back(tool);
|
||||
}
|
||||
if let Some(agent_message) = output_result.last_agent_message {
|
||||
last_agent_message = Some(agent_message);
|
||||
@@ -3291,23 +3316,26 @@ async fn try_run_sampling_request(
|
||||
conversation.id = %sess.thread_id,
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
if step_context.settings.model_info.use_responses_lite
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming
|
||||
&& !(step_context.settings.model_info.use_responses_lite
|
||||
&& stream.interrupt.is_some());
|
||||
} else if step_context.settings.model_info.use_responses_lite
|
||||
&& let Some(interrupt) = stream.interrupt.take()
|
||||
{
|
||||
// A closed receiver can mean completion is already buffered. Drain it
|
||||
// too; a transport failure below still takes the mailbox follow-up.
|
||||
// Completion can already be buffered when the interrupt receiver closes.
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
// Retain completed items and the response ID for incremental continuation.
|
||||
// Further item boundaries must not drop the stream while it drains.
|
||||
*draining_for_mailbox = true;
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
} else {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: true,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: true,
|
||||
last_agent_message,
|
||||
});
|
||||
// Keep the response ID and terminal decision. Later tools receive a denial.
|
||||
*draining_for_mailbox = true;
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ResponseEvent::OutputItemAdded(mut item) => {
|
||||
@@ -3502,7 +3530,10 @@ async fn try_run_sampling_request(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
token_usage.as_ref(),
|
||||
crate::client_common::SamplingUsage {
|
||||
provider: token_usage,
|
||||
charge: hook_output,
|
||||
},
|
||||
if resampling.retry_pending {
|
||||
super::TokenUsageContext::SpendingOnly
|
||||
} else {
|
||||
@@ -3715,7 +3746,7 @@ async fn try_run_sampling_request(
|
||||
conversation.id = %sess.thread_id,
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
drain_in_flight(&mut in_flight, sess.clone(), &step_context).await?;
|
||||
drain_in_flight(&mut in_flight, Arc::clone(&sess), &step_context).await;
|
||||
}
|
||||
|
||||
if should_emit_token_count {
|
||||
|
||||
Reference in New Issue
Block a user