Files
codex/codex-rs/core/src/hook_runtime/sampling.rs
T
tongzhou wang 92c1a4dc2e Add response processing after completion and usage accounting (#61577)
## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
2026-11-14 23:50:12 +00:00

212 lines
8.1 KiB
Rust

//! Each provider attempt pairs its own hooks. Transport recovery within an
//! attempt does not start another hook pair.
use super::emit_hook_completed_events;
use super::emit_hook_started_events;
use super::hooks_for_model_request;
use super::run_model_request_hooks;
use super::thread_spawn_subagent_hook_context;
use super::tool_hook_cwd;
use crate::client::OpenedModelResponse;
use crate::client_common::SamplingStream;
use crate::session::session::Session;
use crate::session::step_context::StepContext;
use codex_hooks::ModelRequestHookEvent;
use codex_hooks::ModelRequestHookInput;
use codex_hooks::SamplingKind;
use codex_protocol::error::Result;
use codex_protocol::protocol::HookCompletedEvent;
use codex_protocol::protocol::HookEventName;
use codex_protocol::protocol::HookExecutionMode;
use codex_protocol::protocol::HookRunStatus;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
pub(crate) struct SamplingRequest {
kind: SamplingKind,
}
impl SamplingRequest {
pub(crate) fn new(kind: SamplingKind) -> Self {
Self { kind }
}
pub(crate) fn open(
&mut self,
session: &Arc<Session>,
step: &Arc<StepContext>,
model: &str,
owner_cancellation: &CancellationToken,
send: impl Future<Output = Result<OpenedModelResponse>> + Send,
) -> impl Future<Output = Result<SamplingStream>> + Send {
// Keep the transport and hook state out of the turn's cancellation and retry futures.
Box::pin(self.open_attempt(session, step, model, owner_cancellation, send))
}
async fn open_attempt(
&mut self,
session: &Arc<Session>,
step: &Arc<StepContext>,
model: &str,
owner_cancellation: &CancellationToken,
send: impl Future<Output = Result<OpenedModelResponse>>,
) -> Result<SamplingStream> {
let send = async {
session.check_rollout_budget()?;
send.await
};
let sampling_turn = match self.kind {
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
};
let settlement = Some((Arc::clone(session), Arc::clone(&step.turn)));
let hooks = hooks_for_model_request(session, step);
let pre_mode = hooks.execution_mode_for_event(HookEventName::PreModelRequest);
let post_mode = hooks.execution_mode_for_event(HookEventName::PostModelRequest);
if pre_mode.is_none() && post_mode.is_none() {
return send.await?.deliver(&mut None, sampling_turn, settlement);
}
let subagent = thread_spawn_subagent_hook_context(session, &step.turn);
let input = ModelRequestHookInput {
lifecycle_id: uuid::Uuid::new_v4(),
kind: self.kind,
session_id: session.session_id().to_string(),
thread_id: session.thread_id().to_string(),
turn_id: step.turn.sub_id.clone(),
cwd: tool_hook_cwd(&step.environments, &step.turn),
transcript_path: session.hook_transcript_path().await,
model: model.to_owned(),
event: ModelRequestHookEvent::PreModelRequest,
agent_id: subagent.as_ref().map(|context| context.agent_id.clone()),
agent_type: subagent.map(|context| context.agent_type),
};
let mut attempt = post_mode.map(|mode| SamplingAttempt {
owner_cancellation: owner_cancellation.clone(),
post: Some(PostHooks {
hooks: Arc::clone(&hooks),
step: Arc::clone(step),
input: input.clone(),
mode,
}),
session: Arc::clone(session),
});
if pre_mode.is_some() {
run_model_request_hooks(session, step, &hooks, input).await;
}
let stream = send
.await
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn, settlement));
if let Err(error) = &stream
&& let Some(mut attempt) = attempt
&& let Some(hooks) = attempt.start_post_hooks(ModelRequestHookEvent::failed(error))
{
hooks.await;
}
stream
}
}
/// The sampling future owns the synchronous handlers. Observer-only and
/// pre-response cancellation notifications run on the session's hook executor.
struct PostHooks {
hooks: Arc<codex_hooks::Hooks>,
step: Arc<StepContext>,
input: ModelRequestHookInput,
mode: HookExecutionMode,
}
impl PostHooks {
async fn run(mut self, session: &Arc<Session>, event: ModelRequestHookEvent) {
self.input.event = event;
run_model_request_hooks(session, &self.step, &self.hooks, self.input).await;
}
async fn run_sync(mut self, session: Arc<Session>, event: ModelRequestHookEvent) {
let (mut done, wait) = oneshot::channel();
self.input.event = event;
// Dropping the waiter cancels execution while the task finishes publishing
// lifecycle events. It must run independently of the async hook limit.
tokio::spawn(async move {
if done.is_closed() {
return;
}
let turn = &self.step.turn;
let runs = self.hooks.preview_model_request(&self.input);
let started = Instant::now();
emit_hook_started_events(&session, turn, runs.clone()).await;
let mut finished: Vec<HookCompletedEvent> = Vec::new();
let events = tokio::select! {
biased;
() = done.closed() => {
let completed_at = chrono::Utc::now().timestamp();
let duration_ms = i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX);
for mut run in runs {
if run.execution_mode != HookExecutionMode::Sync
|| finished.iter().any(|event| event.run.id == run.id)
{
continue;
}
run.status = HookRunStatus::Stopped;
run.status_message = Some("Hook cancelled because the turn was interrupted".into());
run.completed_at = Some(completed_at);
run.duration_ms = Some(duration_ms);
finished.push(HookCompletedEvent {
turn_id: Some(turn.sub_id.clone()),
run,
});
}
finished
}
events = self.hooks.run_model_request(self.input, |event| finished.push(event.clone())) => events,
};
emit_hook_completed_events(&session, turn, events).await;
let _ = done.send(());
});
let _ = wait.await;
}
fn schedule(self, session: Arc<Session>, event: ModelRequestHookEvent) {
Arc::clone(&self.hooks).schedule_async_task(async move {
self.run(&session, event).await;
});
}
}
pub(crate) struct SamplingAttempt {
/// The turn owns foreground responses; the session owns speculative responses.
pub(crate) owner_cancellation: CancellationToken,
session: Arc<Session>,
post: Option<PostHooks>,
}
impl Drop for SamplingAttempt {
fn drop(&mut self) {
if let Some(post) = self.post.take() {
post.schedule(
Arc::clone(&self.session),
ModelRequestHookEvent::cancelled(),
);
}
}
}
impl SamplingAttempt {
/// Observers run on the session executor. The request owns the synchronous
/// group: if the turn stops waiting, dropping this future cancels it.
pub(crate) fn start_post_hooks(
&mut self,
event: ModelRequestHookEvent,
) -> Option<impl Future<Output = ()> + Send + 'static> {
let post = self.post.take()?;
let session = Arc::clone(&self.session);
if post.mode == HookExecutionMode::Sync {
Some(post.run_sync(session, event))
} else {
post.schedule(session, event);
None
}
}
}