Account for synchronous MCP post-hook costs in rollout budgets (#61580)

## What changed

Allow successful synchronous `PostModelRequest` MCP hooks to return `extra_codex_rollout_budget_units` in `structuredContent`. Add these charges to rollout spending while preserving provider usage separately, including completed responses without token usage and incomplete responses with reported usage.

Validate prices as finite and nonnegative. Preserve valid sibling charges when another hook reports an invalid price, and surface accounting failures without retrying inference. Carry hook charges and overflow information through host accounting and enforce local budget limits.

## Testing

Extend coverage for hook-driven budget exhaustion, failed hooks, overflow, interrupted responses, and foreground and background compaction accounting.

GitOrigin-RevId: dc54ba53478d82848bcafb55ecef88215019d09f
This commit is contained in:
tongzhou wang
2026-11-14 23:53:53 +00:00
committed by copyberry
parent c8e4cbcd94
commit c8c3f70893
42 changed files with 595 additions and 156 deletions
+5 -1
View File
@@ -124,7 +124,11 @@ pub trait AgentControl: Send + Sync {
/// Account for one inference response, including compaction. Each call records usage;
/// callers report it once. `SessionBudgetExceeded` means the usage was recorded and
/// the shared budget is now exhausted.
fn record_usage(&self, usage: TokenUsage) -> BoxFuture<'_, Result<()>>;
fn record_usage(
&self,
usage: Option<TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> BoxFuture<'_, Result<()>>;
/// Report the terminal result to the parent and completion activity to the task
/// initiator. Local delivery remains best effort and uses the reporting runtime's
+6 -2
View File
@@ -282,8 +282,12 @@ impl AgentControl for LocalAgentControl {
self.execution_guard(version, source)
}
fn record_usage(&self, usage: TokenUsage) -> BoxFuture<'_, Result<()>> {
Box::pin(async move { self.record_rollout_budget_usage(&usage) })
fn record_usage(
&self,
usage: Option<TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> BoxFuture<'_, Result<()>> {
Box::pin(async move { self.record_rollout_budget_usage(usage.as_ref(), hook_charge) })
}
fn turn_finished<'a>(
+10 -2
View File
@@ -9,8 +9,16 @@ use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
impl LocalAgentControl {
pub(crate) fn record_rollout_budget_usage(&self, usage: &TokenUsage) -> CodexResult<()> {
if self.runtime.rollout_budget.record_usage(usage)? {
pub(crate) fn record_rollout_budget_usage(
&self,
usage: Option<&TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> CodexResult<()> {
if self
.runtime
.rollout_budget
.record_usage(usage, hook_charge)?
{
return Err(CodexErr::SessionBudgetExceeded);
}
Ok(())
+7 -3
View File
@@ -5,14 +5,18 @@ use codex_protocol::error::Result;
use codex_protocol::protocol::TokenUsage;
use futures::future::BoxFuture;
/// Provider-reported spending for one response, including compaction.
#[derive(Clone, Debug, PartialEq, Eq)]
/// Spending reported for one model response, including compaction.
#[derive(Clone, Debug, PartialEq)]
pub struct AgentUsageReport {
pub thread_id: ThreadId,
pub turn_id: String,
/// Per-response usage, not cumulative totals. Remote adapters must explicitly
/// carry `codex_rollout_budget_units`, which `TokenUsage` skips when serialized.
pub usage: TokenUsage,
pub usage: Option<TokenUsage>,
/// Synchronous post-hook contributions, independent of the provider's usage.
pub additional_budget_units: f64,
/// If f64 summation overflowed, this is the largest individual hook price, not the total.
pub additional_budget_units_overflowed: bool,
}
/// Both decisions mean the host recorded the reported spending.
@@ -87,18 +87,23 @@ impl ResponseDelivery {
let owner = attempt.owner_cancellation.clone();
attempt.start_post_hooks(report).map(|hooks| (owner, hooks))
});
if let Some((owner, hooks)) = hooks {
let hook_output = if let Some((owner, hooks)) = hooks {
tokio::select! {
biased;
() = tx.closed() => return false,
() = owner.cancelled() => return false,
() = hooks => {}
output = hooks => output,
}
}
} else {
Default::default()
};
self.attempt = None;
let terminal = SamplingEvent {
event,
hook_output: Default::default(),
hook_output: crate::client_common::SamplingCharge {
hooks: hook_output,
..Default::default()
},
};
let released = self.release(terminal, tx).await;
match released {
+6 -4
View File
@@ -121,13 +121,14 @@ fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelIn
}
}
/// Records whether the response producer has posted the bill.
/// Carries the hook price and whether the response producer has posted the bill.
#[derive(Debug, Default)]
pub(crate) struct SamplingCharge {
pub(crate) hooks: codex_hooks::ModelRequestHookOutcome,
pub(crate) posted: bool,
}
/// Core consumes a terminal event together with its billing status. Other readers
/// Core consumes a terminal event and its hook output together. Other readers
/// of the public response stream still see the provider's unmodified event.
#[derive(Debug)]
pub(crate) struct SamplingEvent {
@@ -135,7 +136,8 @@ pub(crate) struct SamplingEvent {
pub(crate) hook_output: SamplingCharge,
}
/// Provider usage and billing status for one response.
/// The provider's usage and the hook's contribution describe the same
/// response. Keeping them separate preserves the provider's original numbers.
#[derive(Debug)]
pub(crate) struct SamplingUsage {
pub(crate) provider: Option<codex_protocol::protocol::TokenUsage>,
@@ -161,7 +163,7 @@ impl From<Result<ResponseEvent>> for SamplingEvent {
}
}
/// A sampling consumer receives the billing status with the provider event.
/// A sampling consumer receives the hook outcome with the provider event it settles.
pub(crate) struct SamplingStream {
pub(crate) routing_affinity: Option<codex_http_client::ChatGptRoutingAffinity>,
pub(crate) rx_event: mpsc::Receiver<SamplingEvent>,
+75 -40
View File
@@ -1434,16 +1434,26 @@ mod tests {
total_tokens: 120,
..Default::default()
};
let stream = response_stream(vec![Err(CodexErr::new(
CodexErrorDetails::ResponseIncomplete(Box::new(
codex_protocol::error::ResponseIncompleteError {
let stream = response_stream(vec![SamplingEvent {
event: Err(CodexErr::new(CodexErrorDetails::ResponseIncomplete(
Box::new(codex_protocol::error::ResponseIncompleteError {
partial_output: vec![],
reason: "max_output_tokens".to_string(),
response_id: Some("resp-background".to_string()),
token_usage: Some(usage.clone()),
}),
))),
hook_output: crate::client_common::SamplingCharge {
hooks: codex_hooks::ModelRequestHookOutcome {
charge: codex_hooks::HookCharge {
units: 3.0,
overflowed: false,
},
invalid_price: Some("negative sibling price".into()),
},
)),
))]);
..Default::default()
},
}]);
let (mut sess, turn_context, rx) =
crate::session::tests::make_session_and_context_with_rx().await;
let accounting = Arc::new(UsageRecorder {
@@ -1484,65 +1494,90 @@ mod tests {
vec![AgentUsageReport {
thread_id: sess.thread_id(),
turn_id: turn_context.sub_id.clone(),
usage,
usage: Some(usage),
additional_budget_units: 3.0,
additional_budget_units_overflowed: false,
}]
);
}
#[test_case::test_case(true, false; "background bills a complete response even without a summary")]
#[test_case::test_case(false, true; "turn interrupt discards a queued complete response")]
#[test_case::test_case(true, true; "session shutdown bills a queued complete response even without a summary")]
#[test_case::test_case(false, false, false; "foreground bills a complete response even without a summary")]
#[test_case::test_case(true, false, false; "background bills a complete response even without a summary")]
#[test_case::test_case(false, true, false; "turn interrupt discards a queued complete response")]
#[test_case::test_case(true, true, false; "session shutdown bills a queued complete response even without a summary")]
#[test_case::test_case(false, false, true; "a malformed sibling is still reported when valid spending exhausts the account")]
#[tokio::test]
async fn collect_queued_complete_response(background: bool, interrupted: bool) {
async fn collect_queued_complete_response(
background: bool,
interrupted: bool,
bad_sibling: bool,
) {
let (mut sess, turn, _) = crate::session::tests::make_session_and_context_with_rx().await;
let accounting = Arc::new(UsageRecorder {
reports: Mutex::new(Vec::new()),
decision: UsageAccountingDecision::Continue,
decision: if bad_sibling {
UsageAccountingDecision::Exhausted
} else {
UsageAccountingDecision::Continue
},
});
accounting.install(&mut sess);
let usage = TokenUsage {
let provider_usage = interrupted.then(|| TokenUsage {
output_tokens: 42,
..Default::default()
};
let stream = response_stream(vec![Ok(ResponseEvent::Completed {
response_id: "resp-compact".into(),
token_usage: Some(usage.clone()),
usage_metadata: None,
reasoning_removals: None,
end_turn: Some(true),
})]);
});
let stream = response_stream(vec![SamplingEvent {
event: Ok(ResponseEvent::Completed {
response_id: "resp-compact".into(),
token_usage: provider_usage.clone(),
usage_metadata: None,
reasoning_removals: None,
end_turn: Some(true),
}),
hook_output: crate::client_common::SamplingCharge {
hooks: codex_hooks::ModelRequestHookOutcome {
charge: codex_hooks::HookCharge {
units: 42.0,
overflowed: false,
},
invalid_price: bad_sibling.then(|| "negative sibling price".into()),
},
..Default::default()
},
}]);
let cancellation = CancellationToken::new();
if interrupted {
cancellation.cancel();
}
let result = collect_compaction_output(
&sess,
&turn,
stream,
if background {
CompactionRequestKind::Background(&cancellation)
} else {
CompactionRequestKind::Foreground(&cancellation)
},
)
.await;
if !background {
assert!(
matches!(result, Err(error) if matches!(error.details(), CodexErrorDetails::TurnAborted))
);
let kind = if background {
CompactionRequestKind::Background(&cancellation)
} else {
CompactionRequestKind::Foreground(&cancellation)
};
let result = collect_compaction_output(&sess, &turn, stream, kind).await;
let error = result.err().expect("no installable summary");
if interrupted && !background {
assert!(matches!(error.details(), CodexErrorDetails::TurnAborted));
assert!(accounting.reports.lock().unwrap().is_empty());
return;
}
assert!(
matches!(result, Err(error) if matches!(error.details(), CodexErrorDetails::Fatal(_)))
);
if bad_sibling {
assert!(
matches!(error.details(), CodexErrorDetails::UsageAccountingFailed(message) if message.contains("negative sibling price"))
);
} else {
assert!(
matches!(error.details(), CodexErrorDetails::Fatal(message) if message.contains("exactly one compaction output"))
);
}
assert_eq!(
*accounting.reports.lock().unwrap(),
[AgentUsageReport {
thread_id: sess.thread_id(),
turn_id: turn.sub_id.clone(),
usage,
usage: provider_usage,
additional_budget_units: 42.0,
additional_budget_units_overflowed: false,
}]
);
}
+1
View File
@@ -61,6 +61,7 @@ impl HookMcpExecutor for CoreHookMcpExecutor {
}
Ok(HookMcpOutput {
structured_content: result.structured_content,
text,
source_tool_namespace: call_result.source_tool_namespace,
})
+3 -2
View File
@@ -1133,10 +1133,11 @@ async fn run_model_request_hooks(
step: &StepContext,
hooks: &codex_hooks::Hooks,
request: codex_hooks::ModelRequestHookInput,
) {
) -> codex_hooks::ModelRequestHookOutcome {
emit_hook_started_events(session, &step.turn, hooks.preview_model_request(&request)).await;
let events = hooks.run_model_request(request, |_| {}).await;
let (events, outcome) = hooks.run_model_request(request, |_| {}).await;
emit_hook_completed_events(session, &step.turn, events).await;
outcome
}
#[cfg(test)]
+20 -8
View File
@@ -13,6 +13,7 @@ use crate::session::session::Session;
use crate::session::step_context::StepContext;
use codex_hooks::ModelRequestHookEvent;
use codex_hooks::ModelRequestHookInput;
use codex_hooks::ModelRequestHookOutcome;
use codex_hooks::SamplingKind;
use codex_protocol::error::Result;
use codex_protocol::protocol::HookCompletedEvent;
@@ -118,12 +119,20 @@ struct PostHooks {
}
impl PostHooks {
async fn run(mut self, session: &Arc<Session>, event: ModelRequestHookEvent) {
async fn run(
mut self,
session: &Arc<Session>,
event: ModelRequestHookEvent,
) -> ModelRequestHookOutcome {
self.input.event = event;
run_model_request_hooks(session, &self.step, &self.hooks, self.input).await;
run_model_request_hooks(session, &self.step, &self.hooks, self.input).await
}
async fn run_sync(mut self, session: Arc<Session>, event: ModelRequestHookEvent) {
async fn run_sync(
mut self,
session: Arc<Session>,
event: ModelRequestHookEvent,
) -> ModelRequestHookOutcome {
let (mut done, wait) = oneshot::channel();
self.input.event = event;
// Dropping the waiter cancels execution while the task finishes publishing
@@ -137,7 +146,7 @@ impl PostHooks {
let started = Instant::now();
emit_hook_started_events(&session, turn, runs.clone()).await;
let mut finished: Vec<HookCompletedEvent> = Vec::new();
let events = tokio::select! {
let (events, outcome) = tokio::select! {
biased;
() = done.closed() => {
let completed_at = chrono::Utc::now().timestamp();
@@ -157,14 +166,17 @@ impl PostHooks {
run,
});
}
finished
(finished, ModelRequestHookOutcome::default())
}
events = self.hooks.run_model_request(self.input, |event| finished.push(event.clone())) => events,
};
emit_hook_completed_events(&session, turn, events).await;
let _ = done.send(());
let _ = done.send(outcome);
});
let _ = wait.await;
wait.await.unwrap_or_else(|_| ModelRequestHookOutcome {
invalid_price: Some("synchronous hook task exited without reporting its charge".into()),
..Default::default()
})
}
fn schedule(self, session: Arc<Session>, event: ModelRequestHookEvent) {
@@ -198,7 +210,7 @@ impl SamplingAttempt {
pub(crate) fn start_post_hooks(
&mut self,
event: ModelRequestHookEvent,
) -> Option<impl Future<Output = ()> + Send + 'static> {
) -> Option<impl Future<Output = ModelRequestHookOutcome> + Send + 'static> {
let post = self.post.take()?;
let session = Arc::clone(&self.session);
if post.mode == HookExecutionMode::Sync {
+1
View File
@@ -180,6 +180,7 @@ pub use agent::host::PreparedChildContext;
pub use agent::host::RestrictRootCapabilities;
pub use agent::host::ToolInvocationContext;
pub use agent::host::UsageAccountingDecision;
pub use codex_hooks::HookCharge;
pub use thread_spawn::ThreadCreateRequest;
pub use thread_spawn::ThreadSpawnError;
pub use thread_spawn::ThreadSpawnResult;
+1
View File
@@ -36,6 +36,7 @@ pub(crate) fn is_terminal_sampling_outcome(error: &CodexErr) -> bool {
matches!(
error.details(),
CodexErrorDetails::ResponseIncomplete(_)
| CodexErrorDetails::UsageAccountingFailed(_)
| CodexErrorDetails::SessionBudgetExceeded
| CodexErrorDetails::InferenceHookDenied { .. }
)
+34 -6
View File
@@ -51,11 +51,28 @@ impl RolloutBudget {
}
/// Records usage and returns whether an enforced budget is exhausted.
pub(crate) fn record_usage(&self, usage: &TokenUsage) -> CodexResult<bool> {
pub(crate) fn record_usage(
&self,
usage: Option<&TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> CodexResult<bool> {
let Some(mut state) = self.lock() else {
return Ok(false);
};
let units = if let Some(units) = usage.codex_rollout_budget_units.as_ref() {
if !hook_charge.units.is_finite() || hook_charge.units < 0.0 {
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
}
if hook_charge.overflowed {
// All allowances are finite. No future nonnegative receipt can make
// this total representable; the remaining allowance is known to be zero.
state.posted.add(f64::INFINITY);
if state.config.enforce_limit {
self.stop_new_tools();
}
}
let provider_units = if let Some(units) =
usage.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
{
let units = units.as_f64().unwrap_or(f64::NAN);
if !units.is_finite() || units < 0.0 {
return Err(CodexErr::Fatal(
@@ -64,15 +81,26 @@ impl RolloutBudget {
));
}
units
} else {
} else if let Some(usage) = usage {
usage.output_tokens.max(0) as f64 * state.config.sampling_token_weight
+ usage.non_cached_input() as f64 * state.config.prefill_token_weight
} else {
0.0
};
state.posted.add(units);
if state.posted.total().is_infinite() {
return Ok(state.config.enforce_limit);
}
state.posted.add(provider_units);
state.posted.add(hook_charge.units);
if !state.posted.total().is_finite() {
if state.config.enforce_limit {
self.stop_new_tools();
}
return Err(CodexErr::Fatal("invalid rollout budget charge".into()));
}
// Equality is allowed, including one rounding step (0.1 + 0.2 versus 0.3).
let exhausted = state.config.enforce_limit
&& (state.posted.total().is_infinite()
|| state.posted.total() > state.config.limit_budget.next_up());
&& state.posted.total() > state.config.limit_budget.next_up();
if exhausted {
self.stop_new_tools();
}
+32 -9
View File
@@ -3,11 +3,13 @@ use super::turn_context::TurnContext;
use crate::agent::host::AgentRuntimeServices;
use crate::agent::host::AgentUsageReport;
use crate::agent::host::UsageAccountingDecision;
use crate::client_common::SamplingUsage;
use crate::context::ContextualUserFragment;
use crate::rollout_budget::RolloutBudget;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
pub(super) async fn maybe_record_reminder(
sess: &Session,
@@ -55,28 +57,45 @@ impl Session {
pub(crate) async fn record_rollout_budget_usage(
&self,
turn_id: &str,
bill: &crate::client_common::SamplingUsage,
usage: &SamplingUsage,
) -> CodexResult<()> {
if bill.charge.posted {
if !usage.charge.posted {
self.record_numeric_usage(turn_id, usage.provider.as_ref(), usage.charge.hooks.charge)
.await?;
}
if let Some(message) = &usage.charge.hooks.invalid_price {
// A bad price does not erase successful siblings' contributions.
return Err(CodexErr::UsageAccountingFailed(format!(
"Invalid post-hook budget units: {message}"
)));
}
Ok(())
}
async fn record_numeric_usage(
&self,
turn_id: &str,
usage: Option<&TokenUsage>,
hook_charge: codex_hooks::HookCharge,
) -> CodexResult<()> {
if usage.is_none() && hook_charge == codex_hooks::HookCharge::default() {
return Ok(());
}
let Some(usage) = bill.provider.as_ref() else {
return Ok(());
};
let stop_new_work = match &self.agent_runtime {
AgentRuntimeServices::Controller => match self
.services
.agent_control
.record_usage(usage.clone())
.record_usage(usage.cloned(), hook_charge)
.await
{
Ok(()) => false,
// A legacy controller can report a posted bill and a stop through this error.
Err(error)
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
{
true
}
Err(error) => return Err(error),
Err(error) => return Err(CodexErr::UsageAccountingFailed(error.to_string())),
},
AgentRuntimeServices::Host(services) => {
let Some(accounting) = &services.usage_accounting else {
@@ -86,11 +105,15 @@ impl Session {
.report_usage(AgentUsageReport {
thread_id: self.thread_id(),
turn_id: turn_id.to_owned(),
usage: usage.clone(),
usage: usage.cloned(),
additional_budget_units: hook_charge.units,
additional_budget_units_overflowed: hook_charge.overflowed,
})
.await
.map_err(|error| {
CodexErr::Fatal(format!("Host usage accounting failed: {error}"))
CodexErr::UsageAccountingFailed(format!(
"Host usage accounting failed: {error}"
))
})?;
decision == UsageAccountingDecision::Exhausted
}
+7 -4
View File
@@ -1976,10 +1976,13 @@ async fn spawn_internal_session_preserves_parent_lineage_without_forking_history
.session
.services
.agent_control
.record_usage(TokenUsage {
output_tokens: 25,
..Default::default()
})
.record_usage(
Some(TokenUsage {
output_tokens: 25,
..Default::default()
}),
codex_hooks::HookCharge::default(),
)
.await
.expect("record reviewer usage");
let reminder = parent
@@ -258,7 +258,11 @@ impl AgentControl for TestAgentControl {
None
}
fn record_usage(&self, _usage: TokenUsage) -> BoxFuture<'_, CodexResult<()>> {
fn record_usage(
&self,
_usage: Option<TokenUsage>,
_hook_charge: codex_core::HookCharge,
) -> BoxFuture<'_, CodexResult<()>> {
Box::pin(async { Ok(()) })
}
+199 -44
View File
@@ -6,6 +6,7 @@ use anyhow::Result;
use codex_config::HookStateToml;
use codex_config::McpServerConfig;
use codex_config::test_support::CloudConfigBundleFixture;
use codex_core::AgentRuntimeOwnership;
use codex_core::ForkSnapshot;
use codex_core::StartThreadOptions;
use codex_core::TurnInput;
@@ -4593,19 +4594,12 @@ async fn local_bundled_cleanup_hook_runs_without_saved_trust(
.features
.disable(Feature::ExecutorCapabilityDiscovery)
.expect("disable executor capability discovery");
let repl: McpServerConfig = serde_json::from_value(serde_json::json!({
"url": repl_url,
"environment_id": super::rmcp_client::remote_aware_environment_id(),
}))
.expect("valid MCP configuration");
config
.mcp_servers
.set(
[(String::from(mcp_server_name), repl)]
.into_iter()
.collect(),
)
.expect("configure MCP server");
configure_hook_mcp(
config,
mcp_server_name,
&repl_url,
&super::rmcp_client::remote_aware_environment_id(),
);
});
let test = builder.build_with_auto_env(&server).await?;
assert!(!test.config.bypass_hook_trust);
@@ -6058,6 +6052,22 @@ async fn post_tool_use_records_apply_patch_context_with_edit_alias() -> Result<(
Ok(())
}
fn configure_hook_mcp(
config: &mut codex_core::config::Config,
name: &str,
url: &str,
environment_id: &str,
) {
let server: McpServerConfig = serde_json::from_value(json!({
"url": url, "environment_id": environment_id,
}))
.expect("valid MCP configuration");
config
.mcp_servers
.set([(name.to_owned(), server)].into_iter().collect())
.expect("configure MCP server");
}
pub(super) fn write_lifecycle_hooks(home: &Path, events: &[&str]) -> Result<()> {
let mut hooks = serde_json::Map::new();
for event in events {
@@ -6190,6 +6200,7 @@ enum ToolDispatchCase {
SyncHook,
InterruptedHook,
ExhaustedHook,
OverflowHook(AgentRuntimeOwnership),
}
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
@@ -6198,6 +6209,8 @@ enum ToolDispatchCase {
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "the host receives an overflow lower bound and keeps its policy")]
#[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Controller); "a native overflow closes even the largest allowance")]
#[tokio::test]
async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -6206,20 +6219,68 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
use ToolDispatchCase::Eager;
use ToolDispatchCase::ExhaustedHook;
use ToolDispatchCase::InterruptedHook;
use ToolDispatchCase::OverflowHook;
use ToolDispatchCase::SyncHook;
const MARKER: &str = "tool-dispatched";
const STREAMED: &str = "Still streaming.";
let buffered = matches!(case, AfterComplete | InterruptedHook | ExhaustedHook);
let buffered = matches!(
case,
AfterComplete | InterruptedHook | ExhaustedHook | OverflowHook(_)
);
let hook_async = match case {
Eager | AfterComplete => None,
AsyncHook => Some(true),
SyncHook | InterruptedHook | ExhaustedHook => Some(false),
SyncHook | InterruptedHook | ExhaustedHook | OverflowHook(_) => Some(false),
};
// The native controls both spend 98 of 100. Only the successful +3 price
// crosses the limit; the failed sibling's reported 999 never counts.
let price_tools: &[&str] = match case {
SyncHook => &["failed"],
InterruptedHook | ExhaustedHook => &["price", "failed"],
OverflowHook(_) => &["price", "second_price"],
_ => &[],
};
let pricing = if price_tools.is_empty() {
None
} else {
let server = start_mock_server().await;
mount_hook_mcp(&server, "billing", price_tools, move |params| {
if matches!(case, OverflowHook(_)) {
let output = if params["arguments"]["response_id"] == "first" {
json!({"extra_codex_rollout_budget_units":1e308})
} else {
json!({})
};
json!({"content":[], "structuredContent":output})
} else if params["name"] == "failed" {
json!({"isError":true, "content":[{"type":"text","text":"failed"}],
"structuredContent":{"extra_codex_rollout_budget_units":999}})
} else {
// Turn B must report its own 3, not the interrupted first response's 31.
let units =
if case == InterruptedHook && params["arguments"]["response_id"] != "second" {
31
} else {
3
};
json!({"content":[], "structuredContent":{"extra_codex_rollout_budget_units":units}})
}
})
.await;
Some(server)
};
let pricing_url = pricing
.as_ref()
.map(|server| format!("{}/repl", server.uri()));
let (finish_stream, gate) = oneshot::channel();
let mut completed = ev_completed_with_tokens("first", /*total_tokens*/ 42);
completed["response"]["usage"]["codex_rollout_budget_units"] =
json!(if case == ExhaustedHook { 101 } else { 1 });
json!(if matches!(case, SyncHook | ExhaustedHook) {
98
} else {
1
});
let first = vec![
StreamingSseChunk {
gate: None,
@@ -6243,9 +6304,16 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
body: sse_event(completed),
},
];
let second_bill = if case == InterruptedHook {
let mut completed = ev_completed_with_tokens("second", /*total_tokens*/ 7);
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(7);
completed
} else {
ev_completed("second")
};
let second = vec![StreamingSseChunk {
gate: None,
body: sse(vec![ev_response_created("second"), ev_completed("second")]),
body: sse(vec![ev_response_created("second"), second_bill]),
}];
let (server, _) = start_streaming_sse_server(vec![first, second]).await;
let accounting = Arc::new(super::rollout_budget::HostAccounting::default());
@@ -6262,20 +6330,33 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
let post = format!(
r#"python3 -c 'import json, socket, sys; json.load(sys.stdin).get("response_id") == "first" and socket.create_connection(("127.0.0.1", {port})).recv(1)'"#
);
let hooks = json!({"PostModelRequest": [{"hooks": [{
let mut group = vec![json!({
"type": "command", "command": post, "async": hook_async,
}]}]});
})];
group.extend(price_tools.iter().map(|tool| json!({
"type": "mcp_tool", "server": "billing", "tool": tool,
"input": {"response_id": "${response_id}"}
})));
let hooks = json!({"PostModelRequest": [{"hooks": group}]});
fs::write(home.join("hooks.json"), json!({"hooks": hooks}).to_string())
.expect("write hook configuration");
}
})
.with_config(move |config| {
if let Some(url) = &pricing_url {
configure_hook_mcp(config, "billing", url, codex_config::DEFAULT_MCP_SERVER_ENVIRONMENT_ID);
}
if hook_async.is_some() {
trust_discovered_hooks(config);
}
if matches!(case, Eager | SyncHook | ExhaustedHook) {
config.rollout_budget = Some(super::rollout_budget::rollout_budget());
}
if case == OverflowHook(AgentRuntimeOwnership::Controller) {
let mut budget = super::rollout_budget::rollout_budget();
budget.limit_budget = f64::MAX;
config.rollout_budget = Some(budget);
}
// The builder changes cwd after loading config, so bind the write grant to that cwd.
config.workspace_roots = vec![config.cwd.clone()];
config.permissions.set_workspace_roots(config.workspace_roots.clone());
@@ -6283,7 +6364,10 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
config.permissions.set_permission_profile(restrictive_workspace_write_profile())
.expect("allow the fixture's patch inside its temporary workspace");
});
if case == InterruptedHook {
if matches!(
case,
InterruptedHook | OverflowHook(AgentRuntimeOwnership::Host)
) {
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
extensions.thread_lifecycle_contributor(accounting.clone());
builder = builder.with_extensions(Arc::new(extensions.build()));
@@ -6368,39 +6452,105 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
stopped.run.status_message.as_deref(),
Some("Hook cancelled because the turn was interrupted"),
);
drop(held_hook);
assert!(!marker.exists());
let server = start_mock_server().await;
let requests = mount_sse_once(&server, responses::sse_completed("resumed")).await;
let resumed = test_codex().restart(&server, &test).await?;
resumed.submit_turn("continue after interrupt").await?;
assert!(
!requests
.single_request()
.input()
.iter()
.any(|item| item["call_id"] == "call")
);
assert!(accounting.reports.lock().expect("usage reports").is_empty());
return Ok(());
}
drop(held_hook);
if case == ExhaustedHook {
let error = wait_for_event_match(&test.codex, |event| match event {
EventMsg::Error(error) => Some(error.clone()),
// Start and finish B before releasing A's hook gate. A's unread
// response must never publish its output or charge either turn.
let codex_core::TurnInputSubmission::Started { turn_id, .. } = test
.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "continue after interrupt".into(),
text_elements: vec![],
}]))
.await?
else {
anyhow::bail!("expected a new turn B");
};
let done = wait_for_event_match(&test.codex, |event| match event {
EventMsg::TurnComplete(done) if done.turn_id == turn_id => Some(done.clone()),
_ => None,
})
.await;
assert!(done.error.is_none(), "B failed: {:?}", done.error);
drop(held_hook);
assert!(!marker.exists());
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
assert!(!history.items.iter().any(|item| {
let RolloutItem::ResponseItem(envelope) = item else { return false; };
match &envelope.item {
ResponseItem::CustomToolCall { call_id, .. }
| ResponseItem::CustomToolCallOutput { call_id, .. } => call_id == "call",
ResponseItem::Message { role, content, .. } => role == "assistant"
&& content.iter().any(|part| matches!(part, ContentItem::OutputText { text, .. } if text == STREAMED)),
_ => false,
}
}));
let reports = accounting.reports.lock().expect("usage reports");
let [bill] = reports.as_slice() else {
anyhow::bail!("expected only B's bill: {reports:?}");
};
assert_eq!(bill.turn_id, turn_id);
assert_eq!(
error.codex_error_info,
Some(codex_protocol::protocol::CodexErrorInfo::SessionBudgetExceeded)
bill.usage
.as_ref()
.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
.and_then(serde_json::Number::as_f64),
Some(7.0)
);
assert_eq!(bill.additional_budget_units, 3.0);
return Ok(());
}
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
drop(held_hook);
let done = wait_for_event_match(&test.codex, |event| match event {
EventMsg::TurnComplete(done) => Some(done.clone()),
_ => None,
})
.await;
if let OverflowHook(owner) = case {
assert!(
done.error
.is_some_and(|error| error.message.contains("total hook cost exceeds f64 range"))
);
assert!(!marker.exists());
if owner == AgentRuntimeOwnership::Host {
let reports = accounting.reports.lock().expect("usage reports");
let [bill] = reports.as_slice() else {
anyhow::bail!("expected one partial bill: {reports:?}");
};
assert_eq!(bill.additional_budget_units, 1e308);
assert!(bill.additional_budget_units_overflowed);
assert_eq!(
bill.usage
.as_ref()
.and_then(|usage| usage.codex_rollout_budget_units.as_ref())
.and_then(serde_json::Number::as_f64),
Some(1.0)
);
}
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "try the next turn".into(),
text_elements: vec![],
}]))
.await?;
let done = wait_for_event_match(&test.codex, |event| match event {
EventMsg::TurnComplete(done) => Some(done.clone()),
_ => None,
})
.await;
match owner {
AgentRuntimeOwnership::Host => assert!(done.error.is_none()),
AgentRuntimeOwnership::Controller => assert_eq!(
done.error.and_then(|error| error.codex_error_info),
Some(codex_protocol::protocol::CodexErrorInfo::SessionBudgetExceeded),
),
}
return Ok(());
}
if case == ExhaustedHook {
assert_eq!(
done.error.and_then(|error| error.codex_error_info),
Some(codex_protocol::protocol::CodexErrorInfo::SessionBudgetExceeded)
);
assert!(!marker.exists());
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
@@ -6428,6 +6578,11 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
["tool call", "assistant message", "aborted before dispatch"]
);
} else {
assert!(
done.error.is_none(),
"unexpected turn error: {:?}",
done.error
);
assert_eq!(fs::read_to_string(marker)?, "ran\n");
}
Ok(())
+1 -1
View File
@@ -1732,7 +1732,7 @@ async fn steer_interrupts_and_drains_websocket(
.lock()
.expect("usage reports")
.iter()
.filter_map(|report| report.usage.codex_rollout_budget_units.as_ref())
.filter_map(|report| report.usage.as_ref()?.codex_rollout_budget_units.as_ref())
.map(serde_json::Number::as_i64)
.collect::<Vec<_>>();
assert_eq!(billed_units, [Some(17)]);
+17 -5
View File
@@ -378,12 +378,15 @@ async fn subagent_usage_draws_from_the_shared_budget(host_accounting: bool) -> R
.await;
if host_accounting {
let mut reports = accounting.reports.lock().expect("usage reports").clone();
reports.sort_by_key(|report| report.usage.total_tokens);
reports.sort_by_key(|report| report.usage.as_ref().expect("provider usage").total_tokens);
let root_thread_id = test.session_configured.thread_id;
assert_eq!(
reports
.iter()
.map(|report| (report.thread_id, report.usage.total_tokens))
.map(|report| (
report.thread_id,
report.usage.as_ref().expect("provider usage").total_tokens
))
.collect::<Vec<_>>(),
[
(root_thread_id, 10),
@@ -618,7 +621,14 @@ async fn compaction_accounting_errors_fail_without_retry(
let reports = accounting.reports.lock().expect("usage reports");
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].thread_id, test.session_configured.thread_id);
assert_eq!(reports[0].usage.total_tokens, 10);
assert_eq!(
reports[0]
.usage
.as_ref()
.expect("provider usage")
.total_tokens,
10
);
}
Ok(())
@@ -802,7 +812,7 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
vec![AgentUsageReport {
thread_id: test.session_configured.thread_id,
turn_id: completed.turn_id,
usage: TokenUsage {
usage: Some(TokenUsage {
input_tokens: 60,
cached_input_tokens: 40,
output_tokens: 15,
@@ -812,7 +822,9 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics(
serde_json::Number::from_f64(40.5).expect("finite budget units"),
),
..Default::default()
},
}),
additional_budget_units: 0.0,
additional_budget_units_overflowed: false,
}]
);
Ok(())
+12 -2
View File
@@ -4746,7 +4746,12 @@ async fn async_compaction_reuses_pending_work(
.lock()
.expect("usage reports")
.iter()
.any(|report| report.usage.total_tokens == BACKGROUND_TOKENS);
.any(|report| {
report
.usage
.as_ref()
.is_some_and(|usage| usage.total_tokens == BACKGROUND_TOKENS)
});
release_post.send(()).expect("release the compaction hook");
assert!(
!billed_while_held,
@@ -4802,7 +4807,12 @@ async fn async_compaction_reuses_pending_work(
let reports = accounting.reports.lock().expect("usage reports");
let charged_turns: Vec<_> = reports
.iter()
.filter(|report| report.usage.total_tokens == BACKGROUND_TOKENS)
.filter(|report| {
report
.usage
.as_ref()
.is_some_and(|usage| usage.total_tokens == BACKGROUND_TOKENS)
})
.map(|report| &report.turn_id)
.collect();
assert_eq!(charged_turns, [&usage("seed").turn_id]);
@@ -311,7 +311,7 @@ async fn same_manager_creation_preserves_live_bindings_and_attachments(
accounting.reports.lock().expect("usage reports").as_slice(),
[report] if report.thread_id == child_id
&& report.turn_id == turn_id
&& report.usage.total_tokens == 42
&& report.usage.as_ref().is_some_and(|usage| usage.total_tokens == 42)
));
let child_request = child_request.single_request();
assert_eq!(child_request.body_json()["service_tier"], "flex");