Add response processing after completion and usage accounting (#61577)

## Why

Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget.

## What changed

- Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately.
- Track whether response usage has already been charged to avoid duplicate accounting, including during compaction.
- Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation.
- Bound buffered responses to 65,535 events and reject overflow.

## Testing

Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch.

GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
tongzhou wang
2026-11-14 23:50:12 +00:00
committed by copyberry
parent 81b995a6c9
commit 92c1a4dc2e
32 changed files with 780 additions and 259 deletions
+2
View File
@@ -1710,6 +1710,8 @@ async fn foreground_compaction_only_installs_and_bills_after_its_post_hook() ->
fs::write(home.join("config.toml"), toml::to_string(&config).unwrap()).unwrap();
})
.with_config(move |config| {
config.response_process_mode =
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
config.model_provider = provider;
set_test_compact_prompt(config);
trust_discovered_hooks(config);
@@ -383,8 +383,8 @@ enum ProbeCase {
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Escalated, 1; "explicit_escalation_still_reviews")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Rewrite, 1; "direct_hook_rewrite")]
#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Rewrite, 2; "nested_hook_rewrite")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 1; "interrupted_direct_review")]
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 2; "interrupted_cell_review")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 0; "interrupted_direct_review")]
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 0; "interrupted_cell_review")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 0; "catalog_low_without_legacy_flag")]
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_review_without_legacy_flag")]
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::IncompleteAuthorization, 1; "incomplete_authorization_still_reviews")]
@@ -525,6 +525,9 @@ async fn serving_assessment_replaces_async_scoring(
model.auto_review_model_override = Some("gpt-5.5".into());
})
.with_config(move |config| {
if scenario == ProbeCase::Interrupted {
config.response_process_mode = codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
}
if matches!(scenario, ProbeCase::Rewrite | ProbeCase::Interrupted) {
trust_discovered_hooks(config);
}
@@ -621,6 +624,19 @@ async fn serving_assessment_replaces_async_scoring(
test.codex.shutdown_and_wait().await?;
let requests = parent.requests();
assert_eq!(requests.len(), 2);
if scenario == ProbeCase::Interrupted {
// Buffered processing holds partial output. A failed stream must
// not expose the tool call to Guardian or carry a made-up reply into the retry.
let retry = requests[1].body_json();
assert!(
retry["input"]
.as_array()
.expect("request input")
.iter()
.all(|item| item["call_id"] != "call")
);
return Ok(());
}
let output = requests[1].call_output(
"call",
if code_mode {
+92 -11
View File
@@ -6186,32 +6186,40 @@ async fn model_request_observer_reports_cancellation() -> Result<()> {
enum ToolDispatchCase {
Eager,
AsyncHook,
AfterComplete,
SyncHook,
InterruptedHook,
ExhaustedHook,
}
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
#[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")]
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered output waits for completion without hooks")]
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt cancels the synchronous hook before billing")]
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
#[tokio::test]
async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> Result<()> {
skip_if_no_network!(Ok(()));
use ToolDispatchCase::AfterComplete;
use ToolDispatchCase::AsyncHook;
use ToolDispatchCase::Eager;
use ToolDispatchCase::ExhaustedHook;
use ToolDispatchCase::InterruptedHook;
use ToolDispatchCase::SyncHook;
const MARKER: &str = "tool-dispatched";
const STREAMED: &str = "Still streaming.";
let buffered = matches!(case, AfterComplete | InterruptedHook | ExhaustedHook);
let hook_async = match case {
Eager => None,
Eager | AfterComplete => None,
AsyncHook => Some(true),
SyncHook | InterruptedHook => Some(false),
SyncHook | InterruptedHook | ExhaustedHook => Some(false),
};
let (finish_stream, gate) = oneshot::channel();
let mut completed = ev_completed_with_tokens("first", /*total_tokens*/ 42);
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(1);
completed["response"]["usage"]["codex_rollout_budget_units"] =
json!(if case == ExhaustedHook { 101 } else { 1 });
let first = vec![
StreamingSseChunk {
gate: None,
@@ -6246,6 +6254,10 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
let port = hook_gate.local_addr()?.port();
let mut builder = test_codex()
.with_pre_build_hook(move |home| {
if buffered {
fs::write(home.join("config.toml"), "[responses]\nprocess_mode = \"after_complete_response\"\n")
.expect("configure response processing after completion");
}
if let Some(hook_async) = hook_async {
let post = format!(
r#"python3 -c 'import json, socket, sys; json.load(sys.stdin).get("response_id") == "first" and socket.create_connection(("127.0.0.1", {port})).recv(1)'"#
@@ -6261,7 +6273,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
if hook_async.is_some() {
trust_discovered_hooks(config);
}
if matches!(case, Eager | SyncHook) {
if matches!(case, Eager | SyncHook | ExhaustedHook) {
config.rollout_budget = Some(super::rollout_budget::rollout_budget());
}
// The builder changes cwd after loading config, so bind the write grant to that cwd.
@@ -6271,7 +6283,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
config.permissions.set_permission_profile(restrictive_workspace_write_profile())
.expect("allow the fixture's patch inside its temporary workspace");
});
if matches!(case, SyncHook | InterruptedHook) {
if case == InterruptedHook {
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
extensions.thread_lifecycle_contributor(accounting.clone());
builder = builder.with_extensions(Arc::new(extensions.build()));
@@ -6290,8 +6302,12 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
saw_delta |= matches!(event, EventMsg::AgentMessageContentDelta(message) if message.delta == STREAMED);
matches!(event, EventMsg::SafetyBuffering(_))
}).await;
assert!(saw_delta);
fs_wait::wait_for_path_exists(&marker, Duration::from_secs(10)).await?;
assert_eq!(saw_delta, !buffered);
if buffered {
assert!(!marker.exists());
} else {
fs_wait::wait_for_path_exists(&marker, Duration::from_secs(10)).await?;
}
finish_stream.send(()).expect("release response stream");
let held_hook = if hook_async.is_some() {
Some(
@@ -6302,8 +6318,21 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
} else {
None
};
if hook_async == Some(false) {
assert!(accounting.reports.lock().expect("usage reports").is_empty());
if buffered && hook_async == Some(false) {
assert!(!marker.exists());
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
assert!(
!history.items.iter().any(|item| matches!(item,
RolloutItem::ResponseItem(envelope) if match &envelope.item {
ResponseItem::CustomToolCall { call_id, .. } => call_id == "call",
ResponseItem::Message { role, content, .. } => role == "assistant" && content.iter().any(
|part| matches!(part, ContentItem::OutputText { text, .. } if text == STREAMED)),
_ => false,
}
)),
"the completed response is still waiting for its synchronous hook"
);
}
if case == InterruptedHook {
let started = wait_for_event_match(&test.codex, |event| match event {
@@ -6340,15 +6369,67 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
Some("Hook cancelled because the turn was interrupted"),
);
drop(held_hook);
assert!(!marker.exists());
let server = start_mock_server().await;
let requests = mount_sse_once(&server, responses::sse_completed("resumed")).await;
let resumed = test_codex().restart(&server, &test).await?;
resumed.submit_turn("continue after interrupt").await?;
assert!(
!requests
.single_request()
.input()
.iter()
.any(|item| item["call_id"] == "call")
);
assert!(accounting.reports.lock().expect("usage reports").is_empty());
return Ok(());
}
drop(held_hook);
if case == ExhaustedHook {
let error = wait_for_event_match(&test.codex, |event| match event {
EventMsg::Error(error) => Some(error.clone()),
_ => None,
})
.await;
assert_eq!(
error.codex_error_info,
Some(codex_protocol::protocol::CodexErrorInfo::SessionBudgetExceeded)
);
}
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert_eq!(fs::read_to_string(marker)?, "ran\n");
if case == ExhaustedHook {
assert!(!marker.exists());
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
let recorded = history
.items
.iter()
.filter_map(|item| match item {
RolloutItem::ResponseItem(envelope) => match &envelope.item {
ResponseItem::Message { role, content, .. } if role == "assistant" && content.iter().any(
|part| matches!(part, ContentItem::OutputText { text, .. } if text == STREAMED)
) => Some("assistant message"),
ResponseItem::CustomToolCall { call_id, .. } if call_id == "call" => {
Some("tool call")
}
ResponseItem::CustomToolCallOutput {
call_id, output, ..
} if call_id == "call" => output.text_content(),
_ => None,
},
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
recorded,
["tool call", "assistant message", "aborted before dispatch"]
);
} else {
assert_eq!(fs::read_to_string(marker)?, "ran\n");
}
Ok(())
}
+47 -2
View File
@@ -2,6 +2,7 @@ use core_test_support::test_codex::local_requests;
use std::sync::Arc;
use std::time::Duration;
use codex_config::config_toml::ResponseProcessMode;
use codex_core::CodexThread;
use codex_core::InputWaitOutcome;
use codex_core::StartIfIdleSubmission;
@@ -1597,7 +1598,15 @@ async fn steer_interrupts_and_drains_websocket(
) -> anyhow::Result<()> {
core_test_support::skip_if_no_network!(Ok(()));
const CALL_ID: &str = "stale-write";
const FILE_NAME: &str = "stale-write.txt";
let mut write = responses::ev_apply_patch_custom_tool_call(
CALL_ID,
&format!("*** Begin Patch\n*** Add File: {FILE_NAME}\n+stale\n*** End Patch"),
);
write["item"]["id"] = json!("stale-write-item");
let mut interrupted = ev_completed_with_tokens("resp-interrupted", /*total_tokens*/ 42);
interrupted["response"]["usage"]["codex_rollout_budget_units"] = json!(17);
if acknowledgement == Some("response.interrupt.accepted") {
interrupted["type"] = json!("response.incomplete");
interrupted["response"]["status"] = json!("incomplete");
@@ -1607,7 +1616,10 @@ async fn steer_interrupts_and_drains_websocket(
vec![ev_response_created("warm-1"), ev_completed("warm-1")],
vec![
ev_response_created("resp-interrupted"),
write,
ev_reasoning_item_added("reason-1", &["thinking"]),
json!({"type":"response.metadata", "response_id":"resp-interrupted",
"metadata":{"type":"safety_buffering","use_cases":["cyber"],"reasons":["policy-check"]}}),
],
// Finish the active response only after receiving response.interrupt.
[
@@ -1623,7 +1635,7 @@ async fn steer_interrupts_and_drains_websocket(
"type": "response.output_item.interrupted",
"response_id": "resp-interrupted",
"item_id": "reason-1",
"output_index": 0,
"output_index": 1,
"sequence_number": 4,
})
} else {
@@ -1640,12 +1652,27 @@ async fn steer_interrupts_and_drains_websocket(
],
]])
.await;
let accounting = Arc::new(super::rollout_budget::HostAccounting::default());
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
extensions.thread_lifecycle_contributor(accounting.clone());
let test = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_model("gpt-5.4")
.with_model_info_override("gpt-5.4", |model_info| {
model_info.use_responses_lite = true;
})
.with_config(|config| {
config.response_process_mode = ResponseProcessMode::AfterCompleteResponse;
config.workspace_roots = vec![config.cwd.clone()];
config
.permissions
.set_workspace_roots(config.workspace_roots.clone());
config.permissions.approval_policy =
codex_core::config::Constrained::allow_any(AskForApproval::Never);
config
.permissions
.set_permission_profile(PermissionProfile::workspace_write())
.expect("allow the fixture's patch inside its temporary workspace");
config
.features
.enable(Feature::InstantInterrupt)
@@ -1656,7 +1683,11 @@ async fn steer_interrupts_and_drains_websocket(
let codex = &test.codex;
submit_user_input(codex, "first prompt").await;
wait_for_reasoning_item_started(codex).await;
// The safety notification follows the buffered call and reasoning item. The
// websocket is still open, and the provider awaits the client's interrupt.
wait_for_event(codex, |event| matches!(event, EventMsg::SafetyBuffering(_))).await;
let marker = test.workspace_path(FILE_NAME);
assert!(!marker.try_exists()?);
let initial_request = server
.wait_for_request(/*connection_index*/ 0, /*request_index*/ 1)
.await
@@ -1682,6 +1713,10 @@ async fn steer_interrupts_and_drains_websocket(
.filter(|text| text == "first prompt" || text == "second prompt")
.collect::<Vec<_>>();
assert_eq!(prompts, vec!["second prompt"]);
assert_eq!(
call_output_text(&follow_up, "custom_tool_call_output", CALL_ID).as_deref(),
Some("aborted before dispatch")
);
wait_for_event(codex, |event| {
matches!(
event,
@@ -1691,6 +1726,16 @@ async fn steer_interrupts_and_drains_websocket(
})
.await;
wait_for_successful_turn(codex).await;
assert!(!marker.try_exists()?);
let billed_units = accounting
.reports
.lock()
.expect("usage reports")
.iter()
.filter_map(|report| report.usage.codex_rollout_budget_units.as_ref())
.map(serde_json::Number::as_i64)
.collect::<Vec<_>>();
assert_eq!(billed_units, [Some(17)]);
assert_eq!(server.connections().len(), 1);
codex.shutdown_and_wait().await?;
+41 -21
View File
@@ -203,6 +203,10 @@ async fn adds_weighted_initial_and_threshold_reminders(
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()> {
use codex_history::RolloutItem;
use codex_protocol::models::ResponseItem;
use core_test_support::responses::ev_apply_patch_custom_tool_call;
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -210,13 +214,25 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(-1.0);
let responses = mount_sse_sequence(
&server,
vec![sse(vec![ev_response_created("invalid-units"), completed])],
vec![sse(vec![
ev_response_created("invalid-units"),
ev_apply_patch_custom_tool_call(
"unbilled-patch",
"*** Begin Patch\n*** Add File: unbilled-tool-ran\n+unexpected\n*** End Patch",
),
completed,
])],
)
.await;
let test = test_codex()
.with_config(|config| {
config.rollout_budget = Some(rollout_budget());
.with_pre_build_hook(|home| {
std::fs::write(
home.join("config.toml"),
"[responses]\nprocess_mode = \"after_complete_response\"\n",
)
.expect("configure buffered response processing");
})
.with_config(|config| config.rollout_budget = Some(rollout_budget()))
.build(&server)
.await?;
@@ -232,11 +248,8 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
else {
unreachable!();
};
assert_eq!(
error.message,
"Fatal error: response.completed usage.codex_rollout_budget_units must be finite and non-negative"
);
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
assert!(error.message.contains("codex_rollout_budget_units"));
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
@@ -246,6 +259,16 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
1,
"invalid units should not retry"
);
// Invalid accounting rejects the buffered response before any of its items are published.
assert!(!test.workspace_path("unbilled-tool-ran").exists());
test.codex.ensure_rollout_materialized().await;
let history = test.codex.load_history(/*include_archived*/ false).await?;
assert!(!history.items.iter().any(|item| matches!(item,
RolloutItem::ResponseItem(envelope) if matches!(&envelope.item,
ResponseItem::CustomToolCall { call_id, .. }
| ResponseItem::CustomToolCallOutput { call_id, .. } if call_id == "unbilled-patch"
)
)));
Ok(())
}
@@ -403,20 +426,17 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("exhaust-budget"),
ev_completed_with_tokens("exhaust-budget", /*total_tokens*/ 30),
]),
sse(vec![
ev_response_created("already-exhausted"),
ev_completed_with_tokens("already-exhausted", /*total_tokens*/ 1),
]),
],
)
.await;
let mut sequence = vec![sse(vec![
ev_response_created("exhaust-budget"),
ev_completed_with_tokens("exhaust-budget", /*total_tokens*/ 30),
])];
if owner == AgentRuntimeOwnership::Host || !enforce_limit {
sequence.push(sse(vec![
ev_response_created("already-exhausted"),
ev_completed_with_tokens("already-exhausted", /*total_tokens*/ 1),
]));
}
let responses = mount_sse_sequence(&server, sequence).await;
let mut extensions = ExtensionRegistryBuilder::new();
if owner == AgentRuntimeOwnership::Host {
extensions.thread_lifecycle_contributor(Arc::new(HostRuntime));
+2
View File
@@ -4651,6 +4651,8 @@ async fn async_compaction_reuses_pending_work(
env,
move |config| {
if observe_attempts {
config.response_process_mode =
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
core_test_support::hooks::trust_discovered_hooks(config);
}
if over_budget {