Recheck rollout budgets before starting tool work (#61578)

## Why

A rollout budget can be exhausted while a tool call waits for an execution slot, approval, or a lifecycle callback. Checking only when scheduling the call can allow new work to start after the budget is exhausted.

## What changed

- Recheck the budget after acquiring an execution slot, immediately before starting a handler, and immediately before handing an MCP request to the server.
- Report refused calls as aborted in tool outputs, lifecycle callbacks, and dispatch traces, while allowing already-running work to complete.
- Track handler start time separately from dispatch waiting and preserve terminal outcomes when cancellation arrives during completion callbacks.

## Testing

Add regression coverage for queued calls and calls paused in start callbacks when the budget is exhausted. Verify MCP requests are blocked before handoff and retain successful results after handoff, and assert budget-stop state during remote compaction.

GitOrigin-RevId: 573730391452256592d3512f15929a2fbdf03c57
This commit is contained in:
tongzhou wang
2026-11-14 23:52:50 +00:00
committed by copyberry
parent 92c1a4dc2e
commit f226c9ccfb
13 changed files with 590 additions and 191 deletions
+27 -6
View File
@@ -417,6 +417,22 @@ impl PreparedMcpCall {
requested_timeout: Option<Duration>,
prepare: F,
) -> Result<CallToolResult>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<(Option<JsonValue>, Option<JsonValue>)>>,
{
self.call_with_admission(requested_timeout, || Ok(()), prepare)
.await
}
/// Checks admission on the negotiated session immediately before handing a tool request
/// to the peer. An in-flight request can complete even if new work is stopped.
pub async fn call_with_admission<F, Fut>(
&self,
requested_timeout: Option<Duration>,
admit: impl Fn() -> Result<()> + Send + Sync,
prepare: F,
) -> Result<CallToolResult>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<(Option<JsonValue>, Option<JsonValue>)>>,
@@ -466,16 +482,21 @@ impl PreparedMcpCall {
}
None => None,
};
let policy_meta = self.connections.network_policy_meta_for_call(
&self.client,
Some(self.permission_profile()),
self.server_environment_id(),
meta,
);
self.client
.call_tool(
tool_name.clone(),
arguments,
self.connections.network_policy_meta_for_call(
&self.client,
Some(self.permission_profile()),
self.server_environment_id(),
meta,
),
|capabilities| {
let meta = policy_meta(capabilities)?;
admit()?;
Ok(meta)
},
remaining_timeout,
)
.await
+1
View File
@@ -19,6 +19,7 @@ pub struct AgentUsageReport {
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum UsageAccountingDecision {
Continue,
/// Do not start more tool handlers in this local rollout tree. Already-running work continues.
Exhausted,
}
+2
View File
@@ -1472,11 +1472,13 @@ mod tests {
assert!(
matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted))
);
assert!(!sess.rollout_budget().tools_stopped());
assert!(accounting.reports.lock().unwrap().is_empty());
return;
}
assert!(matches!(result, Err(ref err)
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_))));
assert!(sess.rollout_budget().tools_stopped());
assert_eq!(
*accounting.reports.lock().unwrap(),
vec![AgentUsageReport {
+59 -19
View File
@@ -8,6 +8,7 @@ use crate::config::Config;
use crate::config::edit::ConfigEdit;
use crate::config::edit::ConfigEditsBuilder;
use crate::connectors;
use crate::function_tool::FunctionCallError;
use crate::guardian::GuardianApprovalRequest;
use crate::guardian::GuardianMcpAnnotations;
use crate::guardian::GuardianReviewContext;
@@ -167,7 +168,7 @@ pub(crate) async fn handle_mcp_tool_call(
invocation_tool_name: ToolName,
source: &crate::tools::context::ToolCallSource,
arguments: String,
) -> HandledMcpToolCall {
) -> Result<CompletedMcpToolCall, FunctionCallError> {
let turn_context = &step_context.turn;
let server = advertised_tool.server_name().to_owned();
let tool_name = advertised_tool.tool.name.to_string();
@@ -175,10 +176,10 @@ pub(crate) async fn handle_mcp_tool_call(
Ok(value) => value,
Err(e) => {
error!("failed to parse tool call arguments: {e}");
return HandledMcpToolCall {
return Ok(CompletedMcpToolCall {
result: CallToolResult::from_error_text(format!("err: {e}")),
tool_input: JsonValue::Object(serde_json::Map::new()),
};
});
}
};
@@ -230,11 +231,11 @@ pub(crate) async fn handle_mcp_tool_call(
/*already_started*/ false,
)
.await;
return HandledMcpToolCall {
return Ok(CompletedMcpToolCall {
result: CallToolResult::from_result(result),
tool_input: arguments_value
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())),
};
});
}
};
let item_metadata = McpToolCallItemMetadata::from_tool_metadata(&server, Some(&metadata));
@@ -263,11 +264,11 @@ pub(crate) async fn handle_mcp_tool_call(
connector_name.as_deref(),
/*duration*/ None,
);
return HandledMcpToolCall {
return Ok(CompletedMcpToolCall {
result: CallToolResult::from_result(result),
tool_input: arguments_value
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())),
};
});
};
let _approval_metadata = sess.register_mcp_tool_approval_metadata(
&call_id,
@@ -396,11 +397,11 @@ pub(crate) async fn handle_mcp_tool_call(
/*duration*/ None,
);
return HandledMcpToolCall {
return Ok(CompletedMcpToolCall {
result: CallToolResult::from_result(result),
tool_input: arguments_value
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())),
};
});
}
handle_approved_mcp_tool_call(
@@ -418,11 +419,16 @@ pub(crate) async fn handle_mcp_tool_call(
.await
}
pub(crate) struct HandledMcpToolCall {
pub(crate) struct CompletedMcpToolCall {
pub(crate) result: CallToolResult,
pub(crate) tool_input: JsonValue,
}
/// Preserves the local admission failure through RMCP's transport error context.
#[derive(Debug, thiserror::Error)]
#[error(transparent)]
struct McpAdmissionError(FunctionCallError);
#[derive(Clone, Debug, PartialEq, Eq)]
struct McpToolCallItemMetadata {
connector_id: Option<String>,
@@ -480,7 +486,7 @@ async fn handle_approved_mcp_tool_call(
item_metadata: McpToolCallItemMetadata,
source: &crate::tools::context::ToolCallSource,
approval_application: McpToolApprovalApplication,
) -> HandledMcpToolCall {
) -> Result<CompletedMcpToolCall, FunctionCallError> {
let turn_context = step_context.turn.as_ref();
let server = invocation.server.clone();
let tool_name = invocation.tool.clone();
@@ -521,17 +527,31 @@ async fn handle_approved_mcp_tool_call(
/*already_started*/ true,
)
.await;
return HandledMcpToolCall {
return Ok(CompletedMcpToolCall {
result: CallToolResult::from_result(result),
tool_input,
};
});
}
let mut elicitation_type = None;
let mut request_error_code = None;
let mut admission_error = None;
let admit = || {
let reason = sess
.rollout_budget()
.tools_stopped()
.then(|| "rollout budget exhausted".to_owned());
if let Some(reason) = reason {
return Err(McpAdmissionError(FunctionCallError::RespondToModel(format!(
"aborted before MCP handoff: {reason}"
)))
.into());
}
Ok(())
};
let result = async {
let result = async {
let result = prepared_call
.call_with_preparation(/*requested_timeout*/ None, || async {
.call_with_admission(/*requested_timeout*/ None, admit, || async {
if let McpToolApprovalApplication::Apply { decision, policy } =
&approval_application
{
@@ -680,9 +700,16 @@ async fn handle_approved_mcp_tool_call(
Ok((rewritten_arguments, request_meta))
})
.await
.map_err(|error| {
request_error_code = codex_rmcp_client::request_error_code(&error);
format!("tool call error: {error:?}")
.map_err(|error| match error.downcast::<McpAdmissionError>() {
Ok(McpAdmissionError(error)) => {
let message = error.to_string();
admission_error = Some(error);
message
}
Err(error) => {
request_error_code = codex_rmcp_client::request_error_code(&error);
format!("tool call error: {error:?}")
}
});
// Keep the backend response intact, before callbacks or model/UI limits can change it.
sess.services.rollout_thread_trace.record_mcp_tool_result(
@@ -747,6 +774,19 @@ async fn handle_approved_mcp_tool_call(
},
))
.await;
if let Some(error) = admission_error {
notify_mcp_tool_call_completed(
sess,
turn_context,
call_id,
invocation,
item_metadata,
Duration::ZERO,
truncate_mcp_tool_result_for_event(&result),
)
.await;
return Err(error);
}
if let Err(error) = &result {
tracing::warn!("MCP tool call error: {error:?}");
}
@@ -792,10 +832,10 @@ async fn handle_approved_mcp_tool_call(
Some(duration),
);
HandledMcpToolCall {
Ok(CompletedMcpToolCall {
result: CallToolResult::from_result(result),
tool_input,
}
})
}
fn mcp_tool_call_span(
@@ -196,3 +196,171 @@ async fn explicit_cloud_refresh_runs_concurrently_and_awaits_all_results() {
.expect("completed refresh must release the catalog lock"),
);
}
#[derive(Clone, Copy)]
enum McpPause {
Start,
Result,
}
struct PauseMcpCall {
phase: McpPause,
entered: Notify,
release: Notify,
outcome: std::sync::OnceLock<codex_extension_api::ToolCallOutcome>,
}
impl codex_extension_api::ToolLifecycleContributor for PauseMcpCall {
fn on_tool_start<'a>(
&'a self,
input: codex_extension_api::ToolStartInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
Box::pin(async move {
assert_eq!(input.mcp_tool.unwrap().tool_info().tool.name, "probe");
if matches!(self.phase, McpPause::Start) {
self.entered.notify_one();
self.release.notified().await;
}
})
}
fn on_mcp_tool_result<'a>(
&'a self,
_input: codex_extension_api::McpToolResultInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
Box::pin(async move {
if matches!(self.phase, McpPause::Result) {
self.entered.notify_one();
self.release.notified().await;
}
})
}
fn on_tool_finish<'a>(
&'a self,
input: codex_extension_api::ToolFinishInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
Box::pin(async move { self.outcome.set(input.outcome).expect("one finish") })
}
}
#[test_case::test_case(McpPause::Start; "stop_before_send")]
#[test_case::test_case(McpPause::Result; "stop_after_send")]
#[tokio::test]
async fn mcp_budget_stop_at_handoff(phase: McpPause) -> anyhow::Result<()> {
use crate::tools::parallel::ToolCallRuntime;
use crate::tools::router::ToolCall;
use codex_extension_api::ToolCallOutcome;
use core_test_support::hooks::mount_hook_mcp;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
core_test_support::skip_if_no_network!(Ok(()));
let endpoint = wiremock::MockServer::start().await;
let calls = Arc::new(AtomicUsize::new(0));
let received = Arc::clone(&calls);
mount_hook_mcp(&endpoint, "handoff", &["probe"], move |_| {
received.fetch_add(1, Ordering::Relaxed);
json!({"content": [{"type":"text", "text":"peer result"}]})
})
.await;
let observer = Arc::new(PauseMcpCall {
phase,
entered: Notify::new(),
release: Notify::new(),
outcome: Default::default(),
});
let (mut session, _) = make_session_and_context().await;
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
extensions.tool_lifecycle_contributor(observer.clone());
session.services.extensions = Arc::new(extensions.build());
session
.rollout_budget()
.configure(crate::config::RolloutBudgetConfig {
enforce_limit: true,
limit_tokens: 1,
reminder_at_remaining_tokens: Vec::new(),
sampling_token_weight: 1.0,
prefill_token_weight: 1.0,
});
{
let mut state = session.state.lock().await;
let config = Arc::make_mut(&mut state.session_configuration.original_config_do_not_use);
config
.mcp_servers
.set(serde_json::from_value(json!({"handoff": {
"url": format!("{}/repl", endpoint.uri()),
"default_tools_approval_mode": "approve"
}}))?)?;
}
let session = Arc::new(session);
session.mark_mcp_runtime_dirty(McpRefreshReason::ExplicitInvalidation);
session
.wait_for_mcp_server(&codex_mcp::McpServerKey::unscoped("handoff"))
.await;
let turn = session.new_default_turn().await;
let cancel = CancellationToken::new();
let step = session
.capture_step_context_with_required_mcp_servers(
turn,
&cancel,
&["handoff".to_owned()],
&HashSet::new(),
)
.await?;
let tool = step
.mcp
.tools()
.iter()
.find(|tool| tool.server_name() == "handoff" && tool.tool.name == "probe")
.expect("MCP server advertised probe");
let call = ToolCall {
tool_name: tool.canonical_tool_name(),
call_id: "handoff-probe".into(),
payload: ToolPayload::Function {
arguments: "{}".into(),
},
encrypted_function_args: None,
native_call: None,
};
let runtime = ToolCallRuntime::new(
session.clone(),
step,
Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())),
);
let response = runtime.handle_tool_call(call, cancel);
tokio::pin!(response);
tokio::select! {
() = observer.entered.notified() => {}
reply = &mut response => anyhow::bail!("MCP call finished before the observer paused: {reply:?}"),
}
let usage = codex_protocol::protocol::TokenUsage {
output_tokens: 2,
..Default::default()
};
session
.record_rollout_budget_usage("peer-turn", &usage.into())
.await?;
observer.release.notify_one();
let reply = response.await?;
let (count, outcome) = match phase {
McpPause::Start => {
assert_eq!(
reply.item,
ResponseInputItem::FunctionCallOutput {
call_id: "handoff-probe".into(),
output: FunctionCallOutputPayload::from_text(
"aborted before MCP handoff: rollout budget exhausted".into()
),
}
.into()
);
(0, ToolCallOutcome::Aborted)
}
McpPause::Result => (1, ToolCallOutcome::Completed { success: true }),
};
assert_eq!(calls.load(Ordering::Relaxed), count);
assert_eq!(observer.outcome.get(), Some(&outcome));
session.services.mcp_runtime.shutdown().await;
Ok(())
}
+2 -6
View File
@@ -382,14 +382,10 @@ pub(crate) async fn handle_output_item_done(
let cancellation_token = ctx.cancellation_token.child_token();
let runtime = ctx.tool_runtime.clone();
let dispatch = |allowed: bool| -> InFlightFuture<'static> {
if allowed
&& !ctx.sess.rollout_budget().tools_stopped()
&& !cancellation_token.is_cancelled()
{
if allowed && !cancellation_token.is_cancelled() {
Box::pin(runtime.handle_tool_call(call, cancellation_token))
} else {
let output =
ToolCallRuntime::aborted_response(&call, /*elapsed_secs*/ None);
let output = ToolCallRuntime::aborted_before_dispatch(&call);
Box::pin(std::future::ready(Ok(
output.into_response(/*file_backed_tool_audio*/ false)
)))
+8 -2
View File
@@ -6,6 +6,7 @@ use crate::session::turn_context::TurnEnvironment;
use crate::turn_diff_tracker::TurnDiffTracker;
use crate::unified_exec::output::CapturedStream;
use crate::unified_exec::resolve_max_tokens;
use codex_extension_api::ToolCallOutcome;
use codex_protocol::ResponseItemId;
use codex_protocol::mcp::CallToolResult;
use codex_protocol::models::FunctionCallOutputBody;
@@ -25,8 +26,8 @@ use serde::Serialize;
use serde_json::Value as JsonValue;
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
@@ -46,7 +47,8 @@ pub type SharedTurnDiffTracker = Arc<Mutex<TurnDiffTracker>>;
/// Delivery does not finish the call: post-tool hooks can still be cancelled.
#[derive(Default)]
pub(crate) struct ToolCallState {
pub(crate) terminal_outcome_reached: AtomicBool,
pub(crate) terminal_outcome: OnceLock<ToolCallOutcome>,
pub(crate) handler_started_at: OnceLock<Instant>,
pub(crate) delivered_assistant_message: OnceLock<String>,
pub(crate) invocation: crate::agent::host::InvocationReservation,
}
@@ -396,6 +398,10 @@ pub struct AbortedToolOutput {
}
impl ToolOutput for AbortedToolOutput {
fn was_aborted(&self) -> bool {
true
}
fn log_output(&self) -> String {
self.message.clone()
}
+11 -2
View File
@@ -7,9 +7,11 @@ use crate::context::NodeReplReviewEvidenceMode;
use crate::context::node_repl_review_evidence_mode;
use crate::function_tool::FunctionCallError;
use crate::image_preparation::prompt_image_decoder;
use crate::mcp_tool_call::CompletedMcpToolCall;
use crate::mcp_tool_call::handle_mcp_tool_call;
use crate::original_image_detail::can_request_original_image_detail;
use crate::session::session::Session;
use crate::tools::context::AbortedToolOutput;
use crate::tools::context::McpToolOutput;
use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolEnvironments;
@@ -314,9 +316,16 @@ impl McpHandler {
))
.await;
let (result, tool_input) = match result {
Ok(CompletedMcpToolCall { result, tool_input }) => (result, tool_input),
Err(FunctionCallError::RespondToModel(message)) => {
return Ok(boxed_tool_output(AbortedToolOutput { message }));
}
Err(error) => return Err(error),
};
Ok(boxed_tool_output(McpToolOutput {
result: result.result,
tool_input: result.tool_input,
result,
tool_input,
result_metadata_capture_allowed,
wall_time: started.elapsed(),
original_image_detail_supported: can_request_original_image_detail(
+241 -132
View File
@@ -1,6 +1,4 @@
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::Ordering;
use std::time::Instant;
use tokio::sync::RwLock;
@@ -30,6 +28,7 @@ use crate::tools::router::ToolCallSource;
use codex_direct_tool_timing::ToolAdmission;
use codex_direct_tool_timing::ToolBatchTiming;
use codex_direct_tool_timing::ToolGateTiming;
use codex_extension_api::ToolCallOutcome;
use codex_history::ResponseItemEnvelope;
use codex_protocol::error::CodexErr;
use codex_protocol::models::ResponseInputItem;
@@ -37,7 +36,7 @@ use codex_protocol::models::ToolResultMetadata;
struct ToolCallTimingGuard {
started_at: Option<Instant>,
execution_started_at: Arc<OnceLock<Instant>>,
call_state: Arc<ToolCallState>,
conversation_id: String,
turn_id: String,
call_id: String,
@@ -61,6 +60,23 @@ struct DirectToolTiming {
batch: ToolBatchTiming,
}
enum ToolDispatchError {
Call(FunctionCallError),
Aborted(ToolAbort),
}
#[derive(Clone, Copy)]
enum ToolAbort {
BeforeDispatch,
User(f32),
}
impl From<FunctionCallError> for ToolDispatchError {
fn from(error: FunctionCallError) -> Self {
Self::Call(error)
}
}
// Direct spawn and wait calls use this turn's tool time; child work stays on its own turn.
// CodeMode and follow-up agent operations still lack runtime capture.
fn direct_path_supported(tool_name: &codex_tools::ToolName, source: &ToolCallSource) -> bool {
@@ -235,6 +251,7 @@ impl ToolCallRuntime {
&turn.sub_id,
&call,
&source,
Arc::clone(&call_state),
Arc::clone(&session.services.extensions),
);
if let Some(guard) = tool_call_timing_guard.as_mut() {
@@ -247,9 +264,6 @@ impl ToolCallRuntime {
});
}
}
let execution_started_at = tool_call_timing_guard
.as_ref()
.map(|timing| Arc::clone(&timing.execution_started_at));
let abort_session = Arc::clone(&session);
let abort_source = source.clone();
let abort_turn = Arc::clone(&turn);
@@ -264,6 +278,17 @@ impl ToolCallRuntime {
};
let dispatch_tool_name = call.tool_name.clone();
let dispatch_call_id = call.call_id.clone();
let record_result_ready =
move |turn_id: &str, tool_name: &codex_tools::ToolName, call_id: &str| {
call_trace::result_ready(
thread_id,
turn_id,
tool_name,
call_id,
trace_source,
started.elapsed(),
);
};
// Code-mode callbacks can resume outside the turn's local span ancestry.
let dispatch_span = trace_span!(
@@ -312,6 +337,12 @@ impl ToolCallRuntime {
}
.or_cancel_tool(&invocation_cancellation_token)
.await?;
// A peer may have exhausted the account while this call waited for its slot.
if session.rollout_budget().tools_stopped() {
tracing::Span::current().record("aborted", true);
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch));
}
// Publish the completed path before releasing the admission lock,
// including when cancellation drops the handler future.
let _admitted_timing = work_timing.map(|(gate, work)| {
@@ -325,12 +356,6 @@ impl ToolCallRuntime {
admission_started,
)
});
// Admission through the parallel-execution gate marks the end
// of dispatch waiting and the start of handler execution.
if let Some(execution_started_at) = execution_started_at {
let _ = execution_started_at.set(Instant::now());
}
let result = router
.dispatch_tool_call_with_state(
session,
@@ -349,24 +374,17 @@ impl ToolCallRuntime {
// A fatal error still propagates to the caller instead of producing a tool
// result; unlike a normal tool failure, it has no readiness event.
if !matches!(&result, Err(FunctionCallError::Fatal(_))) {
call_trace::result_ready(
thread_id,
&turn.sub_id,
&dispatch_tool_name,
&dispatch_call_id,
trace_source,
started.elapsed(),
);
record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id);
}
// Finished calls release their IDs even if their results have not been collected.
// Otherwise the outer task may still need the ID for its abort callback.
if dispatch_call_state
.terminal_outcome_reached
.load(Ordering::Acquire)
{
if let Some(outcome) = dispatch_call_state.terminal_outcome.get() {
if *outcome == ToolCallOutcome::Aborted {
tracing::Span::current().record("aborted", true);
}
drop(dispatch_call_state.invocation.take());
}
result
result.map_err(ToolDispatchError::Call)
}
.instrument(dispatch_span.clone().or_current()),
));
@@ -382,7 +400,7 @@ impl ToolCallRuntime {
_ = cancellation_token.cancelled() => {
// Dynamic handlers observe the token and publish their terminal item themselves.
if finishes_on_cancellation
|| call_state.terminal_outcome_reached.load(Ordering::Acquire)
|| call_state.terminal_outcome.get().is_some()
|| dispatch_handle.is_finished()
{
dispatch_handle.await.map_err(join_error)?
@@ -393,28 +411,10 @@ impl ToolCallRuntime {
match dispatch_handle.await {
Ok(result) => result,
Err(err) if err.is_cancelled() => {
let _reservation = call_state.invocation.take();
let response = Self::aborted_response(&call, Some(secs));
call_trace::result_ready(
thread_id,
&abort_turn.sub_id,
&call.tool_name,
&call.call_id,
trace_source,
started.elapsed(),
);
call_state.terminal_outcome_reached.store(true, Ordering::Release);
notify_tool_aborted(
abort_session.as_ref(),
abort_turn.as_ref(),
call.call_id.as_str(),
&call.tool_name,
abort_source.clone(),
)
.await;
Ok(response)
record_result_ready(&abort_turn.sub_id, &call.tool_name, &call.call_id);
Err(ToolDispatchError::Aborted(ToolAbort::User(secs)))
}
Err(err) => Err(Self::tool_task_join_error(err)),
Err(err) => Err(Self::tool_task_join_error(err).into()),
}
}
},
@@ -422,25 +422,32 @@ impl ToolCallRuntime {
let _reservation = call_state.invocation.take();
// Cancellation outside the handler can return before the registry reports an outcome.
if cancellation_token.is_cancelled()
&& !call_state
.terminal_outcome_reached
.swap(true, Ordering::AcqRel)
&& !matches!(result, Err(ToolDispatchError::Aborted(_)))
&& call_state.terminal_outcome.get().is_none()
{
notify_tool_aborted(
abort_session.as_ref(),
abort_turn.as_ref(),
call.call_id.as_str(),
&call.tool_name,
abort_source,
)
.await;
result = Ok(Self::aborted_response(
&call,
Some(started.elapsed().as_secs_f32().max(0.1)),
));
result = Err(ToolDispatchError::Aborted(ToolAbort::User(
started.elapsed().as_secs_f32().max(0.1),
)));
}
let mut result = match result {
Ok(output) => Ok(output),
Err(ToolDispatchError::Call(error)) => Err(error),
Err(ToolDispatchError::Aborted(stop)) => {
if call_state.terminal_outcome.set(ToolCallOutcome::Aborted).is_ok() {
notify_tool_aborted(
&abort_session,
&abort_turn,
&call.call_id,
&call.tool_name,
abort_source,
)
.await;
}
Ok(Self::stopped_response(&call, stop))
}
};
// Use one completion measurement for logging and response formatting.
// Measuring inside a handler would omit routing and output processing.
// Completion includes output processing after the handler returns.
if let Some(timing) = tool_call_timing_guard.as_mut()
&& let Some(handler_duration_ms) = timing.finish()
&& let Ok(result) = &mut result
@@ -490,27 +497,28 @@ impl ToolCallRuntime {
}
}
pub(crate) fn aborted_response(call: &ToolCall, elapsed_secs: Option<f32>) -> AnyToolResult {
pub(crate) fn aborted_before_dispatch(call: &ToolCall) -> AnyToolResult {
Self::stopped_response(call, ToolAbort::BeforeDispatch)
}
fn stopped_response(call: &ToolCall, stop: ToolAbort) -> AnyToolResult {
let message = match stop {
ToolAbort::BeforeDispatch => "aborted before dispatch".to_string(),
ToolAbort::User(secs)
if call.tool_name.is_default_namespace()
&& call.tool_name.name == "exec_command" =>
{
format!("Wall time: {secs:.1} seconds\naborted by user")
}
ToolAbort::User(secs) => format!("aborted by user after {secs:.1}s"),
};
AnyToolResult {
call_id: call.call_id.clone(),
payload: call.payload.clone(),
result: Box::new(AbortedToolOutput {
message: Self::abort_message(call, elapsed_secs),
}),
result: Box::new(AbortedToolOutput { message }),
post_tool_use_payload: None,
}
}
fn abort_message(call: &ToolCall, elapsed_secs: Option<f32>) -> String {
let Some(secs) = elapsed_secs else {
return "aborted before dispatch".to_string();
};
if call.tool_name.is_default_namespace() && call.tool_name.name == "exec_command" {
format!("Wall time: {secs:.1} seconds\naborted by user")
} else {
format!("aborted by user after {secs:.1}s")
}
}
}
impl ToolCallTimingGuard {
@@ -520,6 +528,7 @@ impl ToolCallTimingGuard {
turn_id: &str,
call: &ToolCall,
source: &ToolCallSource,
call_state: Arc<ToolCallState>,
extensions: Arc<codex_extension_api::ExtensionRegistry<crate::config::Config>>,
) -> Option<Self> {
// Code-mode calls are nested within a direct code-mode tool call whose
@@ -534,7 +543,7 @@ impl ToolCallTimingGuard {
Some(Self {
started_at: Some(started_at),
execution_started_at: Arc::new(OnceLock::new()),
call_state,
conversation_id: conversation_id.to_string(),
turn_id: turn_id.to_string(),
call_id: call.call_id.clone(),
@@ -551,7 +560,8 @@ impl ToolCallTimingGuard {
// Snapshot once so a concurrently-starting dispatch cannot make one
// event internally inconsistent.
let execution_started_at = self
.execution_started_at
.call_state
.handler_started_at
.get()
.copied()
.filter(|execution_started_at| *execution_started_at <= completed_at);
@@ -622,6 +632,8 @@ impl Drop for ToolCallTimingGuard {
mod tests {
use super::*;
use std::collections::BTreeMap;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use crate::session::step_context::StepContext;
@@ -632,13 +644,11 @@ mod tests {
use crate::tools::registry::ToolRegistry;
use crate::tools::router::ToolRouter;
use crate::turn_diff_tracker::TurnDiffTracker;
use codex_extension_api::ToolCallOutcome;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::openai_models::ToolMode;
use pretty_assertions::assert_eq;
use tokio::sync::Notify;
use tokio::sync::oneshot;
use tracing_test::internal::MockWriter;
#[test]
@@ -704,6 +714,7 @@ mod tests {
"turn-id",
&call,
&ToolCallSource::Direct,
Arc::new(ToolCallState::default()),
codex_extension_api::empty_extension_registry(),
);
assert!(
@@ -721,6 +732,7 @@ mod tests {
cell_id: "cell-1".to_string(),
runtime_tool_call_id: "runtime-call-1".to_string(),
},
Arc::new(ToolCallState::default()),
codex_extension_api::empty_extension_registry(),
);
assert!(
@@ -758,11 +770,13 @@ mod tests {
"turn-id",
&call,
&ToolCallSource::Direct,
Arc::new(ToolCallState::default()),
codex_extension_api::empty_extension_registry(),
)
.expect("model-visible timing must not depend on INFO logging");
timing
.execution_started_at
.call_state
.handler_started_at
.set(started_at + Duration::from_secs(/*secs*/ 4))
.expect("execution start is unset");
let handler_ms = timing.finish().expect("completed handler duration");
@@ -831,6 +845,8 @@ mod tests {
} else {
Arc::new(ImmediateHandler {
tool_name: tool_name.clone(),
supports_parallel: false,
calls: AtomicUsize::default(),
})
};
let step_context = StepContext::for_test(Arc::clone(&turn_context));
@@ -935,9 +951,15 @@ mod tests {
struct ImmediateHandler {
tool_name: codex_tools::ToolName,
supports_parallel: bool,
calls: AtomicUsize,
}
impl ToolExecutor<ToolInvocation> for ImmediateHandler {
fn supports_parallel_tool_calls(&self) -> bool {
self.supports_parallel
}
fn tool_name(&self) -> codex_tools::ToolName {
self.tool_name.clone()
}
@@ -957,6 +979,7 @@ mod tests {
where
ToolInvocation: 'a,
{
self.calls.fetch_add(1, Ordering::Relaxed);
Box::pin(async {
Ok(
Box::new(FunctionToolOutput::from_text("ok".to_string(), Some(true)))
@@ -968,63 +991,82 @@ mod tests {
impl CoreToolRuntime for ImmediateHandler {}
struct BlockingFinishContributor {
records: Arc<std::sync::Mutex<Vec<ToolCallOutcome>>>,
finish_started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
allow_finish: Arc<Notify>,
#[derive(Default)]
struct PausedToolLifecycle {
finishes: std::sync::Mutex<Vec<(String, ToolCallOutcome)>>,
at_start: Notify,
release_start: Notify,
at_finish: Notify,
release_finish: Notify,
}
impl codex_extension_api::ToolLifecycleContributor for BlockingFinishContributor {
impl codex_extension_api::ToolLifecycleContributor for PausedToolLifecycle {
fn on_tool_start<'a>(
&'a self,
input: codex_extension_api::ToolStartInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
Box::pin(async move {
if input.call_id == "held-at-start" {
self.at_start.notify_one();
self.release_start.notified().await;
}
})
}
fn on_tool_finish<'a>(
&'a self,
input: codex_extension_api::ToolFinishInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
let records = Arc::clone(&self.records);
let allow_finish = Arc::clone(&self.allow_finish);
let finish_started = self
.finish_started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
let outcome = input.outcome;
Box::pin(async move {
if let Some(finish_started) = finish_started {
let _ = finish_started.send(());
}
allow_finish.notified().await;
records
self.finishes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(outcome);
.push((input.call_id.to_owned(), input.outcome));
if input.call_id == "call-1" {
self.at_finish.notify_one();
self.release_finish.notified().await;
}
})
}
}
#[tokio::test]
async fn cancellation_after_handler_finishes_preserves_completed_lifecycle()
-> anyhow::Result<()> {
async fn completed_and_budget_stopped_tools_report_their_own_lifecycle() -> anyhow::Result<()> {
let (mut session, turn_context) = crate::session::tests::make_session_and_context().await;
let records = Arc::new(std::sync::Mutex::new(Vec::new()));
let (finish_started_tx, finish_started_rx) = oneshot::channel();
let allow_finish = Arc::new(Notify::new());
session
.rollout_budget()
.configure(crate::config::RolloutBudgetConfig {
enforce_limit: true,
limit_tokens: 1,
reminder_at_remaining_tokens: Vec::new(),
sampling_token_weight: 1.0,
prefill_token_weight: 1.0,
});
let lifecycle = Arc::new(PausedToolLifecycle::default());
let mut builder =
codex_extension_api::ExtensionRegistryBuilder::<crate::config::Config>::new();
builder.tool_lifecycle_contributor(Arc::new(BlockingFinishContributor {
records: Arc::clone(&records),
finish_started: std::sync::Mutex::new(Some(finish_started_tx)),
allow_finish: Arc::clone(&allow_finish),
}));
builder.tool_lifecycle_contributor(lifecycle.clone());
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
let tool_name = codex_tools::ToolName::plain("test_tool");
let handler = Arc::new(ImmediateHandler {
let tool_name = codex_tools::ToolName::plain("parallel_tool");
let queued_name = codex_tools::ToolName::plain("exclusive_tool");
let parallel_handler = Arc::new(ImmediateHandler {
tool_name: tool_name.clone(),
}) as Arc<dyn CoreToolRuntime>;
supports_parallel: true,
calls: AtomicUsize::default(),
});
let queued_handler = Arc::new(ImmediateHandler {
tool_name: queued_name.clone(),
supports_parallel: false,
calls: AtomicUsize::default(),
});
let handlers: [Arc<dyn CoreToolRuntime>; 2] =
[parallel_handler.clone(), queued_handler.clone()];
let step_context = StepContext::for_test(Arc::clone(&turn_context));
let router = Arc::new(ToolRouter::from_parts(
ToolRegistry::from_tools([handler]),
ToolRegistry::from_tools(handlers),
Vec::new(),
ToolMode::Direct,
BTreeMap::new(),
@@ -1034,7 +1076,8 @@ mod tests {
));
let step_context = step_context.with_tool_router_for_test(router);
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
let runtime = ToolCallRuntime::new(session, step_context, tracker);
let runtime = ToolCallRuntime::new(Arc::clone(&session), step_context, tracker);
let queued_runtime = runtime.clone();
let cancellation_token = CancellationToken::new();
let call = ToolCall {
tool_name,
@@ -1046,20 +1089,69 @@ mod tests {
native_call: None,
};
let response_task =
tokio::spawn(runtime.handle_tool_call(call, cancellation_token.clone()));
tokio::time::timeout(Duration::from_secs(1), finish_started_rx)
.await
.expect("timed out waiting for lifecycle notification to start")
.expect("lifecycle notification should start");
cancellation_token.cancel();
tokio::time::sleep(Duration::from_millis(10)).await;
allow_finish.notify_waiters();
let queued_call = ToolCall {
tool_name: queued_name,
call_id: "call-2".to_string(),
..call.clone()
};
let held_call = ToolCall {
call_id: "held-at-start".to_string(),
..call.clone()
};
let response = runtime
.clone()
.handle_tool_call(call, cancellation_token.clone());
tokio::pin!(response);
tokio::time::timeout(Duration::from_secs(1), lifecycle.at_finish.notified()).await?;
let response = tokio::time::timeout(Duration::from_secs(1), response_task)
.await
.expect("timed out waiting for tool response")
.expect("tool response task should join")?;
// This call has a shared slot, but its start observer hasn't released the handler.
let held = runtime.handle_tool_call(held_call, CancellationToken::new());
tokio::time::timeout(Duration::from_secs(1), lifecycle.at_start.notified()).await?;
// The first call keeps its shared slot until its finish callback returns. New
// readers stop getting slots only once the exclusive call has actually queued.
let gate = Arc::clone(&queued_runtime.parallel_execution);
assert!(gate.try_read().is_ok());
let expected_abort = ResponseInputItem::FunctionCallOutput {
call_id: queued_call.call_id.clone(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::Text("aborted before dispatch".to_string()),
success: None,
},
};
let queued = queued_runtime.handle_tool_call(queued_call, CancellationToken::new());
tokio::time::timeout(Duration::from_secs(1), async {
while gate.try_read().is_ok() {
tokio::task::yield_now().await;
}
})
.await
.expect("the tool should be queued behind the held slot");
let usage = codex_protocol::protocol::TokenUsage {
output_tokens: 1,
..Default::default()
};
session
.record_rollout_budget_usage("peer-turn", &usage.into())
.await?;
cancellation_token.cancel();
// Let the caller observe cancellation while the already-finished handler is held.
assert!(futures::poll!(response.as_mut()).is_pending());
lifecycle.release_start.notify_one();
let held_reply = tokio::time::timeout(Duration::from_secs(1), held).await??;
assert_eq!(
held_reply.item,
ResponseInputItem::FunctionCallOutput {
call_id: "held-at-start".into(),
output: FunctionCallOutputPayload::from_text(
"aborted before handler start: rollout budget exhausted".into(),
),
}
.into()
);
lifecycle.release_finish.notify_one();
let response = tokio::time::timeout(Duration::from_secs(1), response).await??;
let expected_response = ResponseInputItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
@@ -1071,13 +1163,30 @@ mod tests {
ResponseItemEnvelope::new(expected_response.into()),
response
);
assert_eq!(
ResponseItemEnvelope::new(expected_abort.into()),
tokio::time::timeout(Duration::from_secs(1), queued).await??
);
assert_eq!(parallel_handler.calls.load(Ordering::Relaxed), 1);
assert_eq!(queued_handler.calls.load(Ordering::Relaxed), 0);
let actual = records
let actual = lifecycle
.finishes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.drain(..)
.collect::<Vec<_>>();
assert_eq!(vec![ToolCallOutcome::Completed { success: true }], actual);
assert_eq!(
vec![
(
"call-1".to_string(),
ToolCallOutcome::Completed { success: true }
),
("held-at-start".to_string(), ToolCallOutcome::Aborted),
("call-2".to_string(), ToolCallOutcome::Aborted),
],
actual
);
Ok(())
}
@@ -36,6 +36,7 @@ fn cancelled_dispatch_reports_zero_handler_work_even_without_tracing() {
"turn",
&call,
&ToolCallSource::Direct,
Arc::new(ToolCallState::default()),
Arc::new(extensions.build()),
)
.unwrap();
+51 -13
View File
@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use crate::function_tool::FunctionCallError;
use crate::function_tool::OrCancelToolExt;
@@ -13,6 +13,7 @@ use crate::memory_usage::emit_metric_for_tool_read;
use crate::memory_usage::shell_script_for_invocation;
use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
use crate::tools::context::AbortedToolOutput;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolCallState;
use crate::tools::context::ToolEnvironments;
@@ -39,6 +40,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::EventMsg;
use codex_rollout::state_db;
use codex_rollout_trace::ExecutionStatus;
use codex_shell_command::parse_command::parse_shell_script;
use codex_tools::FunctionsNamespaceFunctionPrefixes;
use codex_tools::ToolName;
@@ -930,12 +932,40 @@ impl ToolRegistry {
return Err(fail_before_execution(dispatch_trace, &invocation, err).await);
}
let invocation_guard = reservation.activate(&invocation);
if tool.mcp_server_name().is_none() {
notify_tool_start(&invocation, /*mcp_tool*/ None)
.or_cancel_tool(&cancellation_token)
.await?;
// Environment resolution, pre-tool hooks and Guardian may all have waited.
let invocation_guard = if invocation.session.rollout_budget().tools_stopped() {
None
} else {
let guard = reservation.activate(&invocation);
if tool.mcp_server_name().is_none() {
notify_tool_start(&invocation, /*mcp_tool*/ None)
.or_cancel_tool(&cancellation_token)
.await?;
}
guard
};
// A start observer can also wait. This is the last check before polling the
// handler; a stop after admission does not cancel an already-running tool.
if invocation.session.rollout_budget().tools_stopped() {
drop(invocation_guard);
let result = AnyToolResult {
call_id: invocation.call_id.clone(),
payload: invocation.payload.clone(),
result: Box::new(AbortedToolOutput {
message: "aborted before handler start: rollout budget exhausted".into(),
}),
post_tool_use_payload: None,
};
dispatch_trace.record_result(&invocation, &result, ExecutionStatus::Aborted);
notify_tool_finish_if_unclaimed(
&invocation,
call_state.as_deref(),
ToolCallOutcome::Aborted,
)
.await;
return Ok(result);
}
let mut control_tool_analytics = tool
.is_builtin_control_tool()
.then(|| ControlToolCallGuard::new(&invocation));
@@ -1025,6 +1055,7 @@ impl ToolRegistry {
{
ToolCallOutcome::Aborted
}
Ok(result) if result.result.was_aborted() => ToolCallOutcome::Aborted,
Ok(_) => ToolCallOutcome::Completed { success },
Err(_) => ToolCallOutcome::Failed {
handler_executed: true,
@@ -1035,6 +1066,10 @@ impl ToolRegistry {
.await;
match result {
Ok(result) if result.result.was_aborted() => {
dispatch_trace.record_result(&invocation, &result, ExecutionStatus::Aborted);
Ok(result)
}
Ok(mut result) => {
if let Some(outcome) = post_tool_use_outcome {
if outcome.should_block {
@@ -1056,12 +1091,12 @@ impl ToolRegistry {
}
}
tool.on_tool_result_accepted(&invocation, result.result.as_ref());
dispatch_trace.record_completed(
&invocation,
&result.call_id,
&result.payload,
result.result.as_ref(),
);
let status = if success {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
dispatch_trace.record_result(&invocation, &result, status);
Ok(result)
}
Err(err) => {
@@ -1077,7 +1112,7 @@ async fn notify_tool_finish_if_unclaimed(
call_state: Option<&ToolCallState>,
outcome: ToolCallOutcome,
) -> bool {
if call_state.is_some_and(|state| state.terminal_outcome_reached.swap(true, Ordering::AcqRel)) {
if call_state.is_some_and(|state| state.terminal_outcome.set(outcome).is_err()) {
return false;
}
@@ -1092,6 +1127,9 @@ async fn handle_any_tool(
) -> Result<AnyToolResult, FunctionCallError> {
let call_id = invocation.call_id.clone();
let payload = invocation.payload.clone();
if let Some(state) = call_state {
let _ = state.handler_started_at.set(Instant::now());
}
let output = tool.handle(invocation.clone()).await?;
let post_tool_use_payload =
CoreToolRuntime::post_tool_use_payload(tool, &invocation, output.as_ref());
+10 -11
View File
@@ -9,6 +9,7 @@ use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
use crate::tools::registry::AnyToolResult;
use codex_rollout_trace::ExecutionStatus;
use codex_rollout_trace::ToolDispatchInvocation;
use codex_rollout_trace::ToolDispatchPayload;
@@ -31,26 +32,24 @@ impl ToolDispatchTrace {
Self { context }
}
pub(crate) fn record_completed(
pub(crate) fn record_result(
self,
invocation: &ToolInvocation,
call_id: &str,
payload: &ToolPayload,
result: &dyn ToolOutput,
result: &AnyToolResult,
status: ExecutionStatus,
) {
if !self.context.is_enabled() {
return;
}
let Some(result_payload) = tool_dispatch_result(invocation, call_id, payload, result)
else {
let Some(result_payload) = tool_dispatch_result(
invocation,
&result.call_id,
&result.payload,
result.result.as_ref(),
) else {
return;
};
let status = if result.success_for_logging() {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
self.context.record_completed(status, result_payload);
}
+9
View File
@@ -19,6 +19,11 @@ pub trait ToolOutput: Send {
fn success_for_logging(&self) -> bool;
/// Whether the call was refused before its underlying operation started.
fn was_aborted(&self) -> bool {
false
}
/// Finalizes output using the same completed handler duration reported in tool-call logs.
/// Called before recording model-visible history; implementations must not measure time here.
fn set_handler_duration_ms(&mut self, _handler_duration_ms: u64) {}
@@ -92,6 +97,10 @@ where
(**self).success_for_logging()
}
fn was_aborted(&self) -> bool {
(**self).was_aborted()
}
fn set_handler_duration_ms(&mut self, handler_duration_ms: u64) {
(**self).set_handler_duration_ms(handler_duration_ms);
}