diff --git a/codex-rs/core/src/agent/api.rs b/codex-rs/core/src/agent/api.rs index 054770df82..0f46d15de9 100644 --- a/codex-rs/core/src/agent/api.rs +++ b/codex-rs/core/src/agent/api.rs @@ -39,6 +39,12 @@ const _: Option<&dyn AgentControl> = None; /// remote ownership, retries and recovery are separate backend work. /// Boxed Send futures allow callers to use `Arc`. pub trait AgentControl: Send + Sync { + /// Expose the account only when this controller actually owns native billing. + /// External controllers keep their own ledger and need not provide a numeric read. + fn native_rollout_budget(&self) -> Option> { + None + } + fn identity(&self) -> SessionId; /// Resolve an ID or a name relative to the caller's captured source, without loading. diff --git a/codex-rs/core/src/agent/control/api.rs b/codex-rs/core/src/agent/control/api.rs index ce3735f196..22e43a59cb 100644 --- a/codex-rs/core/src/agent/control/api.rs +++ b/codex-rs/core/src/agent/control/api.rs @@ -33,6 +33,10 @@ use futures::future::BoxFuture; use std::collections::HashSet; impl AgentControl for LocalAgentControl { + fn native_rollout_budget(&self) -> Option> { + Some(std::sync::Arc::clone(&self.runtime.rollout_budget)) + } + fn identity(&self) -> SessionId { self.session_id() } diff --git a/codex-rs/core/src/agent/control/budget.rs b/codex-rs/core/src/agent/control/budget.rs index c637750139..f03d77f823 100644 --- a/codex-rs/core/src/agent/control/budget.rs +++ b/codex-rs/core/src/agent/control/budget.rs @@ -4,7 +4,6 @@ use super::LocalAgentControl; use crate::rollout_budget::RolloutBudgetReminder; use codex_protocol::ThreadId; -use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::TokenUsage; @@ -14,14 +13,8 @@ impl LocalAgentControl { usage: Option<&TokenUsage>, hook_charge: codex_hooks::HookCharge, ) -> CodexResult<()> { - if self - .runtime - .rollout_budget - .record_usage(usage, hook_charge)? - { - return Err(CodexErr::SessionBudgetExceeded); - } - Ok(()) + // The caller keeps a consumed response even when its bill stops new work. + self.runtime.rollout_budget.record_usage(usage, hook_charge) } pub(crate) fn pending_budget_reminder( diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index c4e9663e46..2ef62c58aa 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -312,6 +312,7 @@ pub struct ModelClient { http_client_factory: HttpClientFactory, restored_history: bool, request_contributors: Vec>, + budget_policy: Option, executed_tool_calls: Option, // Resolved once when the session is created, like other session feature flags. api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms, @@ -660,6 +661,7 @@ impl ModelClient { http_client_factory, restored_history: false, request_contributors, + budget_policy: None, executed_tool_calls: None, api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms::UnsupportedProvider, @@ -672,6 +674,14 @@ impl ModelClient { self } + pub(crate) fn with_budget_policy( + mut self, + policy: crate::rollout_budget::ThreadBudgetPolicy, + ) -> Self { + self.budget_policy = Some(policy); + self + } + pub(crate) fn reasoning_effort_override_enabled(&self, model_info: &ModelInfo) -> bool { self.state.reasoning_effort_override_enabled && self.state.provider.info().is_openai() @@ -2120,6 +2130,9 @@ impl ResponsesHttpExecutor<'_> { client_setup.api_auth, ) .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); + if let Some(policy) = &self.client.budget_policy { + policy.check_start()?; + } let stream_result = client .stream_request_with_observer(request, options, |wire| { inference_trace_attempt.record_started( @@ -2474,6 +2487,9 @@ impl ModelClientSession { ), ], ); + if !warmup && let Some(policy) = &self.client.budget_policy { + policy.check_start()?; + } let stream_result = websocket_connection .stream_request_with_observer( ws_request, diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 4666ca9c1b..ab673cb222 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -571,8 +571,10 @@ pub(crate) fn compaction_status_from_result(result: &CodexResult) -> Compa match result { Ok(_) => CompactionStatus::Completed, Err(err) - if matches!(err.details(), CodexErrorDetails::Interrupted) - || crate::tasks::ends_turn_as_interrupted(err) => + if matches!( + err.details(), + CodexErrorDetails::Interrupted | CodexErrorDetails::SessionBudgetExceeded + ) || crate::tasks::ends_turn_as_interrupted(err) => { CompactionStatus::Interrupted } diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 637743c6a6..11b8124e81 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -1482,13 +1482,13 @@ mod tests { assert!( matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted)) ); - assert!(!sess.rollout_budget().tools_stopped()); + assert!(!sess.work_budget().is_exhausted()); assert!(accounting.reports.lock().unwrap().is_empty()); return; } assert!(matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_)))); - assert!(sess.rollout_budget().tools_stopped()); + assert!(sess.work_budget().is_exhausted()); assert_eq!( *accounting.reports.lock().unwrap(), vec![AgentUsageReport { diff --git a/codex-rs/core/src/guardian/input_budget.rs b/codex-rs/core/src/guardian/input_budget.rs index 3efdf4ff6f..2a3de9508e 100644 --- a/codex-rs/core/src/guardian/input_budget.rs +++ b/codex-rs/core/src/guardian/input_budget.rs @@ -251,9 +251,7 @@ pub(crate) async fn finalize( .pending_budget_reminder(session.thread_id(), &session.current_window_id().await) .await { - let reminder = ContextualUserFragment::into(crate::context::RolloutBudgetContext { - remaining_units: reminder.remaining_units, - }); + let reminder = session.budget_reminder_item(&reminder); existing = existing.saturating_add( usize::try_from(estimate_item_token_count(&reminder)).unwrap_or(usize::MAX), ); diff --git a/codex-rs/core/src/hook_runtime/sampling.rs b/codex-rs/core/src/hook_runtime/sampling.rs index 9609532301..eddb452f4b 100644 --- a/codex-rs/core/src/hook_runtime/sampling.rs +++ b/codex-rs/core/src/hook_runtime/sampling.rs @@ -54,10 +54,6 @@ impl SamplingRequest { owner_cancellation: &CancellationToken, send: impl Future>, ) -> Result { - let send = async { - session.check_rollout_budget()?; - send.await - }; let sampling_turn = match self.kind { SamplingKind::Sampling => Some(Arc::clone(&step.turn)), SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None, @@ -83,6 +79,9 @@ impl SamplingRequest { agent_id: subagent.as_ref().map(|context| context.agent_id.clone()), agent_type: subagent.map(|context| context.agent_type), }; + // Hook input preparation may wait. Do not start a new hook pair after a stop; + // the HTTP/WS send independently rechecks after a pre-hook has finished. + session.check_rollout_budget()?; let mut attempt = post_mode.map(|mode| SamplingAttempt { owner_cancellation: owner_cancellation.clone(), post: Some(PostHooks { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index a33df95fc9..1e9090ab19 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -85,6 +85,7 @@ pub use agent::types::ResolvedMultiAgentV2UsageHints; pub use agent::types::SpawnAgentForkMode; pub use agent::types::SpawnAgentOptions; pub use agent::types::SpawnProgramSource; +pub use rollout_budget::RolloutBudget; pub use rollout_budget::RolloutBudgetReminder; mod agent_communication; pub use agent_communication::PendingAgentMessage; @@ -202,6 +203,7 @@ pub use thread_manager::StartupModels; pub use thread_manager::SuspendedAgentTree; pub use thread_manager::ThreadManager; pub use thread_manager::ThreadRegistration; +pub use thread_manager::ThreadRegistrationReceipt; pub use thread_manager::ThreadShutdownReport; pub use thread_manager::build_models_manager; pub use thread_manager::build_startup_models; diff --git a/codex-rs/core/src/rollout_budget.rs b/codex-rs/core/src/rollout_budget.rs index 1b13b0fc7a..5a4748b84d 100644 --- a/codex-rs/core/src/rollout_budget.rs +++ b/codex-rs/core/src/rollout_budget.rs @@ -1,3 +1,5 @@ +mod gate; + use crate::config::RolloutBudgetConfig; use crate::config::rollout_budget_threshold_reached; use codex_protocol::BudgetUnitsSum; @@ -5,12 +7,12 @@ use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::TokenUsage; +pub use gate::RolloutWorkGate; use std::collections::HashMap; +use std::sync::Arc; use std::sync::Mutex; use std::sync::MutexGuard; use std::sync::OnceLock; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; /// Budget reminder returned by a controller and acknowledged after history insertion. pub struct RolloutBudgetReminder { @@ -19,14 +21,47 @@ pub struct RolloutBudgetReminder { pub reminder_index: i64, } -/// Shared accounting and reminder state for one root-thread session tree. +/// Native spending and reminders, plus the stop published by the tree's selected +/// accounting owner (native or host). In-flight work can still report its bill. #[derive(Default)] -pub(crate) struct RolloutBudget { +pub struct RolloutBudget { state: OnceLock>, - /// An explicit stop gates new tools across the tree, even with per-thread host accountants. - stop_tools: AtomicBool, + gate: Arc, } +/// A native account owns this thread's bills. A host-owned thread only shares its work gate. +pub(crate) enum ThreadBudgetAccount { + Native(Arc), + Shared(Arc), +} + +/// Every thread consults the shared work gate before starting new work. +#[derive(Clone, Debug, Default)] +pub(crate) struct ThreadBudgetPolicy { + rollout: Arc, +} + +impl ThreadBudgetPolicy { + pub(crate) fn new(account: ThreadBudgetAccount) -> CodexResult { + let (ThreadBudgetAccount::Native(rollout) | ThreadBudgetAccount::Shared(rollout)) = account; + Ok(Self { rollout }) + } + + pub(crate) fn check_start(&self) -> CodexResult<()> { + self.rollout.gate.check_start() + } +} + +impl std::fmt::Debug for RolloutBudget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RolloutBudget") + .field("state", &self.state) + .field("stopped", &self.is_exhausted()) + .finish_non_exhaustive() + } +} + +#[derive(Debug)] struct RolloutBudgetState { config: RolloutBudgetConfig, posted: BudgetUnitsSum, @@ -34,6 +69,7 @@ struct RolloutBudgetState { deliveries: HashMap, } +#[derive(Debug)] struct ThreadBudgetDelivery { window_id: String, reminder_index: i64, @@ -50,25 +86,22 @@ impl RolloutBudget { }); } - /// Records usage and returns whether an enforced budget is exhausted. + /// Post the response cost and enforce the allowance under the same lock. pub(crate) fn record_usage( &self, usage: Option<&TokenUsage>, hook_charge: codex_hooks::HookCharge, - ) -> CodexResult { + ) -> CodexResult<()> { let Some(mut state) = self.lock() else { - return Ok(false); + return Ok(()); }; if !hook_charge.units.is_finite() || hook_charge.units < 0.0 { - return Err(CodexErr::Fatal("invalid rollout budget charge".into())); + return Err(CodexErr::Fatal("invalid additional sampling charge".into())); } - if hook_charge.overflowed { - // All allowances are finite. No future nonnegative receipt can make - // this total representable; the remaining allowance is known to be zero. + if hook_charge.overflowed && state.posted.total().is_finite() { + // Stop new work before validating the rest of an overflowing receipt. state.posted.add(f64::INFINITY); - if state.config.enforce_limit { - self.stop_new_tools(); - } + self.enforce_posted_limit(&state)?; } let provider_units = if let Some(units) = usage.and_then(|usage| usage.codex_rollout_budget_units.as_ref()) @@ -88,31 +121,31 @@ impl RolloutBudget { 0.0 }; if state.posted.total().is_infinite() { - return Ok(state.config.enforce_limit); + // Later valid receipts cannot restore precision. The original + // overflow was reported; it must not veto every advisory turn. + return Ok(()); } + // Keep the full bill even if enforcing the new allowance fails. state.posted.add(provider_units); state.posted.add(hook_charge.units); + // Enforce the allowance before reporting an overflowing sum. + self.enforce_posted_limit(&state)?; if !state.posted.total().is_finite() { - if state.config.enforce_limit { - self.stop_new_tools(); - } - return Err(CodexErr::Fatal("invalid rollout budget charge".into())); + return Err(CodexErr::Fatal( + "rollout sampling cost exceeds f64 range".into(), + )); } - // Equality is allowed, including one rounding step (0.1 + 0.2 versus 0.3). - let exhausted = state.config.enforce_limit - && state.posted.total() > state.config.limit_budget.next_up(); - if exhausted { - self.stop_new_tools(); - } - Ok(exhausted) + Ok(()) } - pub(crate) fn stop_new_tools(&self) { - self.stop_tools.store(true, Ordering::Release); - } - - pub(crate) fn tools_stopped(&self) -> bool { - self.stop_tools.load(Ordering::Acquire) + fn enforce_posted_limit(&self, state: &RolloutBudgetState) -> CodexResult<()> { + let posted = state.posted.total(); + if state.config.enforce_limit + && (!posted.is_finite() || posted > state.config.limit_budget.next_up()) + { + self.stop_new_work(); + } + Ok(()) } pub(crate) fn pending_reminder( @@ -121,7 +154,7 @@ impl RolloutBudget { window_id: &str, ) -> Option { let state = self.lock()?; - let remaining_units = (state.config.limit_budget - state.posted.total()).max(0.0); + let remaining_units = self.remaining_units(&state).ok()?; let reminder_index = state .config .reminder_at_remaining_tokens @@ -167,4 +200,20 @@ impl RolloutBudget { .unwrap_or_else(std::sync::PoisonError::into_inner) }) } + + pub(crate) fn stop_new_work(&self) { + self.gate.stop_budget(); + } + + pub(crate) fn is_exhausted(&self) -> bool { + self.gate.is_stopped() + } + + fn remaining_units(&self, state: &RolloutBudgetState) -> CodexResult { + if state.posted.total().is_infinite() { + return Ok(0.0); + } + let used = state.posted.total(); + Ok((state.config.limit_budget - used).max(0.0)) + } } diff --git a/codex-rs/core/src/rollout_budget/gate.rs b/codex-rs/core/src/rollout_budget/gate.rs new file mode 100644 index 0000000000..608b283dc1 --- /dev/null +++ b/codex-rs/core/src/rollout_budget/gate.rs @@ -0,0 +1,31 @@ +//! The first stop closes every work-start boundary in this rollout tree. + +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result; +use std::sync::OnceLock; + +/// Controls new work across the rollout tree. +#[derive(Debug, Default)] +pub struct RolloutWorkGate(OnceLock); + +#[derive(Debug)] +enum Stop { + Budget, +} + +impl RolloutWorkGate { + pub(super) fn stop_budget(&self) { + let _ = self.0.set(Stop::Budget); + } + + pub(super) fn is_stopped(&self) -> bool { + self.0.get().is_some() + } + + pub(super) fn check_start(&self) -> Result<()> { + match self.0.get() { + None => Ok(()), + Some(Stop::Budget) => Err(CodexErr::SessionBudgetExceeded), + } + } +} diff --git a/codex-rs/core/src/session/async_compaction.rs b/codex-rs/core/src/session/async_compaction.rs index 98a28dba87..f9e4f29418 100644 --- a/codex-rs/core/src/session/async_compaction.rs +++ b/codex-rs/core/src/session/async_compaction.rs @@ -66,6 +66,7 @@ pub(super) async fn maybe_start( return; } if !enabled(turn_context) + || sess.work_budget().is_exhausted() || token_status.token_limit_reached || (matches!(phase, CompactionPhase::PostTurn) && token_status.turn_end_compaction_threshold_reached) diff --git a/codex-rs/core/src/session/rollout_budget.rs b/codex-rs/core/src/session/rollout_budget.rs index 43bd80b6a6..139c5aedb2 100644 --- a/codex-rs/core/src/session/rollout_budget.rs +++ b/codex-rs/core/src/session/rollout_budget.rs @@ -5,7 +5,6 @@ use crate::agent::host::AgentUsageReport; use crate::agent::host::UsageAccountingDecision; use crate::client_common::SamplingUsage; use crate::context::ContextualUserFragment; -use crate::rollout_budget::RolloutBudget; use codex_protocol::error::CodexErr; use codex_protocol::error::CodexErrorDetails; use codex_protocol::error::Result as CodexResult; @@ -27,9 +26,7 @@ pub(super) async fn maybe_record_reminder( else { return; }; - let response_item = ContextualUserFragment::into(crate::context::RolloutBudgetContext { - remaining_units: reminder.remaining_units, - }); + let response_item = sess.budget_reminder_item(&reminder); sess.record_conversation_items( turn_context, turn_context.model_info(), @@ -42,23 +39,41 @@ pub(super) async fn maybe_record_reminder( } impl Session { - pub(crate) fn rollout_budget(&self) -> &RolloutBudget { + pub(crate) fn budget_reminder_item( + &self, + reminder: &crate::RolloutBudgetReminder, + ) -> codex_protocol::models::ResponseItem { + ContextualUserFragment::into(crate::context::RolloutBudgetContext { + remaining_units: reminder.remaining_units, + }) + } + + pub(crate) fn native_rollout_budget(&self) -> Option> { + match &self.agent_runtime { + AgentRuntimeServices::Controller => self.services.agent_control.native_rollout_budget(), + AgentRuntimeServices::Host(_) => None, + } + } + + /// All threads consult the same work gate, even when a host owns one thread's bill. + pub(crate) fn work_budget(&self) -> &crate::RolloutBudget { &self.services.local_agent_runtime.rollout_budget } + pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> { + self.services.budget_policy.check_start() + } + /// Return the diagnostic for a refused tool or propagate an accounting error. pub(crate) fn tool_start_denial(&self) -> CodexResult> { - Ok(self - .rollout_budget() - .tools_stopped() - .then(|| "rollout budget exhausted".into())) - } - - pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> { - if self.rollout_budget().tools_stopped() { - Err(CodexErr::SessionBudgetExceeded) - } else { - Ok(()) + match self.check_rollout_budget() { + Ok(()) => Ok(None), + Err(error) => match error.details() { + CodexErrorDetails::SessionBudgetExceeded => { + Ok(Some("rollout budget exhausted".into())) + } + _ => Err(error), + }, } } @@ -98,10 +113,11 @@ impl Session { { Ok(()) => false, // A legacy controller can report a posted bill and a stop through this error. + // A native account has already chosen its own deadline action. Err(error) if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) => { - true + self.native_rollout_budget().is_none() } Err(error) => return Err(CodexErr::UsageAccountingFailed(error.to_string())), }, @@ -127,7 +143,7 @@ impl Session { } }; if stop_new_work { - self.rollout_budget().stop_new_tools(); + self.work_budget().stop_new_work(); } Ok(()) } diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 97acf034fe..8be322fadc 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1807,6 +1807,24 @@ impl Session { // This placeholder is used only for teardown, before the session is published. Err(err) => (AgentRuntimeServices::Controller, Some(err)), }; + let mut local_agent_runtime = local_agent_runtime; + let mut account = crate::rollout_budget::ThreadBudgetAccount::Shared(Arc::clone( + &local_agent_runtime.rollout_budget, + )); + if startup_error.is_none() && matches!(&agent_runtime, AgentRuntimeServices::Controller) { + if let Some(native) = agent_control.native_rollout_budget() { + local_agent_runtime.rollout_budget = Arc::clone(&native); + account = crate::rollout_budget::ThreadBudgetAccount::Native(native); + } + } + let budget_policy = crate::rollout_budget::ThreadBudgetPolicy::new( + account, + ) + .unwrap_or_else(|err| { + startup_error = Some(err); + // Teardown needs a session. Its empty policy is never published. + Default::default() + }); state.shared_service_tier_pending = parent_thread_id.is_none() && match &agent_runtime { AgentRuntimeServices::Controller => has_provided_agent_control, @@ -1854,6 +1872,7 @@ impl Session { workspace_routing.as_ref().clone(), extensions.model_request_contributors().to_vec(), ) + .with_budget_policy(budget_policy.clone()) .with_executed_tool_calls(executed_tool_calls.clone()) .with_restored_history(matches!( &initial_history, @@ -1946,6 +1965,7 @@ impl Session { mcp_thread_init, client_mcp_extensions, local_agent_runtime, + budget_policy, agent_control, network_proxy: arc_swap::ArcSwapOption::from(network_proxy.map(Arc::new)), network_proxy_audit_metadata, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 5b7f1e272a..0afcca25d7 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -6898,6 +6898,12 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { mcp_thread_init: codex_extension_api::ExtensionDataInit::default(), client_mcp_extensions: ClientMcpExtensions::default(), local_agent_runtime: agent_control.runtime.clone(), + budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new( + crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone( + &agent_control.runtime.rollout_budget, + )), + ) + .expect("test budget policy"), agent_control: Arc::new(agent_control), network_proxy: arc_swap::ArcSwapOption::from(None), network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(), @@ -9249,6 +9255,12 @@ where mcp_thread_init: codex_extension_api::ExtensionDataInit::default(), client_mcp_extensions: ClientMcpExtensions::default(), local_agent_runtime: agent_control.runtime.clone(), + budget_policy: crate::rollout_budget::ThreadBudgetPolicy::new( + crate::rollout_budget::ThreadBudgetAccount::Native(Arc::clone( + &agent_control.runtime.rollout_budget, + )), + ) + .expect("test budget policy"), agent_control: Arc::new(agent_control), network_proxy: arc_swap::ArcSwapOption::from(None), network_proxy_audit_metadata: crate::config::NetworkProxyAuditMetadata::default(), diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 1cfd2a535f..5487d64e16 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -541,7 +541,7 @@ pub(crate) async fn run_turn( // Note that pending_input would be something like a message the user // submitted through the UI while the model was running. Though the UI // may support this, the model might not. - let pending_input = if can_drain_pending_input { + let pending_input = if can_drain_pending_input && !sess.work_budget().is_exhausted() { sess.input_queue .get_pending_input(&sess.active_turn) .await @@ -654,7 +654,8 @@ pub(crate) async fn run_turn( .await } .await; - // Consume the whole response before deciding whether the turn may continue. + // Completed can precede Guardian's accepted-text replay. Preserve the + // entire consumed response before deciding whether the turn may continue. let sampling_request_result = sampling_request_result.and_then(|output| { sess.check_rollout_budget()?; Ok(output) @@ -1694,6 +1695,7 @@ async fn run_auto_compact( candidate: Option>, ) -> CodexResult { let turn_context = &step_context.turn; + sess.check_rollout_budget()?; if matches!(phase, CompactionPhase::PreTurn) && crate::guardian::is_basic_session_source(&turn_context.session_source) && !crate::guardian::should_compact_guardian_input(sess)? @@ -1712,6 +1714,9 @@ async fn run_auto_compact( turn.id = %turn_context.sub_id, ); if turn_context.config.features.enabled(Feature::TokenBudget) { + // Unlike sampling compaction, this reset has no transport admission check. + // World-state preparation above may have waited while a peer posted a bill. + sess.check_rollout_budget()?; // Compaction is the reset request, so force a new context window // instead of consuming a pending `new_context` tool request. crate::compact_token_budget::run_inline_auto_compact_task( diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index e534d9ebc4..e958098928 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -609,6 +609,11 @@ async fn start_if_idle( reason: NotSubmittedReason::NotIdle, }); } + // Idle observers can retry a failed turn. Check before reserving so a permanent + // stop cannot create a loop of new turns that never reach the provider. + if kind != TurnStartKind::User { + session.check_rollout_budget()?; + } let active_turn = active_turn.get_or_insert_with(ActiveTurn::default); Arc::clone(&active_turn.turn_state) }; @@ -843,6 +848,12 @@ impl Session { { return None; } + // Triggered mail and durable sleep bypass start_if_idle. Leave their mail + // pending rather than consuming it to start a turn that the work gate will refuse. + if let Err(error) = self.check_rollout_budget() { + tracing::debug!(%error, "rollout budget stopped a pending-work wake"); + return None; + } let active_turn = active_turn.get_or_insert_with(ActiveTurn::default); let previous_options = if has_trigger_turn { Default::default() diff --git a/codex-rs/core/src/session/turn_start_mcp_tests.rs b/codex-rs/core/src/session/turn_start_mcp_tests.rs index 268639e0ec..1861172f00 100644 --- a/codex-rs/core/src/session/turn_start_mcp_tests.rs +++ b/codex-rs/core/src/session/turn_start_mcp_tests.rs @@ -275,11 +275,11 @@ async fn mcp_budget_stop_at_handoff(phase: McpPause) -> anyhow::Result<()> { extensions.tool_lifecycle_contributor(observer.clone()); session.services.extensions = Arc::new(extensions.build()); session - .rollout_budget() + .work_budget() .configure(crate::config::RolloutBudgetConfig { enforce_limit: true, limit_budget: 1.0, - reminder_at_context_start: true, + reminder_at_context_start: false, reminder_at_remaining_tokens: Vec::new(), sampling_token_weight: 1.0, prefill_token_weight: 1.0, diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 54aee82bb0..0c68039cfe 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -89,6 +89,7 @@ pub(crate) struct SessionServices { pub(crate) mcp_thread_init: ExtensionDataInit, pub(crate) agent_control: Arc, pub(crate) local_agent_runtime: LocalAgentRuntime, + pub(crate) budget_policy: crate::rollout_budget::ThreadBudgetPolicy, pub(crate) network_proxy: ArcSwapOption, pub(crate) network_proxy_audit_metadata: NetworkProxyAuditMetadata, pub(crate) managed_network_requirements_configured: bool, diff --git a/codex-rs/core/src/tasks/compact.rs b/codex-rs/core/src/tasks/compact.rs index 35b249841f..3bb0c61458 100644 --- a/codex-rs/core/src/tasks/compact.rs +++ b/codex-rs/core/src/tasks/compact.rs @@ -41,6 +41,7 @@ impl SessionTask for CompactTask { ); // Preparation errors must reach the task runner, which reports them to the client. session.emit_turn_started(&ctx, TaskKind::Compact).await; + session.check_rollout_budget()?; let step_context = session .capture_step_context(Arc::clone(&ctx), &cancellation_token) .await?; @@ -50,6 +51,8 @@ impl SessionTask for CompactTask { .await?, ); if ctx.config.features.enabled(Feature::TokenBudget) { + // Preparing the context may wait; this reset never sends a model request. + session.check_rollout_budget()?; crate::compact_token_budget::run_manual_compact_task( session, step_context, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index c3c1d20b4c..a00369c736 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -682,7 +682,8 @@ impl Session { reason: TurnAbortReason, error: Option, ) -> bool { - let Some(active_turn) = self.take_named_active_turn(turn_id, &reason).await else { + let active_turn = self.take_named_active_turn(turn_id, &reason).await; + let Some(active_turn) = active_turn else { return false; }; @@ -696,7 +697,8 @@ impl Session { reason: &TurnAbortReason, ) -> Option { let mut active = self.active_turn.lock().await; - if active.as_ref()?.task.as_ref()?.turn_context.sub_id != turn_id { + let turn = &active.as_ref()?.task.as_ref()?.turn_context; + if turn.sub_id != turn_id { return None; } if matches!( @@ -749,8 +751,10 @@ impl Session { (None, None, Some(err)) } }; - let can_continue = - regular_output.is_some() && turn_context.terminal_error.lock().await.is_none(); + // A late steer cannot reopen the stopped rollout after the turn loop has finished. + let can_continue = regular_output.is_some() + && turn_context.terminal_error.lock().await.is_none() + && !self.work_budget().is_exhausted(); let finishing = CancellationToken::new(); let finish_recording = finishing.clone().drop_guard(); let (turn_state, unadmitted_task, mut pending_input, outcome_handoff) = { diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 873f0d8b49..0658a06d4a 100644 --- a/codex-rs/core/src/tasks/regular.rs +++ b/codex-rs/core/src/tasks/regular.rs @@ -163,6 +163,7 @@ impl RegularTask { // Terminal errors are already reported. Let task completion preserve pending // input instead of restarting the failed turn for that same input. if ctx.terminal_error.lock().await.is_some() + || sess.work_budget().is_exhausted() || !sess.input_queue.has_pending_input(&sess.active_turn).await { return Ok(Some(RegularTaskOutput { diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 35e5c84d7a..bf2886028c 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -207,6 +207,16 @@ pub enum ThreadRegistration { Replaced, } +/// A manager-started runtime's receipt stays committed across unload. A runtime +/// started without a manager has no receipt in its thread extension store. +pub struct ThreadRegistrationReceipt(Arc); + +impl ThreadRegistrationReceipt { + pub fn was_registered(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + // TODO(ccunningham): Add an explicit non-interrupting live-turn snapshot once // core can represent sampling boundaries directly instead of relying on // whichever items happened to be persisted mid-turn. @@ -2903,6 +2913,8 @@ impl ThreadManagerState { &session_source, ) => manager, }; + let registered = Arc::new(AtomicBool::new(false)); + thread_extension_init.insert(ThreadRegistrationReceipt(Arc::clone(®istered))); let spawn_result = Session::spawn(SessionSpawnArgs { attached_thread, startup, @@ -2997,7 +3009,7 @@ impl ThreadManagerState { } } let finalize_result = self - .finalize_thread_spawn(session, io, tracked_session_source) + .finalize_thread_spawn(session, io, tracked_session_source, ®istered) .await; let (new_thread, tier_acceptance) = match finalize_result { Ok(registered) => registered, @@ -3040,6 +3052,7 @@ impl ThreadManagerState { session: Arc, io: SessionIo, session_source: SessionSource, + registered: &AtomicBool, ) -> CodexResult<(NewThread, Option)> { let thread_id = session.thread_id(); let event = io.next_event().await?; @@ -3080,6 +3093,8 @@ impl ThreadManagerState { session_configured.rollout_path.clone(), session_source, )); + // All registration checks passed; publish the receipt with this insertion. + registered.store(true, Ordering::Release); e.insert(thread.clone()); let tier_acceptance = pending_service_tier.and_then(|(mut state, update)| { state.service_tier_acceptance = diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 13dcba3fd8..d5f3fcdfdc 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -309,6 +309,13 @@ impl ToolCallRuntime { let mut dispatch_handle = AbortOnDropHandle::new(tokio::spawn( async move { let _message_admission = message_admission?; + // A response replayed after settlement can already have exhausted the budget. + // Record its denial without waiting for a handler that will never start. + if session.work_budget().is_exhausted() { + tracing::Span::current().record("aborted", true); + record_result_ready(&turn.sub_id, &dispatch_tool_name, &dispatch_call_id); + return Err(ToolDispatchError::Aborted(ToolAbort::BeforeDispatch)); + } // Readiness and admission remain cancellable even when the handler must finish. let (guard, admission_started) = async { if let Some(tool_runtime) = tool_runtime @@ -1034,17 +1041,20 @@ mod tests { } } + #[test_case::test_case(false; "posted cost checked after the slot opens")] #[tokio::test] - async fn completed_and_budget_stopped_tools_report_their_own_lifecycle() -> anyhow::Result<()> { + async fn completed_and_budget_stopped_tools_report_their_own_lifecycle( + weighted: bool, + ) -> anyhow::Result<()> { let (mut session, turn_context) = crate::session::tests::make_session_and_context().await; session - .rollout_budget() + .work_budget() .configure(crate::config::RolloutBudgetConfig { enforce_limit: true, limit_budget: 1.0, - reminder_at_context_start: true, + reminder_at_context_start: weighted, reminder_at_remaining_tokens: Vec::new(), - sampling_token_weight: 1.0, + sampling_token_weight: if weighted { f64::MAX } else { 1.0 }, prefill_token_weight: 1.0, }); let lifecycle = Arc::new(PausedToolLifecycle::default()); @@ -1136,25 +1146,33 @@ mod tests { output_tokens: 2, ..Default::default() }; - session + let bill = session .record_rollout_budget_usage("peer-turn", &usage.into()) - .await?; + .await; + bill?; + assert!(session.work_budget().is_exhausted()); cancellation_token.cancel(); // Let the caller observe cancellation while the already-finished handler is held. assert!(futures::poll!(response.as_mut()).is_pending()); + // Both waiting handlers must see the posted stop themselves. lifecycle.release_start.notify_one(); let held_reply = tokio::time::timeout(Duration::from_secs(1), held).await??; + let expected_denial = "rollout budget exhausted"; assert_eq!( held_reply.item, ResponseInputItem::FunctionCallOutput { call_id: "held-at-start".into(), - output: FunctionCallOutputPayload::from_text( - "aborted before handler start: rollout budget exhausted".into(), - ), + output: FunctionCallOutputPayload::from_text(format!( + "aborted before handler start: {expected_denial}" + ),), } .into() ); lifecycle.release_finish.notify_one(); + assert_eq!( + ResponseItemEnvelope::new(expected_abort.into()), + tokio::time::timeout(Duration::from_secs(1), queued).await?? + ); let response = tokio::time::timeout(Duration::from_secs(1), response).await??; let expected_response = ResponseInputItem::FunctionCallOutput { @@ -1168,10 +1186,6 @@ mod tests { ResponseItemEnvelope::new(expected_response.into()), response ); - assert_eq!( - ResponseItemEnvelope::new(expected_abort.into()), - tokio::time::timeout(Duration::from_secs(1), queued).await?? - ); assert_eq!(parallel_handler.calls.load(Ordering::Relaxed), 1); assert_eq!(queued_handler.calls.load(Ordering::Relaxed), 0); diff --git a/codex-rs/core/tests/suite/hooks.rs b/codex-rs/core/tests/suite/hooks.rs index d686d958d5..f821e09476 100644 --- a/codex-rs/core/tests/suite/hooks.rs +++ b/codex-rs/core/tests/suite/hooks.rs @@ -6205,8 +6205,8 @@ enum ToolDispatchCase { #[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")] #[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")] -#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered output waits for completion without hooks")] -#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")] +#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered text and tools wait for completion")] +#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming text and tools do not wait for synchronous pricing")] #[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")] #[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")] #[test_case::test_case(ToolDispatchCase::OverflowHook(AgentRuntimeOwnership::Host); "the host receives an overflow lower bound and keeps its policy")] @@ -6324,7 +6324,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> .with_pre_build_hook(move |home| { if buffered { fs::write(home.join("config.toml"), "[responses]\nprocess_mode = \"after_complete_response\"\n") - .expect("configure response processing after completion"); + .expect("configure buffered response processing"); } if let Some(hook_async) = hook_async { let post = format!( diff --git a/codex-rs/core/tests/suite/rollout_budget.rs b/codex-rs/core/tests/suite/rollout_budget.rs index b84cb8a1b3..6bc386af2c 100644 --- a/codex-rs/core/tests/suite/rollout_budget.rs +++ b/codex-rs/core/tests/suite/rollout_budget.rs @@ -440,6 +440,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( skip_if_no_network!(Ok(())); let server = start_mock_server().await; + const ACCEPTED_REVIEW: &str = r#"{"outcome":"allow"}"#; // Enforced: decimal equality is allowed; the next material charge exceeds it. // Advisory: a real overspend still permits a later request with a zero-balance reminder. let (limit_budget, reminder_at_remaining_tokens, charges): (f64, Vec, &[f64]) = @@ -462,7 +463,12 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( let id = format!("budget-{turn}"); let mut completed = ev_completed_with_tokens(&id, /*total_tokens*/ 1); completed["response"]["usage"]["codex_rollout_budget_units"] = json!(units); - sse(vec![ev_response_created(&id), completed]) + let mut events = vec![ev_response_created(&id)]; + if turn == crossing_turn { + events.push(ev_assistant_message("accepted-review", ACCEPTED_REVIEW)); + } + events.push(completed); + sse(events) }) .collect(); let responses = mount_sse_sequence(&server, sequence).await; @@ -483,6 +489,10 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( .build_with_auto_env(&server) .await?; let late = Arc::new(HostAccounting::default()); + // This real Guardian policy replays accepted text only after Completed. + test.codex + .thread_extension_data() + .insert(codex_guardian_reviewer::DenialResample::default()); if owner == AgentRuntimeOwnership::Host { // Neither the owner nor the accounting provider can change after startup. let data = test.codex.thread_extension_data(); @@ -502,7 +512,7 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( matches!(event, EventMsg::Error(_) | EventMsg::TurnComplete(_)) }) .await; - if turn < crossing_turn || owner == AgentRuntimeOwnership::Host || !enforce_limit { + if turn < 2 || owner == AgentRuntimeOwnership::Host || !enforce_limit { assert!(matches!(event, EventMsg::TurnComplete(done) if done.error.is_none())); continue; } @@ -517,8 +527,14 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( .await; } + test.codex.ensure_rollout_materialized().await; + let history = test.codex.load_history(/*include_archived*/ false).await?; + assert!(history.items.iter().any(|item| matches!(item, codex_history::RolloutItem::ResponseItem(envelope) + if matches!(&envelope.item, codex_protocol::models::ResponseItem::Message { role, content, .. } + if role == "assistant" && content.iter().any(|part| matches!(part, + codex_protocol::models::ContentItem::OutputText { text, .. } if text == ACCEPTED_REVIEW))))), + "a shared stop must not discard Guardian's already-selected final text"); if owner == AgentRuntimeOwnership::Controller && enforce_limit { - let history = test.codex.load_history(/*include_archived*/ false).await?; let warning_roles = history .items .iter() @@ -545,7 +561,6 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns( if owner == AgentRuntimeOwnership::Host { let requests = responses.requests(); - assert_eq!(requests.len(), charges.len()); for request in &requests { assert!(rollout_budget_texts(request).is_empty()); } @@ -770,14 +785,21 @@ async fn host_accounting_reports_raw_usage_and_preserves_error_semantics( extensions.thread_lifecycle_contributor(accounting.clone()); let test = test_codex() .with_extensions(Arc::new(extensions.build())) - .with_config(|config| { - config - .features - .enable(Feature::InstantInterrupt) - .expect("enable instant interrupt"); + .with_config(move |config| { + for feature in [ + Feature::InstantInterrupt, + Feature::DefaultModeRequestUserInput, + ] { + config + .features + .enable(feature) + .expect("enable test feature"); + } config.model_provider.stream_max_retries = Some(2); config.model_provider.request_max_retries = Some(0); - config.rollout_budget = None; + // This allowance would admit the response. The selected host can still stop it. + config.rollout_budget = + matches!(behavior, HostAccountingBehavior::Exhausted).then(rollout_budget); }) .build_with_auto_env(&server) .await?;