mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add response processing after completion and usage accounting (#61577)
## Why Streaming can dispatch tools before a response finishes and its usage is accounted for. Allow callers to wait for completion and synchronous post-hooks so budget enforcement can block tools from the response that exhausts the budget. ## What changed - Add `responses.process_mode = "after_complete_response"` to buffer response items and release them in order after completion, synchronous post-hooks, and usage accounting. Keep `streaming` as the default and deliver safety-buffering notifications immediately. - Track whether response usage has already been charged to avoid duplicate accounting, including during compaction. - Stop new tools across the agent tree when the rollout budget is exhausted. Coordinate steering with accounting and prevent buffered tool calls from dispatching after steering or cancellation. - Bound buffered responses to 65,535 events and reject overflow. ## Testing Add configuration coverage and extend integration tests for buffered output ordering, synchronous hooks, interrupted responses and resume, steering, invalid usage, and budget exhaustion preventing tool dispatch. GitOrigin-RevId: bd06811cfd50cb701babe6353465c57dfed2a37a
This commit is contained in:
@@ -421,6 +421,9 @@ pub struct ConfigToml {
|
||||
/// Optional product SKU forwarded on host-owned Codex Apps MCP requests.
|
||||
pub apps_mcp_product_sku: Option<String>,
|
||||
|
||||
/// Controls how Codex processes model responses.
|
||||
pub responses: Option<ResponsesToml>,
|
||||
|
||||
/// Bounded, product-owned metadata attached to every Responses API request.
|
||||
pub responses_api_metadata: Option<BTreeMap<String, String>>,
|
||||
|
||||
@@ -722,6 +725,24 @@ pub struct CodeModeToml {
|
||||
pub heartbeat_interval_s: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub struct ResponsesToml {
|
||||
/// When to process response items, including text and tool calls. Defaults to `streaming`.
|
||||
pub process_mode: Option<ResponseProcessMode>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ResponseProcessMode {
|
||||
/// Process items as they arrive. A tool can start before its response is complete.
|
||||
#[default]
|
||||
Streaming,
|
||||
/// Process buffered items in order after the response completes, synchronous post-hooks
|
||||
/// finish, and any budget charges are posted.
|
||||
AfterCompleteResponse,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub struct ExperimentalRequestUserInput {
|
||||
|
||||
@@ -4728,6 +4728,38 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ResponseProcessMode": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Process items as they arrive. A tool can start before its response is complete.",
|
||||
"enum": [
|
||||
"streaming"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Process buffered items in order after the response completes, synchronous post-hooks finish, and any budget charges are posted.",
|
||||
"enum": [
|
||||
"after_complete_response"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ResponsesToml": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"process_mode": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ResponseProcessMode"
|
||||
}
|
||||
],
|
||||
"description": "When to process response items, including text and tool calls. Defaults to `streaming`."
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"ResumeCwdMode": {
|
||||
"description": "Working directory to use when resuming or forking a session.",
|
||||
"oneOf": [
|
||||
@@ -9061,6 +9093,14 @@
|
||||
"default": null,
|
||||
"description": "Experimental / do not use. Realtime websocket session selection. `version` controls v1/v2 and `type` controls conversational/transcription."
|
||||
},
|
||||
"responses": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ResponsesToml"
|
||||
}
|
||||
],
|
||||
"description": "Controls how Codex processes model responses."
|
||||
},
|
||||
"responses_api_metadata": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
||||
@@ -229,7 +229,7 @@ pub(crate) struct LocalAgentRuntime {
|
||||
pub(super) thread_id_generator: ThreadIdGenerator,
|
||||
pub(super) agent_execution_limiter: Arc<AgentExecutionLimiter>,
|
||||
/// Session-scoped state shared by the root thread and every cloned sub-agent control handle.
|
||||
pub(super) rollout_budget: Arc<RolloutBudget>,
|
||||
pub(crate) rollout_budget: Arc<RolloutBudget>,
|
||||
/// The user-selected root routing tier, shared by the entire agent tree.
|
||||
pub(super) root_service_tier: Arc<ArcSwapOption<String>>,
|
||||
/// Retains the root's opt-in instruction provider even when the root is unloaded.
|
||||
|
||||
+27
-10
@@ -129,6 +129,7 @@ use crate::attestation::X_OAI_ATTESTATION_HEADER;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::context::BaseInstructionsFragment;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::cyber_access_program;
|
||||
@@ -2614,7 +2615,7 @@ impl ModelClientSession {
|
||||
.await
|
||||
{
|
||||
Ok(WebsocketStreamOutcome::Stream(opened)) => {
|
||||
let mut stream = (*opened).into_stream()?;
|
||||
let mut stream = (*opened).into_raw_stream()?;
|
||||
// Wait for the v2 warmup request to complete before sending the first turn request.
|
||||
while let Some(event) = stream.next().await {
|
||||
match event {
|
||||
@@ -2670,7 +2671,7 @@ impl ModelClientSession {
|
||||
inference_trace,
|
||||
)
|
||||
.await?
|
||||
.into_stream()
|
||||
.into_raw_stream()
|
||||
}
|
||||
|
||||
pub(crate) async fn open_model_response(
|
||||
@@ -2814,16 +2815,20 @@ pub(crate) struct OpenedModelResponse {
|
||||
}
|
||||
|
||||
impl OpenedModelResponse {
|
||||
pub(crate) fn into_stream(self) -> Result<ResponseStream> {
|
||||
self.deliver(&mut None, /*sampling_turn*/ None)
|
||||
fn into_raw_stream(self) -> Result<ResponseStream> {
|
||||
self.deliver(
|
||||
&mut None, /*sampling_turn*/ None, /*settlement*/ None,
|
||||
)
|
||||
.map(ResponseStream)
|
||||
}
|
||||
|
||||
pub(crate) fn deliver(
|
||||
mut self,
|
||||
attempt: &mut Option<crate::hook_runtime::sampling::SamplingAttempt>,
|
||||
sampling_turn: Option<Arc<TurnContext>>,
|
||||
) -> Result<ResponseStream> {
|
||||
let delivery = ResponseDelivery::new(attempt);
|
||||
settlement: Option<(Arc<crate::session::session::Session>, Arc<TurnContext>)>,
|
||||
) -> Result<SamplingStream> {
|
||||
let delivery = ResponseDelivery::new(attempt, settlement);
|
||||
let routing_affinity = self.stream.routing_affinity.take();
|
||||
let request_id = self.stream.upstream_request_id.take();
|
||||
let interrupt = self.stream.interrupt.take();
|
||||
@@ -2865,17 +2870,17 @@ fn map_response_events<S>(
|
||||
client: &ModelClientState,
|
||||
mut last_response_tx: Option<oneshot::Sender<LastResponse>>,
|
||||
mut delivery: ResponseDelivery,
|
||||
) -> ResponseStream
|
||||
) -> SamplingStream
|
||||
where
|
||||
S: futures::Stream<Item = std::result::Result<ResponseEvent, ApiError>>
|
||||
+ Unpin
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let (tx_event, rx_event) =
|
||||
mpsc::channel::<Result<ResponseEvent>>(RESPONSE_STREAM_CHANNEL_CAPACITY);
|
||||
let (tx_event, rx_event) = mpsc::channel(RESPONSE_STREAM_CHANNEL_CAPACITY);
|
||||
let consumer_dropped = CancellationToken::new();
|
||||
let consumer_dropped_for_stream = consumer_dropped.clone();
|
||||
let settlement_started = Arc::clone(&delivery.settlement_started);
|
||||
let provider = Arc::clone(&client.provider);
|
||||
let item_id_generator = Arc::clone(&client.item_id_generator);
|
||||
|
||||
@@ -2906,6 +2911,14 @@ where
|
||||
let Some(event) = event else {
|
||||
break;
|
||||
};
|
||||
let overflow = delivery.would_overflow(&event);
|
||||
let event = if overflow {
|
||||
Err(ApiError::Stream(
|
||||
"sampling response exceeded event buffer limit".to_owned(),
|
||||
))
|
||||
} else {
|
||||
event
|
||||
};
|
||||
let event = match event {
|
||||
Ok(ResponseEvent::OutputItemDone(mut item)) => {
|
||||
// Assign IDs after interceptors, which may replace provider items.
|
||||
@@ -3020,6 +3033,9 @@ where
|
||||
}
|
||||
return;
|
||||
}
|
||||
if overflow {
|
||||
return;
|
||||
}
|
||||
}
|
||||
inference_trace_attempt.record_failed(
|
||||
"stream closed before response.completed",
|
||||
@@ -3029,11 +3045,12 @@ where
|
||||
delivery.finish_eof(&tx_event).await;
|
||||
});
|
||||
|
||||
ResponseStream {
|
||||
SamplingStream {
|
||||
routing_affinity: None,
|
||||
rx_event,
|
||||
interrupt: None,
|
||||
consumer_dropped: consumer_dropped_for_stream,
|
||||
settlement_started,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,34 +1,73 @@
|
||||
//! Synchronous post-hooks finish before the turn receives response completion.
|
||||
//! Settles completed responses and releases output according to the configured processing mode.
|
||||
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::hook_runtime::sampling::SamplingAttempt;
|
||||
use codex_api::ApiError;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const MAX_BUFFERED_EVENTS: usize = 65_535;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ResponseDelivery {
|
||||
attempt: Option<SamplingAttempt>,
|
||||
held: Vec<ResponseEvent>,
|
||||
pub(super) settlement_started: Arc<AtomicBool>,
|
||||
settlement: Option<(
|
||||
Arc<crate::session::session::Session>,
|
||||
Arc<crate::session::turn_context::TurnContext>,
|
||||
)>,
|
||||
}
|
||||
|
||||
impl ResponseDelivery {
|
||||
pub(super) fn new(attempt: &mut Option<SamplingAttempt>) -> Self {
|
||||
pub(super) fn new(
|
||||
attempt: &mut Option<SamplingAttempt>,
|
||||
settlement: Option<(
|
||||
Arc<crate::session::session::Session>,
|
||||
Arc<crate::session::turn_context::TurnContext>,
|
||||
)>,
|
||||
) -> Self {
|
||||
Self {
|
||||
attempt: attempt.take(),
|
||||
held: Vec::new(),
|
||||
settlement_started: Default::default(),
|
||||
settlement,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn would_overflow(
|
||||
&self,
|
||||
event: &std::result::Result<ResponseEvent, ApiError>,
|
||||
) -> bool {
|
||||
self.held.len() >= MAX_BUFFERED_EVENTS && event.as_ref().is_ok_and(|item| self.holds(item))
|
||||
}
|
||||
|
||||
fn holds(&self, item: &ResponseEvent) -> bool {
|
||||
!matches!(
|
||||
item,
|
||||
ResponseEvent::Completed { .. } | ResponseEvent::SafetyBuffering(_)
|
||||
) && self.settlement.as_ref().is_some_and(|(_, turn)| {
|
||||
turn.config.response_process_mode
|
||||
== codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse
|
||||
})
|
||||
}
|
||||
|
||||
/// Return false when the turn has stopped receiving this attempt's output.
|
||||
pub(super) async fn send_or_hold(
|
||||
&mut self,
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
tx: &mpsc::Sender<SamplingEvent>,
|
||||
) -> bool {
|
||||
let Some(attempt) = self.attempt.as_mut() else {
|
||||
return tx.send(event).await.is_ok();
|
||||
};
|
||||
let passthrough = self.attempt.is_none() && self.settlement.is_none();
|
||||
if passthrough {
|
||||
return tx.send(event.into()).await.is_ok();
|
||||
}
|
||||
let report = match &event {
|
||||
Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
@@ -36,40 +75,72 @@ impl ResponseDelivery {
|
||||
..
|
||||
}) => ModelRequestHookEvent::completed(response_id, token_usage.as_ref()),
|
||||
Err(error) => ModelRequestHookEvent::failed(error),
|
||||
Ok(_) => return tx.send(event).await.is_ok(),
|
||||
Ok(item) if self.holds(item) => {
|
||||
if let Ok(item) = event {
|
||||
self.held.push(item);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Ok(_) => return tx.send(event.into()).await.is_ok(),
|
||||
};
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
if let Some(hooks) = attempt.start_post_hooks(report) {
|
||||
let hooks = self.attempt.as_mut().and_then(|attempt| {
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
attempt.start_post_hooks(report).map(|hooks| (owner, hooks))
|
||||
});
|
||||
if let Some((owner, hooks)) = hooks {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = tx.closed() => return false,
|
||||
() = owner.cancelled() => return false,
|
||||
() = hooks => {}
|
||||
}
|
||||
// The owner can interrupt before the reader is scheduled to close its channel.
|
||||
if owner.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
self.attempt = None;
|
||||
let released = Self::release(event, tx).await;
|
||||
let terminal = SamplingEvent {
|
||||
event,
|
||||
hook_output: Default::default(),
|
||||
};
|
||||
let released = self.release(terminal, tx).await;
|
||||
match released {
|
||||
Ok(delivered) => delivered,
|
||||
Err(error) => tx.send(Err(codex_api::map_api_error(error))).await.is_ok(),
|
||||
Err(error) => tx.send(Err(error).into()).await.is_ok(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
) -> std::result::Result<bool, ApiError> {
|
||||
if tx.is_closed() {
|
||||
return Ok(false);
|
||||
&mut self,
|
||||
mut terminal: SamplingEvent,
|
||||
tx: &mpsc::Sender<SamplingEvent>,
|
||||
) -> Result<bool> {
|
||||
if self.settlement.is_some()
|
||||
&& matches!(terminal.event, Ok(ResponseEvent::Completed { .. }))
|
||||
{
|
||||
self.settlement_started.store(true, Ordering::Release);
|
||||
}
|
||||
Ok(tx.send(event).await.is_ok())
|
||||
if let Some((session, turn)) = &self.settlement
|
||||
&& let Ok(ResponseEvent::Completed { token_usage, .. }) = &terminal.event
|
||||
{
|
||||
let usage = crate::client_common::SamplingUsage {
|
||||
provider: token_usage.clone(),
|
||||
charge: std::mem::take(&mut terminal.hook_output),
|
||||
};
|
||||
session
|
||||
.record_rollout_budget_usage(&turn.sub_id, &usage)
|
||||
.await?;
|
||||
terminal.hook_output = usage.charge;
|
||||
terminal.hook_output.posted = true;
|
||||
}
|
||||
if terminal.event.is_ok() {
|
||||
for item in std::mem::take(&mut self.held) {
|
||||
if tx.send(Ok(item).into()).await.is_err() {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(tx.send(terminal).await.is_ok())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<Result<ResponseEvent>>) {
|
||||
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<SamplingEvent>) {
|
||||
let Some(mut attempt) = self.attempt.take() else {
|
||||
return;
|
||||
};
|
||||
@@ -81,7 +152,7 @@ impl ResponseDelivery {
|
||||
biased;
|
||||
() = tx.closed() => {}
|
||||
() = owner.cancelled() => {}
|
||||
() = hooks => {}
|
||||
_ = hooks => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,23 +121,66 @@ fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelIn
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ResponseStream {
|
||||
/// Records whether the response producer has posted the bill.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct SamplingCharge {
|
||||
pub(crate) posted: bool,
|
||||
}
|
||||
|
||||
/// Core consumes a terminal event together with its billing status. Other readers
|
||||
/// of the public response stream still see the provider's unmodified event.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SamplingEvent {
|
||||
pub(crate) event: Result<ResponseEvent>,
|
||||
pub(crate) hook_output: SamplingCharge,
|
||||
}
|
||||
|
||||
/// Provider usage and billing status for one response.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SamplingUsage {
|
||||
pub(crate) provider: Option<codex_protocol::protocol::TokenUsage>,
|
||||
pub(crate) charge: SamplingCharge,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl From<codex_protocol::protocol::TokenUsage> for SamplingUsage {
|
||||
fn from(provider: codex_protocol::protocol::TokenUsage) -> Self {
|
||||
Self {
|
||||
provider: Some(provider),
|
||||
charge: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Result<ResponseEvent>> for SamplingEvent {
|
||||
fn from(event: Result<ResponseEvent>) -> Self {
|
||||
Self {
|
||||
event,
|
||||
hook_output: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A sampling consumer receives the billing status with the provider event.
|
||||
pub(crate) struct SamplingStream {
|
||||
pub(crate) routing_affinity: Option<codex_http_client::ChatGptRoutingAffinity>,
|
||||
pub(crate) rx_event: mpsc::Receiver<Result<ResponseEvent>>,
|
||||
pub(crate) rx_event: mpsc::Receiver<SamplingEvent>,
|
||||
pub(crate) interrupt: Option<oneshot::Sender<codex_api::ResponseInterrupt>>,
|
||||
/// Signals the mapper task that the consumer stopped polling before the
|
||||
/// provider stream reached its own terminal event.
|
||||
pub(crate) consumer_dropped: CancellationToken,
|
||||
/// Once settlement begins, a steer waits for its terminal accounting decision.
|
||||
pub(crate) settlement_started: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl ResponseStream {
|
||||
impl SamplingStream {
|
||||
/// A cancelled turn discards any queued event, including an incomplete response's usage.
|
||||
pub(crate) fn accept_for_turn(
|
||||
event: Option<Result<ResponseEvent>>,
|
||||
event: Option<SamplingEvent>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Option<Result<ResponseEvent>> {
|
||||
) -> Option<SamplingEvent> {
|
||||
if cancellation.is_cancelled() {
|
||||
Some(Err(CodexErr::TurnAborted))
|
||||
Some(Err(CodexErr::TurnAborted).into())
|
||||
} else {
|
||||
event
|
||||
}
|
||||
@@ -146,30 +189,43 @@ impl ResponseStream {
|
||||
pub(crate) async fn next_for_turn(
|
||||
&mut self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Option<Result<ResponseEvent>> {
|
||||
) -> Option<SamplingEvent> {
|
||||
let event = tokio::select! {
|
||||
biased;
|
||||
event = self.rx_event.recv() => event,
|
||||
() = cancellation.cancelled() => return Some(Err(CodexErr::TurnAborted)),
|
||||
() = cancellation.cancelled() => return Some(Err(CodexErr::TurnAborted).into()),
|
||||
};
|
||||
Self::accept_for_turn(event, cancellation)
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for ResponseStream {
|
||||
type Item = Result<ResponseEvent>;
|
||||
impl Stream for SamplingStream {
|
||||
type Item = SamplingEvent;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
self.rx_event.poll_recv(cx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponseStream {
|
||||
impl Drop for SamplingStream {
|
||||
fn drop(&mut self) {
|
||||
self.consumer_dropped.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
/// Response events for low-level client requests that do not run sampling hooks.
|
||||
pub struct ResponseStream(pub(crate) SamplingStream);
|
||||
|
||||
impl Stream for ResponseStream {
|
||||
type Item = Result<ResponseEvent>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
Pin::new(&mut self.0)
|
||||
.poll_next(cx)
|
||||
.map(|event| event.map(|event| event.event))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "client_common_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -1695,7 +1695,7 @@ async fn untraced_http_stream_releases_completed_output_before_response_ends() {
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
stream.next().await,
|
||||
stream.next().await.map(|item| item.event),
|
||||
Some(Ok(ResponseEvent::OutputItemDone(_)))
|
||||
));
|
||||
assert_eq!(payload.strong_count(), 0);
|
||||
@@ -1736,7 +1736,7 @@ async fn incomplete_response_traces_attempted_call_without_streaming_it() -> any
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
assert!(matches!(stream.next().await, Some(Err(_))));
|
||||
assert!(stream.next().await.is_some_and(|item| item.event.is_err()));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
@@ -1786,7 +1786,8 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re
|
||||
let observed = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("mapped stream should yield output item")?;
|
||||
.expect("mapped stream should yield output item")
|
||||
.event?;
|
||||
assert!(matches!(observed, ResponseEvent::OutputItemDone(_)));
|
||||
|
||||
// Dropping the consumer is how turn interruption/preemption stops polling
|
||||
@@ -2894,7 +2895,7 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow:
|
||||
interceptors: vec![Box::new(ReplaceOutput)],
|
||||
continuation: Some(last_response_tx),
|
||||
}
|
||||
.into_stream()?;
|
||||
.into_raw_stream()?;
|
||||
let mut delivered = Vec::new();
|
||||
while let Some(event) = stream.next().await {
|
||||
if let ResponseEvent::OutputItemDone(item) = event? {
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::Prompt;
|
||||
use crate::client::ModelClientSession;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::compact_pending_input::is_retained_pending_mail;
|
||||
use crate::compact_remote_v2::retained_client_items;
|
||||
use crate::context::BaseInstructionsFragment;
|
||||
@@ -250,6 +251,8 @@ async fn run_compact_task_inner(
|
||||
CompactionAnalyticsDetails::default(),
|
||||
)
|
||||
.await;
|
||||
// Keep an installed checkpoint and its analytics; stop before continuing the turn.
|
||||
let result = result.and_then(|_| sess.check_rollout_budget());
|
||||
if let Err(err) = &result
|
||||
&& !matches!(phase, CompactionPhase::PostTurn)
|
||||
&& !matches!(
|
||||
@@ -264,7 +267,7 @@ async fn run_compact_task_inner(
|
||||
sess.send_event(&turn_context, event).await;
|
||||
}
|
||||
}
|
||||
result.map(|_| ())
|
||||
result
|
||||
}
|
||||
|
||||
async fn run_compact_task_inner_impl(
|
||||
@@ -899,8 +902,8 @@ async fn drain_to_completed(
|
||||
.await??;
|
||||
let mut output = Vec::new();
|
||||
loop {
|
||||
let maybe_event = stream.next_for_turn(cancellation).await;
|
||||
let Some(event) = maybe_event else {
|
||||
let Some(SamplingEvent { event, hook_output }) = stream.next_for_turn(cancellation).await
|
||||
else {
|
||||
return Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
));
|
||||
@@ -945,7 +948,10 @@ async fn drain_to_completed(
|
||||
.await;
|
||||
sess.update_token_usage_info(
|
||||
turn_context,
|
||||
token_usage.as_ref(),
|
||||
crate::client_common::SamplingUsage {
|
||||
provider: token_usage,
|
||||
charge: hook_output,
|
||||
},
|
||||
includes_reasoning,
|
||||
)
|
||||
.await?;
|
||||
@@ -961,6 +967,7 @@ async fn drain_to_completed(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
incomplete,
|
||||
hook_output,
|
||||
)
|
||||
.await;
|
||||
sess.send_token_count_event(turn_context).await;
|
||||
|
||||
@@ -7,11 +7,13 @@ use crate::compact_retained_tool_outputs::is_retained_tool_output;
|
||||
use crate::session::FileAudioDurations;
|
||||
|
||||
use crate::Prompt;
|
||||
use crate::ResponseStream;
|
||||
use crate::client::ModelClientSession;
|
||||
use crate::client::ResponsesHttpExecutor;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::client_common::SamplingUsage;
|
||||
use crate::compact::CompactedHistoryMetadata;
|
||||
use crate::compact::CompactionAnalyticsAttempt;
|
||||
use crate::compact::CompactionAnalyticsDetails;
|
||||
@@ -262,6 +264,7 @@ async fn run_remote_compact_task_inner(
|
||||
attempt
|
||||
.track(sess.as_ref(), status, codex_error, analytics_details)
|
||||
.await;
|
||||
let result = result.and_then(|installed| sess.check_rollout_budget().map(|()| installed));
|
||||
match result {
|
||||
Ok(installed) => Ok(installed),
|
||||
Err(err)
|
||||
@@ -411,7 +414,7 @@ async fn run_remote_compaction_request_v2(
|
||||
async fn collect_compaction_output(
|
||||
sess: &Session,
|
||||
turn_context: &TurnContext,
|
||||
mut stream: ResponseStream,
|
||||
mut stream: SamplingStream,
|
||||
request_kind: CompactionRequestKind<'_>,
|
||||
) -> CodexResult<RemoteCompactionV2Output> {
|
||||
let mut output_item_count = 0usize;
|
||||
@@ -431,31 +434,40 @@ async fn collect_compaction_output(
|
||||
_ = cancellation.cancelled() => return Err(CodexErr::TurnAborted),
|
||||
},
|
||||
};
|
||||
let Some(event) = event else { break };
|
||||
if let Err(err) = &event
|
||||
&& let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details()
|
||||
{
|
||||
match request_kind {
|
||||
CompactionRequestKind::Background(_) => {
|
||||
if let Some(usage) = incomplete.token_usage.as_ref() {
|
||||
// Match completed background requests without replacing the terminal error.
|
||||
let _ = sess
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, usage)
|
||||
let Some(SamplingEvent { event, hook_output }) = event else {
|
||||
break;
|
||||
};
|
||||
let event = match event {
|
||||
Ok(event) => event,
|
||||
Err(err) => {
|
||||
if let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details() {
|
||||
match request_kind {
|
||||
CompactionRequestKind::Background(_) => {
|
||||
// Match completed background requests without replacing the terminal error.
|
||||
let usage = SamplingUsage {
|
||||
provider: incomplete.token_usage.clone(),
|
||||
charge: hook_output,
|
||||
};
|
||||
let _ = sess
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, &usage)
|
||||
.await;
|
||||
}
|
||||
CompactionRequestKind::Foreground(_) => {
|
||||
sess.record_incomplete_response_usage(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
incomplete,
|
||||
hook_output,
|
||||
)
|
||||
.await;
|
||||
sess.send_token_count_event(turn_context).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
CompactionRequestKind::Foreground(_) => {
|
||||
sess.record_incomplete_response_usage(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
incomplete,
|
||||
)
|
||||
.await;
|
||||
sess.send_token_count_event(turn_context).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
match event? {
|
||||
};
|
||||
match event {
|
||||
ResponseEvent::OutputItemDone(item) => {
|
||||
output_item_count += 1;
|
||||
if let ResponseItem::Compaction { .. } = item {
|
||||
@@ -471,25 +483,25 @@ async fn collect_compaction_output(
|
||||
usage_metadata,
|
||||
..
|
||||
} => {
|
||||
match request_kind {
|
||||
CompactionRequestKind::Background(_) => {
|
||||
if let Some(usage) = token_usage.as_ref() {
|
||||
sess.record_rollout_budget_usage(&turn_context.sub_id, usage)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
CompactionRequestKind::Foreground(_) => {
|
||||
sess.record_observed_response_completed(
|
||||
turn_context,
|
||||
&response_id,
|
||||
token_usage.as_ref(),
|
||||
usage_metadata.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let usage = SamplingUsage {
|
||||
provider: token_usage,
|
||||
charge: hook_output,
|
||||
};
|
||||
if matches!(request_kind, CompactionRequestKind::Foreground(_)) {
|
||||
sess.record_observed_response_completed(
|
||||
turn_context,
|
||||
&response_id,
|
||||
usage.provider.as_ref(),
|
||||
usage_metadata.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// Both callers consumed this response. Whether it contains an
|
||||
// installable summary is a separate decision below.
|
||||
sess.record_rollout_budget_usage(&turn_context.sub_id, &usage)
|
||||
.await?;
|
||||
completed_response_id = Some(response_id);
|
||||
completed_token_usage = token_usage;
|
||||
completed_token_usage = usage.provider;
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
@@ -983,19 +995,20 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn response_stream(events: Vec<CodexResult<ResponseEvent>>) -> ResponseStream {
|
||||
fn response_stream(events: Vec<impl Into<SamplingEvent>>) -> SamplingStream {
|
||||
let (tx_event, rx_event) = mpsc::channel(events.len().max(1));
|
||||
for event in events {
|
||||
tx_event
|
||||
.try_send(event)
|
||||
.try_send(event.into())
|
||||
.expect("response stream test channel should have capacity");
|
||||
}
|
||||
drop(tx_event);
|
||||
ResponseStream {
|
||||
SamplingStream {
|
||||
routing_affinity: None,
|
||||
rx_event,
|
||||
interrupt: None,
|
||||
consumer_dropped: CancellationToken::new(),
|
||||
settlement_started: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -198,11 +198,9 @@ async fn build_and_install_compaction(
|
||||
owned_client_session: _owned_client_session,
|
||||
} = attempt;
|
||||
if prefix.is_none()
|
||||
&& let Some(token_usage) = token_usage.as_ref()
|
||||
&& let Some(usage) = &token_usage
|
||||
{
|
||||
sess.record_rollout_budget_usage(&compaction_turn_context.sub_id, token_usage)
|
||||
.await?;
|
||||
analytics_details.active_context_tokens_before = Some(token_usage.input_tokens);
|
||||
analytics_details.active_context_tokens_before = Some(usage.input_tokens);
|
||||
}
|
||||
let (compacted_history, retained_images) = build_v2_compacted_history(
|
||||
prompt_input,
|
||||
|
||||
@@ -708,6 +708,28 @@ disable_in_process_fallback = true
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_config_resolves_response_process_mode() -> std::io::Result<()> {
|
||||
let streaming = r#"[responses]
|
||||
process_mode = "streaming"
|
||||
"#;
|
||||
let buffered = r#"[responses]
|
||||
process_mode = "after_complete_response"
|
||||
"#;
|
||||
for (toml, expected) in [
|
||||
("", ResponseProcessMode::Streaming),
|
||||
(streaming, ResponseProcessMode::Streaming),
|
||||
(buffered, ResponseProcessMode::AfterCompleteResponse),
|
||||
] {
|
||||
assert_eq!(
|
||||
load_config_from_toml(toml).await?.response_process_mode,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_config_resolves_tool_registry_config() -> std::io::Result<()> {
|
||||
let codex_home = tempdir()?;
|
||||
@@ -1009,14 +1031,14 @@ sleep_tool = true
|
||||
},
|
||||
),
|
||||
] {
|
||||
let config = load_current_time_reminder_config(config_toml).await?;
|
||||
let config = load_config_from_toml(config_toml).await?;
|
||||
assert!(config.features.enabled(Feature::CurrentTimeReminder));
|
||||
assert_eq!(config.current_time_reminder, Some(expected));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_current_time_reminder_config(config_toml: &str) -> std::io::Result<Config> {
|
||||
async fn load_config_from_toml(config_toml: &str) -> std::io::Result<Config> {
|
||||
let codex_home = tempdir()?;
|
||||
let config_toml = toml::from_str(config_toml).expect("TOML should deserialize");
|
||||
Config::load_from_base_config_with_overrides(
|
||||
|
||||
@@ -33,6 +33,7 @@ use codex_config::config_toml::DEFAULT_PROJECT_DOC_MAX_BYTES;
|
||||
use codex_config::config_toml::ProjectConfig;
|
||||
use codex_config::config_toml::RealtimeAudioConfig;
|
||||
use codex_config::config_toml::RealtimeConfig;
|
||||
use codex_config::config_toml::ResponseProcessMode;
|
||||
use codex_config::config_toml::ThreadStoreToml;
|
||||
use codex_config::config_toml::validate_model_providers;
|
||||
use codex_config::loader::load_config_layers_state;
|
||||
@@ -1167,6 +1168,9 @@ pub struct Config {
|
||||
/// Policy for collecting and validating tool runtimes.
|
||||
pub tool_registry: ToolRegistryConfig,
|
||||
|
||||
/// When to process model output, including text and tool calls.
|
||||
pub response_process_mode: ResponseProcessMode,
|
||||
|
||||
/// Configuration for the experimental code-mode tool surface.
|
||||
pub code_mode: CodeModeConfig,
|
||||
|
||||
@@ -4053,6 +4057,8 @@ impl Config {
|
||||
let experimental_request_user_input_enabled =
|
||||
resolve_experimental_request_user_input_enabled(&cfg);
|
||||
let update_plan_enabled = resolve_update_plan_enabled(&cfg);
|
||||
let response_process_mode = cfg.responses.as_ref().and_then(|responses| responses.process_mode);
|
||||
let response_process_mode = response_process_mode.unwrap_or_default();
|
||||
let tool_registry = ToolRegistryConfig {
|
||||
error_on_tool_collisions: cfg
|
||||
.features
|
||||
@@ -4810,6 +4816,7 @@ impl Config {
|
||||
experimental_request_user_input_enabled,
|
||||
update_plan_enabled,
|
||||
tool_registry,
|
||||
response_process_mode,
|
||||
code_mode,
|
||||
background_terminal_max_timeout,
|
||||
thread_unload_delay,
|
||||
|
||||
@@ -14,7 +14,8 @@ use futures::StreamExt;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingEvent;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::turn::get_last_assistant_message_from_turn;
|
||||
use crate::tools::ToolRouter;
|
||||
@@ -41,12 +42,9 @@ impl ResponseResampling {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn next(
|
||||
&mut self,
|
||||
stream: &mut ResponseStream,
|
||||
) -> Option<CodexResult<ResponseEvent>> {
|
||||
pub(crate) async fn next(&mut self, stream: &mut SamplingStream) -> Option<SamplingEvent> {
|
||||
match self.accepted_output.pop_front() {
|
||||
Some(item) => Some(Ok(ResponseEvent::OutputItemDone(item))),
|
||||
Some(item) => Some(Ok(ResponseEvent::OutputItemDone(item)).into()),
|
||||
None => stream.next().await,
|
||||
}
|
||||
}
|
||||
@@ -113,9 +111,9 @@ impl ResponseResampling {
|
||||
/// Called after the denied completion has been accounted for, with the unchanged request.
|
||||
pub(crate) async fn retry(
|
||||
&mut self,
|
||||
request: impl Future<Output = CodexResult<ResponseStream>> + Send,
|
||||
request: impl Future<Output = CodexResult<SamplingStream>> + Send,
|
||||
cancellation_token: &CancellationToken,
|
||||
) -> CodexResult<ResponseStream> {
|
||||
) -> CodexResult<SamplingStream> {
|
||||
self.retry_pending = false;
|
||||
self.buffered_output = Some(Vec::new());
|
||||
request.or_cancel(cancellation_token).await?
|
||||
|
||||
@@ -8,7 +8,7 @@ use super::run_model_request_hooks;
|
||||
use super::thread_spawn_subagent_hook_context;
|
||||
use super::tool_hook_cwd;
|
||||
use crate::client::OpenedModelResponse;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::step_context::StepContext;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
@@ -40,7 +40,7 @@ impl SamplingRequest {
|
||||
model: &str,
|
||||
owner_cancellation: &CancellationToken,
|
||||
send: impl Future<Output = Result<OpenedModelResponse>> + Send,
|
||||
) -> impl Future<Output = Result<ResponseStream>> + Send {
|
||||
) -> impl Future<Output = Result<SamplingStream>> + Send {
|
||||
// Keep the transport and hook state out of the turn's cancellation and retry futures.
|
||||
Box::pin(self.open_attempt(session, step, model, owner_cancellation, send))
|
||||
}
|
||||
@@ -52,16 +52,21 @@ impl SamplingRequest {
|
||||
model: &str,
|
||||
owner_cancellation: &CancellationToken,
|
||||
send: impl Future<Output = Result<OpenedModelResponse>>,
|
||||
) -> Result<ResponseStream> {
|
||||
) -> Result<SamplingStream> {
|
||||
let send = async {
|
||||
session.check_rollout_budget()?;
|
||||
send.await
|
||||
};
|
||||
let sampling_turn = match self.kind {
|
||||
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
|
||||
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
|
||||
};
|
||||
let settlement = Some((Arc::clone(session), Arc::clone(&step.turn)));
|
||||
let hooks = hooks_for_model_request(session, step);
|
||||
let pre_mode = hooks.execution_mode_for_event(HookEventName::PreModelRequest);
|
||||
let post_mode = hooks.execution_mode_for_event(HookEventName::PostModelRequest);
|
||||
if pre_mode.is_none() && post_mode.is_none() {
|
||||
return send.await?.deliver(&mut None, sampling_turn);
|
||||
return send.await?.deliver(&mut None, sampling_turn, settlement);
|
||||
}
|
||||
let subagent = thread_spawn_subagent_hook_context(session, &step.turn);
|
||||
let input = ModelRequestHookInput {
|
||||
@@ -92,7 +97,7 @@ impl SamplingRequest {
|
||||
}
|
||||
let stream = send
|
||||
.await
|
||||
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn));
|
||||
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn, settlement));
|
||||
if let Err(error) = &stream
|
||||
&& let Some(mut attempt) = attempt
|
||||
&& let Some(hooks) = attempt.start_post_hooks(ModelRequestHookEvent::failed(error))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Interceptors can transform incremental events and retain upstream cancellation.
|
||||
use super::*;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use codex_extension_api::ResponseEvent;
|
||||
use futures::StreamExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -29,14 +29,15 @@ async fn interceptors_forward_before_completion_in_order_and_cancel_upstream() {
|
||||
let upstream_cancelled = consumer_dropped.clone();
|
||||
let mut stream = intercept_stream(
|
||||
Box::pin(
|
||||
ResponseStream {
|
||||
SamplingStream {
|
||||
routing_affinity: None,
|
||||
rx_event,
|
||||
interrupt: None,
|
||||
consumer_dropped,
|
||||
settlement_started: Default::default(),
|
||||
}
|
||||
.map(|event| {
|
||||
event.map_err(|error| {
|
||||
event.event.map_err(|error| {
|
||||
codex_extension_api::ModelResponseError::Stream(error.to_string())
|
||||
})
|
||||
}),
|
||||
@@ -44,7 +45,7 @@ async fn interceptors_forward_before_completion_in_order_and_cancel_upstream() {
|
||||
vec![Box::new(Append("a")), Box::new(Append("b"))],
|
||||
);
|
||||
tx_event
|
||||
.send(Ok(ResponseEvent::OutputTextDelta("text".into())))
|
||||
.send(Ok(ResponseEvent::OutputTextDelta("text".into())).into())
|
||||
.await
|
||||
.unwrap();
|
||||
// Upstream remains open and has not sent Completed.
|
||||
|
||||
@@ -7,6 +7,8 @@ use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::MutexGuard;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
/// Budget reminder returned by a controller and acknowledged after history insertion.
|
||||
pub struct RolloutBudgetReminder {
|
||||
@@ -19,6 +21,8 @@ pub struct RolloutBudgetReminder {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RolloutBudget {
|
||||
state: OnceLock<Mutex<RolloutBudgetState>>,
|
||||
/// An explicit stop gates new tools across the tree, even with per-thread host accountants.
|
||||
stop_tools: AtomicBool,
|
||||
}
|
||||
|
||||
struct RolloutBudgetState {
|
||||
@@ -63,8 +67,20 @@ impl RolloutBudget {
|
||||
+ usage.non_cached_input() as f64 * state.config.prefill_token_weight
|
||||
};
|
||||
state.weighted_tokens_used += units;
|
||||
Ok(state.config.enforce_limit
|
||||
&& state.weighted_tokens_used >= state.config.limit_tokens as f64)
|
||||
let exhausted = state.config.enforce_limit
|
||||
&& state.weighted_tokens_used >= state.config.limit_tokens as f64;
|
||||
if exhausted {
|
||||
self.stop_new_tools();
|
||||
}
|
||||
Ok(exhausted)
|
||||
}
|
||||
|
||||
pub(crate) fn stop_new_tools(&self) {
|
||||
self.stop_tools.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn tools_stopped(&self) -> bool {
|
||||
self.stop_tools.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn pending_reminder(
|
||||
|
||||
@@ -106,11 +106,12 @@ async fn cancelled_turn_skips_launch_but_preserves_existing_work(trigger: Compac
|
||||
]));
|
||||
sess.update_token_usage_info(
|
||||
&turn,
|
||||
Some(&TokenUsage {
|
||||
TokenUsage {
|
||||
input_tokens: 60_000,
|
||||
total_tokens: 60_000,
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
.into(),
|
||||
/*includes_reasoning*/ false,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -540,7 +540,7 @@ impl InputQueue {
|
||||
}
|
||||
|
||||
impl TurnInputQueue {
|
||||
fn has_user_input(&self) -> bool {
|
||||
pub(crate) fn has_user_input(&self) -> bool {
|
||||
self.items
|
||||
.iter()
|
||||
.any(|input| matches!(input, TurnInput::UserInput { .. }))
|
||||
|
||||
@@ -5654,6 +5654,7 @@ impl Session {
|
||||
turn_context: &TurnContext,
|
||||
settings: &ResolvedStepSettings,
|
||||
incomplete: &ResponseIncompleteError,
|
||||
charge: crate::client_common::SamplingCharge,
|
||||
) {
|
||||
if let Some(response_id) = &incomplete.response_id {
|
||||
self.record_response_token_usage(
|
||||
@@ -5668,7 +5669,10 @@ impl Session {
|
||||
.record_token_usage_info(
|
||||
turn_context,
|
||||
settings,
|
||||
incomplete.token_usage.as_ref(),
|
||||
crate::client_common::SamplingUsage {
|
||||
provider: incomplete.token_usage.clone(),
|
||||
charge,
|
||||
},
|
||||
TokenUsageContext::SpendingOnly,
|
||||
)
|
||||
.await;
|
||||
@@ -5677,14 +5681,14 @@ impl Session {
|
||||
pub(crate) async fn update_token_usage_info(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
usage: crate::client_common::SamplingUsage,
|
||||
includes_reasoning: bool,
|
||||
) -> CodexResult<()> {
|
||||
let result = self
|
||||
.record_token_usage_info(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
token_usage,
|
||||
usage,
|
||||
TokenUsageContext::Sampling {
|
||||
includes_reasoning,
|
||||
input_item_count: 0,
|
||||
@@ -5743,10 +5747,10 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
settings: &ResolvedStepSettings,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
usage: crate::client_common::SamplingUsage,
|
||||
context: TokenUsageContext,
|
||||
) -> CodexResult<()> {
|
||||
if let Some(token_usage) = token_usage {
|
||||
let token_info = if let Some(token_usage) = usage.provider.as_ref() {
|
||||
let token_info = {
|
||||
let mut state = self.state.lock().await;
|
||||
match context {
|
||||
@@ -5799,24 +5803,26 @@ impl Session {
|
||||
token_usage,
|
||||
);
|
||||
}
|
||||
let budget_result = self
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, token_usage)
|
||||
.await;
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
contributor
|
||||
.on_token_usage(
|
||||
&self.services.session_extension_data,
|
||||
&self.services.thread_extension_data,
|
||||
turn_context.extension_data.as_ref(),
|
||||
token_info,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
token_info
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let budget_result = self
|
||||
.record_rollout_budget_usage(&turn_context.sub_id, &usage)
|
||||
.await;
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
contributor
|
||||
.on_token_usage(
|
||||
&self.services.session_extension_data,
|
||||
&self.services.thread_extension_data,
|
||||
turn_context.extension_data.as_ref(),
|
||||
token_info,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
budget_result?;
|
||||
}
|
||||
Ok(())
|
||||
budget_result
|
||||
}
|
||||
|
||||
pub(crate) async fn recompute_token_usage(&self, turn_context: &TurnContext) {
|
||||
|
||||
@@ -4,9 +4,10 @@ use crate::agent::host::AgentRuntimeServices;
|
||||
use crate::agent::host::AgentUsageReport;
|
||||
use crate::agent::host::UsageAccountingDecision;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::rollout_budget::RolloutBudget;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
pub(super) async fn maybe_record_reminder(
|
||||
sess: &Session,
|
||||
@@ -39,18 +40,44 @@ pub(super) async fn maybe_record_reminder(
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) fn rollout_budget(&self) -> &RolloutBudget {
|
||||
&self.services.local_agent_runtime.rollout_budget
|
||||
}
|
||||
|
||||
pub(crate) fn check_rollout_budget(&self) -> CodexResult<()> {
|
||||
if self.rollout_budget().tools_stopped() {
|
||||
Err(CodexErr::SessionBudgetExceeded)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn record_rollout_budget_usage(
|
||||
&self,
|
||||
turn_id: &str,
|
||||
usage: &TokenUsage,
|
||||
bill: &crate::client_common::SamplingUsage,
|
||||
) -> CodexResult<()> {
|
||||
match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => {
|
||||
self.services
|
||||
.agent_control
|
||||
.record_usage(usage.clone())
|
||||
.await
|
||||
}
|
||||
if bill.charge.posted {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(usage) = bill.provider.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let stop_new_work = match &self.agent_runtime {
|
||||
AgentRuntimeServices::Controller => match self
|
||||
.services
|
||||
.agent_control
|
||||
.record_usage(usage.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => false,
|
||||
Err(error)
|
||||
if matches!(error.details(), CodexErrorDetails::SessionBudgetExceeded) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
},
|
||||
AgentRuntimeServices::Host(services) => {
|
||||
let Some(accounting) = &services.usage_accounting else {
|
||||
return Ok(());
|
||||
@@ -65,11 +92,12 @@ impl Session {
|
||||
.map_err(|error| {
|
||||
CodexErr::Fatal(format!("Host usage accounting failed: {error}"))
|
||||
})?;
|
||||
match decision {
|
||||
UsageAccountingDecision::Continue => Ok(()),
|
||||
UsageAccountingDecision::Exhausted => Err(CodexErr::SessionBudgetExceeded),
|
||||
}
|
||||
decision == UsageAccountingDecision::Exhausted
|
||||
}
|
||||
};
|
||||
if stop_new_work {
|
||||
self.rollout_budget().stop_new_tools();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3398,7 +3398,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&first_usage),
|
||||
first_usage.clone().into(),
|
||||
crate::session::TokenUsageContext::Sampling {
|
||||
includes_reasoning: true,
|
||||
input_item_count: 0,
|
||||
@@ -3410,7 +3410,7 @@ async fn record_token_usage_info_notifies_extension_contributors(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&second_usage),
|
||||
second_usage.clone().into(),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
@@ -13173,7 +13173,6 @@ async fn tool_calls_reopen_mailbox_delivery_for_current_turn() {
|
||||
step_context: StepContext::for_test(Arc::clone(&tc)),
|
||||
turn_store: Arc::new(codex_extension_api::ExtensionData::new(tc.sub_id.clone())),
|
||||
tool_runtime: test_tool_runtime(Arc::clone(&sess), Arc::clone(&tc)).await,
|
||||
defer_tool_dispatch: false,
|
||||
cancellation_token: CancellationToken::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::client::ModelClientSession;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::client_common::SamplingStream;
|
||||
use crate::compact::AutoCompactionDisabled;
|
||||
use crate::compact::run_inline_auto_compact_task;
|
||||
use crate::compact_remote_v2::BackgroundCompactionCandidate;
|
||||
@@ -654,6 +654,11 @@ pub(crate) async fn run_turn(
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
// Consume the whole response before deciding whether the turn may continue.
|
||||
let sampling_request_result = sampling_request_result.and_then(|output| {
|
||||
sess.check_rollout_budget()?;
|
||||
Ok(output)
|
||||
});
|
||||
match sampling_request_result {
|
||||
Ok((sampling_request_output, notify_messages)) => {
|
||||
guardian_budget_compacted = false;
|
||||
@@ -2827,7 +2832,7 @@ async fn drain_in_flight(
|
||||
in_flight: &mut FuturesOrdered<InFlightFuture<'static>>,
|
||||
sess: Arc<Session>,
|
||||
step_context: &StepContext,
|
||||
) -> CodexResult<()> {
|
||||
) {
|
||||
let turn_context = &step_context.turn;
|
||||
while let Some(res) = in_flight.next().await {
|
||||
match res {
|
||||
@@ -2850,7 +2855,6 @@ async fn drain_in_flight(
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2977,7 +2981,10 @@ async fn try_run_sampling_request(
|
||||
if cancellation_token.is_cancelled() && !can_interrupt {
|
||||
return Err(CodexErr::TurnAborted);
|
||||
}
|
||||
let mut in_flight: FuturesOrdered<InFlightFuture<'static>> = FuturesOrdered::new();
|
||||
let streaming = turn_context.config.response_process_mode
|
||||
== codex_config::config_toml::ResponseProcessMode::Streaming;
|
||||
let mut wait_for_bill_only = false;
|
||||
let mut in_flight = FuturesOrdered::new();
|
||||
let mut needs_follow_up = false;
|
||||
let mut last_agent_message: Option<String> = None;
|
||||
let mut active_item: Option<TurnItem> = None;
|
||||
@@ -3053,31 +3060,36 @@ async fn try_run_sampling_request(
|
||||
preempt = CancellationToken::new();
|
||||
continue;
|
||||
}
|
||||
if let Some(interrupt) = stream.interrupt.take() {
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming && stream.interrupt.is_none();
|
||||
} else {
|
||||
let Some(interrupt) = stream.interrupt.take() else {
|
||||
// TODO: Reconcile any response item already being presented to the client.
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: false,
|
||||
last_agent_message,
|
||||
});
|
||||
};
|
||||
if step_context.settings.model_info.use_responses_lite {
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
}
|
||||
// Drain the response normally before reusing its connection and history.
|
||||
preempt = CancellationToken::new();
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
// TODO: Reconcile any response item already being presented to the client.
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: false,
|
||||
last_agent_message,
|
||||
});
|
||||
// A steer cannot start another request before the accounting owner returns its decision.
|
||||
preempt = CancellationToken::new();
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
Err(_) => break Err(CodexErr::TurnAborted),
|
||||
};
|
||||
|
||||
let event = ResponseStream::accept_for_turn(event, &cancellation_token);
|
||||
let event = match event.unwrap_or_else(|| {
|
||||
Err(CodexErr::Stream(
|
||||
let event = SamplingStream::accept_for_turn(event, &cancellation_token);
|
||||
let Some(crate::client_common::SamplingEvent { event, hook_output }) = event else {
|
||||
break Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
))
|
||||
}) {
|
||||
));
|
||||
};
|
||||
let event = match event {
|
||||
Ok(event) => event,
|
||||
Err(err) => {
|
||||
if let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details() {
|
||||
@@ -3085,6 +3097,7 @@ async fn try_run_sampling_request(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
incomplete,
|
||||
hook_output,
|
||||
)
|
||||
.await;
|
||||
should_emit_token_count |= incomplete.token_usage.is_some();
|
||||
@@ -3105,6 +3118,19 @@ async fn try_run_sampling_request(
|
||||
}
|
||||
};
|
||||
|
||||
// A steer can cut off unread items on an HTTP stream while the owner posts its bill.
|
||||
// Consume the terminal accounting decision before opening the follow-up request.
|
||||
if wait_for_bill_only {
|
||||
if matches!(event, ResponseEvent::Completed { .. }) {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: *draining_for_mailbox,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(event) = resampling.filter(event, &step_context.tool_router) else {
|
||||
continue;
|
||||
};
|
||||
@@ -3215,7 +3241,6 @@ async fn try_run_sampling_request(
|
||||
step_context: Arc::clone(&step_context),
|
||||
turn_store: Arc::clone(&turn_store),
|
||||
tool_runtime,
|
||||
defer_tool_dispatch: *draining_for_mailbox,
|
||||
cancellation_token: cancellation_token.child_token(),
|
||||
};
|
||||
if *draining_for_mailbox {
|
||||
@@ -3263,8 +3288,8 @@ async fn try_run_sampling_request(
|
||||
Ok(output_result) => output_result,
|
||||
Err(err) => break Err(err),
|
||||
};
|
||||
if let Some(tool_future) = output_result.tool_future {
|
||||
in_flight.push_back(tool_future);
|
||||
if let Some(tool) = output_result.tool_future {
|
||||
in_flight.push_back(tool);
|
||||
}
|
||||
if let Some(agent_message) = output_result.last_agent_message {
|
||||
last_agent_message = Some(agent_message);
|
||||
@@ -3291,23 +3316,26 @@ async fn try_run_sampling_request(
|
||||
conversation.id = %sess.thread_id,
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
if step_context.settings.model_info.use_responses_lite
|
||||
if stream.settlement_started.load(Ordering::Acquire) {
|
||||
wait_for_bill_only = streaming
|
||||
&& !(step_context.settings.model_info.use_responses_lite
|
||||
&& stream.interrupt.is_some());
|
||||
} else if step_context.settings.model_info.use_responses_lite
|
||||
&& let Some(interrupt) = stream.interrupt.take()
|
||||
{
|
||||
// A closed receiver can mean completion is already buffered. Drain it
|
||||
// too; a transport failure below still takes the mailbox follow-up.
|
||||
// Completion can already be buffered when the interrupt receiver closes.
|
||||
let _ = interrupt.send(codex_api::ResponseInterrupt::Drain);
|
||||
// Retain completed items and the response ID for incremental continuation.
|
||||
// Further item boundaries must not drop the stream while it drains.
|
||||
*draining_for_mailbox = true;
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
} else {
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: true,
|
||||
last_agent_message,
|
||||
});
|
||||
}
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
mailbox_preempted: true,
|
||||
last_agent_message,
|
||||
});
|
||||
// Keep the response ID and terminal decision. Later tools receive a denial.
|
||||
*draining_for_mailbox = true;
|
||||
needs_follow_up = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ResponseEvent::OutputItemAdded(mut item) => {
|
||||
@@ -3502,7 +3530,10 @@ async fn try_run_sampling_request(
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
token_usage.as_ref(),
|
||||
crate::client_common::SamplingUsage {
|
||||
provider: token_usage,
|
||||
charge: hook_output,
|
||||
},
|
||||
if resampling.retry_pending {
|
||||
super::TokenUsageContext::SpendingOnly
|
||||
} else {
|
||||
@@ -3715,7 +3746,7 @@ async fn try_run_sampling_request(
|
||||
conversation.id = %sess.thread_id,
|
||||
turn.id = %turn_context.sub_id,
|
||||
);
|
||||
drain_in_flight(&mut in_flight, sess.clone(), &step_context).await?;
|
||||
drain_in_flight(&mut in_flight, Arc::clone(&sess), &step_context).await;
|
||||
}
|
||||
|
||||
if should_emit_token_count {
|
||||
|
||||
@@ -221,9 +221,6 @@ async fn record_stage1_output_usage_for_memory_citation(
|
||||
true
|
||||
}
|
||||
|
||||
/// Handle a completed output item from the model stream, recording it and
|
||||
/// queuing any tool execution futures. This records items immediately so
|
||||
/// history and rollout stay in sync even if the turn is later cancelled.
|
||||
pub(crate) type InFlightFuture<'f> =
|
||||
Pin<Box<dyn Future<Output = Result<ResponseItemEnvelope>> + Send + 'f>>;
|
||||
|
||||
@@ -239,7 +236,6 @@ pub(crate) struct HandleOutputCtx {
|
||||
pub step_context: Arc<StepContext>,
|
||||
pub turn_store: Arc<ExtensionData>,
|
||||
pub tool_runtime: ToolCallRuntime,
|
||||
pub defer_tool_dispatch: bool,
|
||||
pub cancellation_token: CancellationToken,
|
||||
}
|
||||
|
||||
@@ -385,24 +381,44 @@ pub(crate) async fn handle_output_item_done(
|
||||
|
||||
let cancellation_token = ctx.cancellation_token.child_token();
|
||||
let runtime = ctx.tool_runtime.clone();
|
||||
// handle_tool_call dispatches immediately, not when its future is polled.
|
||||
// Defer for post-model hooks or to cancel buffered calls during mailbox drains.
|
||||
let tool_future: InFlightFuture<'static> = if ctx.defer_tool_dispatch {
|
||||
Box::pin(async move {
|
||||
if cancellation_token.is_cancelled() {
|
||||
// The call is already in history; resume needs its output even if we never dispatch.
|
||||
let output =
|
||||
ToolCallRuntime::aborted_response(&call, /*elapsed_secs*/ None);
|
||||
return Ok(output.into_response(/*file_backed_tool_audio*/ false));
|
||||
}
|
||||
runtime.handle_tool_call(call, cancellation_token).await
|
||||
})
|
||||
} else {
|
||||
Box::pin(runtime.handle_tool_call(call, cancellation_token))
|
||||
let dispatch = |allowed: bool| -> InFlightFuture<'static> {
|
||||
if allowed
|
||||
&& !ctx.sess.rollout_budget().tools_stopped()
|
||||
&& !cancellation_token.is_cancelled()
|
||||
{
|
||||
Box::pin(runtime.handle_tool_call(call, cancellation_token))
|
||||
} else {
|
||||
let output =
|
||||
ToolCallRuntime::aborted_response(&call, /*elapsed_secs*/ None);
|
||||
Box::pin(std::future::ready(Ok(
|
||||
output.into_response(/*file_backed_tool_audio*/ false)
|
||||
)))
|
||||
}
|
||||
};
|
||||
// Steer acceptance appends input under this lock. A buffered request
|
||||
// loses admission if the user steered while its response was settling.
|
||||
output.tool_future = Some(
|
||||
if ctx.step_context.turn.config.response_process_mode
|
||||
== codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse
|
||||
&& let Some(preempt) = &ctx.step_context.preempt
|
||||
{
|
||||
if let Some(state) = ctx
|
||||
.sess
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&ctx.sess.active_turn, &ctx.step_context.turn.sub_id)
|
||||
.await
|
||||
{
|
||||
let state = state.lock().await;
|
||||
dispatch(!state.pending_input.has_user_input() && !preempt.is_cancelled())
|
||||
} else {
|
||||
dispatch(false)
|
||||
}
|
||||
} else {
|
||||
dispatch(true)
|
||||
},
|
||||
);
|
||||
|
||||
output.needs_follow_up = true;
|
||||
output.tool_future = Some(tool_future);
|
||||
}
|
||||
// No tool call: convert messages/reasoning into turn items and mark them as complete.
|
||||
Ok(None) => {
|
||||
|
||||
@@ -315,7 +315,6 @@ async fn output_context(session: Arc<Session>, turn_context: Arc<TurnContext>) -
|
||||
step_context,
|
||||
turn_store: Arc::new(ExtensionData::new(turn_context.sub_id.clone())),
|
||||
tool_runtime,
|
||||
defer_tool_dispatch: false,
|
||||
cancellation_token: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,11 +257,12 @@ async fn turn_completion_metrics_follow_model_switch(scenario: UsageScenario) {
|
||||
session
|
||||
.update_token_usage_info(
|
||||
&previous_context,
|
||||
Some(&TokenUsage {
|
||||
TokenUsage {
|
||||
input_tokens: 1_000,
|
||||
total_tokens: 1_000,
|
||||
..TokenUsage::default()
|
||||
}),
|
||||
}
|
||||
.into(),
|
||||
/*includes_reasoning*/ false,
|
||||
)
|
||||
.await
|
||||
@@ -287,7 +288,7 @@ async fn turn_completion_metrics_follow_model_switch(scenario: UsageScenario) {
|
||||
session
|
||||
.update_token_usage_info(
|
||||
&previous_context,
|
||||
Some(&compaction_usage),
|
||||
compaction_usage.clone().into(),
|
||||
/*includes_reasoning*/ false,
|
||||
)
|
||||
.await
|
||||
@@ -320,7 +321,7 @@ async fn turn_completion_metrics_follow_model_switch(scenario: UsageScenario) {
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&response_context.initial_settings,
|
||||
Some(&response_usage),
|
||||
response_usage.clone().into(),
|
||||
crate::session::TokenUsageContext::Sampling {
|
||||
includes_reasoning: false,
|
||||
input_item_count: 0,
|
||||
|
||||
@@ -1710,6 +1710,8 @@ async fn foreground_compaction_only_installs_and_bills_after_its_post_hook() ->
|
||||
fs::write(home.join("config.toml"), toml::to_string(&config).unwrap()).unwrap();
|
||||
})
|
||||
.with_config(move |config| {
|
||||
config.response_process_mode =
|
||||
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
|
||||
config.model_provider = provider;
|
||||
set_test_compact_prompt(config);
|
||||
trust_discovered_hooks(config);
|
||||
|
||||
@@ -383,8 +383,8 @@ enum ProbeCase {
|
||||
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Escalated, 1; "explicit_escalation_still_reviews")]
|
||||
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::Rewrite, 1; "direct_hook_rewrite")]
|
||||
#[test_case::test_case(Some(false), true, "probe-item", ProbeCase::Rewrite, 2; "nested_hook_rewrite")]
|
||||
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 1; "interrupted_direct_review")]
|
||||
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 2; "interrupted_cell_review")]
|
||||
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::Interrupted, 0; "interrupted_direct_review")]
|
||||
#[test_case::test_case(Some(true), true, "probe-item", ProbeCase::Interrupted, 0; "interrupted_cell_review")]
|
||||
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::CatalogPolicy, 0; "catalog_low_without_legacy_flag")]
|
||||
#[test_case::test_case(Some(true), false, "probe-item", ProbeCase::CatalogPolicy, 1; "catalog_review_without_legacy_flag")]
|
||||
#[test_case::test_case(Some(false), false, "probe-item", ProbeCase::IncompleteAuthorization, 1; "incomplete_authorization_still_reviews")]
|
||||
@@ -525,6 +525,9 @@ async fn serving_assessment_replaces_async_scoring(
|
||||
model.auto_review_model_override = Some("gpt-5.5".into());
|
||||
})
|
||||
.with_config(move |config| {
|
||||
if scenario == ProbeCase::Interrupted {
|
||||
config.response_process_mode = codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
|
||||
}
|
||||
if matches!(scenario, ProbeCase::Rewrite | ProbeCase::Interrupted) {
|
||||
trust_discovered_hooks(config);
|
||||
}
|
||||
@@ -621,6 +624,19 @@ async fn serving_assessment_replaces_async_scoring(
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
let requests = parent.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
if scenario == ProbeCase::Interrupted {
|
||||
// Buffered processing holds partial output. A failed stream must
|
||||
// not expose the tool call to Guardian or carry a made-up reply into the retry.
|
||||
let retry = requests[1].body_json();
|
||||
assert!(
|
||||
retry["input"]
|
||||
.as_array()
|
||||
.expect("request input")
|
||||
.iter()
|
||||
.all(|item| item["call_id"] != "call")
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let output = requests[1].call_output(
|
||||
"call",
|
||||
if code_mode {
|
||||
|
||||
@@ -6186,32 +6186,40 @@ async fn model_request_observer_reports_cancellation() -> Result<()> {
|
||||
enum ToolDispatchCase {
|
||||
Eager,
|
||||
AsyncHook,
|
||||
AfterComplete,
|
||||
SyncHook,
|
||||
InterruptedHook,
|
||||
ExhaustedHook,
|
||||
}
|
||||
|
||||
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
|
||||
#[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")]
|
||||
#[test_case::test_case(ToolDispatchCase::AfterComplete; "buffered output waits for completion without hooks")]
|
||||
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
|
||||
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt cancels the synchronous hook before billing")]
|
||||
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt discards the held response and its charge")]
|
||||
#[test_case::test_case(ToolDispatchCase::ExhaustedHook; "the response bill blocks its own tool")]
|
||||
#[tokio::test]
|
||||
async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
use ToolDispatchCase::AfterComplete;
|
||||
use ToolDispatchCase::AsyncHook;
|
||||
use ToolDispatchCase::Eager;
|
||||
use ToolDispatchCase::ExhaustedHook;
|
||||
use ToolDispatchCase::InterruptedHook;
|
||||
use ToolDispatchCase::SyncHook;
|
||||
const MARKER: &str = "tool-dispatched";
|
||||
const STREAMED: &str = "Still streaming.";
|
||||
|
||||
let buffered = matches!(case, AfterComplete | InterruptedHook | ExhaustedHook);
|
||||
let hook_async = match case {
|
||||
Eager => None,
|
||||
Eager | AfterComplete => None,
|
||||
AsyncHook => Some(true),
|
||||
SyncHook | InterruptedHook => Some(false),
|
||||
SyncHook | InterruptedHook | ExhaustedHook => Some(false),
|
||||
};
|
||||
let (finish_stream, gate) = oneshot::channel();
|
||||
let mut completed = ev_completed_with_tokens("first", /*total_tokens*/ 42);
|
||||
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(1);
|
||||
completed["response"]["usage"]["codex_rollout_budget_units"] =
|
||||
json!(if case == ExhaustedHook { 101 } else { 1 });
|
||||
let first = vec![
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
@@ -6246,6 +6254,10 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
let port = hook_gate.local_addr()?.port();
|
||||
let mut builder = test_codex()
|
||||
.with_pre_build_hook(move |home| {
|
||||
if buffered {
|
||||
fs::write(home.join("config.toml"), "[responses]\nprocess_mode = \"after_complete_response\"\n")
|
||||
.expect("configure response processing after completion");
|
||||
}
|
||||
if let Some(hook_async) = hook_async {
|
||||
let post = format!(
|
||||
r#"python3 -c 'import json, socket, sys; json.load(sys.stdin).get("response_id") == "first" and socket.create_connection(("127.0.0.1", {port})).recv(1)'"#
|
||||
@@ -6261,7 +6273,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
if hook_async.is_some() {
|
||||
trust_discovered_hooks(config);
|
||||
}
|
||||
if matches!(case, Eager | SyncHook) {
|
||||
if matches!(case, Eager | SyncHook | ExhaustedHook) {
|
||||
config.rollout_budget = Some(super::rollout_budget::rollout_budget());
|
||||
}
|
||||
// The builder changes cwd after loading config, so bind the write grant to that cwd.
|
||||
@@ -6271,7 +6283,7 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
config.permissions.set_permission_profile(restrictive_workspace_write_profile())
|
||||
.expect("allow the fixture's patch inside its temporary workspace");
|
||||
});
|
||||
if matches!(case, SyncHook | InterruptedHook) {
|
||||
if case == InterruptedHook {
|
||||
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
builder = builder.with_extensions(Arc::new(extensions.build()));
|
||||
@@ -6290,8 +6302,12 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
saw_delta |= matches!(event, EventMsg::AgentMessageContentDelta(message) if message.delta == STREAMED);
|
||||
matches!(event, EventMsg::SafetyBuffering(_))
|
||||
}).await;
|
||||
assert!(saw_delta);
|
||||
fs_wait::wait_for_path_exists(&marker, Duration::from_secs(10)).await?;
|
||||
assert_eq!(saw_delta, !buffered);
|
||||
if buffered {
|
||||
assert!(!marker.exists());
|
||||
} else {
|
||||
fs_wait::wait_for_path_exists(&marker, Duration::from_secs(10)).await?;
|
||||
}
|
||||
finish_stream.send(()).expect("release response stream");
|
||||
let held_hook = if hook_async.is_some() {
|
||||
Some(
|
||||
@@ -6302,8 +6318,21 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if hook_async == Some(false) {
|
||||
assert!(accounting.reports.lock().expect("usage reports").is_empty());
|
||||
if buffered && hook_async == Some(false) {
|
||||
assert!(!marker.exists());
|
||||
test.codex.ensure_rollout_materialized().await;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
assert!(
|
||||
!history.items.iter().any(|item| matches!(item,
|
||||
RolloutItem::ResponseItem(envelope) if match &envelope.item {
|
||||
ResponseItem::CustomToolCall { call_id, .. } => call_id == "call",
|
||||
ResponseItem::Message { role, content, .. } => role == "assistant" && content.iter().any(
|
||||
|part| matches!(part, ContentItem::OutputText { text, .. } if text == STREAMED)),
|
||||
_ => false,
|
||||
}
|
||||
)),
|
||||
"the completed response is still waiting for its synchronous hook"
|
||||
);
|
||||
}
|
||||
if case == InterruptedHook {
|
||||
let started = wait_for_event_match(&test.codex, |event| match event {
|
||||
@@ -6340,15 +6369,67 @@ async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) ->
|
||||
Some("Hook cancelled because the turn was interrupted"),
|
||||
);
|
||||
drop(held_hook);
|
||||
assert!(!marker.exists());
|
||||
let server = start_mock_server().await;
|
||||
let requests = mount_sse_once(&server, responses::sse_completed("resumed")).await;
|
||||
let resumed = test_codex().restart(&server, &test).await?;
|
||||
resumed.submit_turn("continue after interrupt").await?;
|
||||
assert!(
|
||||
!requests
|
||||
.single_request()
|
||||
.input()
|
||||
.iter()
|
||||
.any(|item| item["call_id"] == "call")
|
||||
);
|
||||
assert!(accounting.reports.lock().expect("usage reports").is_empty());
|
||||
return Ok(());
|
||||
}
|
||||
drop(held_hook);
|
||||
if case == ExhaustedHook {
|
||||
let error = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::Error(error) => Some(error.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
error.codex_error_info,
|
||||
Some(codex_protocol::protocol::CodexErrorInfo::SessionBudgetExceeded)
|
||||
);
|
||||
}
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(fs::read_to_string(marker)?, "ran\n");
|
||||
if case == ExhaustedHook {
|
||||
assert!(!marker.exists());
|
||||
test.codex.ensure_rollout_materialized().await;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
let recorded = history
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
RolloutItem::ResponseItem(envelope) => match &envelope.item {
|
||||
ResponseItem::Message { role, content, .. } if role == "assistant" && content.iter().any(
|
||||
|part| matches!(part, ContentItem::OutputText { text, .. } if text == STREAMED)
|
||||
) => Some("assistant message"),
|
||||
ResponseItem::CustomToolCall { call_id, .. } if call_id == "call" => {
|
||||
Some("tool call")
|
||||
}
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id, output, ..
|
||||
} if call_id == "call" => output.text_content(),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
recorded,
|
||||
["tool call", "assistant message", "aborted before dispatch"]
|
||||
);
|
||||
} else {
|
||||
assert_eq!(fs::read_to_string(marker)?, "ran\n");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ use core_test_support::test_codex::local_requests;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_config::config_toml::ResponseProcessMode;
|
||||
use codex_core::CodexThread;
|
||||
use codex_core::InputWaitOutcome;
|
||||
use codex_core::StartIfIdleSubmission;
|
||||
@@ -1597,7 +1598,15 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
) -> anyhow::Result<()> {
|
||||
core_test_support::skip_if_no_network!(Ok(()));
|
||||
|
||||
const CALL_ID: &str = "stale-write";
|
||||
const FILE_NAME: &str = "stale-write.txt";
|
||||
let mut write = responses::ev_apply_patch_custom_tool_call(
|
||||
CALL_ID,
|
||||
&format!("*** Begin Patch\n*** Add File: {FILE_NAME}\n+stale\n*** End Patch"),
|
||||
);
|
||||
write["item"]["id"] = json!("stale-write-item");
|
||||
let mut interrupted = ev_completed_with_tokens("resp-interrupted", /*total_tokens*/ 42);
|
||||
interrupted["response"]["usage"]["codex_rollout_budget_units"] = json!(17);
|
||||
if acknowledgement == Some("response.interrupt.accepted") {
|
||||
interrupted["type"] = json!("response.incomplete");
|
||||
interrupted["response"]["status"] = json!("incomplete");
|
||||
@@ -1607,7 +1616,10 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
vec![ev_response_created("warm-1"), ev_completed("warm-1")],
|
||||
vec![
|
||||
ev_response_created("resp-interrupted"),
|
||||
write,
|
||||
ev_reasoning_item_added("reason-1", &["thinking"]),
|
||||
json!({"type":"response.metadata", "response_id":"resp-interrupted",
|
||||
"metadata":{"type":"safety_buffering","use_cases":["cyber"],"reasons":["policy-check"]}}),
|
||||
],
|
||||
// Finish the active response only after receiving response.interrupt.
|
||||
[
|
||||
@@ -1623,7 +1635,7 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
"type": "response.output_item.interrupted",
|
||||
"response_id": "resp-interrupted",
|
||||
"item_id": "reason-1",
|
||||
"output_index": 0,
|
||||
"output_index": 1,
|
||||
"sequence_number": 4,
|
||||
})
|
||||
} else {
|
||||
@@ -1640,12 +1652,27 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
],
|
||||
]])
|
||||
.await;
|
||||
let accounting = Arc::new(super::rollout_budget::HostAccounting::default());
|
||||
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_model("gpt-5.4")
|
||||
.with_model_info_override("gpt-5.4", |model_info| {
|
||||
model_info.use_responses_lite = true;
|
||||
})
|
||||
.with_config(|config| {
|
||||
config.response_process_mode = ResponseProcessMode::AfterCompleteResponse;
|
||||
config.workspace_roots = vec![config.cwd.clone()];
|
||||
config
|
||||
.permissions
|
||||
.set_workspace_roots(config.workspace_roots.clone());
|
||||
config.permissions.approval_policy =
|
||||
codex_core::config::Constrained::allow_any(AskForApproval::Never);
|
||||
config
|
||||
.permissions
|
||||
.set_permission_profile(PermissionProfile::workspace_write())
|
||||
.expect("allow the fixture's patch inside its temporary workspace");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::InstantInterrupt)
|
||||
@@ -1656,7 +1683,11 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
let codex = &test.codex;
|
||||
|
||||
submit_user_input(codex, "first prompt").await;
|
||||
wait_for_reasoning_item_started(codex).await;
|
||||
// The safety notification follows the buffered call and reasoning item. The
|
||||
// websocket is still open, and the provider awaits the client's interrupt.
|
||||
wait_for_event(codex, |event| matches!(event, EventMsg::SafetyBuffering(_))).await;
|
||||
let marker = test.workspace_path(FILE_NAME);
|
||||
assert!(!marker.try_exists()?);
|
||||
let initial_request = server
|
||||
.wait_for_request(/*connection_index*/ 0, /*request_index*/ 1)
|
||||
.await
|
||||
@@ -1682,6 +1713,10 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
.filter(|text| text == "first prompt" || text == "second prompt")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(prompts, vec!["second prompt"]);
|
||||
assert_eq!(
|
||||
call_output_text(&follow_up, "custom_tool_call_output", CALL_ID).as_deref(),
|
||||
Some("aborted before dispatch")
|
||||
);
|
||||
wait_for_event(codex, |event| {
|
||||
matches!(
|
||||
event,
|
||||
@@ -1691,6 +1726,16 @@ async fn steer_interrupts_and_drains_websocket(
|
||||
})
|
||||
.await;
|
||||
wait_for_successful_turn(codex).await;
|
||||
assert!(!marker.try_exists()?);
|
||||
let billed_units = accounting
|
||||
.reports
|
||||
.lock()
|
||||
.expect("usage reports")
|
||||
.iter()
|
||||
.filter_map(|report| report.usage.codex_rollout_budget_units.as_ref())
|
||||
.map(serde_json::Number::as_i64)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(billed_units, [Some(17)]);
|
||||
|
||||
assert_eq!(server.connections().len(), 1);
|
||||
codex.shutdown_and_wait().await?;
|
||||
|
||||
@@ -203,6 +203,10 @@ async fn adds_weighted_initial_and_threshold_reminders(
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()> {
|
||||
use codex_history::RolloutItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use core_test_support::responses::ev_apply_patch_custom_tool_call;
|
||||
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
@@ -210,13 +214,25 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
|
||||
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(-1.0);
|
||||
let responses = mount_sse_sequence(
|
||||
&server,
|
||||
vec![sse(vec![ev_response_created("invalid-units"), completed])],
|
||||
vec![sse(vec![
|
||||
ev_response_created("invalid-units"),
|
||||
ev_apply_patch_custom_tool_call(
|
||||
"unbilled-patch",
|
||||
"*** Begin Patch\n*** Add File: unbilled-tool-ran\n+unexpected\n*** End Patch",
|
||||
),
|
||||
completed,
|
||||
])],
|
||||
)
|
||||
.await;
|
||||
let test = test_codex()
|
||||
.with_config(|config| {
|
||||
config.rollout_budget = Some(rollout_budget());
|
||||
.with_pre_build_hook(|home| {
|
||||
std::fs::write(
|
||||
home.join("config.toml"),
|
||||
"[responses]\nprocess_mode = \"after_complete_response\"\n",
|
||||
)
|
||||
.expect("configure buffered response processing");
|
||||
})
|
||||
.with_config(|config| config.rollout_budget = Some(rollout_budget()))
|
||||
.build(&server)
|
||||
.await?;
|
||||
|
||||
@@ -232,11 +248,8 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
assert_eq!(
|
||||
error.message,
|
||||
"Fatal error: response.completed usage.codex_rollout_budget_units must be finite and non-negative"
|
||||
);
|
||||
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
|
||||
assert!(error.message.contains("codex_rollout_budget_units"));
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
@@ -246,6 +259,16 @@ async fn invalid_provider_rollout_budget_units_fail_without_retry() -> Result<()
|
||||
1,
|
||||
"invalid units should not retry"
|
||||
);
|
||||
// Invalid accounting rejects the buffered response before any of its items are published.
|
||||
assert!(!test.workspace_path("unbilled-tool-ran").exists());
|
||||
test.codex.ensure_rollout_materialized().await;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
assert!(!history.items.iter().any(|item| matches!(item,
|
||||
RolloutItem::ResponseItem(envelope) if matches!(&envelope.item,
|
||||
ResponseItem::CustomToolCall { call_id, .. }
|
||||
| ResponseItem::CustomToolCallOutput { call_id, .. } if call_id == "unbilled-patch"
|
||||
)
|
||||
)));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -403,20 +426,17 @@ async fn exhausted_budget_respects_enforcement_on_current_and_later_turns(
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let responses = mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_response_created("exhaust-budget"),
|
||||
ev_completed_with_tokens("exhaust-budget", /*total_tokens*/ 30),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("already-exhausted"),
|
||||
ev_completed_with_tokens("already-exhausted", /*total_tokens*/ 1),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let mut sequence = vec![sse(vec![
|
||||
ev_response_created("exhaust-budget"),
|
||||
ev_completed_with_tokens("exhaust-budget", /*total_tokens*/ 30),
|
||||
])];
|
||||
if owner == AgentRuntimeOwnership::Host || !enforce_limit {
|
||||
sequence.push(sse(vec![
|
||||
ev_response_created("already-exhausted"),
|
||||
ev_completed_with_tokens("already-exhausted", /*total_tokens*/ 1),
|
||||
]));
|
||||
}
|
||||
let responses = mount_sse_sequence(&server, sequence).await;
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
if owner == AgentRuntimeOwnership::Host {
|
||||
extensions.thread_lifecycle_contributor(Arc::new(HostRuntime));
|
||||
|
||||
@@ -4651,6 +4651,8 @@ async fn async_compaction_reuses_pending_work(
|
||||
env,
|
||||
move |config| {
|
||||
if observe_attempts {
|
||||
config.response_process_mode =
|
||||
codex_config::config_toml::ResponseProcessMode::AfterCompleteResponse;
|
||||
core_test_support::hooks::trust_discovered_hooks(config);
|
||||
}
|
||||
if over_budget {
|
||||
|
||||
@@ -394,6 +394,7 @@ async fn new_config(
|
||||
experimental_request_user_input_enabled: true,
|
||||
update_plan_enabled: true,
|
||||
tool_registry: Default::default(),
|
||||
response_process_mode: Default::default(),
|
||||
code_mode: Default::default(),
|
||||
background_terminal_max_timeout: 300_000,
|
||||
thread_unload_delay: std::time::Duration::from_secs(/*secs*/ 60),
|
||||
|
||||
Reference in New Issue
Block a user