mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Unify model request hook lifecycles across sampling and compaction (#61562)
## What changed - Pair `PreModelRequest` and `PostModelRequest` hooks per sampling or compaction attempt, with distinct lifecycle IDs and cancellation reporting. - Add `thread_id` and `kind` to hook inputs, distinguishing `sampling`, `foreground_compaction`, and `background_compaction`. Include the provider's `response_id` in post-hook inputs when available. - Wait for synchronous post-hooks before releasing terminal response events, accounting for usage, or installing compaction results. Allow streaming output and tool execution to proceed before the hook finishes, and cancel synchronous post-hooks when their owning request is interrupted. - Preserve incomplete-response, budget-exceeded, and hook-denied outcomes across sampling retries and compaction model fallback. ## Testing Add coverage for streaming and tool ordering, interrupted post-hooks, per-attempt metadata, and foreground and background compaction accounting and installation ordering. GitOrigin-RevId: 3e808c5b3bf2d5a6a326d9f5be5bbd0359b2d16e
This commit is contained in:
+147
-98
@@ -33,9 +33,12 @@ use std::sync::OnceLock;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
mod response_delivery;
|
||||
#[path = "client_tool_metadata.rs"]
|
||||
mod tool_metadata;
|
||||
|
||||
use response_delivery::ResponseDelivery;
|
||||
|
||||
use crate::CodexResponsesHeaders;
|
||||
use crate::PromptCacheKey;
|
||||
use crate::tools::ExecutedToolCalls;
|
||||
@@ -132,6 +135,7 @@ use crate::cyber_access_program;
|
||||
use crate::feedback_tags;
|
||||
use crate::responses_metadata::CodexResponsesMetadata;
|
||||
use crate::responses_metadata::subagent_header_value;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::util::emit_feedback_auth_recovery_tags;
|
||||
use codex_feedback::FeedbackRequestTags;
|
||||
use codex_feedback::emit_feedback_request_tags_with_auth_env;
|
||||
@@ -552,8 +556,16 @@ impl WebsocketSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Generation settings shared by HTTP and WebSocket, including transport fallback.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SamplingOptions {
|
||||
pub(crate) reasoning_effort: Option<ReasoningEffortConfig>,
|
||||
pub(crate) reasoning_summary: ReasoningSummaryConfig,
|
||||
pub(crate) service_tier: Option<String>,
|
||||
}
|
||||
|
||||
enum WebsocketStreamOutcome {
|
||||
Stream(ResponseStream),
|
||||
Stream(Box<OpenedModelResponse>),
|
||||
FallbackToHttp,
|
||||
}
|
||||
|
||||
@@ -1978,12 +1990,10 @@ impl ResponsesHttpExecutor<'_> {
|
||||
prompt: &Arc<Prompt>,
|
||||
model_info: &ModelInfo,
|
||||
session_telemetry: &SessionTelemetry,
|
||||
effort: Option<ReasoningEffortConfig>,
|
||||
summary: ReasoningSummaryConfig,
|
||||
service_tier: Option<String>,
|
||||
sampling: SamplingOptions,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<ResponseStream> {
|
||||
) -> Result<OpenedModelResponse> {
|
||||
let auth_manager = self.client.state.provider.auth_manager();
|
||||
let mut auth_recovery = auth_manager
|
||||
.as_ref()
|
||||
@@ -2036,9 +2046,9 @@ impl ResponsesHttpExecutor<'_> {
|
||||
let mut request = self.client.build_responses_request(
|
||||
prompt,
|
||||
model_info,
|
||||
effort.clone(),
|
||||
summary,
|
||||
service_tier.clone(),
|
||||
sampling.reasoning_effort.clone(),
|
||||
sampling.reasoning_summary,
|
||||
sampling.service_tier.clone(),
|
||||
responses_metadata,
|
||||
include_internal,
|
||||
)?;
|
||||
@@ -2121,14 +2131,14 @@ impl ResponsesHttpExecutor<'_> {
|
||||
|
||||
match stream_result {
|
||||
Ok(stream) => {
|
||||
return Ok(map_response_stream(
|
||||
return Ok(OpenedModelResponse {
|
||||
stream,
|
||||
request_session_telemetry,
|
||||
inference_trace_attempt,
|
||||
&self.client.state,
|
||||
telemetry: request_session_telemetry,
|
||||
trace: inference_trace_attempt,
|
||||
client: Arc::clone(&self.client.state),
|
||||
interceptors,
|
||||
/*last_response_tx*/ None,
|
||||
));
|
||||
continuation: None,
|
||||
});
|
||||
}
|
||||
Err(ApiError::Transport(unauthorized_transport))
|
||||
if self
|
||||
@@ -2195,9 +2205,7 @@ impl ModelClientSession {
|
||||
prompt: &Arc<Prompt>,
|
||||
model_info: &ModelInfo,
|
||||
session_telemetry: &SessionTelemetry,
|
||||
effort: Option<ReasoningEffortConfig>,
|
||||
summary: ReasoningSummaryConfig,
|
||||
service_tier: Option<String>,
|
||||
sampling: SamplingOptions,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
warmup: bool,
|
||||
request_trace: Option<W3cTraceContext>,
|
||||
@@ -2244,9 +2252,9 @@ impl ModelClientSession {
|
||||
let mut request = self.client.build_responses_request(
|
||||
prompt,
|
||||
model_info,
|
||||
effort.clone(),
|
||||
summary,
|
||||
service_tier.clone(),
|
||||
sampling.reasoning_effort.clone(),
|
||||
sampling.reasoning_summary,
|
||||
sampling.service_tier.clone(),
|
||||
responses_metadata,
|
||||
include_internal,
|
||||
)?;
|
||||
@@ -2509,16 +2517,16 @@ impl ModelClientSession {
|
||||
err
|
||||
})?;
|
||||
let (last_response_tx, last_response_rx) = oneshot::channel();
|
||||
let stream = map_response_stream(
|
||||
stream_result,
|
||||
request_session_telemetry,
|
||||
inference_trace_attempt,
|
||||
&self.client.state,
|
||||
let stream = OpenedModelResponse {
|
||||
stream: stream_result,
|
||||
telemetry: request_session_telemetry,
|
||||
trace: inference_trace_attempt,
|
||||
client: Arc::clone(&self.client.state),
|
||||
interceptors,
|
||||
Some(last_response_tx),
|
||||
);
|
||||
continuation: Some(last_response_tx),
|
||||
};
|
||||
self.websocket_session.last_response_rx = Some(last_response_rx);
|
||||
return Ok(WebsocketStreamOutcome::Stream(stream));
|
||||
return Ok(WebsocketStreamOutcome::Stream(Box::new(stream)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2593,9 +2601,11 @@ impl ModelClientSession {
|
||||
prompt,
|
||||
model_info,
|
||||
session_telemetry,
|
||||
effort,
|
||||
summary,
|
||||
service_tier,
|
||||
SamplingOptions {
|
||||
reasoning_effort: effort,
|
||||
reasoning_summary: summary,
|
||||
service_tier,
|
||||
},
|
||||
responses_metadata,
|
||||
/*warmup*/ true,
|
||||
current_span_w3c_trace_context(),
|
||||
@@ -2603,7 +2613,8 @@ impl ModelClientSession {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(WebsocketStreamOutcome::Stream(mut stream)) => {
|
||||
Ok(WebsocketStreamOutcome::Stream(opened)) => {
|
||||
let mut stream = (*opened).into_stream()?;
|
||||
// Wait for the v2 warmup request to complete before sending the first turn request.
|
||||
while let Some(event) = stream.next().await {
|
||||
match event {
|
||||
@@ -2646,6 +2657,31 @@ impl ModelClientSession {
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<ResponseStream> {
|
||||
self.open_model_response(
|
||||
prompt,
|
||||
model_info,
|
||||
session_telemetry,
|
||||
SamplingOptions {
|
||||
reasoning_effort: effort,
|
||||
reasoning_summary: summary,
|
||||
service_tier,
|
||||
},
|
||||
responses_metadata,
|
||||
inference_trace,
|
||||
)
|
||||
.await?
|
||||
.into_stream()
|
||||
}
|
||||
|
||||
pub(crate) async fn open_model_response(
|
||||
&mut self,
|
||||
prompt: &Arc<Prompt>,
|
||||
model_info: &ModelInfo,
|
||||
session_telemetry: &SessionTelemetry,
|
||||
sampling: SamplingOptions,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<OpenedModelResponse> {
|
||||
let wire_api = self.client.state.provider.info().wire_api;
|
||||
match wire_api {
|
||||
WireApi::Responses => {
|
||||
@@ -2656,9 +2692,7 @@ impl ModelClientSession {
|
||||
prompt,
|
||||
model_info,
|
||||
session_telemetry,
|
||||
effort.clone(),
|
||||
summary,
|
||||
service_tier.clone(),
|
||||
sampling.clone(),
|
||||
responses_metadata,
|
||||
/*warmup*/ false,
|
||||
request_trace,
|
||||
@@ -2666,7 +2700,7 @@ impl ModelClientSession {
|
||||
)
|
||||
.await?
|
||||
{
|
||||
WebsocketStreamOutcome::Stream(stream) => return Ok(stream),
|
||||
WebsocketStreamOutcome::Stream(stream) => return Ok(*stream),
|
||||
WebsocketStreamOutcome::FallbackToHttp => {
|
||||
self.try_switch_fallback_transport(session_telemetry, model_info);
|
||||
}
|
||||
@@ -2681,9 +2715,7 @@ impl ModelClientSession {
|
||||
prompt,
|
||||
model_info,
|
||||
session_telemetry,
|
||||
effort,
|
||||
summary,
|
||||
service_tier,
|
||||
sampling,
|
||||
responses_metadata,
|
||||
inference_trace,
|
||||
)
|
||||
@@ -2770,29 +2802,59 @@ fn add_responses_lite_header(headers: &mut ApiHeaderMap, use_responses_lite: boo
|
||||
const RESPONSE_STREAM_CHANNEL_CAPACITY: usize = 1600;
|
||||
const STREAM_DROPPED_REASON: &str = "response stream dropped before provider terminal event";
|
||||
|
||||
fn map_response_stream(
|
||||
mut api_stream: codex_api::ResponseStream,
|
||||
session_telemetry: SessionTelemetry,
|
||||
inference_trace_attempt: InferenceTraceAttempt,
|
||||
client: &ModelClientState,
|
||||
/// The transport has opened the provider stream. Its caller now chooses whether
|
||||
/// a synchronous post-hook must finish before Core can use the response.
|
||||
pub(crate) struct OpenedModelResponse {
|
||||
stream: codex_api::ResponseStream,
|
||||
telemetry: SessionTelemetry,
|
||||
trace: InferenceTraceAttempt,
|
||||
client: Arc<ModelClientState>,
|
||||
interceptors: Vec<Box<dyn codex_extension_api::ModelResponseInterceptor>>,
|
||||
last_response_tx: Option<oneshot::Sender<LastResponse>>,
|
||||
) -> ResponseStream {
|
||||
let routing_affinity = api_stream.routing_affinity.take();
|
||||
let upstream_request_id = api_stream.upstream_request_id.take();
|
||||
let interrupt = api_stream.interrupt.take();
|
||||
let events = crate::model_request::intercept_stream(Box::pin(api_stream), interceptors);
|
||||
let mut stream = map_response_events(
|
||||
upstream_request_id,
|
||||
events,
|
||||
session_telemetry,
|
||||
inference_trace_attempt,
|
||||
client,
|
||||
last_response_tx,
|
||||
);
|
||||
stream.interrupt = interrupt;
|
||||
stream.routing_affinity = routing_affinity;
|
||||
stream
|
||||
continuation: Option<oneshot::Sender<LastResponse>>,
|
||||
}
|
||||
|
||||
impl OpenedModelResponse {
|
||||
pub(crate) fn into_stream(self) -> Result<ResponseStream> {
|
||||
self.deliver(&mut None, /*sampling_turn*/ None)
|
||||
}
|
||||
|
||||
pub(crate) fn deliver(
|
||||
mut self,
|
||||
attempt: &mut Option<crate::hook_runtime::sampling::SamplingAttempt>,
|
||||
sampling_turn: Option<Arc<TurnContext>>,
|
||||
) -> Result<ResponseStream> {
|
||||
let delivery = ResponseDelivery::new(attempt);
|
||||
let routing_affinity = self.stream.routing_affinity.take();
|
||||
let request_id = self.stream.upstream_request_id.take();
|
||||
let interrupt = self.stream.interrupt.take();
|
||||
// A response interceptor can hold even the first token until completion. Measure
|
||||
// ordinary turn TTFT on the decoded provider stream, before any such wait.
|
||||
let provider_stream = if let Some(turn) = sampling_turn {
|
||||
futures::stream::unfold((self.stream, turn), |(mut stream, turn)| async move {
|
||||
let event = stream.next().await?;
|
||||
if let Ok(event) = &event {
|
||||
crate::turn_timing::record_turn_ttft_metric(&turn, event).await;
|
||||
}
|
||||
Some((event, (stream, turn)))
|
||||
})
|
||||
.boxed()
|
||||
} else {
|
||||
self.stream.boxed()
|
||||
};
|
||||
let events = crate::model_request::intercept_stream(provider_stream, self.interceptors);
|
||||
let mut stream = map_response_events(
|
||||
request_id,
|
||||
events,
|
||||
self.telemetry,
|
||||
self.trace,
|
||||
&self.client,
|
||||
self.continuation,
|
||||
delivery,
|
||||
);
|
||||
stream.interrupt = interrupt;
|
||||
stream.routing_affinity = routing_affinity;
|
||||
Ok(stream)
|
||||
}
|
||||
}
|
||||
|
||||
fn map_response_events<S>(
|
||||
@@ -2802,6 +2864,7 @@ fn map_response_events<S>(
|
||||
inference_trace_attempt: InferenceTraceAttempt,
|
||||
client: &ModelClientState,
|
||||
mut last_response_tx: Option<oneshot::Sender<LastResponse>>,
|
||||
mut delivery: ResponseDelivery,
|
||||
) -> ResponseStream
|
||||
where
|
||||
S: futures::Stream<Item = std::result::Result<ResponseEvent, ApiError>>
|
||||
@@ -2843,7 +2906,7 @@ where
|
||||
let Some(event) = event else {
|
||||
break;
|
||||
};
|
||||
match event {
|
||||
let event = match event {
|
||||
Ok(ResponseEvent::OutputItemDone(mut item)) => {
|
||||
// Assign IDs after interceptors, which may replace provider items.
|
||||
// Reuse the added event's ID so tracing and the harness agree.
|
||||
@@ -2858,18 +2921,7 @@ where
|
||||
if collect_output_items {
|
||||
items_added.push(item.clone());
|
||||
}
|
||||
if tx_event
|
||||
.send(Ok(ResponseEvent::OutputItemDone(item)))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
inference_trace_attempt.record_cancelled(
|
||||
STREAM_DROPPED_REASON,
|
||||
upstream_request_id,
|
||||
&items_added,
|
||||
);
|
||||
return;
|
||||
}
|
||||
Ok(ResponseEvent::OutputItemDone(item))
|
||||
}
|
||||
Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
@@ -2895,19 +2947,13 @@ where
|
||||
});
|
||||
}
|
||||
items_added = Vec::new();
|
||||
if tx_event
|
||||
.send(Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
token_usage,
|
||||
usage_metadata,
|
||||
reasoning_removals,
|
||||
end_turn,
|
||||
}))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
token_usage,
|
||||
usage_metadata,
|
||||
reasoning_removals,
|
||||
end_turn,
|
||||
})
|
||||
}
|
||||
Ok(mut event) => {
|
||||
if let ResponseEvent::OutputItemAdded(item) = &mut event {
|
||||
@@ -2921,14 +2967,7 @@ where
|
||||
i64::try_from(request_start.elapsed().as_millis()).unwrap_or(i64::MAX),
|
||||
);
|
||||
}
|
||||
if tx_event.send(Ok(event)).await.is_err() {
|
||||
inference_trace_attempt.record_cancelled(
|
||||
STREAM_DROPPED_REASON,
|
||||
upstream_request_id,
|
||||
&items_added,
|
||||
);
|
||||
return;
|
||||
}
|
||||
Ok(event)
|
||||
}
|
||||
Err(err) => {
|
||||
let response_debug_context =
|
||||
@@ -2967,10 +3006,19 @@ where
|
||||
session_telemetry.see_event_completed_failed(&mapped);
|
||||
logged_error = true;
|
||||
}
|
||||
if tx_event.send(Err(mapped)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
Err(mapped)
|
||||
}
|
||||
};
|
||||
let terminal = matches!(&event, Ok(ResponseEvent::Completed { .. }) | Err(_));
|
||||
if !delivery.send_or_hold(event, &tx_event).await {
|
||||
if !terminal {
|
||||
inference_trace_attempt.record_cancelled(
|
||||
STREAM_DROPPED_REASON,
|
||||
upstream_request_id,
|
||||
&items_added,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
inference_trace_attempt.record_failed(
|
||||
@@ -2978,6 +3026,7 @@ where
|
||||
upstream_request_id,
|
||||
&items_added,
|
||||
);
|
||||
delivery.finish_eof(&tx_event).await;
|
||||
});
|
||||
|
||||
ResponseStream {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Synchronous post-hooks finish before the turn receives response completion.
|
||||
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::hook_runtime::sampling::SamplingAttempt;
|
||||
use codex_api::ApiError;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ResponseDelivery {
|
||||
attempt: Option<SamplingAttempt>,
|
||||
}
|
||||
|
||||
impl ResponseDelivery {
|
||||
pub(super) fn new(attempt: &mut Option<SamplingAttempt>) -> Self {
|
||||
Self {
|
||||
attempt: attempt.take(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return false when the turn has stopped receiving this attempt's output.
|
||||
pub(super) async fn send_or_hold(
|
||||
&mut self,
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
) -> bool {
|
||||
let Some(attempt) = self.attempt.as_mut() else {
|
||||
return tx.send(event).await.is_ok();
|
||||
};
|
||||
let report = match &event {
|
||||
Ok(ResponseEvent::Completed {
|
||||
response_id,
|
||||
token_usage,
|
||||
..
|
||||
}) => ModelRequestHookEvent::completed(response_id, token_usage.as_ref()),
|
||||
Err(error) => ModelRequestHookEvent::failed(error),
|
||||
Ok(_) => return tx.send(event).await.is_ok(),
|
||||
};
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
if let Some(hooks) = attempt.start_post_hooks(report) {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = tx.closed() => return false,
|
||||
() = owner.cancelled() => return false,
|
||||
() = hooks => {}
|
||||
}
|
||||
// The owner can interrupt before the reader is scheduled to close its channel.
|
||||
if owner.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
self.attempt = None;
|
||||
let released = Self::release(event, tx).await;
|
||||
match released {
|
||||
Ok(delivered) => delivered,
|
||||
Err(error) => tx.send(Err(codex_api::map_api_error(error))).await.is_ok(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(
|
||||
event: Result<ResponseEvent>,
|
||||
tx: &mpsc::Sender<Result<ResponseEvent>>,
|
||||
) -> std::result::Result<bool, ApiError> {
|
||||
if tx.is_closed() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(tx.send(event).await.is_ok())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_eof(&mut self, tx: &mpsc::Sender<Result<ResponseEvent>>) {
|
||||
let Some(mut attempt) = self.attempt.take() else {
|
||||
return;
|
||||
};
|
||||
let owner = attempt.owner_cancellation.clone();
|
||||
if let Some(hooks) = attempt.start_post_hooks(ModelRequestHookEvent::failed(
|
||||
&CodexErr::Stream("stream closed before response.completed".into()),
|
||||
)) {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = tx.closed() => {}
|
||||
() = owner.cancelled() => {}
|
||||
() = hooks => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub use codex_api::ResponseEvent;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
@@ -129,6 +130,32 @@ pub struct ResponseStream {
|
||||
pub(crate) consumer_dropped: CancellationToken,
|
||||
}
|
||||
|
||||
impl ResponseStream {
|
||||
/// A cancelled turn discards any queued event, including an incomplete response's usage.
|
||||
pub(crate) fn accept_for_turn(
|
||||
event: Option<Result<ResponseEvent>>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Option<Result<ResponseEvent>> {
|
||||
if cancellation.is_cancelled() {
|
||||
Some(Err(CodexErr::TurnAborted))
|
||||
} else {
|
||||
event
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn next_for_turn(
|
||||
&mut self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Option<Result<ResponseEvent>> {
|
||||
let event = tokio::select! {
|
||||
biased;
|
||||
event = self.rx_event.recv() => event,
|
||||
() = cancellation.cancelled() => return Some(Err(CodexErr::TurnAborted)),
|
||||
};
|
||||
Self::accept_for_turn(event, cancellation)
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for ResponseStream {
|
||||
type Item = Result<ResponseEvent>;
|
||||
|
||||
|
||||
@@ -1691,6 +1691,7 @@ async fn untraced_http_stream_releases_completed_output_before_response_ends() {
|
||||
InferenceTraceAttempt::disabled(),
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
@@ -1733,6 +1734,7 @@ async fn incomplete_response_traces_attempted_call_without_streaming_it() -> any
|
||||
attempt,
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
assert!(matches!(stream.next().await, Some(Err(_))));
|
||||
assert!(stream.next().await.is_none());
|
||||
@@ -1778,6 +1780,7 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re
|
||||
attempt,
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
|
||||
let observed = stream
|
||||
@@ -1836,6 +1839,7 @@ async fn response_stream_records_last_model_feedback_ids() {
|
||||
InferenceTraceAttempt::disabled(),
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
|
||||
while stream.next().await.is_some() {}
|
||||
@@ -2060,6 +2064,7 @@ async fn dropped_backpressured_response_stream_traces_cancelled_partial_output()
|
||||
attempt,
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
/*last_response_tx*/ None,
|
||||
super::ResponseDelivery::default(),
|
||||
);
|
||||
|
||||
// Fill the mapper channel with non-terminal events, then yield one output
|
||||
@@ -2876,19 +2881,20 @@ async fn intercepted_output_reaches_trace_and_websocket_bookkeeping() -> anyhow:
|
||||
.await?;
|
||||
drop(tx_event);
|
||||
let (last_response_tx, last_response_rx) = tokio::sync::oneshot::channel();
|
||||
let mut stream = super::map_response_stream(
|
||||
codex_api::ResponseStream {
|
||||
let mut stream = super::OpenedModelResponse {
|
||||
stream: codex_api::ResponseStream {
|
||||
routing_affinity: None,
|
||||
rx_event,
|
||||
upstream_request_id: None,
|
||||
interrupt: None,
|
||||
},
|
||||
test_session_telemetry(),
|
||||
attempt,
|
||||
&test_model_client(SessionSource::Cli).state,
|
||||
vec![Box::new(ReplaceOutput)],
|
||||
Some(last_response_tx),
|
||||
);
|
||||
telemetry: test_session_telemetry(),
|
||||
trace: attempt,
|
||||
client: Arc::clone(&test_model_client(SessionSource::Cli).state),
|
||||
interceptors: vec![Box::new(ReplaceOutput)],
|
||||
continuation: Some(last_response_tx),
|
||||
}
|
||||
.into_stream()?;
|
||||
let mut delivered = Vec::new();
|
||||
while let Some(event) = stream.next().await {
|
||||
if let ResponseEvent::OutputItemDone(item) = event? {
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::time::Instant;
|
||||
|
||||
use crate::Prompt;
|
||||
use crate::client::ModelClientSession;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::compact_pending_input::is_retained_pending_mail;
|
||||
use crate::compact_remote_v2::retained_client_items;
|
||||
@@ -37,6 +38,7 @@ use codex_analytics::CompactionStatus;
|
||||
use codex_analytics::CompactionStrategy;
|
||||
use codex_analytics::CompactionTrigger;
|
||||
use codex_analytics::now_unix_seconds;
|
||||
use codex_async_utils::OrCancelExt;
|
||||
use codex_context_fragments::AnnotatedContent;
|
||||
use codex_context_fragments::set_annotated_content;
|
||||
use codex_context_fragments::truncate_annotated_text;
|
||||
@@ -59,7 +61,6 @@ use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_rollout_trace::InferenceTraceContext;
|
||||
use codex_utils_output_truncation::approx_token_count;
|
||||
use futures::prelude::*;
|
||||
use tracing::error;
|
||||
|
||||
pub use codex_prompts::SUMMARIZATION_PROMPT;
|
||||
@@ -275,6 +276,9 @@ async fn run_compact_task_inner_impl(
|
||||
compaction_metadata: CompactionTurnMetadata,
|
||||
) -> CodexResult<String> {
|
||||
let turn_context = Arc::clone(&step_context.turn);
|
||||
let mut request = crate::hook_runtime::sampling::SamplingRequest::new(
|
||||
codex_hooks::SamplingKind::ForegroundCompaction,
|
||||
);
|
||||
let compaction_item = TurnItem::ContextCompaction(ContextCompactionItem::new());
|
||||
sess.emit_turn_item_started(&turn_context, &compaction_item)
|
||||
.await;
|
||||
@@ -337,8 +341,9 @@ async fn run_compact_task_inner_impl(
|
||||
.compaction_responses_metadata(&step_context, compaction_metadata)
|
||||
.await;
|
||||
let attempt_result = drain_to_completed(
|
||||
&mut request,
|
||||
&sess,
|
||||
turn_context.as_ref(),
|
||||
&step_context,
|
||||
&mut client_session,
|
||||
&responses_metadata,
|
||||
&Arc::new(prompt),
|
||||
@@ -853,35 +858,48 @@ struct CompactionResponse {
|
||||
}
|
||||
|
||||
async fn drain_to_completed(
|
||||
sess: &Session,
|
||||
turn_context: &TurnContext,
|
||||
request: &mut crate::hook_runtime::sampling::SamplingRequest,
|
||||
sess: &Arc<Session>,
|
||||
step_context: &Arc<StepContext>,
|
||||
client_session: &mut ModelClientSession,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
prompt: &Arc<Prompt>,
|
||||
phase: CompactionPhase,
|
||||
) -> CodexResult<CompactionResponse> {
|
||||
let turn_context = step_context.turn.as_ref();
|
||||
let mut includes_reasoning = turn_context.model_info().use_responses_lite;
|
||||
let mut stream = client_session
|
||||
.stream(
|
||||
prompt,
|
||||
turn_context.model_info(),
|
||||
&turn_context.session_telemetry,
|
||||
sess.reasoning_effort_for_request(
|
||||
&turn_context.initial_settings,
|
||||
RequestEffortUsage::Compaction,
|
||||
)
|
||||
.await,
|
||||
turn_context.reasoning_summary(),
|
||||
turn_context.config.service_tier.clone(),
|
||||
responses_metadata,
|
||||
// Rollout tracing currently models remote compaction only; local compaction streams
|
||||
// are left untraced until the reducer has a first-class local compaction lifecycle.
|
||||
&InferenceTraceContext::disabled(),
|
||||
let cancellation = &step_context.turn_cancellation;
|
||||
let mut stream = request
|
||||
.open(
|
||||
sess,
|
||||
step_context,
|
||||
&turn_context.model_info().slug,
|
||||
cancellation,
|
||||
client_session.open_model_response(
|
||||
prompt,
|
||||
turn_context.model_info(),
|
||||
&turn_context.session_telemetry,
|
||||
SamplingOptions {
|
||||
reasoning_effort: sess
|
||||
.reasoning_effort_for_request(
|
||||
&turn_context.initial_settings,
|
||||
RequestEffortUsage::Compaction,
|
||||
)
|
||||
.await,
|
||||
reasoning_summary: turn_context.reasoning_summary(),
|
||||
service_tier: turn_context.config.service_tier.clone(),
|
||||
},
|
||||
responses_metadata,
|
||||
// Rollout tracing currently models remote compaction only; local compaction streams
|
||||
// are left untraced until the reducer has a first-class local compaction lifecycle.
|
||||
&InferenceTraceContext::disabled(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
.or_cancel(cancellation)
|
||||
.await??;
|
||||
let mut output = Vec::new();
|
||||
loop {
|
||||
let maybe_event = stream.next().await;
|
||||
let maybe_event = stream.next_for_turn(cancellation).await;
|
||||
let Some(event) = maybe_event else {
|
||||
return Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
|
||||
@@ -12,17 +12,16 @@ pub(crate) fn should_retry_with_current_model(
|
||||
previous: &TurnContext,
|
||||
current: &TurnContext,
|
||||
) -> bool {
|
||||
if matches!(
|
||||
error.details(),
|
||||
CodexErrorDetails::TurnAborted
|
||||
| CodexErrorDetails::Interrupted
|
||||
| CodexErrorDetails::SessionBudgetExceeded
|
||||
| CodexErrorDetails::InferenceHookDenied { .. }
|
||||
| CodexErrorDetails::WorkspaceAccessDenied(_)
|
||||
| CodexErrorDetails::ContentPolicyViolation { .. }
|
||||
| CodexErrorDetails::ResponseIncomplete(_)
|
||||
) || (previous.model_info().slug == current.model_info().slug
|
||||
&& previous.cyber_access_program == current.cyber_access_program)
|
||||
if crate::responses_retry::is_terminal_sampling_outcome(error)
|
||||
|| matches!(
|
||||
error.details(),
|
||||
CodexErrorDetails::TurnAborted
|
||||
| CodexErrorDetails::Interrupted
|
||||
| CodexErrorDetails::WorkspaceAccessDenied(_)
|
||||
| CodexErrorDetails::ContentPolicyViolation { .. }
|
||||
)
|
||||
|| (previous.model_info().slug == current.model_info().slug
|
||||
&& previous.cyber_access_program == current.cyber_access_program)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::Prompt;
|
||||
use crate::ResponseStream;
|
||||
use crate::client::ModelClientSession;
|
||||
use crate::client::ResponsesHttpExecutor;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::compact::CompactedHistoryMetadata;
|
||||
use crate::compact::CompactionAnalyticsAttempt;
|
||||
@@ -40,6 +41,7 @@ use codex_analytics::CompactionImplementation;
|
||||
use codex_analytics::CompactionPhase;
|
||||
use codex_analytics::CompactionReason;
|
||||
use codex_analytics::CompactionTrigger;
|
||||
use codex_async_utils::OrCancelExt;
|
||||
use codex_context_fragments::set_annotated_content;
|
||||
use codex_context_fragments::to_annotated_content;
|
||||
use codex_features::Feature;
|
||||
@@ -97,7 +99,7 @@ const MAX_REMOTE_COMPACTION_V2_STREAM_RETRIES: u64 = 2;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum CompactionRequestKind<'a> {
|
||||
Foreground,
|
||||
Foreground(&'a CancellationToken),
|
||||
Background(&'a CancellationToken),
|
||||
}
|
||||
|
||||
@@ -289,8 +291,9 @@ struct RemoteCompactionV2Output {
|
||||
}
|
||||
|
||||
async fn run_remote_compaction_request_v2(
|
||||
sess: &Session,
|
||||
step_context: &StepContext,
|
||||
request: &mut crate::hook_runtime::sampling::SamplingRequest,
|
||||
sess: &Arc<Session>,
|
||||
step_context: &Arc<StepContext>,
|
||||
prompt: &Arc<Prompt>,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
transport: &mut CompactionRequestTransport<'_>,
|
||||
@@ -308,14 +311,25 @@ async fn run_remote_compaction_request_v2(
|
||||
// Replay the routing token without letting this request change foreground routing.
|
||||
let turn_state = Arc::new(turn_state.clone().map(OnceLock::from).unwrap_or_default());
|
||||
let http_executor = ResponsesHttpExecutor::new(&sess.services.model_client);
|
||||
let send = http_executor.stream_responses_http(
|
||||
&turn_state,
|
||||
auth_owner.clone(),
|
||||
/*event_sender*/ None,
|
||||
prompt,
|
||||
&settings.model_info,
|
||||
&step_context.session_telemetry,
|
||||
SamplingOptions {
|
||||
reasoning_effort: effort,
|
||||
reasoning_summary: settings.reasoning_summary,
|
||||
service_tier: settings.service_tier.clone(),
|
||||
},
|
||||
responses_metadata,
|
||||
&trace,
|
||||
);
|
||||
let stream = tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return Err(CodexErr::TurnAborted),
|
||||
stream = http_executor.stream_responses_http(
|
||||
&turn_state, auth_owner.clone(), /*event_sender*/ None, prompt, &settings.model_info, &step_context.session_telemetry, effort,
|
||||
settings.reasoning_summary, settings.service_tier.clone(), responses_metadata,
|
||||
&trace,
|
||||
) => stream?,
|
||||
stream = request.open(sess, step_context, &settings.model_info.slug, &cancellation, send) => stream?,
|
||||
};
|
||||
return collect_compaction_output(
|
||||
sess,
|
||||
@@ -334,27 +348,38 @@ async fn run_remote_compaction_request_v2(
|
||||
.stream_max_retries()
|
||||
.min(MAX_REMOTE_COMPACTION_V2_STREAM_RETRIES);
|
||||
let mut retry_state = ResponsesStreamRetryState::default();
|
||||
let cancellation = &step_context.turn_cancellation;
|
||||
loop {
|
||||
let result = match client_session
|
||||
.stream(
|
||||
prompt,
|
||||
&settings.model_info,
|
||||
&turn_context.session_telemetry,
|
||||
sess.reasoning_effort_for_request(settings, RequestEffortUsage::Compaction)
|
||||
.await,
|
||||
settings.reasoning_summary,
|
||||
step_context.settings.service_tier.clone(),
|
||||
responses_metadata,
|
||||
&InferenceTraceContext::disabled(),
|
||||
let result = match request
|
||||
.open(
|
||||
sess,
|
||||
step_context,
|
||||
&settings.model_info.slug,
|
||||
cancellation,
|
||||
client_session.open_model_response(
|
||||
prompt,
|
||||
&settings.model_info,
|
||||
&turn_context.session_telemetry,
|
||||
SamplingOptions {
|
||||
reasoning_effort: sess
|
||||
.reasoning_effort_for_request(settings, RequestEffortUsage::Compaction)
|
||||
.await,
|
||||
reasoning_summary: settings.reasoning_summary,
|
||||
service_tier: step_context.settings.service_tier.clone(),
|
||||
},
|
||||
responses_metadata,
|
||||
&InferenceTraceContext::disabled(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.or_cancel(cancellation)
|
||||
.await?
|
||||
{
|
||||
Ok(stream) => {
|
||||
collect_compaction_output(
|
||||
sess,
|
||||
turn_context,
|
||||
stream,
|
||||
CompactionRequestKind::Foreground,
|
||||
CompactionRequestKind::Foreground(cancellation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -396,7 +421,9 @@ async fn collect_compaction_output(
|
||||
let mut completed_token_usage = None;
|
||||
loop {
|
||||
let event = match request_kind {
|
||||
CompactionRequestKind::Foreground => stream.next().await,
|
||||
CompactionRequestKind::Foreground(cancellation) => {
|
||||
stream.next_for_turn(cancellation).await
|
||||
}
|
||||
CompactionRequestKind::Background(cancellation) => tokio::select! {
|
||||
biased;
|
||||
// Charge completed usage even when cancellation arrives at the same time.
|
||||
@@ -417,7 +444,7 @@ async fn collect_compaction_output(
|
||||
.await;
|
||||
}
|
||||
}
|
||||
CompactionRequestKind::Foreground => {
|
||||
CompactionRequestKind::Foreground(_) => {
|
||||
sess.record_incomplete_response_usage(
|
||||
turn_context,
|
||||
&turn_context.initial_settings,
|
||||
@@ -451,7 +478,7 @@ async fn collect_compaction_output(
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
CompactionRequestKind::Foreground => {
|
||||
CompactionRequestKind::Foreground(_) => {
|
||||
sess.record_observed_response_completed(
|
||||
turn_context,
|
||||
&response_id,
|
||||
@@ -1384,8 +1411,10 @@ mod tests {
|
||||
assert_eq!(truncated, vec![newest]);
|
||||
}
|
||||
|
||||
#[test_case::test_case(false; "turn interrupt discards queued incomplete usage")]
|
||||
#[test_case::test_case(true; "session shutdown preserves queued incomplete usage without foreground updates")]
|
||||
#[tokio::test]
|
||||
async fn collect_background_incomplete_does_not_publish_foreground_usage() {
|
||||
async fn collect_queued_incomplete_usage_on_cancellation(background: bool) {
|
||||
let usage = TokenUsage {
|
||||
input_tokens: 100,
|
||||
output_tokens: 20,
|
||||
@@ -1416,14 +1445,25 @@ mod tests {
|
||||
&sess,
|
||||
&turn_context,
|
||||
stream,
|
||||
CompactionRequestKind::Background(&cancellation),
|
||||
if background {
|
||||
CompactionRequestKind::Background(&cancellation)
|
||||
} else {
|
||||
CompactionRequestKind::Foreground(&cancellation)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ref err)
|
||||
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_))));
|
||||
assert_eq!(sess.token_usage_info().await, None);
|
||||
assert!(rx.is_empty());
|
||||
if !background {
|
||||
assert!(
|
||||
matches!(result, Err(ref err) if matches!(err.details(), CodexErrorDetails::TurnAborted))
|
||||
);
|
||||
assert!(accounting.reports.lock().unwrap().is_empty());
|
||||
return;
|
||||
}
|
||||
assert!(matches!(result, Err(ref err)
|
||||
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_))));
|
||||
assert_eq!(
|
||||
*accounting.reports.lock().unwrap(),
|
||||
vec![AgentUsageReport {
|
||||
@@ -1434,8 +1474,11 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case::test_case(true, false; "background bills a complete response even without a summary")]
|
||||
#[test_case::test_case(false, true; "turn interrupt discards a queued complete response")]
|
||||
#[test_case::test_case(true, true; "session shutdown bills a queued complete response even without a summary")]
|
||||
#[tokio::test]
|
||||
async fn background_compaction_charges_usage_even_if_output_is_missing() {
|
||||
async fn collect_queued_complete_response(background: bool, interrupted: bool) {
|
||||
let (mut sess, turn, _) = crate::session::tests::make_session_and_context_with_rx().await;
|
||||
let accounting = Arc::new(UsageRecorder {
|
||||
reports: Mutex::new(Vec::new()),
|
||||
@@ -1453,14 +1496,29 @@ mod tests {
|
||||
reasoning_removals: None,
|
||||
end_turn: Some(true),
|
||||
})]);
|
||||
let cancellation = CancellationToken::new();
|
||||
if interrupted {
|
||||
cancellation.cancel();
|
||||
}
|
||||
let result = collect_compaction_output(
|
||||
&sess,
|
||||
&turn,
|
||||
stream,
|
||||
CompactionRequestKind::Background(&CancellationToken::new()),
|
||||
if background {
|
||||
CompactionRequestKind::Background(&cancellation)
|
||||
} else {
|
||||
CompactionRequestKind::Foreground(&cancellation)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
if !background {
|
||||
assert!(
|
||||
matches!(result, Err(error) if matches!(error.details(), CodexErrorDetails::TurnAborted))
|
||||
);
|
||||
assert!(accounting.reports.lock().unwrap().is_empty());
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
matches!(result, Err(error) if matches!(error.details(), CodexErrorDetails::Fatal(_)))
|
||||
);
|
||||
@@ -1514,7 +1572,7 @@ mod tests {
|
||||
&sess,
|
||||
&turn_context,
|
||||
stream,
|
||||
CompactionRequestKind::Foreground,
|
||||
CompactionRequestKind::Foreground(&CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("compaction should be collected");
|
||||
|
||||
@@ -73,6 +73,7 @@ pub(crate) struct BackgroundCompactionInput {
|
||||
}
|
||||
|
||||
pub(super) async fn run_remote_compact_v2_attempt(
|
||||
request: &mut crate::hook_runtime::sampling::SamplingRequest,
|
||||
sess: &Arc<Session>,
|
||||
step_context: &Arc<StepContext>,
|
||||
compaction_trace: &CompactionTraceContext,
|
||||
@@ -240,6 +241,7 @@ pub(super) async fn run_remote_compact_v2_attempt(
|
||||
})
|
||||
});
|
||||
let compaction_output_result = run_remote_compaction_request_v2(
|
||||
request,
|
||||
sess,
|
||||
step_context,
|
||||
&prompt,
|
||||
@@ -352,7 +354,11 @@ pub(crate) async fn run_background_compaction(
|
||||
CompactionImplementation::ResponsesCompactionV2,
|
||||
phase,
|
||||
);
|
||||
let mut request = crate::hook_runtime::sampling::SamplingRequest::new(
|
||||
codex_hooks::SamplingKind::BackgroundCompaction,
|
||||
);
|
||||
let attempt = run_remote_compact_v2_attempt(
|
||||
&mut request,
|
||||
sess,
|
||||
&step_context,
|
||||
&CompactionTraceContext::disabled(),
|
||||
|
||||
@@ -124,7 +124,11 @@ async fn build_and_install_compaction(
|
||||
)),
|
||||
)
|
||||
} else {
|
||||
let mut request = crate::hook_runtime::sampling::SamplingRequest::new(
|
||||
codex_hooks::SamplingKind::ForegroundCompaction,
|
||||
);
|
||||
let attempt = run_remote_compact_v2_attempt(
|
||||
&mut request,
|
||||
sess,
|
||||
step_context,
|
||||
&compaction_trace,
|
||||
@@ -150,6 +154,7 @@ async fn build_and_install_compaction(
|
||||
fallback_turn_context.provider.info().name.as_str(),
|
||||
);
|
||||
let fallback_result = run_remote_compact_v2_attempt(
|
||||
&mut request,
|
||||
sess,
|
||||
replacement_step_context,
|
||||
&fallback_compaction_trace,
|
||||
@@ -168,6 +173,13 @@ async fn build_and_install_compaction(
|
||||
);
|
||||
match fallback_result {
|
||||
Ok(attempt) => (attempt, fallback_turn_context),
|
||||
Err(fallback_error)
|
||||
if crate::responses_retry::is_terminal_sampling_outcome(
|
||||
&fallback_error,
|
||||
) =>
|
||||
{
|
||||
return Err(fallback_error);
|
||||
}
|
||||
Err(_) => return Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod executor;
|
||||
pub(crate) mod sampling;
|
||||
|
||||
pub(crate) use executor::prepare_executor_hook_sources;
|
||||
|
||||
@@ -1111,10 +1112,7 @@ fn compaction_trigger_label(value: CompactionTrigger) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn hooks_for_model_request(
|
||||
session: &Arc<Session>,
|
||||
step: &StepContext,
|
||||
) -> Arc<codex_hooks::Hooks> {
|
||||
fn hooks_for_model_request(session: &Arc<Session>, step: &StepContext) -> Arc<codex_hooks::Hooks> {
|
||||
match step.turn.session_source {
|
||||
SessionSource::Internal(InternalSessionSource::MemoryConsolidation)
|
||||
| SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) => Arc::new(
|
||||
@@ -1130,25 +1128,14 @@ pub(crate) fn hooks_for_model_request(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn run_model_request_hooks(
|
||||
async fn run_model_request_hooks(
|
||||
session: &Arc<Session>,
|
||||
step: &StepContext,
|
||||
hooks: &codex_hooks::Hooks,
|
||||
event: codex_hooks::ModelRequestHookEvent,
|
||||
request: codex_hooks::ModelRequestHookInput,
|
||||
) {
|
||||
let subagent = thread_spawn_subagent_hook_context(session, &step.turn);
|
||||
let request = codex_hooks::ModelRequestHookInput {
|
||||
session_id: session.session_id().to_string(),
|
||||
turn_id: step.turn.sub_id.clone(),
|
||||
cwd: tool_hook_cwd(&step.environments, &step.turn),
|
||||
transcript_path: session.hook_transcript_path().await,
|
||||
model: step.settings.model_info.slug.clone(),
|
||||
event,
|
||||
agent_id: subagent.as_ref().map(|context| context.agent_id.clone()),
|
||||
agent_type: subagent.map(|context| context.agent_type),
|
||||
};
|
||||
emit_hook_started_events(session, &step.turn, hooks.preview_model_request(&request)).await;
|
||||
let events = hooks.run_model_request(request).await;
|
||||
let events = hooks.run_model_request(request, |_| {}).await;
|
||||
emit_hook_completed_events(session, &step.turn, events).await;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
//! Each provider attempt pairs its own hooks. Transport recovery within an
|
||||
//! attempt does not start another hook pair.
|
||||
|
||||
use super::emit_hook_completed_events;
|
||||
use super::emit_hook_started_events;
|
||||
use super::hooks_for_model_request;
|
||||
use super::run_model_request_hooks;
|
||||
use super::thread_spawn_subagent_hook_context;
|
||||
use super::tool_hook_cwd;
|
||||
use crate::client::OpenedModelResponse;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::step_context::StepContext;
|
||||
use codex_hooks::ModelRequestHookEvent;
|
||||
use codex_hooks::ModelRequestHookInput;
|
||||
use codex_hooks::SamplingKind;
|
||||
use codex_protocol::error::Result;
|
||||
use codex_protocol::protocol::HookCompletedEvent;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookExecutionMode;
|
||||
use codex_protocol::protocol::HookRunStatus;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub(crate) struct SamplingRequest {
|
||||
kind: SamplingKind,
|
||||
}
|
||||
|
||||
impl SamplingRequest {
|
||||
pub(crate) fn new(kind: SamplingKind) -> Self {
|
||||
Self { kind }
|
||||
}
|
||||
|
||||
pub(crate) fn open(
|
||||
&mut self,
|
||||
session: &Arc<Session>,
|
||||
step: &Arc<StepContext>,
|
||||
model: &str,
|
||||
owner_cancellation: &CancellationToken,
|
||||
send: impl Future<Output = Result<OpenedModelResponse>> + Send,
|
||||
) -> impl Future<Output = Result<ResponseStream>> + Send {
|
||||
// Keep the transport and hook state out of the turn's cancellation and retry futures.
|
||||
Box::pin(self.open_attempt(session, step, model, owner_cancellation, send))
|
||||
}
|
||||
|
||||
async fn open_attempt(
|
||||
&mut self,
|
||||
session: &Arc<Session>,
|
||||
step: &Arc<StepContext>,
|
||||
model: &str,
|
||||
owner_cancellation: &CancellationToken,
|
||||
send: impl Future<Output = Result<OpenedModelResponse>>,
|
||||
) -> Result<ResponseStream> {
|
||||
let sampling_turn = match self.kind {
|
||||
SamplingKind::Sampling => Some(Arc::clone(&step.turn)),
|
||||
SamplingKind::ForegroundCompaction | SamplingKind::BackgroundCompaction => None,
|
||||
};
|
||||
let hooks = hooks_for_model_request(session, step);
|
||||
let pre_mode = hooks.execution_mode_for_event(HookEventName::PreModelRequest);
|
||||
let post_mode = hooks.execution_mode_for_event(HookEventName::PostModelRequest);
|
||||
if pre_mode.is_none() && post_mode.is_none() {
|
||||
return send.await?.deliver(&mut None, sampling_turn);
|
||||
}
|
||||
let subagent = thread_spawn_subagent_hook_context(session, &step.turn);
|
||||
let input = ModelRequestHookInput {
|
||||
lifecycle_id: uuid::Uuid::new_v4(),
|
||||
kind: self.kind,
|
||||
session_id: session.session_id().to_string(),
|
||||
thread_id: session.thread_id().to_string(),
|
||||
turn_id: step.turn.sub_id.clone(),
|
||||
cwd: tool_hook_cwd(&step.environments, &step.turn),
|
||||
transcript_path: session.hook_transcript_path().await,
|
||||
model: model.to_owned(),
|
||||
event: ModelRequestHookEvent::PreModelRequest,
|
||||
agent_id: subagent.as_ref().map(|context| context.agent_id.clone()),
|
||||
agent_type: subagent.map(|context| context.agent_type),
|
||||
};
|
||||
let mut attempt = post_mode.map(|mode| SamplingAttempt {
|
||||
owner_cancellation: owner_cancellation.clone(),
|
||||
post: Some(PostHooks {
|
||||
hooks: Arc::clone(&hooks),
|
||||
step: Arc::clone(step),
|
||||
input: input.clone(),
|
||||
mode,
|
||||
}),
|
||||
session: Arc::clone(session),
|
||||
});
|
||||
if pre_mode.is_some() {
|
||||
run_model_request_hooks(session, step, &hooks, input).await;
|
||||
}
|
||||
let stream = send
|
||||
.await
|
||||
.and_then(|opened| opened.deliver(&mut attempt, sampling_turn));
|
||||
if let Err(error) = &stream
|
||||
&& let Some(mut attempt) = attempt
|
||||
&& let Some(hooks) = attempt.start_post_hooks(ModelRequestHookEvent::failed(error))
|
||||
{
|
||||
hooks.await;
|
||||
}
|
||||
stream
|
||||
}
|
||||
}
|
||||
|
||||
/// The sampling future owns the synchronous handlers. Observer-only and
|
||||
/// pre-response cancellation notifications run on the session's hook executor.
|
||||
struct PostHooks {
|
||||
hooks: Arc<codex_hooks::Hooks>,
|
||||
step: Arc<StepContext>,
|
||||
input: ModelRequestHookInput,
|
||||
mode: HookExecutionMode,
|
||||
}
|
||||
|
||||
impl PostHooks {
|
||||
async fn run(mut self, session: &Arc<Session>, event: ModelRequestHookEvent) {
|
||||
self.input.event = event;
|
||||
run_model_request_hooks(session, &self.step, &self.hooks, self.input).await;
|
||||
}
|
||||
|
||||
async fn run_sync(mut self, session: Arc<Session>, event: ModelRequestHookEvent) {
|
||||
let (mut done, wait) = oneshot::channel();
|
||||
self.input.event = event;
|
||||
// Dropping the waiter cancels execution while the task finishes publishing
|
||||
// lifecycle events. It must run independently of the async hook limit.
|
||||
tokio::spawn(async move {
|
||||
if done.is_closed() {
|
||||
return;
|
||||
}
|
||||
let turn = &self.step.turn;
|
||||
let runs = self.hooks.preview_model_request(&self.input);
|
||||
let started = Instant::now();
|
||||
emit_hook_started_events(&session, turn, runs.clone()).await;
|
||||
let mut finished: Vec<HookCompletedEvent> = Vec::new();
|
||||
let events = tokio::select! {
|
||||
biased;
|
||||
() = done.closed() => {
|
||||
let completed_at = chrono::Utc::now().timestamp();
|
||||
let duration_ms = i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX);
|
||||
for mut run in runs {
|
||||
if run.execution_mode != HookExecutionMode::Sync
|
||||
|| finished.iter().any(|event| event.run.id == run.id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
run.status = HookRunStatus::Stopped;
|
||||
run.status_message = Some("Hook cancelled because the turn was interrupted".into());
|
||||
run.completed_at = Some(completed_at);
|
||||
run.duration_ms = Some(duration_ms);
|
||||
finished.push(HookCompletedEvent {
|
||||
turn_id: Some(turn.sub_id.clone()),
|
||||
run,
|
||||
});
|
||||
}
|
||||
finished
|
||||
}
|
||||
events = self.hooks.run_model_request(self.input, |event| finished.push(event.clone())) => events,
|
||||
};
|
||||
emit_hook_completed_events(&session, turn, events).await;
|
||||
let _ = done.send(());
|
||||
});
|
||||
let _ = wait.await;
|
||||
}
|
||||
|
||||
fn schedule(self, session: Arc<Session>, event: ModelRequestHookEvent) {
|
||||
Arc::clone(&self.hooks).schedule_async_task(async move {
|
||||
self.run(&session, event).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct SamplingAttempt {
|
||||
/// The turn owns foreground responses; the session owns speculative responses.
|
||||
pub(crate) owner_cancellation: CancellationToken,
|
||||
session: Arc<Session>,
|
||||
post: Option<PostHooks>,
|
||||
}
|
||||
|
||||
impl Drop for SamplingAttempt {
|
||||
fn drop(&mut self) {
|
||||
if let Some(post) = self.post.take() {
|
||||
post.schedule(
|
||||
Arc::clone(&self.session),
|
||||
ModelRequestHookEvent::cancelled(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SamplingAttempt {
|
||||
/// Observers run on the session executor. The request owns the synchronous
|
||||
/// group: if the turn stops waiting, dropping this future cancels it.
|
||||
pub(crate) fn start_post_hooks(
|
||||
&mut self,
|
||||
event: ModelRequestHookEvent,
|
||||
) -> Option<impl Future<Output = ()> + Send + 'static> {
|
||||
let post = self.post.take()?;
|
||||
let session = Arc::clone(&self.session);
|
||||
if post.mode == HookExecutionMode::Sync {
|
||||
Some(post.run_sync(session, event))
|
||||
} else {
|
||||
post.schedule(session, event);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,18 @@ pub(crate) enum ResponsesStreamRequest {
|
||||
RemoteCompactionV2,
|
||||
}
|
||||
|
||||
/// A new generation cannot repair a terminal provider response or settle the
|
||||
/// previous attempt's bill. Preserve these outcomes across transport retries,
|
||||
/// same-turn steering, and compaction's fallback to another model.
|
||||
pub(crate) fn is_terminal_sampling_outcome(error: &CodexErr) -> bool {
|
||||
matches!(
|
||||
error.details(),
|
||||
CodexErrorDetails::ResponseIncomplete(_)
|
||||
| CodexErrorDetails::SessionBudgetExceeded
|
||||
| CodexErrorDetails::InferenceHookDenied { .. }
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) struct ResponsesStreamRetryState {
|
||||
retries: u64,
|
||||
connection_retries: u64,
|
||||
|
||||
@@ -4654,6 +4654,7 @@ impl Session {
|
||||
Ok(Arc::new(StepContext {
|
||||
sampled_through_sequence: None,
|
||||
serving_assessment: None,
|
||||
turn_cancellation: cancellation_token.clone(),
|
||||
preempt: turn_context
|
||||
.config
|
||||
.features
|
||||
|
||||
@@ -31,6 +31,8 @@ pub(crate) struct StepContext {
|
||||
/// Final serving value for this invocation; retained by its code cell across waits.
|
||||
pub(crate) serving_assessment: Option<Arc<crate::guardian::ServingAssessment>>,
|
||||
pub(crate) turn: Arc<TurnContext>,
|
||||
/// Cancels work owned by this turn, independently of steers or session shutdown.
|
||||
pub(crate) turn_cancellation: CancellationToken,
|
||||
/// Preempts this request and yields its code-mode observations when user input arrives.
|
||||
pub(crate) preempt: Option<CancellationToken>,
|
||||
/// Realtime call activity and instructions captured for this sampling request.
|
||||
|
||||
@@ -305,6 +305,7 @@ impl StepContext {
|
||||
settings.service_tier = turn.config.service_tier.clone();
|
||||
Arc::new(Self {
|
||||
sampled_through_sequence: None,
|
||||
turn_cancellation: tokio_util::sync::CancellationToken::new(),
|
||||
preempt: turn
|
||||
.config
|
||||
.features
|
||||
|
||||
@@ -7,8 +7,10 @@ use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::client::ModelClientSession;
|
||||
use crate::client::SamplingOptions;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::compact::AutoCompactionDisabled;
|
||||
use crate::compact::run_inline_auto_compact_task;
|
||||
use crate::compact_remote_v2::BackgroundCompactionCandidate;
|
||||
@@ -21,15 +23,14 @@ use crate::cyber_access_program;
|
||||
use crate::environment_selection::TurnEnvironmentSnapshot;
|
||||
use crate::feedback_tags;
|
||||
use crate::hook_runtime::drain_async_hook_results;
|
||||
use crate::hook_runtime::hooks_for_model_request;
|
||||
use crate::hook_runtime::inspect_pending_input;
|
||||
use crate::hook_runtime::legacy_notify_input_messages;
|
||||
use crate::hook_runtime::record_additional_contexts;
|
||||
use crate::hook_runtime::record_pending_input;
|
||||
use crate::hook_runtime::run_legacy_after_agent_hook;
|
||||
use crate::hook_runtime::run_model_request_hooks;
|
||||
use crate::hook_runtime::run_pending_session_start_hooks;
|
||||
use crate::hook_runtime::run_turn_stop_hooks;
|
||||
use crate::hook_runtime::sampling::SamplingRequest;
|
||||
use crate::mcp_skill_dependencies::maybe_prompt_and_install_mcp_dependencies;
|
||||
use crate::mentions::build_connector_slug_counts;
|
||||
use crate::mentions::collect_explicit_app_ids;
|
||||
@@ -68,7 +69,6 @@ use crate::tools::router::ToolSuggestPresentation;
|
||||
use crate::tools::router::record_registration_metrics;
|
||||
use crate::tools::spec_plan::build_tool_router;
|
||||
use crate::turn_diff_tracker::TurnDiffTracker;
|
||||
use crate::turn_timing::record_turn_ttft_metric;
|
||||
use crate::util::error_or_panic;
|
||||
use codex_analytics::AppInvocation;
|
||||
use codex_analytics::CompactionPhase;
|
||||
@@ -86,9 +86,7 @@ use codex_features::Feature;
|
||||
use codex_file_system::EnvironmentAccess;
|
||||
use codex_file_system::FindUpErrorPolicy;
|
||||
use codex_file_system::find_nearest_ancestor_with_markers;
|
||||
use codex_hooks::ModelRequestHookEvent::PostModelRequest;
|
||||
use codex_hooks::ModelRequestHookEvent::PreModelRequest;
|
||||
use codex_hooks::ModelRequestStatus;
|
||||
use codex_hooks::SamplingKind;
|
||||
use codex_login::CodexAuth;
|
||||
use codex_model_provider::RemoteCompactionSupport;
|
||||
use codex_protocol::ResponseItemId;
|
||||
@@ -113,8 +111,6 @@ use codex_protocol::protocol::AgentReasoningSectionBreakEvent;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::ErrorEvent;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookExecutionMode;
|
||||
use codex_protocol::protocol::InternalSessionSource;
|
||||
use codex_protocol::protocol::PlanDeltaEvent;
|
||||
use codex_protocol::protocol::ReasoningContentDeltaEvent;
|
||||
@@ -1936,6 +1932,7 @@ async fn run_sampling_request(
|
||||
let mut initial_input = Some(input);
|
||||
let mut notify_messages = Vec::new();
|
||||
let mut executed_tool_calls_by_output = HashMap::new();
|
||||
let mut sampling_request = SamplingRequest::new(SamplingKind::Sampling);
|
||||
loop {
|
||||
// Running code-mode cells can request review while this response is in flight.
|
||||
// Keep the latest received ID until response.created replaces it.
|
||||
@@ -2011,6 +2008,7 @@ async fn run_sampling_request(
|
||||
};
|
||||
let mut draining_for_mailbox = false;
|
||||
let err = match try_run_sampling_request(
|
||||
&mut sampling_request,
|
||||
tool_runtime.clone(),
|
||||
Arc::clone(&sess),
|
||||
sampling_step_context,
|
||||
@@ -2042,11 +2040,8 @@ async fn run_sampling_request(
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
CodexErrorDetails::SessionBudgetExceeded
|
||||
| CodexErrorDetails::InferenceHookDenied { .. }
|
||||
| CodexErrorDetails::Fatal(_) => {
|
||||
return Err(err);
|
||||
}
|
||||
CodexErrorDetails::Fatal(_) => return Err(err),
|
||||
_ if crate::responses_retry::is_terminal_sampling_outcome(&err) => return Err(err),
|
||||
_ => err,
|
||||
},
|
||||
};
|
||||
@@ -2867,6 +2862,7 @@ async fn drain_in_flight(
|
||||
)
|
||||
)]
|
||||
async fn try_run_sampling_request(
|
||||
sampling_request: &mut SamplingRequest,
|
||||
tool_runtime: ToolCallRuntime,
|
||||
sess: Arc<Session>,
|
||||
step_context: Arc<StepContext>,
|
||||
@@ -2925,47 +2921,11 @@ async fn try_run_sampling_request(
|
||||
let effort = sess
|
||||
.reasoning_effort_for_request(&step_context.settings, super::RequestEffortUsage::Sampling)
|
||||
.await;
|
||||
let hooks = hooks_for_model_request(&sess, &step_context);
|
||||
let post_mode = hooks.execution_mode_for_event(HookEventName::PostModelRequest);
|
||||
let defer_tool_dispatch = post_mode == Some(HookExecutionMode::Sync);
|
||||
let finish_post_request = post_mode.map(|_| {
|
||||
let (post_request, completed) = tokio::sync::oneshot::channel();
|
||||
let (hook_done, wait_for_hook) = tokio::sync::oneshot::channel();
|
||||
hooks.schedule_async_task({
|
||||
let hooks = Arc::clone(&hooks);
|
||||
let session = Arc::clone(&sess);
|
||||
let step = Arc::clone(&step_context);
|
||||
async move {
|
||||
let event = completed.await.unwrap_or(PostModelRequest {
|
||||
status: ModelRequestStatus::Cancelled,
|
||||
error: None,
|
||||
token_usage: None,
|
||||
});
|
||||
run_model_request_hooks(&session, &step, &hooks, event).await;
|
||||
let _ = hook_done.send(());
|
||||
}
|
||||
});
|
||||
let cancellation_token = cancellation_token.clone();
|
||||
async move |event| {
|
||||
let _ = post_request.send(event);
|
||||
if defer_tool_dispatch {
|
||||
// Interrupt the wait, not the session-owned hook or final token accounting.
|
||||
let _ = wait_for_hook.or_cancel(&cancellation_token).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
if hooks
|
||||
.execution_mode_for_event(HookEventName::PreModelRequest)
|
||||
.is_some()
|
||||
{
|
||||
run_model_request_hooks(&sess, &step_context, &hooks, PreModelRequest).await;
|
||||
}
|
||||
let sampling_timing_guard = turn_context.turn_timing_state.begin_sampling();
|
||||
let mut response_id_for_assessment = None;
|
||||
let mut answer_moderation: Option<codex_protocol::protocol::TargetedAnswerModeration> = None;
|
||||
let mut pending_assessment: Option<codex_api::GuardianAssessment> = None;
|
||||
let mut response_completed = false;
|
||||
let mut request_token_usage: Option<codex_protocol::protocol::TokenUsage> = None;
|
||||
let mut includes_reasoning = step_context.settings.model_info.use_responses_lite;
|
||||
// Catalog policies can enable adaptive review independently of the legacy scorer flag.
|
||||
// Capture evidence whenever Guardian is available; the reviewer resolves the policy.
|
||||
@@ -2981,37 +2941,33 @@ async fn try_run_sampling_request(
|
||||
let mut resampling = crate::guardian::ResponseResampling::new(&sess);
|
||||
let prompt = Arc::new(prompt);
|
||||
let mut stream_request = async |prompt: &Arc<Prompt>| {
|
||||
client_session
|
||||
.stream(
|
||||
let send = client_session
|
||||
.open_model_response(
|
||||
prompt,
|
||||
&step_context.settings.model_info,
|
||||
&step_context.session_telemetry,
|
||||
effort.clone(),
|
||||
step_context.settings.reasoning_summary,
|
||||
step_context.settings.service_tier.clone(),
|
||||
SamplingOptions {
|
||||
reasoning_effort: effort.clone(),
|
||||
reasoning_summary: step_context.settings.reasoning_summary,
|
||||
service_tier: step_context.settings.service_tier.clone(),
|
||||
},
|
||||
responses_metadata,
|
||||
&inference_trace,
|
||||
)
|
||||
.instrument(trace_span!("stream_request"))
|
||||
.instrument(trace_span!("stream_request"));
|
||||
sampling_request
|
||||
.open(
|
||||
&sess,
|
||||
&step_context,
|
||||
&step_context.settings.model_info.slug,
|
||||
&cancellation_token,
|
||||
send,
|
||||
)
|
||||
.await
|
||||
};
|
||||
let stream = stream_request(&prompt)
|
||||
let mut stream = stream_request(&prompt)
|
||||
.or_cancel(&cancellation_token)
|
||||
.await?;
|
||||
let mut stream = match stream {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if let Some(finish_post_request) = finish_post_request {
|
||||
finish_post_request(PostModelRequest {
|
||||
status: ModelRequestStatus::Failed,
|
||||
error: Some(error.to_string()),
|
||||
token_usage: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
.await??;
|
||||
let can_interrupt = turn_context
|
||||
.config
|
||||
.features
|
||||
@@ -3090,8 +3046,6 @@ async fn try_run_sampling_request(
|
||||
"codex.response.receive_wait_ms",
|
||||
receive_started.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
// Account for an already-received terminal error even if cancellation raced it.
|
||||
// Cancellation still stops ordinary stream events before they can run tools.
|
||||
let event = match event {
|
||||
Ok(Ok(event)) => event,
|
||||
Ok(Err(_)) => {
|
||||
@@ -3118,13 +3072,7 @@ async fn try_run_sampling_request(
|
||||
Err(_) => break Err(CodexErr::TurnAborted),
|
||||
};
|
||||
|
||||
if cancellation_token.is_cancelled()
|
||||
&& !matches!(event.as_ref(), Some(Err(err))
|
||||
if matches!(err.details(), CodexErrorDetails::ResponseIncomplete(_)))
|
||||
{
|
||||
break Err(CodexErr::TurnAborted);
|
||||
}
|
||||
|
||||
let event = ResponseStream::accept_for_turn(event, &cancellation_token);
|
||||
let event = match event.unwrap_or_else(|| {
|
||||
Err(CodexErr::Stream(
|
||||
"stream closed before response.completed".into(),
|
||||
@@ -3133,11 +3081,6 @@ async fn try_run_sampling_request(
|
||||
Ok(event) => event,
|
||||
Err(err) => {
|
||||
if let CodexErrorDetails::ResponseIncomplete(incomplete) = err.details() {
|
||||
if let Some(usage) = &incomplete.token_usage {
|
||||
request_token_usage
|
||||
.get_or_insert_default()
|
||||
.add_assign(usage);
|
||||
}
|
||||
sess.record_incomplete_response_usage(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
@@ -3162,7 +3105,6 @@ async fn try_run_sampling_request(
|
||||
}
|
||||
};
|
||||
|
||||
record_turn_ttft_metric(&turn_context, &event).await;
|
||||
let Some(event) = resampling.filter(event, &step_context.tool_router) else {
|
||||
continue;
|
||||
};
|
||||
@@ -3273,7 +3215,7 @@ async fn try_run_sampling_request(
|
||||
step_context: Arc::clone(&step_context),
|
||||
turn_store: Arc::clone(&turn_store),
|
||||
tool_runtime,
|
||||
defer_tool_dispatch: defer_tool_dispatch || *draining_for_mailbox,
|
||||
defer_tool_dispatch: *draining_for_mailbox,
|
||||
cancellation_token: cancellation_token.child_token(),
|
||||
};
|
||||
if *draining_for_mailbox {
|
||||
@@ -3532,11 +3474,6 @@ async fn try_run_sampling_request(
|
||||
.await;
|
||||
}
|
||||
response_completed = true;
|
||||
if let Some(usage) = &token_usage {
|
||||
request_token_usage
|
||||
.get_or_insert_default()
|
||||
.add_assign(usage);
|
||||
}
|
||||
sess.services
|
||||
.analytics_events_client
|
||||
.track_code_mode_tool_call(
|
||||
@@ -3757,21 +3694,6 @@ async fn try_run_sampling_request(
|
||||
drop(stream);
|
||||
drop(sampling_span);
|
||||
drop(sampling_timing_guard);
|
||||
if let Some(finish_post_request) = finish_post_request {
|
||||
let (status, error) = match (response_completed, &outcome) {
|
||||
(true, _) => (ModelRequestStatus::Completed, None),
|
||||
(false, Err(error)) if !matches!(error.details(), CodexErrorDetails::TurnAborted) => {
|
||||
(ModelRequestStatus::Failed, Some(error.to_string()))
|
||||
}
|
||||
_ => (ModelRequestStatus::Cancelled, None),
|
||||
};
|
||||
finish_post_request(PostModelRequest {
|
||||
status,
|
||||
error,
|
||||
token_usage: request_token_usage,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
flush_assistant_text_segments_all(
|
||||
&sess,
|
||||
&turn_context,
|
||||
|
||||
@@ -5,6 +5,56 @@ use codex_core::config::Config;
|
||||
use codex_features::Feature;
|
||||
use codex_hooks::HookListEntry;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use serde_json::json;
|
||||
|
||||
/// Build the MCP reply for a loopback hook fixture; notifications have no body.
|
||||
pub fn hook_mcp_reply(
|
||||
request: &serde_json::Value,
|
||||
server: &str,
|
||||
tools: &[&str],
|
||||
call: impl FnOnce(&serde_json::Value) -> serde_json::Value,
|
||||
) -> Option<serde_json::Value> {
|
||||
let method = request["method"].as_str().expect("MCP method");
|
||||
let Some(id) = request.get("id") else {
|
||||
assert!(
|
||||
method.starts_with("notifications/"),
|
||||
"missing ID for {method}"
|
||||
);
|
||||
return None;
|
||||
};
|
||||
let result = match method {
|
||||
"initialize" => json!({
|
||||
"protocolVersion": request["params"]["protocolVersion"],
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": server, "version": "1.0.0"},
|
||||
}),
|
||||
"tools/list" => {
|
||||
json!({"tools":tools.iter().map(|name| json!({"name":name,"inputSchema":{"type":"object"}})).collect::<Vec<_>>()})
|
||||
}
|
||||
"tools/call" => call(&request["params"]),
|
||||
method => panic!("unexpected MCP request: {method}"),
|
||||
};
|
||||
Some(json!({"jsonrpc":"2.0", "id":id, "result":result}))
|
||||
}
|
||||
|
||||
pub async fn mount_hook_mcp(
|
||||
server: &wiremock::MockServer,
|
||||
name: &'static str,
|
||||
tools: &'static [&'static str],
|
||||
respond: impl Fn(&serde_json::Value) -> serde_json::Value + Send + Sync + 'static,
|
||||
) {
|
||||
wiremock::Mock::given(wiremock::matchers::method("POST"))
|
||||
.and(wiremock::matchers::path("/repl"))
|
||||
.respond_with(move |req: &wiremock::Request| {
|
||||
let request = serde_json::from_slice(&req.body).expect("MCP request");
|
||||
match hook_mcp_reply(&request, name, tools, &respond) {
|
||||
Some(reply) => wiremock::ResponseTemplate::new(200).set_body_json(reply),
|
||||
None => wiremock::ResponseTemplate::new(202),
|
||||
}
|
||||
})
|
||||
.mount(server)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub fn trust_discovered_hooks(config: &mut Config) {
|
||||
config
|
||||
|
||||
@@ -1643,6 +1643,146 @@ async fn failed_compaction_preserves_history_after_restart() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn foreground_compaction_only_installs_and_bills_after_its_post_hook() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let responses = ["seed", "interrupted", "accepted"].map(|id| {
|
||||
sse(vec![
|
||||
ev_assistant_message(id, id),
|
||||
ev_completed_with_tokens(id, /*total_tokens*/ 42),
|
||||
])
|
||||
});
|
||||
let model = mount_sse_sequence(&server, responses.to_vec()).await;
|
||||
let hook_server = start_mock_server().await;
|
||||
let calls = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let held = Arc::new(tokio::sync::Notify::new());
|
||||
let (release, gate) = std::sync::mpsc::channel();
|
||||
let gate = std::sync::Mutex::new(gate);
|
||||
core_test_support::hooks::mount_hook_mcp(&hook_server, "observer", &["pre", "post"], {
|
||||
let calls = Arc::clone(&calls);
|
||||
let held = Arc::clone(&held);
|
||||
move |params| {
|
||||
let input = ¶ms["arguments"];
|
||||
if input["kind"] == "foreground_compaction" {
|
||||
calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((params["name"].clone(), input["response_id"].clone()));
|
||||
if input["response_id"] == "interrupted" {
|
||||
held.notify_one();
|
||||
// Dropping the sender also releases the mock if an assertion fails.
|
||||
let _ = gate.lock().unwrap().recv();
|
||||
}
|
||||
}
|
||||
json!({"content":[]})
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let url = format!("{}/repl", hook_server.uri());
|
||||
let provider = non_openai_model_provider(&server);
|
||||
let accounting = Arc::new(super::rollout_budget::HostAccounting::default());
|
||||
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
let test = test_codex()
|
||||
.with_pre_build_hook(move |home| {
|
||||
let hook = |tool, input| {
|
||||
json!([{ "hooks": [{
|
||||
"type": "mcp_tool",
|
||||
"server": "observer",
|
||||
"tool": tool,
|
||||
"input": input
|
||||
}] }])
|
||||
};
|
||||
let config = json!({
|
||||
"hooks": {
|
||||
"PreModelRequest": hook("pre", json!({ "kind": "${kind}" })),
|
||||
"PostModelRequest": hook("post", json!({
|
||||
"kind": "${kind}",
|
||||
"response_id": "${response_id}"
|
||||
}))
|
||||
},
|
||||
"mcp_servers": { "observer": {
|
||||
"url": url,
|
||||
"environment_id": super::rmcp_client::remote_aware_environment_id()
|
||||
} }
|
||||
});
|
||||
fs::write(home.join("config.toml"), toml::to_string(&config).unwrap()).unwrap();
|
||||
})
|
||||
.with_config(move |config| {
|
||||
config.model_provider = provider;
|
||||
set_test_compact_prompt(config);
|
||||
trust_discovered_hooks(config);
|
||||
})
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
test.submit_turn("seed history").await?;
|
||||
let charged_turns = || {
|
||||
accounting
|
||||
.reports
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|r| r.turn_id.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let seed_charges = charged_turns();
|
||||
assert_eq!(seed_charges.len(), 1);
|
||||
|
||||
test.codex.submit(Op::Compact).await?;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(15), held.notified()).await?;
|
||||
test.codex.flush_rollout().await?;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
assert!(
|
||||
!history
|
||||
.items
|
||||
.iter()
|
||||
.any(|item| matches!(item, RolloutItem::Compacted(_)))
|
||||
);
|
||||
assert_eq!(charged_turns(), seed_charges);
|
||||
test.codex.submit(Op::Interrupt).await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnAborted(_))
|
||||
})
|
||||
.await;
|
||||
release.send(())?;
|
||||
|
||||
let accepted_turn = test.codex.submit(Op::Compact).await?;
|
||||
wait_for_event(
|
||||
&test.codex,
|
||||
|event| matches!(event, EventMsg::TurnComplete(done) if done.turn_id == accepted_turn),
|
||||
)
|
||||
.await;
|
||||
test.codex.flush_rollout().await?;
|
||||
let history = test.codex.load_history(/*include_archived*/ false).await?;
|
||||
let checkpoints: Vec<_> = history
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
RolloutItem::Compacted(checkpoint) => {
|
||||
Some(checkpoint.compaction_response_id.as_deref())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(checkpoints, [Some("accepted")]);
|
||||
let mut expected_charges = seed_charges;
|
||||
expected_charges.push(accepted_turn);
|
||||
assert_eq!(charged_turns(), expected_charges);
|
||||
assert_eq!(
|
||||
*calls.lock().unwrap(),
|
||||
[
|
||||
(json!("pre"), Value::Null),
|
||||
(json!("post"), json!("interrupted")),
|
||||
(json!("pre"), Value::Null),
|
||||
(json!("post"), json!("accepted")),
|
||||
]
|
||||
);
|
||||
assert_eq!(model.requests().len(), 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn manual_compact_records_durable_and_local_token_usage() {
|
||||
skip_if_no_network!();
|
||||
|
||||
+149
-135
@@ -31,6 +31,7 @@ use codex_protocol::permissions::NetworkSandboxPolicy;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookRunStatus;
|
||||
use codex_protocol::protocol::InternalSessionSource;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
@@ -47,6 +48,7 @@ use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::InMemoryThreadStore;
|
||||
use core_test_support::TestTargetOs;
|
||||
use core_test_support::fs_wait;
|
||||
use core_test_support::hooks::mount_hook_mcp;
|
||||
use core_test_support::hooks::trust_discovered_hooks;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::responses::ev_apply_patch_custom_tool_call;
|
||||
@@ -4503,32 +4505,13 @@ async fn local_bundled_cleanup_hook_runs_without_saved_trust(
|
||||
}),
|
||||
};
|
||||
let server = start_mock_server().await;
|
||||
wiremock::Mock::given(wiremock::matchers::method("POST"))
|
||||
.and(wiremock::matchers::path("/repl"))
|
||||
.respond_with(move |request: &wiremock::Request| {
|
||||
let request: Value = serde_json::from_slice(&request.body).expect("MCP request");
|
||||
let result = match request["method"].as_str() {
|
||||
Some("initialize") => serde_json::json!({
|
||||
"protocolVersion": request["params"]["protocolVersion"],
|
||||
"capabilities": { "tools": {} },
|
||||
"serverInfo": { "name": mcp_server_name, "version": "1.0.0" },
|
||||
}),
|
||||
Some("notifications/initialized") => return wiremock::ResponseTemplate::new(202),
|
||||
Some("tools/list") => serde_json::json!({ "tools": [
|
||||
{ "name": "turn_ended", "inputSchema": { "type": "object" } },
|
||||
{ "name": "regular_stop", "inputSchema": { "type": "object" } },
|
||||
] }),
|
||||
Some("tools/call") => hook_response.clone(),
|
||||
method => panic!("unexpected MCP request: {method:?}"),
|
||||
};
|
||||
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": request["id"],
|
||||
"result": result,
|
||||
}))
|
||||
})
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_hook_mcp(
|
||||
&server,
|
||||
mcp_server_name,
|
||||
&["turn_ended", "regular_stop"],
|
||||
move |_| hook_response.clone(),
|
||||
)
|
||||
.await;
|
||||
let response = mount_sse_once(
|
||||
&server,
|
||||
sse(vec![
|
||||
@@ -6142,33 +6125,18 @@ async fn model_request_hooks_report_failures_and_retries_but_cannot_veto() -> Re
|
||||
wait_for_hook_inputs(test.codex_home_path(), "PostModelRequest", /*count*/ 2).await?;
|
||||
assert_eq!(pre.len(), 2);
|
||||
assert_eq!(post.len(), 2);
|
||||
let thread_id = test.session_configured.thread_id.to_string();
|
||||
for hook in pre.iter().chain(&post) {
|
||||
assert_eq!(hook["thread_id"], thread_id);
|
||||
assert_eq!(hook["kind"], "sampling");
|
||||
}
|
||||
assert_eq!(post[0]["status"], "failed");
|
||||
assert!(post[0]["error"].is_string());
|
||||
assert_eq!(post[1]["status"], "completed");
|
||||
assert_eq!(post[1]["error"], Value::Null);
|
||||
assert_eq!(post[0]["response_id"], Value::Null);
|
||||
assert_eq!(post[0]["token_usage"], Value::Null);
|
||||
assert_eq!(
|
||||
post[1]["token_usage"],
|
||||
json!({
|
||||
"input_tokens": 42,
|
||||
"cached_input_tokens": 0,
|
||||
"cache_write_input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"reasoning_output_tokens": 0,
|
||||
"total_tokens": 42,
|
||||
})
|
||||
);
|
||||
for (before, after) in pre.iter().zip(&post) {
|
||||
let mut common = after.clone();
|
||||
common["hook_event_name"] = json!("PreModelRequest");
|
||||
let fields = common.as_object_mut().unwrap();
|
||||
fields.remove("status");
|
||||
fields.remove("error");
|
||||
fields.remove("token_usage");
|
||||
assert_eq!(before, &common);
|
||||
assert!(before["session_id"].is_string());
|
||||
assert!(before["cwd"].is_string());
|
||||
}
|
||||
assert_eq!(post[1]["status"], "completed");
|
||||
assert_eq!(post[1]["response_id"], "successful-attempt");
|
||||
assert_eq!(post[1]["token_usage"]["total_tokens"], 42);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6214,33 +6182,57 @@ async fn model_request_observer_reports_cancellation() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case::test_case(Some(false), false; "sync finishes before dispatch")]
|
||||
#[test_case::test_case(Some(false), true; "resume after interrupt preserves aborted custom call and usage")]
|
||||
#[test_case::test_case(Some(true), false; "async dispatches before stream ends")]
|
||||
#[test_case::test_case(None, false; "no post hook dispatches before stream ends")]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum ToolDispatchCase {
|
||||
Eager,
|
||||
AsyncHook,
|
||||
SyncHook,
|
||||
InterruptedHook,
|
||||
}
|
||||
|
||||
#[test_case::test_case(ToolDispatchCase::Eager; "default runs the tool before the response ends")]
|
||||
#[test_case::test_case(ToolDispatchCase::AsyncHook; "an observer does not hold the tool")]
|
||||
#[test_case::test_case(ToolDispatchCase::SyncHook; "streaming output and tools precede a synchronous hook")]
|
||||
#[test_case::test_case(ToolDispatchCase::InterruptedHook; "interrupt cancels the synchronous hook before billing")]
|
||||
#[tokio::test]
|
||||
async fn model_request_post_hook_orders_tool_dispatch(
|
||||
hook_async: Option<bool>,
|
||||
interrupt: bool,
|
||||
) -> Result<()> {
|
||||
async fn model_request_post_hook_orders_tool_dispatch(case: ToolDispatchCase) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let (finish_stream, gate) = oneshot::channel();
|
||||
let call = if interrupt {
|
||||
ev_apply_patch_custom_tool_call(
|
||||
"call",
|
||||
"*** Begin Patch\n*** Add File: should-not-exist\n+unexpected\n*** End Patch",
|
||||
)
|
||||
} else {
|
||||
ev_function_call("call", "test_sync_tool", "{}")
|
||||
use ToolDispatchCase::AsyncHook;
|
||||
use ToolDispatchCase::Eager;
|
||||
use ToolDispatchCase::InterruptedHook;
|
||||
use ToolDispatchCase::SyncHook;
|
||||
const MARKER: &str = "tool-dispatched";
|
||||
const STREAMED: &str = "Still streaming.";
|
||||
|
||||
let hook_async = match case {
|
||||
Eager => None,
|
||||
AsyncHook => Some(true),
|
||||
SyncHook | InterruptedHook => Some(false),
|
||||
};
|
||||
let (finish_stream, gate) = oneshot::channel();
|
||||
let mut completed = ev_completed_with_tokens("first", /*total_tokens*/ 42);
|
||||
completed["response"]["usage"]["codex_rollout_budget_units"] = json!(1);
|
||||
let first = vec![
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse(vec![ev_response_created("first"), call]),
|
||||
body: sse(vec![
|
||||
ev_response_created("first"),
|
||||
ev_apply_patch_custom_tool_call(
|
||||
"call",
|
||||
&format!("*** Begin Patch\n*** Add File: {MARKER}\n+ran\n*** End Patch"),
|
||||
),
|
||||
ev_message_item_added("progress", ""),
|
||||
ev_output_text_delta(STREAMED),
|
||||
ev_message_item_done("progress", STREAMED),
|
||||
// Safety notifications pass through in either mode. Observing this one
|
||||
// proves the reader consumed the preceding items while completion was held.
|
||||
json!({"type":"response.metadata", "response_id":"first",
|
||||
"metadata":{"type":"safety_buffering","use_cases":["cyber"],"reasons":["policy-check"]}}),
|
||||
]),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: Some(gate),
|
||||
body: sse_event(ev_completed_with_tokens("first", /*total_tokens*/ 42)),
|
||||
body: sse_event(completed),
|
||||
},
|
||||
];
|
||||
let second = vec![StreamingSseChunk {
|
||||
@@ -6248,31 +6240,43 @@ async fn model_request_post_hook_orders_tool_dispatch(
|
||||
body: sse(vec![ev_response_created("second"), ev_completed("second")]),
|
||||
}];
|
||||
let (server, _) = start_streaming_sse_server(vec![first, second]).await;
|
||||
let test = test_codex()
|
||||
.with_model("test-gpt-5.1-codex")
|
||||
let accounting = Arc::new(super::rollout_budget::HostAccounting::default());
|
||||
// The first post-hook blocks on this connection until the test closes it.
|
||||
let hook_gate = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
|
||||
let port = hook_gate.local_addr()?.port();
|
||||
let mut builder = test_codex()
|
||||
.with_pre_build_hook(move |home| {
|
||||
let order = home.join("order").display().to_string();
|
||||
let tool = format!("echo tool >> '{order}'");
|
||||
let mut hooks =
|
||||
json!({"PreToolUse": [{"hooks": [{"type": "command", "command": tool}]}]});
|
||||
if let Some(hook_async) = hook_async {
|
||||
let started = home.join("post-started");
|
||||
let release = home.join("release-post");
|
||||
let post = format!(
|
||||
"touch '{}'; while test ! -e '{}'; do sleep 0.02; done; echo post >> '{order}'",
|
||||
started.display(),
|
||||
release.display(),
|
||||
r#"python3 -c 'import json, socket, sys; json.load(sys.stdin).get("response_id") == "first" and socket.create_connection(("127.0.0.1", {port})).recv(1)'"#
|
||||
);
|
||||
hooks["PostModelRequest"] = json!([{"hooks": [{
|
||||
let hooks = json!({"PostModelRequest": [{"hooks": [{
|
||||
"type": "command", "command": post, "async": hook_async,
|
||||
}]}]);
|
||||
}]}]});
|
||||
fs::write(home.join("hooks.json"), json!({"hooks": hooks}).to_string())
|
||||
.expect("write hook configuration");
|
||||
}
|
||||
fs::write(home.join("hooks.json"), json!({"hooks": hooks}).to_string())
|
||||
.expect("write hook configuration");
|
||||
})
|
||||
.with_config(trust_discovered_hooks)
|
||||
.build_with_streaming_server(&server)
|
||||
.await?;
|
||||
.with_config(move |config| {
|
||||
if hook_async.is_some() {
|
||||
trust_discovered_hooks(config);
|
||||
}
|
||||
if matches!(case, Eager | SyncHook) {
|
||||
config.rollout_budget = Some(super::rollout_budget::rollout_budget());
|
||||
}
|
||||
// The builder changes cwd after loading config, so bind the write grant to that cwd.
|
||||
config.workspace_roots = vec![config.cwd.clone()];
|
||||
config.permissions.set_workspace_roots(config.workspace_roots.clone());
|
||||
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::Never);
|
||||
config.permissions.set_permission_profile(restrictive_workspace_write_profile())
|
||||
.expect("allow the fixture's patch inside its temporary workspace");
|
||||
});
|
||||
if matches!(case, SyncHook | InterruptedHook) {
|
||||
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
builder = builder.with_extensions(Arc::new(extensions.build()));
|
||||
}
|
||||
let test = builder.build_with_streaming_server(&server).await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "use the tool".into(),
|
||||
@@ -6280,61 +6284,71 @@ async fn model_request_post_hook_orders_tool_dispatch(
|
||||
}]))
|
||||
.await?;
|
||||
|
||||
let home = test.codex_home_path();
|
||||
let order = home.join("order");
|
||||
let sync_post = hook_async == Some(false);
|
||||
if !sync_post {
|
||||
// Dispatch must not depend on the terminal response event, which we still hold.
|
||||
fs_wait::wait_for_path_exists(&order, Duration::from_secs(10)).await?;
|
||||
}
|
||||
let marker = test.workspace_path(MARKER);
|
||||
let mut saw_delta = false;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
saw_delta |= matches!(event, EventMsg::AgentMessageContentDelta(message) if message.delta == STREAMED);
|
||||
matches!(event, EventMsg::SafetyBuffering(_))
|
||||
}).await;
|
||||
assert!(saw_delta);
|
||||
fs_wait::wait_for_path_exists(&marker, Duration::from_secs(10)).await?;
|
||||
finish_stream.send(()).expect("release response stream");
|
||||
let release = home.join("release-post");
|
||||
if sync_post {
|
||||
fs_wait::wait_for_path_exists(home.join("post-started"), Duration::from_secs(10)).await?;
|
||||
}
|
||||
if interrupt {
|
||||
test.codex.submit(Op::Interrupt).await?;
|
||||
wait_for_event(&test.codex, |event| matches!(event,
|
||||
EventMsg::TokenCount(ev) if ev.info.as_ref().is_some_and(|info| info.total_token_usage.total_tokens == 42)
|
||||
)).await;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnAborted(_))
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
!order.exists(),
|
||||
"interrupt must not dispatch the deferred tool"
|
||||
);
|
||||
fs::write(release, "")?;
|
||||
assert!(!test.workspace_path("should-not-exist").exists());
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let requests = mount_sse_once(&server, responses::sse_completed("resumed")).await;
|
||||
let resumed = test_codex().restart(&server, &test).await?;
|
||||
let usage = resumed
|
||||
.codex
|
||||
.token_usage_info()
|
||||
.await
|
||||
.context("restored usage")?;
|
||||
assert_eq!(usage.total_token_usage.total_tokens, 42);
|
||||
resumed.submit_turn("continue after interrupt").await?;
|
||||
let (output, _) = requests
|
||||
.single_request()
|
||||
.custom_tool_call_output_content_and_success("call")
|
||||
.context("aborted custom tool output")?;
|
||||
assert_eq!(output.as_deref(), Some("aborted before dispatch"));
|
||||
let held_hook = if hook_async.is_some() {
|
||||
Some(
|
||||
timeout(Duration::from_secs(10), hook_gate.accept())
|
||||
.await??
|
||||
.0,
|
||||
)
|
||||
} else {
|
||||
fs::write(release, "")?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
None
|
||||
};
|
||||
if hook_async == Some(false) {
|
||||
assert!(accounting.reports.lock().expect("usage reports").is_empty());
|
||||
}
|
||||
if case == InterruptedHook {
|
||||
let started = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::HookStarted(started)
|
||||
if started.run.event_name == HookEventName::PostModelRequest
|
||||
&& started.run.handler_type
|
||||
== codex_protocol::protocol::HookHandlerType::Command =>
|
||||
{
|
||||
Some(started.clone())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
let events = fs::read_to_string(order)?;
|
||||
assert!(
|
||||
events.starts_with(if sync_post { "post\ntool\n" } else { "tool\n" }),
|
||||
"{events}"
|
||||
test.codex.submit(Op::Interrupt).await?;
|
||||
let mut stopped = None;
|
||||
let mut aborted = false;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
match event {
|
||||
EventMsg::HookCompleted(completed) if completed.run.id == started.run.id => {
|
||||
stopped = Some(completed.clone());
|
||||
}
|
||||
EventMsg::TurnAborted(_) => aborted = true,
|
||||
_ => {}
|
||||
}
|
||||
aborted && stopped.is_some()
|
||||
})
|
||||
.await;
|
||||
let stopped = stopped.expect("post-hook must emit a terminal event after interruption");
|
||||
assert_eq!(stopped.turn_id, started.turn_id);
|
||||
assert_eq!(stopped.run.id, started.run.id);
|
||||
assert_eq!(stopped.run.status, HookRunStatus::Stopped);
|
||||
assert_eq!(
|
||||
stopped.run.status_message.as_deref(),
|
||||
Some("Hook cancelled because the turn was interrupted"),
|
||||
);
|
||||
drop(held_hook);
|
||||
assert!(accounting.reports.lock().expect("usage reports").is_empty());
|
||||
return Ok(());
|
||||
}
|
||||
drop(held_hook);
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(fs::read_to_string(marker)?, "ran\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -4581,6 +4581,59 @@ async fn async_compaction_reuses_pending_work(
|
||||
let accounting =
|
||||
(outcome == PendingCompactionOutcome::Success).then(|| Arc::new(HostAccounting::default()));
|
||||
let mut builder = test_codex();
|
||||
let observe_attempts = outcome == PendingCompactionOutcome::Success && !manual;
|
||||
let timeout = Duration::from_secs(/*secs*/ 20);
|
||||
let post_started = Arc::new(tokio::sync::Notify::new());
|
||||
let (release_post, wait_for_release) = std::sync::mpsc::channel();
|
||||
let (record, recorded) = std::sync::mpsc::channel();
|
||||
// The harness owns the loopback MCP server. Hook inputs and waits do not
|
||||
// depend on a shell or filesystem shared with the tool environment.
|
||||
let _hook_server = if observe_attempts {
|
||||
let server = start_mock_server().await;
|
||||
core_test_support::hooks::mount_hook_mcp(
|
||||
&server,
|
||||
"compaction-observer",
|
||||
&["pre", "post"],
|
||||
{
|
||||
let started = Arc::clone(&post_started);
|
||||
let release = std::sync::Mutex::new(wait_for_release);
|
||||
move |params| {
|
||||
record.send(params.clone()).expect("record hook call");
|
||||
if observe_attempts
|
||||
&& params["name"] == "post"
|
||||
&& params["arguments"]["kind"] == "background_compaction"
|
||||
{
|
||||
started.notify_one();
|
||||
release
|
||||
.lock()
|
||||
.expect("post-hook gate")
|
||||
.recv_timeout(timeout)
|
||||
.expect("release background post-hook");
|
||||
}
|
||||
json!({"content": []})
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let url = format!("{}/repl", server.uri());
|
||||
builder = builder.with_pre_build_hook(move |home| {
|
||||
let hook = |tool, input| json!([{"hooks": [{
|
||||
"type": "mcp_tool", "server": "compaction-observer", "tool": tool, "input": input
|
||||
}]}]);
|
||||
let hooks = json!({
|
||||
"PreModelRequest": hook("pre", json!({"kind": "${kind}"})),
|
||||
"PostModelRequest": hook("post", json!({"kind": "${kind}", "response_id": "${response_id}", "token_usage": "${token_usage}"})),
|
||||
});
|
||||
let config = json!({"hooks": hooks, "mcp_servers": {"compaction-observer": {
|
||||
"url": url, "environment_id": super::rmcp_client::remote_aware_environment_id()
|
||||
}}});
|
||||
std::fs::write(home.join("config.toml"), toml::to_string(&config).expect("serialize hook config"))
|
||||
.expect("install compaction hooks");
|
||||
});
|
||||
Some(server)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(accounting) = &accounting {
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.thread_lifecycle_contributor(accounting.clone());
|
||||
@@ -4597,6 +4650,9 @@ async fn async_compaction_reuses_pending_work(
|
||||
},
|
||||
env,
|
||||
move |config| {
|
||||
if observe_attempts {
|
||||
core_test_support::hooks::trust_discovered_hooks(config);
|
||||
}
|
||||
if over_budget {
|
||||
config.async_compaction_threshold_percent = 1;
|
||||
config.model_auto_compact_token_limit = Some(5_000);
|
||||
@@ -4605,7 +4661,6 @@ async fn async_compaction_reuses_pending_work(
|
||||
)
|
||||
.await?;
|
||||
test.codex.ensure_rollout_materialized().await;
|
||||
let timeout = Duration::from_secs(/*secs*/ 20);
|
||||
test.submit_text_turn("first").await?;
|
||||
if has_background {
|
||||
tokio::time::timeout(timeout, streaming.wait_for_request_count(/*count*/ 2)).await?;
|
||||
@@ -4678,6 +4733,24 @@ async fn async_compaction_reuses_pending_work(
|
||||
if has_background && let Some(release_background) = release_background {
|
||||
release_background.send(()).expect("release pending result");
|
||||
}
|
||||
if observe_attempts {
|
||||
tokio::time::timeout(timeout, post_started.notified()).await?;
|
||||
// The collector bills a background terminal event before returning its
|
||||
// candidate. A blocked post-hook must still hold that event.
|
||||
let billed_while_held = accounting
|
||||
.as_ref()
|
||||
.expect("successful background attempt")
|
||||
.reports
|
||||
.lock()
|
||||
.expect("usage reports")
|
||||
.iter()
|
||||
.any(|report| report.usage.total_tokens == BACKGROUND_TOKENS);
|
||||
release_post.send(()).expect("release the compaction hook");
|
||||
assert!(
|
||||
!billed_while_held,
|
||||
"background usage reached the collector before its post-hook finished"
|
||||
);
|
||||
}
|
||||
let EventMsg::ItemStarted(started) = started else {
|
||||
unreachable!();
|
||||
};
|
||||
@@ -4733,6 +4806,28 @@ async fn async_compaction_reuses_pending_work(
|
||||
assert_eq!(charged_turns, [&usage("seed").turn_id]);
|
||||
}
|
||||
let continuation = usage("continuation");
|
||||
if observe_attempts {
|
||||
let calls: Vec<_> = recorded.try_iter().collect();
|
||||
let compactions = |tool: &str| {
|
||||
calls
|
||||
.iter()
|
||||
.filter(|call| call["name"] == tool)
|
||||
.map(|call| &call["arguments"])
|
||||
.filter(|input| input["kind"] != "sampling")
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let pre = compactions("pre");
|
||||
let post = compactions("post");
|
||||
assert_eq!(
|
||||
(pre.len(), post.len()),
|
||||
(1, 1),
|
||||
"adopting the candidate must not run a second attempt"
|
||||
);
|
||||
assert_eq!(pre[0]["kind"], "background_compaction");
|
||||
assert_eq!(post[0]["kind"], pre[0]["kind"]);
|
||||
assert_eq!(post[0]["response_id"], "background");
|
||||
assert_eq!(post[0]["token_usage"]["total_tokens"], BACKGROUND_TOKENS);
|
||||
}
|
||||
assert_eq!(
|
||||
(
|
||||
continuation.turn_token_usage.total_tokens,
|
||||
|
||||
@@ -19,6 +19,14 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"SamplingKind": {
|
||||
"enum": [
|
||||
"sampling",
|
||||
"foreground_compaction",
|
||||
"background_compaction"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"TokenUsage": {
|
||||
"properties": {
|
||||
"cache_write_input_tokens": {
|
||||
@@ -83,6 +91,14 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"response_id": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/NullableString"
|
||||
}
|
||||
],
|
||||
"description": "The provider's identifier for the completed or incomplete response, if available."
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/ModelRequestStatus"
|
||||
},
|
||||
@@ -100,6 +116,7 @@
|
||||
"required": [
|
||||
"error",
|
||||
"hook_event_name",
|
||||
"response_id",
|
||||
"status",
|
||||
"token_usage"
|
||||
],
|
||||
@@ -116,12 +133,19 @@
|
||||
"cwd": {
|
||||
"$ref": "#/definitions/AbsolutePathBuf"
|
||||
},
|
||||
"kind": {
|
||||
"$ref": "#/definitions/SamplingKind"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"thread_id": {
|
||||
"description": "Identifies the issuing thread within its shared session.",
|
||||
"type": "string"
|
||||
},
|
||||
"transcript_path": {
|
||||
"$ref": "#/definitions/NullableString"
|
||||
},
|
||||
@@ -131,8 +155,10 @@
|
||||
},
|
||||
"required": [
|
||||
"cwd",
|
||||
"kind",
|
||||
"model",
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"transcript_path",
|
||||
"turn_id"
|
||||
],
|
||||
|
||||
@@ -123,7 +123,9 @@ impl CommandHookRuntime {
|
||||
event_input: Value,
|
||||
cwd: std::path::PathBuf,
|
||||
turn_id: Option<String>,
|
||||
parse: fn(&ConfiguredHandler, HandlerRunResult, Option<String>) -> ParsedHandler<T>,
|
||||
parse: impl Fn(&ConfiguredHandler, HandlerRunResult, Option<String>) -> ParsedHandler<T>
|
||||
+ Send
|
||||
+ 'static,
|
||||
) {
|
||||
if self.result_sender.is_closed() {
|
||||
return;
|
||||
|
||||
@@ -128,6 +128,34 @@ pub(crate) async fn execute_handlers_with_metadata<T: 'static>(
|
||||
turn_id: Option<String>,
|
||||
metadata: Option<&Map<String, Value>>,
|
||||
parse: fn(&ConfiguredHandler, HandlerRunResult, Option<String>) -> ParsedHandler<T>,
|
||||
) -> Vec<ParsedHandler<T>> {
|
||||
execute_handlers_with_progress(
|
||||
engine,
|
||||
handlers,
|
||||
input,
|
||||
cwd,
|
||||
turn_id,
|
||||
metadata,
|
||||
parse,
|
||||
|_| {},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn execute_handlers_with_progress<T: 'static>(
|
||||
engine: &ClaudeHooksEngine,
|
||||
handlers: Vec<ConfiguredHandler>,
|
||||
input: &Value,
|
||||
cwd: &Path,
|
||||
turn_id: Option<String>,
|
||||
metadata: Option<&Map<String, Value>>,
|
||||
parse: impl Fn(&ConfiguredHandler, HandlerRunResult, Option<String>) -> ParsedHandler<T>
|
||||
+ Copy
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
mut on_completed: impl FnMut(&HookCompletedEvent) + Send,
|
||||
) -> Vec<ParsedHandler<T>> {
|
||||
let mut executor_handlers = Vec::new();
|
||||
let mut pending = FuturesUnordered::new();
|
||||
@@ -169,6 +197,7 @@ pub(crate) async fn execute_handlers_with_metadata<T: 'static>(
|
||||
should_block |= parsed.completed.run.status == HookRunStatus::Blocked;
|
||||
parsed.completion_order = completion_order;
|
||||
completion_order += 1;
|
||||
on_completed(&parsed.completed);
|
||||
completed.push((configured_order, parsed));
|
||||
}
|
||||
completed.sort_by_key(|(configured_order, _)| *configured_order);
|
||||
|
||||
@@ -24,19 +24,22 @@ pub enum ModelRequestHookEvent {
|
||||
status: ModelRequestStatus,
|
||||
#[schemars(with = "crate::schema::NullableString")]
|
||||
error: Option<String>,
|
||||
#[schemars(required, schema_with = "token_usage_schema")]
|
||||
#[schemars(required, schema_with = "nullable_schema::<TokenUsage>")]
|
||||
token_usage: Option<TokenUsage>,
|
||||
/// The provider's identifier for the completed or incomplete response, if available.
|
||||
#[schemars(with = "crate::schema::NullableString")]
|
||||
response_id: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
// Hook schemas otherwise omit null from Option fields, but usage is always present on post.
|
||||
fn token_usage_schema(
|
||||
fn nullable_schema<T: JsonSchema>(
|
||||
generator: &mut schemars::r#gen::SchemaGenerator,
|
||||
) -> schemars::schema::Schema {
|
||||
schemars::schema::SchemaObject {
|
||||
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
|
||||
any_of: Some(vec![
|
||||
generator.subschema_for::<TokenUsage>(),
|
||||
generator.subschema_for::<T>(),
|
||||
generator.subschema_for::<()>(),
|
||||
]),
|
||||
..Default::default()
|
||||
@@ -55,6 +58,44 @@ pub enum ModelRequestStatus {
|
||||
}
|
||||
|
||||
impl ModelRequestHookEvent {
|
||||
pub fn completed(response_id: &str, token_usage: Option<&TokenUsage>) -> Self {
|
||||
Self::PostModelRequest {
|
||||
status: ModelRequestStatus::Completed,
|
||||
error: None,
|
||||
token_usage: token_usage.cloned(),
|
||||
response_id: Some(response_id.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn failed(error: &codex_protocol::error::CodexErr) -> Self {
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
let (response_id, token_usage) = match error.details() {
|
||||
CodexErrorDetails::TurnAborted | CodexErrorDetails::Interrupted => {
|
||||
return Self::cancelled();
|
||||
}
|
||||
CodexErrorDetails::ResponseIncomplete(incomplete) => (
|
||||
incomplete.response_id.clone(),
|
||||
incomplete.token_usage.clone(),
|
||||
),
|
||||
_ => (None, None),
|
||||
};
|
||||
Self::PostModelRequest {
|
||||
status: ModelRequestStatus::Failed,
|
||||
error: Some(error.to_string()),
|
||||
token_usage,
|
||||
response_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cancelled() -> Self {
|
||||
Self::PostModelRequest {
|
||||
status: ModelRequestStatus::Cancelled,
|
||||
error: None,
|
||||
token_usage: None,
|
||||
response_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn hook_event(&self) -> HookEventName {
|
||||
match self {
|
||||
Self::PreModelRequest => HookEventName::PreModelRequest,
|
||||
@@ -63,10 +104,24 @@ impl ModelRequestHookEvent {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SamplingKind {
|
||||
Sampling,
|
||||
ForegroundCompaction,
|
||||
BackgroundCompaction,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, JsonSchema)]
|
||||
#[schemars(rename = "model-request.command.input")]
|
||||
pub struct ModelRequestHookInput {
|
||||
/// Names the hook lifecycle for this attempt, including asynchronous completions.
|
||||
#[serde(skip)]
|
||||
pub lifecycle_id: uuid::Uuid,
|
||||
pub kind: SamplingKind,
|
||||
pub session_id: String,
|
||||
/// Identifies the issuing thread within its shared session.
|
||||
pub thread_id: String,
|
||||
pub turn_id: String,
|
||||
pub cwd: AbsolutePathBuf,
|
||||
#[schemars(with = "crate::schema::NullableString")]
|
||||
@@ -80,6 +135,10 @@ pub struct ModelRequestHookInput {
|
||||
pub event: ModelRequestHookEvent,
|
||||
}
|
||||
|
||||
fn identify_run(run: &mut HookRunSummary, lifecycle_id: uuid::Uuid) {
|
||||
run.id = format!("{}:{lifecycle_id}", run.id);
|
||||
}
|
||||
|
||||
pub(crate) fn preview(
|
||||
handlers: &[ConfiguredHandler],
|
||||
request: &ModelRequestHookInput,
|
||||
@@ -90,13 +149,18 @@ pub(crate) fn preview(
|
||||
/*matcher_input*/ None,
|
||||
)
|
||||
.iter()
|
||||
.map(dispatcher::running_summary)
|
||||
.map(|handler| {
|
||||
let mut run = dispatcher::running_summary(handler);
|
||||
identify_run(&mut run, request.lifecycle_id);
|
||||
run
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn run(
|
||||
engine: &ClaudeHooksEngine,
|
||||
request: ModelRequestHookInput,
|
||||
on_completed: impl FnMut(&HookCompletedEvent) + Send,
|
||||
) -> Vec<HookCompletedEvent> {
|
||||
let matched = dispatcher::select_handlers(
|
||||
&engine.handlers,
|
||||
@@ -109,20 +173,30 @@ pub(crate) async fn run(
|
||||
let input = match serde_json::to_value(&request) {
|
||||
Ok(input) => input,
|
||||
Err(error) => {
|
||||
return common::serialization_failure_hook_events(
|
||||
let mut events = common::serialization_failure_hook_events(
|
||||
matched,
|
||||
Some(request.turn_id),
|
||||
error.to_string(),
|
||||
);
|
||||
for event in &mut events {
|
||||
identify_run(&mut event.run, request.lifecycle_id);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
};
|
||||
dispatcher::execute_handlers(
|
||||
dispatcher::execute_handlers_with_progress(
|
||||
engine,
|
||||
matched,
|
||||
&input,
|
||||
request.cwd.as_path(),
|
||||
Some(request.turn_id),
|
||||
parse_completed,
|
||||
/*metadata*/ None,
|
||||
move |handler, result, turn_id| {
|
||||
let mut parsed = parse_completed(handler, result, turn_id);
|
||||
identify_run(&mut parsed.completed.run, request.lifecycle_id);
|
||||
parsed
|
||||
},
|
||||
on_completed,
|
||||
)
|
||||
.await
|
||||
.into_iter()
|
||||
|
||||
@@ -137,3 +137,4 @@ pub fn hook_key(
|
||||
pub use events::model_request::ModelRequestHookEvent;
|
||||
pub use events::model_request::ModelRequestHookInput;
|
||||
pub use events::model_request::ModelRequestStatus;
|
||||
pub use events::model_request::SamplingKind;
|
||||
|
||||
@@ -33,6 +33,7 @@ use codex_config::ConfigLayerStack;
|
||||
use codex_plugin::ExecutorPluginHookSource;
|
||||
use codex_plugin::PluginHookSource;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::HookCompletedEvent;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookExecutionMode;
|
||||
use codex_protocol::shell_environment::scrub_non_inheritable_env_vars;
|
||||
@@ -380,8 +381,9 @@ impl Hooks {
|
||||
pub async fn run_model_request(
|
||||
&self,
|
||||
request: crate::ModelRequestHookInput,
|
||||
) -> Vec<codex_protocol::protocol::HookCompletedEvent> {
|
||||
crate::events::model_request::run(&self.engine, request).await
|
||||
on_completed: impl FnMut(&HookCompletedEvent) + Send,
|
||||
) -> Vec<HookCompletedEvent> {
|
||||
crate::events::model_request::run(&self.engine, request, on_completed).await
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user