codex: simplify capacity retry handling (#31176)

This commit is contained in:
Eric Traut
2026-07-08 19:21:14 -07:00
parent 135c262a7a
commit ab636f44d1
8 changed files with 120 additions and 128 deletions
+7 -4
View File
@@ -118,10 +118,13 @@ impl Session {
));
}
// Keep automatic input task-owned so a pre-turn failure drops this
// attempt instead of immediately replaying it from the pending queue.
let input = input.into_iter().map(TurnInput::ResponseItem).collect();
self.start_task(turn_context, input, RegularTask::new())
self.input_queue
.extend_pending_input_for_turn_state(
turn_state.as_ref(),
input.into_iter().map(TurnInput::ResponseItem).collect(),
)
.await;
self.start_task(turn_context, Vec::new(), RegularTask::new())
.await;
Ok(())
}
+1 -2
View File
@@ -2533,7 +2533,7 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
fn on_turn_error<'a>(
&'a self,
input: codex_extension_api::TurnErrorInput<'a>,
) -> codex_extension_api::ExtensionFuture<'a, Option<std::time::Duration>> {
) -> codex_extension_api::ExtensionFuture<'a, ()> {
Box::pin(async move {
self.records
.lock()
@@ -2553,7 +2553,6 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
.get::<ThreadTurnErrorMarker>()
.is_some(),
});
None
})
}
}
+47 -51
View File
@@ -3,7 +3,6 @@ use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use crate::SkillInjections;
use crate::build_skill_injections;
@@ -152,30 +151,19 @@ pub(crate) async fn run_turn(
) -> CodexResult<Option<String>> {
let mut client_session =
prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session());
let turn_state = sess
.input_queue
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
.await;
let (mut retry_activity_rx, _) = sess
.input_queue
.subscribe_activity(turn_state.as_deref())
.await;
// TODO(ccunningham): Pre-turn compaction runs before context updates and the
// new user message are recorded. Estimate pending incoming items (context
// diffs/full reinjection + user input) and trigger compaction preemptively
// when they would push the thread over the compaction threshold.
while let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
if matches!(err, CodexErr::TurnAborted) {
return Err(err);
}
let retry_delay = sess
.emit_turn_error_lifecycle(turn_context.as_ref(), err.to_codex_protocol_error())
let error = err.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
error!("Failed to run pre-sampling compact");
let Some(delay) = retry_delay else {
return Ok(None);
};
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
return Ok(None);
}
// run_turn owns the step used to seed context and make the first sampling request.
@@ -200,9 +188,7 @@ pub(crate) async fn run_turn(
if run_pending_session_start_hooks(&sess, &turn_context).await {
return Ok(None);
}
let mut can_drain_pending_input = !input
.iter()
.any(|item| matches!(item, TurnInput::UserInput { .. }));
let mut can_drain_pending_input = input.is_empty();
if run_hooks_and_record_inputs(&sess, &turn_context, &input).await {
return Ok(None);
}
@@ -234,7 +220,7 @@ pub(crate) async fn run_turn(
// one instance across retries within this turn.
// Pending input is drained into history before building the next model request.
// However, we defer that drain until after sampling in two cases:
// 1. At the start of a user turn, so the submitted user input gets sampled first.
// 1. At the start of a turn, so the fresh turn input in `input` gets sampled first.
// 2. After auto-compact, when model/tool continuation needs to resume before any steer.
let mut next_step_context = Some(first_step_context);
@@ -362,7 +348,7 @@ pub(crate) async fn run_turn(
if needs_follow_up
&& (sess.take_new_context_window_request().await || token_limit_reached)
{
while let Err(err) = run_auto_compact(
if let Err(err) = run_auto_compact(
&sess,
Arc::clone(&step_context),
/*fallback_step_context*/ None,
@@ -376,16 +362,10 @@ pub(crate) async fn run_turn(
if matches!(err, CodexErr::TurnAborted) {
return Err(err);
}
let retry_delay = sess
.emit_turn_error_lifecycle(
turn_context.as_ref(),
err.to_codex_protocol_error(),
)
let error = err.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
let Some(delay) = retry_delay else {
return Ok(None);
};
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
return Ok(None);
}
can_drain_pending_input = !model_needs_follow_up;
continue;
@@ -467,9 +447,21 @@ pub(crate) async fn run_turn(
Err(e) => {
info!("Turn error: {e:#}");
let error = e.to_codex_protocol_error();
let retry_delay = sess
.emit_turn_error_lifecycle(turn_context.as_ref(), error)
.await;
let mut retry_delay = None;
for contributor in sess.services.extensions.turn_lifecycle_contributors() {
retry_delay = contributor
.retry_delay_for_turn_error(codex_extension_api::TurnErrorInput {
turn_id: turn_context.sub_id.as_str(),
error: error.clone(),
session_store: &sess.services.session_extension_data,
thread_store: &sess.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await;
if retry_delay.is_some() {
break;
}
}
sess.track_turn_codex_error(turn_context.as_ref(), &e);
let error_event = e.to_error_event(/*message_prefix*/ None);
if let Some(delay) = retry_delay {
@@ -483,11 +475,31 @@ pub(crate) async fn run_turn(
)
.await;
// Keep this turn alive so its recorded input is not appended again.
// Let queued input interrupt the backoff instead of parking the live turn.
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
// Let queued user input interrupt the backoff instead of parking the turn.
let turn_state = sess
.input_queue
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
.await;
let (mut activity_rx, pending_activity) = sess
.input_queue
.subscribe_activity(turn_state.as_deref())
.await;
if pending_activity != Some(InputQueueActivity::Steer) {
let _ = tokio::time::timeout(delay, async {
while activity_rx.changed().await.is_ok() {
// Mailbox activity does not represent user intent to retry now.
if *activity_rx.borrow_and_update() == InputQueueActivity::Steer {
break;
}
}
})
.await;
}
can_drain_pending_input = true;
continue;
}
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error)
.await;
sess.send_event(&turn_context, EventMsg::Error(error_event))
.await;
// let the user continue the conversation
@@ -499,22 +511,6 @@ pub(crate) async fn run_turn(
Ok(last_agent_message)
}
async fn wait_for_retry_delay(
sess: &Session,
delay: Duration,
activity_rx: &mut tokio::sync::watch::Receiver<InputQueueActivity>,
) {
let _ = tokio::time::timeout(delay, async {
while activity_rx.changed().await.is_ok() {
// Ignore mailbox activity that is deferred to the next turn.
if sess.input_queue.has_pending_input(&sess.active_turn).await {
break;
}
}
})
.await;
}
#[instrument(level = "trace", skip_all)]
async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, PathBuf)> {
let mut display_roots = Vec::new();
+3 -7
View File
@@ -1,5 +1,3 @@
use std::time::Duration;
use codex_extension_api::ExtensionData;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::TokenUsage;
@@ -78,10 +76,9 @@ impl Session {
&self,
turn_context: &TurnContext,
error: CodexErrorInfo,
) -> Option<Duration> {
let mut retry_delay: Option<Duration> = None;
) {
for contributor in self.services.extensions.turn_lifecycle_contributors() {
retry_delay = retry_delay.or(contributor
contributor
.on_turn_error(codex_extension_api::TurnErrorInput {
turn_id: turn_context.sub_id.as_str(),
error: error.clone(),
@@ -89,8 +86,7 @@ impl Session {
thread_store: &self.services.thread_extension_data,
turn_store: turn_context.extension_data.as_ref(),
})
.await);
.await;
}
retry_delay
}
}
+9 -21
View File
@@ -29,7 +29,6 @@ use core_test_support::responses::ev_output_text_delta;
use core_test_support::responses::ev_reasoning_item;
use core_test_support::responses::ev_reasoning_item_added;
use core_test_support::responses::ev_response_created;
use core_test_support::skip_if_no_network;
use core_test_support::streaming_sse::StreamingSseChunk;
use core_test_support::streaming_sse::StreamingSseServer;
use core_test_support::streaming_sse::start_streaming_sse_server;
@@ -43,14 +42,14 @@ use serde_json::from_slice;
use serde_json::json;
use tokio::sync::oneshot;
struct RetryTurnErrors(std::time::Duration);
struct RetryTurnErrors;
impl TurnLifecycleContributor for RetryTurnErrors {
fn on_turn_error<'a>(
fn retry_delay_for_turn_error<'a>(
&'a self,
_input: codex_extension_api::TurnErrorInput<'a>,
) -> codex_extension_api::ExtensionFuture<'a, Option<std::time::Duration>> {
Box::pin(async { Some(self.0) })
Box::pin(async { Some(std::time::Duration::from_secs(60)) })
}
}
@@ -302,11 +301,8 @@ async fn wait_for_sleep_item_completed(codex: &CodexThread, call_id: &str, durat
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
skip_if_no_network!();
const CONTINUATION: &str = "GOAL_CONTINUATION";
const MAIL: &str = "CHILD_UPDATE";
async fn sampling_error_retries_in_place_and_steer_interrupts_delay() {
const INITIAL: &str = "INITIAL_PROMPT";
const STEER: &str = "USER_STEER";
let server = responses::start_mock_server().await;
let response_mock = responses::mount_sse_sequence(
@@ -322,9 +318,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
)
.await;
let mut extensions = ExtensionRegistryBuilder::new();
extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors(
std::time::Duration::from_secs(60),
)));
extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors));
let test = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_config(|config| config.model_provider.stream_max_retries = Some(0))
@@ -332,11 +326,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
.await
.expect("build Codex test session");
submit_queue_only_agent_mail(&test.codex, MAIL).await;
test.codex
.try_start_turn_if_idle(vec![responses::user_message_item(CONTINUATION)])
.await
.expect("start automatic turn");
submit_user_input(&test.codex, INITIAL).await;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::StreamError(_))
})
@@ -345,18 +335,16 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
wait_for_turn_complete(&test.codex).await;
let requests = response_mock.requests();
let [first, second] = requests.as_slice() else {
let [_, second] = requests.as_slice() else {
panic!("expected two requests, got {}", requests.len());
};
assert_eq!(1, first.inputs_of_type("agent_message").len());
assert!(first.body_contains_text(MAIL));
assert!(second.body_contains_text(STEER));
assert_eq!(
1,
second
.message_input_texts("user")
.iter()
.filter(|text| text.as_str() == CONTINUATION)
.filter(|text| text.as_str() == INITIAL)
.count()
);
}
+11 -3
View File
@@ -185,9 +185,9 @@ pub trait TurnLifecycleContributor: Send + Sync {
})
}
/// Called when the host observes an error for a running turn. Returning a
/// delay asks the host to retry the failed operation without ending the turn.
fn on_turn_error<'a>(
/// Returns how long the host should wait before retrying a failed sampling
/// request. Returning `None` leaves the error terminal.
fn retry_delay_for_turn_error<'a>(
&'a self,
input: TurnErrorInput<'a>,
) -> ExtensionFuture<'a, Option<Duration>> {
@@ -197,6 +197,14 @@ pub trait TurnLifecycleContributor: Send + Sync {
None
})
}
/// Called when the host observes a terminal error for a running turn.
fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
let _self = self;
let _input = input;
})
}
}
/// Extension contribution that can add turn-local model input.
+16 -10
View File
@@ -47,11 +47,6 @@ use crate::spec::UPDATE_GOAL_TOOL_NAME;
use crate::steering::budget_limit_steering_item;
use crate::tool::GoalToolExecutor;
// Capacity failures do not consume user tokens. Add jitter around a five-minute
// average so clients that hit capacity together do not retry in lockstep.
const DEFERRED_GOAL_RETRY_MIN_SECS: u64 = 4 * 60;
const DEFERRED_GOAL_RETRY_MAX_SECS: u64 = 6 * 60;
#[derive(Clone, Debug)]
pub struct GoalExtensionConfig {
pub enabled: bool,
@@ -303,7 +298,7 @@ where
})
}
fn on_turn_error<'a>(
fn retry_delay_for_turn_error<'a>(
&'a self,
input: TurnErrorInput<'a>,
) -> ExtensionFuture<'a, Option<Duration>> {
@@ -316,10 +311,22 @@ where
.accounting_state()
.turn_is_current_active_goal(input.turn_id)
{
return Some(Duration::from_secs(rand::rng().random_range(
DEFERRED_GOAL_RETRY_MIN_SECS..=DEFERRED_GOAL_RETRY_MAX_SECS,
)));
// Capacity failures do not consume tokens. Jitter around five minutes
// prevents clients that hit capacity together from retrying in lockstep.
return Some(Duration::from_secs(
rand::rng().random_range(4 * 60..=6 * 60),
));
}
None
})
}
fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
let Some(runtime) = goal_runtime_handle(input.thread_store) else {
return;
};
let reason = match input.error {
CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit,
// The turn has ended because the error was non-retryable or its
@@ -337,7 +344,6 @@ where
"failed to stop active goal after turn error: {err}"
);
}
None
})
}
}
@@ -574,26 +574,7 @@ async fn turn_error_usage_limit_accounts_progress_and_clears_accounting() -> any
}
#[tokio::test]
async fn turn_error_blocks_goal() -> anyhow::Result<()> {
assert_eq!(
(codex_state::ThreadGoalStatus::Blocked, false),
goal_status_after_turn_error(CodexErrorInfo::Other).await?
);
Ok(())
}
#[tokio::test]
async fn server_overloaded_error_keeps_goal_active() -> anyhow::Result<()> {
assert_eq!(
(codex_state::ThreadGoalStatus::Active, true),
goal_status_after_turn_error(CodexErrorInfo::ServerOverloaded).await?
);
Ok(())
}
async fn goal_status_after_turn_error(
error: CodexErrorInfo,
) -> anyhow::Result<(codex_state::ThreadGoalStatus, bool)> {
async fn retryable_error_keeps_goal_active_and_terminal_error_blocks_it() -> anyhow::Result<()> {
let runtime = test_runtime().await?;
let thread_id = test_thread_id()?;
seed_thread_metadata(runtime.as_ref(), thread_id).await?;
@@ -609,14 +590,35 @@ async fn goal_status_after_turn_error(
))
.await?;
let retry_delay = harness.notify_turn_error("turn-1", error).await;
let turn_store = ExtensionData::new("turn-1");
let retry_delay = harness.registry.turn_lifecycle_contributors()[0]
.retry_delay_for_turn_error(TurnErrorInput {
turn_id: "turn-1",
error: CodexErrorInfo::ServerOverloaded,
session_store: &harness.session_store,
thread_store: &harness.thread_store,
turn_store: &turn_store,
})
.await;
assert!(retry_delay.is_some());
let goal = runtime
.thread_goals()
.get_thread_goal(thread_id)
.await?
.ok_or_else(|| anyhow::anyhow!("goal should exist"))?;
assert_eq!(codex_state::ThreadGoalStatus::Active, goal.status);
harness
.notify_turn_error("turn-1", CodexErrorInfo::Other)
.await;
let goal = runtime
.thread_goals()
.get_thread_goal(thread_id)
.await?
.ok_or_else(|| anyhow::anyhow!("goal should exist"))?;
Ok((goal.status, retry_delay.is_some()))
assert_eq!(codex_state::ThreadGoalStatus::Blocked, goal.status);
Ok(())
}
#[tokio::test]
@@ -1321,15 +1323,10 @@ impl GoalExtensionHarness {
}
}
async fn notify_turn_error(
&self,
turn_id: &str,
error: CodexErrorInfo,
) -> Option<std::time::Duration> {
async fn notify_turn_error(&self, turn_id: &str, error: CodexErrorInfo) {
let turn_store = ExtensionData::new(turn_id);
let mut retry_delay = None;
for contributor in self.registry.turn_lifecycle_contributors() {
retry_delay = contributor
contributor
.on_turn_error(TurnErrorInput {
turn_id,
error: error.clone(),
@@ -1339,7 +1336,6 @@ impl GoalExtensionHarness {
})
.await;
}
retry_delay
}
fn runtime_handle(&self) -> Arc<GoalRuntimeHandle> {