mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
codex: simplify capacity retry handling (#31176)
This commit is contained in:
@@ -118,10 +118,13 @@ impl Session {
|
||||
));
|
||||
}
|
||||
|
||||
// Keep automatic input task-owned so a pre-turn failure drops this
|
||||
// attempt instead of immediately replaying it from the pending queue.
|
||||
let input = input.into_iter().map(TurnInput::ResponseItem).collect();
|
||||
self.start_task(turn_context, input, RegularTask::new())
|
||||
self.input_queue
|
||||
.extend_pending_input_for_turn_state(
|
||||
turn_state.as_ref(),
|
||||
input.into_iter().map(TurnInput::ResponseItem).collect(),
|
||||
)
|
||||
.await;
|
||||
self.start_task(turn_context, Vec::new(), RegularTask::new())
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2533,7 +2533,7 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
|
||||
fn on_turn_error<'a>(
|
||||
&'a self,
|
||||
input: codex_extension_api::TurnErrorInput<'a>,
|
||||
) -> codex_extension_api::ExtensionFuture<'a, Option<std::time::Duration>> {
|
||||
) -> codex_extension_api::ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
self.records
|
||||
.lock()
|
||||
@@ -2553,7 +2553,6 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
|
||||
.get::<ThreadTurnErrorMarker>()
|
||||
.is_some(),
|
||||
});
|
||||
None
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::SkillInjections;
|
||||
use crate::build_skill_injections;
|
||||
@@ -152,30 +151,19 @@ pub(crate) async fn run_turn(
|
||||
) -> CodexResult<Option<String>> {
|
||||
let mut client_session =
|
||||
prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session());
|
||||
let turn_state = sess
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
|
||||
.await;
|
||||
let (mut retry_activity_rx, _) = sess
|
||||
.input_queue
|
||||
.subscribe_activity(turn_state.as_deref())
|
||||
.await;
|
||||
// TODO(ccunningham): Pre-turn compaction runs before context updates and the
|
||||
// new user message are recorded. Estimate pending incoming items (context
|
||||
// diffs/full reinjection + user input) and trigger compaction preemptively
|
||||
// when they would push the thread over the compaction threshold.
|
||||
while let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
|
||||
if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(turn_context.as_ref(), err.to_codex_protocol_error())
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
error!("Failed to run pre-sampling compact");
|
||||
let Some(delay) = retry_delay else {
|
||||
return Ok(None);
|
||||
};
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// run_turn owns the step used to seed context and make the first sampling request.
|
||||
@@ -200,9 +188,7 @@ pub(crate) async fn run_turn(
|
||||
if run_pending_session_start_hooks(&sess, &turn_context).await {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut can_drain_pending_input = !input
|
||||
.iter()
|
||||
.any(|item| matches!(item, TurnInput::UserInput { .. }));
|
||||
let mut can_drain_pending_input = input.is_empty();
|
||||
if run_hooks_and_record_inputs(&sess, &turn_context, &input).await {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -234,7 +220,7 @@ pub(crate) async fn run_turn(
|
||||
// one instance across retries within this turn.
|
||||
// Pending input is drained into history before building the next model request.
|
||||
// However, we defer that drain until after sampling in two cases:
|
||||
// 1. At the start of a user turn, so the submitted user input gets sampled first.
|
||||
// 1. At the start of a turn, so the fresh turn input in `input` gets sampled first.
|
||||
// 2. After auto-compact, when model/tool continuation needs to resume before any steer.
|
||||
|
||||
let mut next_step_context = Some(first_step_context);
|
||||
@@ -362,7 +348,7 @@ pub(crate) async fn run_turn(
|
||||
if needs_follow_up
|
||||
&& (sess.take_new_context_window_request().await || token_limit_reached)
|
||||
{
|
||||
while let Err(err) = run_auto_compact(
|
||||
if let Err(err) = run_auto_compact(
|
||||
&sess,
|
||||
Arc::clone(&step_context),
|
||||
/*fallback_step_context*/ None,
|
||||
@@ -376,16 +362,10 @@ pub(crate) async fn run_turn(
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(
|
||||
turn_context.as_ref(),
|
||||
err.to_codex_protocol_error(),
|
||||
)
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
let Some(delay) = retry_delay else {
|
||||
return Ok(None);
|
||||
};
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
return Ok(None);
|
||||
}
|
||||
can_drain_pending_input = !model_needs_follow_up;
|
||||
continue;
|
||||
@@ -467,9 +447,21 @@ pub(crate) async fn run_turn(
|
||||
Err(e) => {
|
||||
info!("Turn error: {e:#}");
|
||||
let error = e.to_codex_protocol_error();
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(turn_context.as_ref(), error)
|
||||
.await;
|
||||
let mut retry_delay = None;
|
||||
for contributor in sess.services.extensions.turn_lifecycle_contributors() {
|
||||
retry_delay = contributor
|
||||
.retry_delay_for_turn_error(codex_extension_api::TurnErrorInput {
|
||||
turn_id: turn_context.sub_id.as_str(),
|
||||
error: error.clone(),
|
||||
session_store: &sess.services.session_extension_data,
|
||||
thread_store: &sess.services.thread_extension_data,
|
||||
turn_store: turn_context.extension_data.as_ref(),
|
||||
})
|
||||
.await;
|
||||
if retry_delay.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sess.track_turn_codex_error(turn_context.as_ref(), &e);
|
||||
let error_event = e.to_error_event(/*message_prefix*/ None);
|
||||
if let Some(delay) = retry_delay {
|
||||
@@ -483,11 +475,31 @@ pub(crate) async fn run_turn(
|
||||
)
|
||||
.await;
|
||||
// Keep this turn alive so its recorded input is not appended again.
|
||||
// Let queued input interrupt the backoff instead of parking the live turn.
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
// Let queued user input interrupt the backoff instead of parking the turn.
|
||||
let turn_state = sess
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
|
||||
.await;
|
||||
let (mut activity_rx, pending_activity) = sess
|
||||
.input_queue
|
||||
.subscribe_activity(turn_state.as_deref())
|
||||
.await;
|
||||
if pending_activity != Some(InputQueueActivity::Steer) {
|
||||
let _ = tokio::time::timeout(delay, async {
|
||||
while activity_rx.changed().await.is_ok() {
|
||||
// Mailbox activity does not represent user intent to retry now.
|
||||
if *activity_rx.borrow_and_update() == InputQueueActivity::Steer {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
can_drain_pending_input = true;
|
||||
continue;
|
||||
}
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error)
|
||||
.await;
|
||||
sess.send_event(&turn_context, EventMsg::Error(error_event))
|
||||
.await;
|
||||
// let the user continue the conversation
|
||||
@@ -499,22 +511,6 @@ pub(crate) async fn run_turn(
|
||||
Ok(last_agent_message)
|
||||
}
|
||||
|
||||
async fn wait_for_retry_delay(
|
||||
sess: &Session,
|
||||
delay: Duration,
|
||||
activity_rx: &mut tokio::sync::watch::Receiver<InputQueueActivity>,
|
||||
) {
|
||||
let _ = tokio::time::timeout(delay, async {
|
||||
while activity_rx.changed().await.is_ok() {
|
||||
// Ignore mailbox activity that is deferred to the next turn.
|
||||
if sess.input_queue.has_pending_input(&sess.active_turn).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, PathBuf)> {
|
||||
let mut display_roots = Vec::new();
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
@@ -78,10 +76,9 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
error: CodexErrorInfo,
|
||||
) -> Option<Duration> {
|
||||
let mut retry_delay: Option<Duration> = None;
|
||||
) {
|
||||
for contributor in self.services.extensions.turn_lifecycle_contributors() {
|
||||
retry_delay = retry_delay.or(contributor
|
||||
contributor
|
||||
.on_turn_error(codex_extension_api::TurnErrorInput {
|
||||
turn_id: turn_context.sub_id.as_str(),
|
||||
error: error.clone(),
|
||||
@@ -89,8 +86,7 @@ impl Session {
|
||||
thread_store: &self.services.thread_extension_data,
|
||||
turn_store: turn_context.extension_data.as_ref(),
|
||||
})
|
||||
.await);
|
||||
.await;
|
||||
}
|
||||
retry_delay
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ use core_test_support::responses::ev_output_text_delta;
|
||||
use core_test_support::responses::ev_reasoning_item;
|
||||
use core_test_support::responses::ev_reasoning_item_added;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::streaming_sse::StreamingSseChunk;
|
||||
use core_test_support::streaming_sse::StreamingSseServer;
|
||||
use core_test_support::streaming_sse::start_streaming_sse_server;
|
||||
@@ -43,14 +42,14 @@ use serde_json::from_slice;
|
||||
use serde_json::json;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
struct RetryTurnErrors(std::time::Duration);
|
||||
struct RetryTurnErrors;
|
||||
|
||||
impl TurnLifecycleContributor for RetryTurnErrors {
|
||||
fn on_turn_error<'a>(
|
||||
fn retry_delay_for_turn_error<'a>(
|
||||
&'a self,
|
||||
_input: codex_extension_api::TurnErrorInput<'a>,
|
||||
) -> codex_extension_api::ExtensionFuture<'a, Option<std::time::Duration>> {
|
||||
Box::pin(async { Some(self.0) })
|
||||
Box::pin(async { Some(std::time::Duration::from_secs(60)) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,11 +301,8 @@ async fn wait_for_sleep_item_completed(codex: &CodexThread, call_id: &str, durat
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
|
||||
skip_if_no_network!();
|
||||
|
||||
const CONTINUATION: &str = "GOAL_CONTINUATION";
|
||||
const MAIL: &str = "CHILD_UPDATE";
|
||||
async fn sampling_error_retries_in_place_and_steer_interrupts_delay() {
|
||||
const INITIAL: &str = "INITIAL_PROMPT";
|
||||
const STEER: &str = "USER_STEER";
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock = responses::mount_sse_sequence(
|
||||
@@ -322,9 +318,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
|
||||
)
|
||||
.await;
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors(
|
||||
std::time::Duration::from_secs(60),
|
||||
)));
|
||||
extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors));
|
||||
let test = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_config(|config| config.model_provider.stream_max_retries = Some(0))
|
||||
@@ -332,11 +326,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
|
||||
.await
|
||||
.expect("build Codex test session");
|
||||
|
||||
submit_queue_only_agent_mail(&test.codex, MAIL).await;
|
||||
test.codex
|
||||
.try_start_turn_if_idle(vec![responses::user_message_item(CONTINUATION)])
|
||||
.await
|
||||
.expect("start automatic turn");
|
||||
submit_user_input(&test.codex, INITIAL).await;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::StreamError(_))
|
||||
})
|
||||
@@ -345,18 +335,16 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() {
|
||||
wait_for_turn_complete(&test.codex).await;
|
||||
|
||||
let requests = response_mock.requests();
|
||||
let [first, second] = requests.as_slice() else {
|
||||
let [_, second] = requests.as_slice() else {
|
||||
panic!("expected two requests, got {}", requests.len());
|
||||
};
|
||||
assert_eq!(1, first.inputs_of_type("agent_message").len());
|
||||
assert!(first.body_contains_text(MAIL));
|
||||
assert!(second.body_contains_text(STEER));
|
||||
assert_eq!(
|
||||
1,
|
||||
second
|
||||
.message_input_texts("user")
|
||||
.iter()
|
||||
.filter(|text| text.as_str() == CONTINUATION)
|
||||
.filter(|text| text.as_str() == INITIAL)
|
||||
.count()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,9 +185,9 @@ pub trait TurnLifecycleContributor: Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
/// Called when the host observes an error for a running turn. Returning a
|
||||
/// delay asks the host to retry the failed operation without ending the turn.
|
||||
fn on_turn_error<'a>(
|
||||
/// Returns how long the host should wait before retrying a failed sampling
|
||||
/// request. Returning `None` leaves the error terminal.
|
||||
fn retry_delay_for_turn_error<'a>(
|
||||
&'a self,
|
||||
input: TurnErrorInput<'a>,
|
||||
) -> ExtensionFuture<'a, Option<Duration>> {
|
||||
@@ -197,6 +197,14 @@ pub trait TurnLifecycleContributor: Send + Sync {
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
/// Called when the host observes a terminal error for a running turn.
|
||||
fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
let _self = self;
|
||||
let _input = input;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Extension contribution that can add turn-local model input.
|
||||
|
||||
@@ -47,11 +47,6 @@ use crate::spec::UPDATE_GOAL_TOOL_NAME;
|
||||
use crate::steering::budget_limit_steering_item;
|
||||
use crate::tool::GoalToolExecutor;
|
||||
|
||||
// Capacity failures do not consume user tokens. Add jitter around a five-minute
|
||||
// average so clients that hit capacity together do not retry in lockstep.
|
||||
const DEFERRED_GOAL_RETRY_MIN_SECS: u64 = 4 * 60;
|
||||
const DEFERRED_GOAL_RETRY_MAX_SECS: u64 = 6 * 60;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GoalExtensionConfig {
|
||||
pub enabled: bool,
|
||||
@@ -303,7 +298,7 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
fn on_turn_error<'a>(
|
||||
fn retry_delay_for_turn_error<'a>(
|
||||
&'a self,
|
||||
input: TurnErrorInput<'a>,
|
||||
) -> ExtensionFuture<'a, Option<Duration>> {
|
||||
@@ -316,10 +311,22 @@ where
|
||||
.accounting_state()
|
||||
.turn_is_current_active_goal(input.turn_id)
|
||||
{
|
||||
return Some(Duration::from_secs(rand::rng().random_range(
|
||||
DEFERRED_GOAL_RETRY_MIN_SECS..=DEFERRED_GOAL_RETRY_MAX_SECS,
|
||||
)));
|
||||
// Capacity failures do not consume tokens. Jitter around five minutes
|
||||
// prevents clients that hit capacity together from retrying in lockstep.
|
||||
return Some(Duration::from_secs(
|
||||
rand::rng().random_range(4 * 60..=6 * 60),
|
||||
));
|
||||
}
|
||||
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
let Some(runtime) = goal_runtime_handle(input.thread_store) else {
|
||||
return;
|
||||
};
|
||||
let reason = match input.error {
|
||||
CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit,
|
||||
// The turn has ended because the error was non-retryable or its
|
||||
@@ -337,7 +344,6 @@ where
|
||||
"failed to stop active goal after turn error: {err}"
|
||||
);
|
||||
}
|
||||
None
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,26 +574,7 @@ async fn turn_error_usage_limit_accounts_progress_and_clears_accounting() -> any
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_error_blocks_goal() -> anyhow::Result<()> {
|
||||
assert_eq!(
|
||||
(codex_state::ThreadGoalStatus::Blocked, false),
|
||||
goal_status_after_turn_error(CodexErrorInfo::Other).await?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_overloaded_error_keeps_goal_active() -> anyhow::Result<()> {
|
||||
assert_eq!(
|
||||
(codex_state::ThreadGoalStatus::Active, true),
|
||||
goal_status_after_turn_error(CodexErrorInfo::ServerOverloaded).await?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn goal_status_after_turn_error(
|
||||
error: CodexErrorInfo,
|
||||
) -> anyhow::Result<(codex_state::ThreadGoalStatus, bool)> {
|
||||
async fn retryable_error_keeps_goal_active_and_terminal_error_blocks_it() -> anyhow::Result<()> {
|
||||
let runtime = test_runtime().await?;
|
||||
let thread_id = test_thread_id()?;
|
||||
seed_thread_metadata(runtime.as_ref(), thread_id).await?;
|
||||
@@ -609,14 +590,35 @@ async fn goal_status_after_turn_error(
|
||||
))
|
||||
.await?;
|
||||
|
||||
let retry_delay = harness.notify_turn_error("turn-1", error).await;
|
||||
let turn_store = ExtensionData::new("turn-1");
|
||||
let retry_delay = harness.registry.turn_lifecycle_contributors()[0]
|
||||
.retry_delay_for_turn_error(TurnErrorInput {
|
||||
turn_id: "turn-1",
|
||||
error: CodexErrorInfo::ServerOverloaded,
|
||||
session_store: &harness.session_store,
|
||||
thread_store: &harness.thread_store,
|
||||
turn_store: &turn_store,
|
||||
})
|
||||
.await;
|
||||
assert!(retry_delay.is_some());
|
||||
let goal = runtime
|
||||
.thread_goals()
|
||||
.get_thread_goal(thread_id)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow::anyhow!("goal should exist"))?;
|
||||
assert_eq!(codex_state::ThreadGoalStatus::Active, goal.status);
|
||||
|
||||
harness
|
||||
.notify_turn_error("turn-1", CodexErrorInfo::Other)
|
||||
.await;
|
||||
|
||||
let goal = runtime
|
||||
.thread_goals()
|
||||
.get_thread_goal(thread_id)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow::anyhow!("goal should exist"))?;
|
||||
Ok((goal.status, retry_delay.is_some()))
|
||||
assert_eq!(codex_state::ThreadGoalStatus::Blocked, goal.status);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1321,15 +1323,10 @@ impl GoalExtensionHarness {
|
||||
}
|
||||
}
|
||||
|
||||
async fn notify_turn_error(
|
||||
&self,
|
||||
turn_id: &str,
|
||||
error: CodexErrorInfo,
|
||||
) -> Option<std::time::Duration> {
|
||||
async fn notify_turn_error(&self, turn_id: &str, error: CodexErrorInfo) {
|
||||
let turn_store = ExtensionData::new(turn_id);
|
||||
let mut retry_delay = None;
|
||||
for contributor in self.registry.turn_lifecycle_contributors() {
|
||||
retry_delay = contributor
|
||||
contributor
|
||||
.on_turn_error(TurnErrorInput {
|
||||
turn_id,
|
||||
error: error.clone(),
|
||||
@@ -1339,7 +1336,6 @@ impl GoalExtensionHarness {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
retry_delay
|
||||
}
|
||||
|
||||
fn runtime_handle(&self) -> Arc<GoalRuntimeHandle> {
|
||||
|
||||
Reference in New Issue
Block a user