mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
codex: simplify capacity retry handling (#31176)
This commit is contained in:
@@ -118,10 +118,13 @@ impl Session {
|
||||
));
|
||||
}
|
||||
|
||||
// Keep automatic input task-owned so a pre-turn failure drops this
|
||||
// attempt instead of immediately replaying it from the pending queue.
|
||||
let input = input.into_iter().map(TurnInput::ResponseItem).collect();
|
||||
self.start_task(turn_context, input, RegularTask::new())
|
||||
self.input_queue
|
||||
.extend_pending_input_for_turn_state(
|
||||
turn_state.as_ref(),
|
||||
input.into_iter().map(TurnInput::ResponseItem).collect(),
|
||||
)
|
||||
.await;
|
||||
self.start_task(turn_context, Vec::new(), RegularTask::new())
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2533,7 +2533,7 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
|
||||
fn on_turn_error<'a>(
|
||||
&'a self,
|
||||
input: codex_extension_api::TurnErrorInput<'a>,
|
||||
) -> codex_extension_api::ExtensionFuture<'a, Option<std::time::Duration>> {
|
||||
) -> codex_extension_api::ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
self.records
|
||||
.lock()
|
||||
@@ -2553,7 +2553,6 @@ async fn turn_error_lifecycle_exposes_error_and_stores() {
|
||||
.get::<ThreadTurnErrorMarker>()
|
||||
.is_some(),
|
||||
});
|
||||
None
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::SkillInjections;
|
||||
use crate::build_skill_injections;
|
||||
@@ -152,30 +151,19 @@ pub(crate) async fn run_turn(
|
||||
) -> CodexResult<Option<String>> {
|
||||
let mut client_session =
|
||||
prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session());
|
||||
let turn_state = sess
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
|
||||
.await;
|
||||
let (mut retry_activity_rx, _) = sess
|
||||
.input_queue
|
||||
.subscribe_activity(turn_state.as_deref())
|
||||
.await;
|
||||
// TODO(ccunningham): Pre-turn compaction runs before context updates and the
|
||||
// new user message are recorded. Estimate pending incoming items (context
|
||||
// diffs/full reinjection + user input) and trigger compaction preemptively
|
||||
// when they would push the thread over the compaction threshold.
|
||||
while let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
|
||||
if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(turn_context.as_ref(), err.to_codex_protocol_error())
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
error!("Failed to run pre-sampling compact");
|
||||
let Some(delay) = retry_delay else {
|
||||
return Ok(None);
|
||||
};
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// run_turn owns the step used to seed context and make the first sampling request.
|
||||
@@ -200,9 +188,7 @@ pub(crate) async fn run_turn(
|
||||
if run_pending_session_start_hooks(&sess, &turn_context).await {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut can_drain_pending_input = !input
|
||||
.iter()
|
||||
.any(|item| matches!(item, TurnInput::UserInput { .. }));
|
||||
let mut can_drain_pending_input = input.is_empty();
|
||||
if run_hooks_and_record_inputs(&sess, &turn_context, &input).await {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -234,7 +220,7 @@ pub(crate) async fn run_turn(
|
||||
// one instance across retries within this turn.
|
||||
// Pending input is drained into history before building the next model request.
|
||||
// However, we defer that drain until after sampling in two cases:
|
||||
// 1. At the start of a user turn, so the submitted user input gets sampled first.
|
||||
// 1. At the start of a turn, so the fresh turn input in `input` gets sampled first.
|
||||
// 2. After auto-compact, when model/tool continuation needs to resume before any steer.
|
||||
|
||||
let mut next_step_context = Some(first_step_context);
|
||||
@@ -362,7 +348,7 @@ pub(crate) async fn run_turn(
|
||||
if needs_follow_up
|
||||
&& (sess.take_new_context_window_request().await || token_limit_reached)
|
||||
{
|
||||
while let Err(err) = run_auto_compact(
|
||||
if let Err(err) = run_auto_compact(
|
||||
&sess,
|
||||
Arc::clone(&step_context),
|
||||
/*fallback_step_context*/ None,
|
||||
@@ -376,16 +362,10 @@ pub(crate) async fn run_turn(
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(
|
||||
turn_context.as_ref(),
|
||||
err.to_codex_protocol_error(),
|
||||
)
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
let Some(delay) = retry_delay else {
|
||||
return Ok(None);
|
||||
};
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
return Ok(None);
|
||||
}
|
||||
can_drain_pending_input = !model_needs_follow_up;
|
||||
continue;
|
||||
@@ -467,9 +447,21 @@ pub(crate) async fn run_turn(
|
||||
Err(e) => {
|
||||
info!("Turn error: {e:#}");
|
||||
let error = e.to_codex_protocol_error();
|
||||
let retry_delay = sess
|
||||
.emit_turn_error_lifecycle(turn_context.as_ref(), error)
|
||||
.await;
|
||||
let mut retry_delay = None;
|
||||
for contributor in sess.services.extensions.turn_lifecycle_contributors() {
|
||||
retry_delay = contributor
|
||||
.retry_delay_for_turn_error(codex_extension_api::TurnErrorInput {
|
||||
turn_id: turn_context.sub_id.as_str(),
|
||||
error: error.clone(),
|
||||
session_store: &sess.services.session_extension_data,
|
||||
thread_store: &sess.services.thread_extension_data,
|
||||
turn_store: turn_context.extension_data.as_ref(),
|
||||
})
|
||||
.await;
|
||||
if retry_delay.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sess.track_turn_codex_error(turn_context.as_ref(), &e);
|
||||
let error_event = e.to_error_event(/*message_prefix*/ None);
|
||||
if let Some(delay) = retry_delay {
|
||||
@@ -483,11 +475,31 @@ pub(crate) async fn run_turn(
|
||||
)
|
||||
.await;
|
||||
// Keep this turn alive so its recorded input is not appended again.
|
||||
// Let queued input interrupt the backoff instead of parking the live turn.
|
||||
wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await;
|
||||
// Let queued user input interrupt the backoff instead of parking the turn.
|
||||
let turn_state = sess
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id)
|
||||
.await;
|
||||
let (mut activity_rx, pending_activity) = sess
|
||||
.input_queue
|
||||
.subscribe_activity(turn_state.as_deref())
|
||||
.await;
|
||||
if pending_activity != Some(InputQueueActivity::Steer) {
|
||||
let _ = tokio::time::timeout(delay, async {
|
||||
while activity_rx.changed().await.is_ok() {
|
||||
// Mailbox activity does not represent user intent to retry now.
|
||||
if *activity_rx.borrow_and_update() == InputQueueActivity::Steer {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
can_drain_pending_input = true;
|
||||
continue;
|
||||
}
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error)
|
||||
.await;
|
||||
sess.send_event(&turn_context, EventMsg::Error(error_event))
|
||||
.await;
|
||||
// let the user continue the conversation
|
||||
@@ -499,22 +511,6 @@ pub(crate) async fn run_turn(
|
||||
Ok(last_agent_message)
|
||||
}
|
||||
|
||||
async fn wait_for_retry_delay(
|
||||
sess: &Session,
|
||||
delay: Duration,
|
||||
activity_rx: &mut tokio::sync::watch::Receiver<InputQueueActivity>,
|
||||
) {
|
||||
let _ = tokio::time::timeout(delay, async {
|
||||
while activity_rx.changed().await.is_ok() {
|
||||
// Ignore mailbox activity that is deferred to the next turn.
|
||||
if sess.input_queue.has_pending_input(&sess.active_turn).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, PathBuf)> {
|
||||
let mut display_roots = Vec::new();
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
@@ -78,10 +76,9 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
error: CodexErrorInfo,
|
||||
) -> Option<Duration> {
|
||||
let mut retry_delay: Option<Duration> = None;
|
||||
) {
|
||||
for contributor in self.services.extensions.turn_lifecycle_contributors() {
|
||||
retry_delay = retry_delay.or(contributor
|
||||
contributor
|
||||
.on_turn_error(codex_extension_api::TurnErrorInput {
|
||||
turn_id: turn_context.sub_id.as_str(),
|
||||
error: error.clone(),
|
||||
@@ -89,8 +86,7 @@ impl Session {
|
||||
thread_store: &self.services.thread_extension_data,
|
||||
turn_store: turn_context.extension_data.as_ref(),
|
||||
})
|
||||
.await);
|
||||
.await;
|
||||
}
|
||||
retry_delay
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user