diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rs index a22da3297d..5c5b7d1feb 100644 --- a/codex-rs/core/src/session/inject.rs +++ b/codex-rs/core/src/session/inject.rs @@ -118,10 +118,13 @@ impl Session { )); } - // Keep automatic input task-owned so a pre-turn failure drops this - // attempt instead of immediately replaying it from the pending queue. - let input = input.into_iter().map(TurnInput::ResponseItem).collect(); - self.start_task(turn_context, input, RegularTask::new()) + self.input_queue + .extend_pending_input_for_turn_state( + turn_state.as_ref(), + input.into_iter().map(TurnInput::ResponseItem).collect(), + ) + .await; + self.start_task(turn_context, Vec::new(), RegularTask::new()) .await; Ok(()) } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 0ceda1007a..ac8f9d049e 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2533,7 +2533,7 @@ async fn turn_error_lifecycle_exposes_error_and_stores() { fn on_turn_error<'a>( &'a self, input: codex_extension_api::TurnErrorInput<'a>, - ) -> codex_extension_api::ExtensionFuture<'a, Option> { + ) -> codex_extension_api::ExtensionFuture<'a, ()> { Box::pin(async move { self.records .lock() @@ -2553,7 +2553,6 @@ async fn turn_error_lifecycle_exposes_error_and_stores() { .get::() .is_some(), }); - None }) } } diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 90ddf963fe..46aa709789 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -3,7 +3,6 @@ use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::Ordering; -use std::time::Duration; use crate::SkillInjections; use crate::build_skill_injections; @@ -152,30 +151,19 @@ pub(crate) async fn run_turn( ) -> CodexResult> { let mut client_session = prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session()); - let turn_state = sess - .input_queue - .turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id) - .await; - let (mut retry_activity_rx, _) = sess - .input_queue - .subscribe_activity(turn_state.as_deref()) - .await; // TODO(ccunningham): Pre-turn compaction runs before context updates and the // new user message are recorded. Estimate pending incoming items (context // diffs/full reinjection + user input) and trigger compaction preemptively // when they would push the thread over the compaction threshold. - while let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await { + if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await { if matches!(err, CodexErr::TurnAborted) { return Err(err); } - let retry_delay = sess - .emit_turn_error_lifecycle(turn_context.as_ref(), err.to_codex_protocol_error()) + let error = err.to_codex_protocol_error(); + sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone()) .await; error!("Failed to run pre-sampling compact"); - let Some(delay) = retry_delay else { - return Ok(None); - }; - wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await; + return Ok(None); } // run_turn owns the step used to seed context and make the first sampling request. @@ -200,9 +188,7 @@ pub(crate) async fn run_turn( if run_pending_session_start_hooks(&sess, &turn_context).await { return Ok(None); } - let mut can_drain_pending_input = !input - .iter() - .any(|item| matches!(item, TurnInput::UserInput { .. })); + let mut can_drain_pending_input = input.is_empty(); if run_hooks_and_record_inputs(&sess, &turn_context, &input).await { return Ok(None); } @@ -234,7 +220,7 @@ pub(crate) async fn run_turn( // one instance across retries within this turn. // Pending input is drained into history before building the next model request. // However, we defer that drain until after sampling in two cases: - // 1. At the start of a user turn, so the submitted user input gets sampled first. + // 1. At the start of a turn, so the fresh turn input in `input` gets sampled first. // 2. After auto-compact, when model/tool continuation needs to resume before any steer. let mut next_step_context = Some(first_step_context); @@ -362,7 +348,7 @@ pub(crate) async fn run_turn( if needs_follow_up && (sess.take_new_context_window_request().await || token_limit_reached) { - while let Err(err) = run_auto_compact( + if let Err(err) = run_auto_compact( &sess, Arc::clone(&step_context), /*fallback_step_context*/ None, @@ -376,16 +362,10 @@ pub(crate) async fn run_turn( if matches!(err, CodexErr::TurnAborted) { return Err(err); } - let retry_delay = sess - .emit_turn_error_lifecycle( - turn_context.as_ref(), - err.to_codex_protocol_error(), - ) + let error = err.to_codex_protocol_error(); + sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone()) .await; - let Some(delay) = retry_delay else { - return Ok(None); - }; - wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await; + return Ok(None); } can_drain_pending_input = !model_needs_follow_up; continue; @@ -467,9 +447,21 @@ pub(crate) async fn run_turn( Err(e) => { info!("Turn error: {e:#}"); let error = e.to_codex_protocol_error(); - let retry_delay = sess - .emit_turn_error_lifecycle(turn_context.as_ref(), error) - .await; + let mut retry_delay = None; + for contributor in sess.services.extensions.turn_lifecycle_contributors() { + retry_delay = contributor + .retry_delay_for_turn_error(codex_extension_api::TurnErrorInput { + turn_id: turn_context.sub_id.as_str(), + error: error.clone(), + session_store: &sess.services.session_extension_data, + thread_store: &sess.services.thread_extension_data, + turn_store: turn_context.extension_data.as_ref(), + }) + .await; + if retry_delay.is_some() { + break; + } + } sess.track_turn_codex_error(turn_context.as_ref(), &e); let error_event = e.to_error_event(/*message_prefix*/ None); if let Some(delay) = retry_delay { @@ -483,11 +475,31 @@ pub(crate) async fn run_turn( ) .await; // Keep this turn alive so its recorded input is not appended again. - // Let queued input interrupt the backoff instead of parking the live turn. - wait_for_retry_delay(&sess, delay, &mut retry_activity_rx).await; + // Let queued user input interrupt the backoff instead of parking the turn. + let turn_state = sess + .input_queue + .turn_state_for_sub_id(&sess.active_turn, &turn_context.sub_id) + .await; + let (mut activity_rx, pending_activity) = sess + .input_queue + .subscribe_activity(turn_state.as_deref()) + .await; + if pending_activity != Some(InputQueueActivity::Steer) { + let _ = tokio::time::timeout(delay, async { + while activity_rx.changed().await.is_ok() { + // Mailbox activity does not represent user intent to retry now. + if *activity_rx.borrow_and_update() == InputQueueActivity::Steer { + break; + } + } + }) + .await; + } can_drain_pending_input = true; continue; } + sess.emit_turn_error_lifecycle(turn_context.as_ref(), error) + .await; sess.send_event(&turn_context, EventMsg::Error(error_event)) .await; // let the user continue the conversation @@ -499,22 +511,6 @@ pub(crate) async fn run_turn( Ok(last_agent_message) } -async fn wait_for_retry_delay( - sess: &Session, - delay: Duration, - activity_rx: &mut tokio::sync::watch::Receiver, -) { - let _ = tokio::time::timeout(delay, async { - while activity_rx.changed().await.is_ok() { - // Ignore mailbox activity that is deferred to the next turn. - if sess.input_queue.has_pending_input(&sess.active_turn).await { - break; - } - } - }) - .await; -} - #[instrument(level = "trace", skip_all)] async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, PathBuf)> { let mut display_roots = Vec::new(); diff --git a/codex-rs/core/src/tasks/lifecycle.rs b/codex-rs/core/src/tasks/lifecycle.rs index 785051253d..782cc8c7de 100644 --- a/codex-rs/core/src/tasks/lifecycle.rs +++ b/codex-rs/core/src/tasks/lifecycle.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use codex_extension_api::ExtensionData; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::TokenUsage; @@ -78,10 +76,9 @@ impl Session { &self, turn_context: &TurnContext, error: CodexErrorInfo, - ) -> Option { - let mut retry_delay: Option = None; + ) { for contributor in self.services.extensions.turn_lifecycle_contributors() { - retry_delay = retry_delay.or(contributor + contributor .on_turn_error(codex_extension_api::TurnErrorInput { turn_id: turn_context.sub_id.as_str(), error: error.clone(), @@ -89,8 +86,7 @@ impl Session { thread_store: &self.services.thread_extension_data, turn_store: turn_context.extension_data.as_ref(), }) - .await); + .await; } - retry_delay } } diff --git a/codex-rs/core/tests/suite/pending_input.rs b/codex-rs/core/tests/suite/pending_input.rs index 2c74d2d9e5..9cdfc1794e 100644 --- a/codex-rs/core/tests/suite/pending_input.rs +++ b/codex-rs/core/tests/suite/pending_input.rs @@ -29,7 +29,6 @@ use core_test_support::responses::ev_output_text_delta; use core_test_support::responses::ev_reasoning_item; use core_test_support::responses::ev_reasoning_item_added; use core_test_support::responses::ev_response_created; -use core_test_support::skip_if_no_network; use core_test_support::streaming_sse::StreamingSseChunk; use core_test_support::streaming_sse::StreamingSseServer; use core_test_support::streaming_sse::start_streaming_sse_server; @@ -43,14 +42,14 @@ use serde_json::from_slice; use serde_json::json; use tokio::sync::oneshot; -struct RetryTurnErrors(std::time::Duration); +struct RetryTurnErrors; impl TurnLifecycleContributor for RetryTurnErrors { - fn on_turn_error<'a>( + fn retry_delay_for_turn_error<'a>( &'a self, _input: codex_extension_api::TurnErrorInput<'a>, ) -> codex_extension_api::ExtensionFuture<'a, Option> { - Box::pin(async { Some(self.0) }) + Box::pin(async { Some(std::time::Duration::from_secs(60)) }) } } @@ -302,11 +301,8 @@ async fn wait_for_sleep_item_completed(codex: &CodexThread, call_id: &str, durat } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() { - skip_if_no_network!(); - - const CONTINUATION: &str = "GOAL_CONTINUATION"; - const MAIL: &str = "CHILD_UPDATE"; +async fn sampling_error_retries_in_place_and_steer_interrupts_delay() { + const INITIAL: &str = "INITIAL_PROMPT"; const STEER: &str = "USER_STEER"; let server = responses::start_mock_server().await; let response_mock = responses::mount_sse_sequence( @@ -322,9 +318,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() { ) .await; let mut extensions = ExtensionRegistryBuilder::new(); - extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors( - std::time::Duration::from_secs(60), - ))); + extensions.turn_lifecycle_contributor(Arc::new(RetryTurnErrors)); let test = test_codex() .with_extensions(Arc::new(extensions.build())) .with_config(|config| config.model_provider.stream_max_retries = Some(0)) @@ -332,11 +326,7 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() { .await .expect("build Codex test session"); - submit_queue_only_agent_mail(&test.codex, MAIL).await; - test.codex - .try_start_turn_if_idle(vec![responses::user_message_item(CONTINUATION)]) - .await - .expect("start automatic turn"); + submit_user_input(&test.codex, INITIAL).await; wait_for_event(&test.codex, |event| { matches!(event, EventMsg::StreamError(_)) }) @@ -345,18 +335,16 @@ async fn automatic_turn_retries_in_place_with_queued_mail_in_first_request() { wait_for_turn_complete(&test.codex).await; let requests = response_mock.requests(); - let [first, second] = requests.as_slice() else { + let [_, second] = requests.as_slice() else { panic!("expected two requests, got {}", requests.len()); }; - assert_eq!(1, first.inputs_of_type("agent_message").len()); - assert!(first.body_contains_text(MAIL)); assert!(second.body_contains_text(STEER)); assert_eq!( 1, second .message_input_texts("user") .iter() - .filter(|text| text.as_str() == CONTINUATION) + .filter(|text| text.as_str() == INITIAL) .count() ); } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 3fb30e56ff..d81ddadc43 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -185,9 +185,9 @@ pub trait TurnLifecycleContributor: Send + Sync { }) } - /// Called when the host observes an error for a running turn. Returning a - /// delay asks the host to retry the failed operation without ending the turn. - fn on_turn_error<'a>( + /// Returns how long the host should wait before retrying a failed sampling + /// request. Returning `None` leaves the error terminal. + fn retry_delay_for_turn_error<'a>( &'a self, input: TurnErrorInput<'a>, ) -> ExtensionFuture<'a, Option> { @@ -197,6 +197,14 @@ pub trait TurnLifecycleContributor: Send + Sync { None }) } + + /// Called when the host observes a terminal error for a running turn. + fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } } /// Extension contribution that can add turn-local model input. diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 7e5f2a85c9..d65139fe5e 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -47,11 +47,6 @@ use crate::spec::UPDATE_GOAL_TOOL_NAME; use crate::steering::budget_limit_steering_item; use crate::tool::GoalToolExecutor; -// Capacity failures do not consume user tokens. Add jitter around a five-minute -// average so clients that hit capacity together do not retry in lockstep. -const DEFERRED_GOAL_RETRY_MIN_SECS: u64 = 4 * 60; -const DEFERRED_GOAL_RETRY_MAX_SECS: u64 = 6 * 60; - #[derive(Clone, Debug)] pub struct GoalExtensionConfig { pub enabled: bool, @@ -303,7 +298,7 @@ where }) } - fn on_turn_error<'a>( + fn retry_delay_for_turn_error<'a>( &'a self, input: TurnErrorInput<'a>, ) -> ExtensionFuture<'a, Option> { @@ -316,10 +311,22 @@ where .accounting_state() .turn_is_current_active_goal(input.turn_id) { - return Some(Duration::from_secs(rand::rng().random_range( - DEFERRED_GOAL_RETRY_MIN_SECS..=DEFERRED_GOAL_RETRY_MAX_SECS, - ))); + // Capacity failures do not consume tokens. Jitter around five minutes + // prevents clients that hit capacity together from retrying in lockstep. + return Some(Duration::from_secs( + rand::rng().random_range(4 * 60..=6 * 60), + )); } + + None + }) + } + + fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; let reason = match input.error { CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit, // The turn has ended because the error was non-retryable or its @@ -337,7 +344,6 @@ where "failed to stop active goal after turn error: {err}" ); } - None }) } } diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index f28bb88817..30b3a2ec4f 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -574,26 +574,7 @@ async fn turn_error_usage_limit_accounts_progress_and_clears_accounting() -> any } #[tokio::test] -async fn turn_error_blocks_goal() -> anyhow::Result<()> { - assert_eq!( - (codex_state::ThreadGoalStatus::Blocked, false), - goal_status_after_turn_error(CodexErrorInfo::Other).await? - ); - Ok(()) -} - -#[tokio::test] -async fn server_overloaded_error_keeps_goal_active() -> anyhow::Result<()> { - assert_eq!( - (codex_state::ThreadGoalStatus::Active, true), - goal_status_after_turn_error(CodexErrorInfo::ServerOverloaded).await? - ); - Ok(()) -} - -async fn goal_status_after_turn_error( - error: CodexErrorInfo, -) -> anyhow::Result<(codex_state::ThreadGoalStatus, bool)> { +async fn retryable_error_keeps_goal_active_and_terminal_error_blocks_it() -> anyhow::Result<()> { let runtime = test_runtime().await?; let thread_id = test_thread_id()?; seed_thread_metadata(runtime.as_ref(), thread_id).await?; @@ -609,14 +590,35 @@ async fn goal_status_after_turn_error( )) .await?; - let retry_delay = harness.notify_turn_error("turn-1", error).await; + let turn_store = ExtensionData::new("turn-1"); + let retry_delay = harness.registry.turn_lifecycle_contributors()[0] + .retry_delay_for_turn_error(TurnErrorInput { + turn_id: "turn-1", + error: CodexErrorInfo::ServerOverloaded, + session_store: &harness.session_store, + thread_store: &harness.thread_store, + turn_store: &turn_store, + }) + .await; + assert!(retry_delay.is_some()); + let goal = runtime + .thread_goals() + .get_thread_goal(thread_id) + .await? + .ok_or_else(|| anyhow::anyhow!("goal should exist"))?; + assert_eq!(codex_state::ThreadGoalStatus::Active, goal.status); + + harness + .notify_turn_error("turn-1", CodexErrorInfo::Other) + .await; let goal = runtime .thread_goals() .get_thread_goal(thread_id) .await? .ok_or_else(|| anyhow::anyhow!("goal should exist"))?; - Ok((goal.status, retry_delay.is_some())) + assert_eq!(codex_state::ThreadGoalStatus::Blocked, goal.status); + Ok(()) } #[tokio::test] @@ -1321,15 +1323,10 @@ impl GoalExtensionHarness { } } - async fn notify_turn_error( - &self, - turn_id: &str, - error: CodexErrorInfo, - ) -> Option { + async fn notify_turn_error(&self, turn_id: &str, error: CodexErrorInfo) { let turn_store = ExtensionData::new(turn_id); - let mut retry_delay = None; for contributor in self.registry.turn_lifecycle_contributors() { - retry_delay = contributor + contributor .on_turn_error(TurnErrorInput { turn_id, error: error.clone(), @@ -1339,7 +1336,6 @@ impl GoalExtensionHarness { }) .await; } - retry_delay } fn runtime_handle(&self) -> Arc {