From 54717e13bbadb52d0707d4bd3306b6f257645540 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Mon, 16 Mar 2026 01:13:36 -0700 Subject: [PATCH] Clarify stalled auto-compaction safeguard Make the mid-turn compaction cap explicit as a stalled auto-compaction safeguard with dedicated error text, telemetry, and regression coverage. Co-authored-by: Codex --- codex-rs/core/src/codex.rs | 29 ++++++--- codex-rs/core/tests/suite/compact.rs | 97 ++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 62120f6431..50a2440a01 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -379,7 +379,7 @@ pub(crate) const SUBMISSION_CHANNEL_CAPACITY: usize = 512; const CYBER_VERIFY_URL: &str = "https://chatgpt.com/cyber"; const CYBER_SAFETY_URL: &str = "https://developers.openai.com/codex/concepts/cyber-safety"; const DIRECT_APP_TOOL_EXPOSURE_THRESHOLD: usize = 100; -const MAX_MID_TURN_COMPACTION_ATTEMPTS: u32 = 3; +const MAX_MID_TURN_AUTO_COMPACTION_ATTEMPTS: u32 = 3; impl Codex { /// Spawn a new [`Codex`] and initialize the session. @@ -5680,7 +5680,7 @@ pub(crate) async fn run_turn( // many turns, from the perspective of the user, it is a single turn. let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); let mut server_model_warning_emitted_for_turn = false; - let mut mid_turn_compaction_attempts = 0_u32; + let mut mid_turn_auto_compaction_attempts = 0_u32; // `ModelClientSession` is turn-scoped and caches WebSocket + sticky routing state, so we reuse // one instance across retries within this turn. @@ -5818,19 +5818,30 @@ pub(crate) async fn run_turn( "post sampling token usage" ); - // as long as compaction works well in getting us way below the token limit, we shouldn't worry about being in an infinite loop. + // Guard against a pathological loop where mid-turn auto-compaction + // keeps succeeding, but the turn still remains over the token + // threshold and still needs follow-up. Treat that as stalled + // auto-compaction so we fail explicitly instead of hot-looping. if token_limit_reached && needs_follow_up { - mid_turn_compaction_attempts += 1; - if mid_turn_compaction_attempts > MAX_MID_TURN_COMPACTION_ATTEMPTS { + mid_turn_auto_compaction_attempts += 1; + if mid_turn_auto_compaction_attempts > MAX_MID_TURN_AUTO_COMPACTION_ATTEMPTS { + let _ = sess.services.session_telemetry.counter( + "codex.auto_compaction.stalled", + 1, + &[("phase", "mid_turn")], + ); error!( turn_id = %turn_context.sub_id, - mid_turn_compaction_attempts, - "mid-turn compaction exceeded retry limit" + mid_turn_auto_compaction_attempts, + total_usage_tokens, + estimated_token_count = ?estimated_token_count, + auto_compact_limit, + "mid-turn auto-compaction stalled" ); let event = EventMsg::Error(CodexErr::ContextWindowExceeded.to_error_event( Some( format!( - "Mid-turn compaction exceeded retry limit ({MAX_MID_TURN_COMPACTION_ATTEMPTS} attempts); start a new thread or compact manually" + "Mid-turn auto-compaction stalled after {MAX_MID_TURN_AUTO_COMPACTION_ATTEMPTS} attempts without getting this turn back under the token threshold. Start a new thread or compact manually." ), ), )); @@ -5849,7 +5860,7 @@ pub(crate) async fn run_turn( } continue; } - mid_turn_compaction_attempts = 0; + mid_turn_auto_compaction_attempts = 0; if !needs_follow_up { last_agent_message = sampling_request_last_agent_message; diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 6cf1c275a7..20967b55f5 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -2588,6 +2588,103 @@ async fn auto_compact_allows_multiple_attempts_when_interleaved_with_other_turn_ ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mid_turn_auto_compaction_stall_emits_specific_error_after_limit() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let context_window = 100; + let limit = context_window * 90 / 100; + let over_limit_tokens = context_window * 95 / 100 + 1; + let responses = vec![ + sse(vec![ + ev_function_call("call-stall-1", DUMMY_FUNCTION_NAME, "{}"), + ev_completed_with_tokens("r1", over_limit_tokens), + ]), + sse(vec![ + ev_assistant_message("m2", &auto_summary("STALL_SUMMARY_1")), + ev_completed_with_tokens("r2", 10), + ]), + sse(vec![ + ev_function_call("call-stall-2", DUMMY_FUNCTION_NAME, "{}"), + ev_completed_with_tokens("r3", over_limit_tokens), + ]), + sse(vec![ + ev_assistant_message("m4", &auto_summary("STALL_SUMMARY_2")), + ev_completed_with_tokens("r4", 10), + ]), + sse(vec![ + ev_function_call("call-stall-3", DUMMY_FUNCTION_NAME, "{}"), + ev_completed_with_tokens("r5", over_limit_tokens), + ]), + sse(vec![ + ev_assistant_message("m6", &auto_summary("STALL_SUMMARY_3")), + ev_completed_with_tokens("r6", 10), + ]), + sse(vec![ + ev_function_call("call-stall-4", DUMMY_FUNCTION_NAME, "{}"), + ev_completed_with_tokens("r7", over_limit_tokens), + ]), + ]; + let request_log = mount_sse_sequence(&server, responses).await; + + let model_provider = non_openai_model_provider(&server); + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_context_window = Some(context_window); + config.model_auto_compact_token_limit = Some(limit); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "STALL_AUTO_COMPACT".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit stalled auto compact turn"); + + let error_message = wait_for_event_match(&codex, |event| match event { + EventMsg::Error(err) => Some(err.message.clone()), + _ => None, + }) + .await; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + assert!( + error_message.contains("Mid-turn auto-compaction stalled after 3 attempts"), + "expected stalled auto-compaction error, got {error_message}" + ); + + let request_bodies: Vec = request_log + .requests() + .into_iter() + .map(|request| request.body_json().to_string()) + .collect(); + assert_eq!( + request_bodies.len(), + 7, + "expected three auto compactions and no fourth compaction request" + ); + assert_eq!( + request_bodies + .iter() + .filter(|body| body_contains_text(body, SUMMARIZATION_PROMPT)) + .count(), + 3, + "stalled safeguard should stop before a fourth auto compaction request" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_request_shape_mid_turn_continuation_compaction() { skip_if_no_network!();