mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
fix(core): preserve input across capacity recovery
This commit is contained in:
@@ -611,6 +611,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
+7
@@ -7109,6 +7109,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
+7
@@ -3349,6 +3349,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -20,6 +20,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -43,6 +43,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -124,6 +124,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
+7
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -124,6 +124,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -124,6 +124,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -46,6 +46,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -43,6 +43,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -43,6 +43,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -43,6 +43,13 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Capacity failure before the incoming user input was recorded in thread history.",
|
||||
"enum": [
|
||||
"serverOverloadedBeforeInput"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
|
||||
@@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind";
|
||||
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
|
||||
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
|
||||
*/
|
||||
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "serverOverloaded" | "cyberPolicy" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
||||
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "serverOverloaded" | "serverOverloadedBeforeInput" | "cyberPolicy" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
||||
|
||||
@@ -73,6 +73,8 @@ pub enum CodexErrorInfo {
|
||||
SessionBudgetExceeded,
|
||||
UsageLimitExceeded,
|
||||
ServerOverloaded,
|
||||
/// Capacity failure before the incoming user input was recorded in thread history.
|
||||
ServerOverloadedBeforeInput,
|
||||
CyberPolicy,
|
||||
HttpConnectionFailed {
|
||||
#[serde(rename = "httpStatusCode")]
|
||||
@@ -119,6 +121,9 @@ impl From<CoreCodexErrorInfo> for CodexErrorInfo {
|
||||
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
||||
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
|
||||
CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
|
||||
CoreCodexErrorInfo::ServerOverloadedBeforeInput => {
|
||||
CodexErrorInfo::ServerOverloadedBeforeInput
|
||||
}
|
||||
CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy,
|
||||
CoreCodexErrorInfo::HttpConnectionFailed { http_status_code } => {
|
||||
CodexErrorInfo::HttpConnectionFailed { http_status_code }
|
||||
|
||||
@@ -34,7 +34,9 @@ use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::CompactedItem;
|
||||
use codex_protocol::protocol::ErrorEvent;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
@@ -188,6 +190,7 @@ async fn run_compact_task_inner(
|
||||
input,
|
||||
initial_context_injection,
|
||||
compaction_metadata,
|
||||
phase,
|
||||
)
|
||||
.await;
|
||||
let status = compaction_status_from_result(&result);
|
||||
@@ -223,6 +226,7 @@ async fn run_compact_task_inner_impl(
|
||||
input: Vec<UserInput>,
|
||||
initial_context_injection: InitialContextInjection,
|
||||
compaction_metadata: CompactionTurnMetadata,
|
||||
phase: CompactionPhase,
|
||||
) -> CodexResult<String> {
|
||||
let compaction_item = TurnItem::ContextCompaction(ContextCompactionItem::new());
|
||||
sess.emit_turn_item_started(&turn_context, &compaction_item)
|
||||
@@ -311,7 +315,9 @@ async fn run_compact_task_inner_impl(
|
||||
continue;
|
||||
} else {
|
||||
sess.track_turn_codex_error(turn_context.as_ref(), &e);
|
||||
let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None));
|
||||
let event = EventMsg::Error(compaction_error_event(
|
||||
&e, phase, /*message_prefix*/ None,
|
||||
));
|
||||
sess.send_event(&turn_context, event).await;
|
||||
return Err(e);
|
||||
}
|
||||
@@ -471,6 +477,18 @@ pub(crate) fn compaction_status_from_result<T>(result: &CodexResult<T>) -> Compa
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn compaction_error_event(
|
||||
error: &CodexErr,
|
||||
phase: CompactionPhase,
|
||||
message_prefix: Option<String>,
|
||||
) -> ErrorEvent {
|
||||
let mut event = error.to_error_event(message_prefix);
|
||||
if matches!(phase, CompactionPhase::PreTurn) && matches!(error, CodexErr::ServerOverloaded) {
|
||||
event.codex_error_info = Some(CodexErrorInfo::ServerOverloadedBeforeInput);
|
||||
}
|
||||
event
|
||||
}
|
||||
|
||||
pub fn content_items_to_text(content: &[ContentItem]) -> Option<String> {
|
||||
let mut pieces = Vec::new();
|
||||
for item in content {
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::compact::CompactionAnalyticsAttempt;
|
||||
use crate::compact::CompactionAnalyticsDetails;
|
||||
use crate::compact::InitialContextInjection;
|
||||
use crate::compact::build_compaction_initial_context;
|
||||
use crate::compact::compaction_error_event;
|
||||
use crate::compact::compaction_status_from_result;
|
||||
use crate::compact::insert_initial_context_before_last_real_user_or_summary;
|
||||
use crate::compact_model_fallback::record_model_fallback;
|
||||
@@ -171,9 +172,11 @@ async fn run_remote_compact_task_inner(
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
sess.track_turn_codex_error(turn_context, &err);
|
||||
let event = EventMsg::Error(
|
||||
err.to_error_event(Some("Error running remote compact task".to_string())),
|
||||
);
|
||||
let event = EventMsg::Error(compaction_error_event(
|
||||
&err,
|
||||
phase,
|
||||
Some("Error running remote compact task".to_string()),
|
||||
));
|
||||
sess.send_event(turn_context, event).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::client_common::ResponseEvent;
|
||||
use crate::compact::CompactionAnalyticsAttempt;
|
||||
use crate::compact::CompactionAnalyticsDetails;
|
||||
use crate::compact::InitialContextInjection;
|
||||
use crate::compact::compaction_error_event;
|
||||
use crate::compact::compaction_status_from_result;
|
||||
use crate::compact_model_fallback::record_model_fallback;
|
||||
use crate::compact_remote::process_compacted_history;
|
||||
@@ -193,9 +194,11 @@ async fn run_remote_compact_task_inner(
|
||||
Err(err @ CodexErr::TurnAborted) => Err(err),
|
||||
Err(err) => {
|
||||
sess.track_turn_codex_error(turn_context, &err);
|
||||
let event = EventMsg::Error(
|
||||
err.to_error_event(Some("Error running remote compact task".to_string())),
|
||||
);
|
||||
let event = EventMsg::Error(compaction_error_event(
|
||||
&err,
|
||||
phase,
|
||||
Some("Error running remote compact task".to_string()),
|
||||
));
|
||||
sess.send_event(turn_context, event).await;
|
||||
Err(err)
|
||||
}
|
||||
|
||||
@@ -926,6 +926,7 @@ fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool {
|
||||
GuardianReviewError::Session {
|
||||
error_info: Some(
|
||||
CodexErrorInfo::ServerOverloaded
|
||||
| CodexErrorInfo::ServerOverloadedBeforeInput
|
||||
| CodexErrorInfo::HttpConnectionFailed { .. }
|
||||
| CodexErrorInfo::ResponseStreamConnectionFailed { .. }
|
||||
| CodexErrorInfo::InternalServerError
|
||||
@@ -981,6 +982,7 @@ mod review_tests {
|
||||
};
|
||||
let transient_error_info = [
|
||||
CodexErrorInfo::ServerOverloaded,
|
||||
CodexErrorInfo::ServerOverloadedBeforeInput,
|
||||
CodexErrorInfo::HttpConnectionFailed {
|
||||
http_status_code: Some(502),
|
||||
},
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::client::ModelClientSession;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::util::backoff;
|
||||
use crate::util::jitter;
|
||||
use crate::util::positive_jitter;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
@@ -67,9 +67,9 @@ pub(crate) async fn handle_retryable_response_stream_error(
|
||||
*retries += 1;
|
||||
let retry_count = *retries;
|
||||
let delay = match &err {
|
||||
CodexErr::ServerOverloaded => {
|
||||
jitter(SERVER_OVERLOADED_RETRY_DELAYS[retry_count.saturating_sub(1) as usize])
|
||||
}
|
||||
CodexErr::ServerOverloaded => positive_jitter(
|
||||
SERVER_OVERLOADED_RETRY_DELAYS[retry_count.saturating_sub(1) as usize],
|
||||
),
|
||||
CodexErr::Stream(_, requested_delay) => {
|
||||
requested_delay.unwrap_or_else(|| backoff(retry_count))
|
||||
}
|
||||
|
||||
@@ -164,7 +164,11 @@ pub(crate) async fn run_turn(
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
let error = if matches!(err, CodexErr::ServerOverloaded) {
|
||||
CodexErrorInfo::ServerOverloadedBeforeInput
|
||||
} else {
|
||||
err.to_codex_protocol_error()
|
||||
};
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
error!("Failed to run pre-sampling compact");
|
||||
|
||||
@@ -93,6 +93,12 @@ pub(crate) fn jitter(delay: Duration) -> Duration {
|
||||
delay.mul_f64(jitter)
|
||||
}
|
||||
|
||||
/// Adds positive jitter without allowing a retry earlier than the base delay.
|
||||
pub(crate) fn positive_jitter(delay: Duration) -> Duration {
|
||||
let jitter = rand::rng().random_range(1.0..2.0);
|
||||
delay.mul_f64(jitter)
|
||||
}
|
||||
|
||||
pub(crate) fn error_or_panic(message: impl std::string::ToString) {
|
||||
if cfg!(debug_assertions) {
|
||||
panic!("{}", message.to_string());
|
||||
|
||||
@@ -3587,7 +3587,7 @@ async fn server_overload_retries_use_a_separate_budget_on_the_same_turn() -> any
|
||||
if event.codex_error_info == Some(CodexErrorInfo::ServerOverloaded) {
|
||||
server_overload_retry_messages.push(event.message);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(360)).await;
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
}
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
|
||||
@@ -14,6 +14,7 @@ use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
use codex_protocol::openai_models::ModelsResponse;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookRunStatus;
|
||||
@@ -746,6 +747,86 @@ async fn local_compact_keeps_provider_server_overload_retries() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_pre_turn_capacity_exhaustion_marks_input_uncommitted() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let overloaded = || {
|
||||
sse_response(sse_failed(
|
||||
"compact-overloaded",
|
||||
"server_is_overloaded",
|
||||
"selected model is at capacity",
|
||||
))
|
||||
};
|
||||
let responses = mount_response_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse_response(sse(vec![
|
||||
ev_assistant_message("m1", FIRST_REPLY),
|
||||
ev_completed_with_tokens("r1", /*total_tokens*/ 500),
|
||||
])),
|
||||
overloaded(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut model_provider = non_openai_model_provider(&server);
|
||||
model_provider.request_max_retries = Some(0);
|
||||
model_provider.stream_max_retries = Some(0);
|
||||
let codex = test_codex()
|
||||
.with_config(move |config| {
|
||||
config.model_provider = model_provider;
|
||||
set_test_compact_prompt(config);
|
||||
config.model_auto_compact_token_limit = Some(200);
|
||||
})
|
||||
.build(&server)
|
||||
.await?
|
||||
.codex;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "first turn".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let incoming = "prompt that must survive local capacity exhaustion";
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: incoming.into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
|
||||
let error_info = wait_for_event_match(&codex, |event| match event {
|
||||
EventMsg::Error(error) => error.codex_error_info.clone(),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(error_info, CodexErrorInfo::ServerOverloadedBeforeInput);
|
||||
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let requests = responses.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in &requests[1..] {
|
||||
assert!(!request.body_json().to_string().contains(incoming));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn manual_pre_compact_block_decision_does_not_block_compaction() {
|
||||
skip_if_no_network!();
|
||||
|
||||
@@ -1108,7 +1108,7 @@ async fn remote_compact_v2_retries_capacity_after_stream_retry_budget() -> Resul
|
||||
if event.codex_error_info == Some(CodexErrorInfo::ServerOverloaded) {
|
||||
server_overload_retry_messages.push(event.message);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(360)).await;
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
}
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
@@ -2214,6 +2214,205 @@ async fn auto_remote_compact_failure_stops_agent_loop() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_turn_capacity_exhaustion_marks_input_uncommitted() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let harness = TestCodexHarness::with_builder(
|
||||
test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(|config| {
|
||||
let _ = config.features.enable(Feature::RemoteCompactionV2);
|
||||
config.model_auto_compact_token_limit = Some(120);
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
let codex = harness.test().codex.clone();
|
||||
|
||||
let overloaded = || {
|
||||
ResponseTemplate::new(503).set_body_json(serde_json::json!({
|
||||
"error": {
|
||||
"code": "server_is_overloaded",
|
||||
"message": "selected model is at capacity"
|
||||
}
|
||||
}))
|
||||
};
|
||||
let responses_mock = responses::mount_response_sequence(
|
||||
harness.server(),
|
||||
vec![
|
||||
responses::sse_response(sse(vec![
|
||||
responses::ev_assistant_message("initial-assistant", "initial turn complete"),
|
||||
responses::ev_completed_with_tokens(
|
||||
"initial-response",
|
||||
/*total_tokens*/ 500_000,
|
||||
),
|
||||
])),
|
||||
overloaded(),
|
||||
overloaded(),
|
||||
overloaded(),
|
||||
overloaded(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "turn that exceeds token threshold".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
wait_for_turn_complete(&codex).await;
|
||||
|
||||
tokio::time::pause();
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "prompt that must survive capacity exhaustion".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
|
||||
let mut terminal_error_info = None;
|
||||
loop {
|
||||
match codex.next_event().await?.msg {
|
||||
EventMsg::StreamError(_) => tokio::time::advance(Duration::from_secs(600)).await,
|
||||
EventMsg::Error(event) => terminal_error_info = event.codex_error_info,
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
tokio::time::resume();
|
||||
|
||||
assert_eq!(
|
||||
terminal_error_info,
|
||||
Some(CodexErrorInfo::ServerOverloadedBeforeInput)
|
||||
);
|
||||
let requests = responses_mock.requests();
|
||||
assert_eq!(requests.len(), 5);
|
||||
for request in &requests[1..] {
|
||||
let body = request.body_json().to_string();
|
||||
assert!(body.contains("\"type\":\"compaction_trigger\""));
|
||||
assert!(!body.contains("prompt that must survive capacity exhaustion"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_turn_capacity_recovery_records_incoming_input_once() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let harness = TestCodexHarness::with_builder(
|
||||
test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(|config| {
|
||||
let _ = config.features.enable(Feature::RemoteCompactionV2);
|
||||
config.model_auto_compact_token_limit = Some(120);
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
let codex = harness.test().codex.clone();
|
||||
|
||||
let overloaded = || {
|
||||
ResponseTemplate::new(503).set_body_json(serde_json::json!({
|
||||
"error": {
|
||||
"code": "server_is_overloaded",
|
||||
"message": "selected model is at capacity"
|
||||
}
|
||||
}))
|
||||
};
|
||||
let responses_mock = responses::mount_response_sequence(
|
||||
harness.server(),
|
||||
vec![
|
||||
responses::sse_response(sse(vec![
|
||||
responses::ev_assistant_message("initial-assistant", "initial turn complete"),
|
||||
responses::ev_completed_with_tokens(
|
||||
"initial-response",
|
||||
/*total_tokens*/ 500_000,
|
||||
),
|
||||
])),
|
||||
overloaded(),
|
||||
overloaded(),
|
||||
overloaded(),
|
||||
responses::sse_response(responses::sse(vec![
|
||||
serde_json::json!({
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "compaction",
|
||||
"encrypted_content": "RECOVERED_COMPACT_SUMMARY",
|
||||
}
|
||||
}),
|
||||
responses::ev_completed("resp-compact-recovered"),
|
||||
])),
|
||||
responses::sse_response(responses::sse(vec![
|
||||
responses::ev_assistant_message("after-retry", "done"),
|
||||
responses::ev_completed("resp-after-retry"),
|
||||
])),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "turn that exceeds token threshold".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
wait_for_turn_complete(&codex).await;
|
||||
|
||||
let incoming = "prompt recorded exactly once after capacity recovery";
|
||||
tokio::time::pause();
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: incoming.into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
loop {
|
||||
match codex.next_event().await?.msg {
|
||||
EventMsg::StreamError(_) => tokio::time::advance(Duration::from_secs(600)).await,
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
tokio::time::resume();
|
||||
|
||||
let requests = responses_mock.requests();
|
||||
assert_eq!(requests.len(), 6);
|
||||
for compact_request in &requests[1..5] {
|
||||
let body = compact_request.body_json().to_string();
|
||||
assert!(body.contains("\"type\":\"compaction_trigger\""));
|
||||
assert!(!body.contains(incoming));
|
||||
}
|
||||
let sampling_body = requests[5].body_json().to_string();
|
||||
assert_eq!(sampling_body.matches(incoming).count(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg_attr(target_os = "windows", ignore)]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result<()> {
|
||||
|
||||
@@ -1741,6 +1741,8 @@ pub enum CodexErrorInfo {
|
||||
SessionBudgetExceeded,
|
||||
UsageLimitExceeded,
|
||||
ServerOverloaded,
|
||||
/// Capacity failure before the incoming user input was recorded in thread history.
|
||||
ServerOverloadedBeforeInput,
|
||||
CyberPolicy,
|
||||
HttpConnectionFailed {
|
||||
http_status_code: Option<u16>,
|
||||
@@ -1779,6 +1781,7 @@ impl CodexErrorInfo {
|
||||
| Self::SessionBudgetExceeded
|
||||
| Self::UsageLimitExceeded
|
||||
| Self::ServerOverloaded
|
||||
| Self::ServerOverloadedBeforeInput
|
||||
| Self::CyberPolicy
|
||||
| Self::HttpConnectionFailed { .. }
|
||||
| Self::ResponseStreamConnectionFailed { .. }
|
||||
|
||||
@@ -138,7 +138,10 @@ pub(super) fn app_server_rate_limit_error_kind(
|
||||
info: &AppServerCodexErrorInfo,
|
||||
) -> Option<RateLimitErrorKind> {
|
||||
match info {
|
||||
AppServerCodexErrorInfo::ServerOverloaded => Some(RateLimitErrorKind::ServerOverloaded),
|
||||
AppServerCodexErrorInfo::ServerOverloaded
|
||||
| AppServerCodexErrorInfo::ServerOverloadedBeforeInput => {
|
||||
Some(RateLimitErrorKind::ServerOverloaded)
|
||||
}
|
||||
AppServerCodexErrorInfo::UsageLimitExceeded => Some(RateLimitErrorKind::UsageLimit),
|
||||
AppServerCodexErrorInfo::ResponseTooManyFailedAttempts {
|
||||
http_status_code: Some(429),
|
||||
|
||||
Reference in New Issue
Block a user