mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Honor Retry-After and preserve server retry deadlines (#47641)
## Why HTTP retries used local backoff even when the server supplied `Retry-After`. Relative retry delays also failed to account for time spent propagating errors or reporting retries. ## What changed - Parse `Retry-After` delay seconds and HTTP dates into a monotonic deadline captured when response headers arrive. - Honor that deadline for HTTP retries, falling back to local backoff when valid advice is absent. - Preserve deadlines through API and Bedrock error mapping, stream retries, and exhausted retry state. Expired advice remains a zero delay instead of triggering local backoff. ## Testing Add coverage for header parsing, deadline countdown and expiry, error mapping after exhausted HTTP retries, and delayed stream retry notifications. Update Responses and remote compaction tests to verify that HTTP retries honor `Retry-After`. GitOrigin-RevId: a2eff02946e5524f29e06f7ba1431e3d339261a4
This commit is contained in:
committed by
copyberry
parent
ab021e9f13
commit
9d8de19674
@@ -214,6 +214,7 @@ async fn workspace_routed_http_rejects_redirects_without_a_routing_header() {
|
||||
matches!(
|
||||
result,
|
||||
Err(TransportError::Http {
|
||||
retry_after: None,
|
||||
status: http::StatusCode::TEMPORARY_REDIRECT,
|
||||
..
|
||||
})
|
||||
@@ -1411,6 +1412,7 @@ async fn bedrock_unauthorized_error_uses_provider_mapping() {
|
||||
let url = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses";
|
||||
let error = super::handle_unauthorized(
|
||||
TransportError::Http {
|
||||
retry_after: None,
|
||||
status: http::StatusCode::UNAUTHORIZED,
|
||||
url: Some(url.to_string()),
|
||||
headers: None,
|
||||
@@ -1504,6 +1506,7 @@ async fn provider_owned_auth_recovery_is_bounded_and_preserves_unauthorized_fail
|
||||
assert!(provider.auth_manager().is_none());
|
||||
|
||||
let unauthorized = || TransportError::Http {
|
||||
retry_after: None,
|
||||
status: http::StatusCode::UNAUTHORIZED,
|
||||
url: Some("https://example.com/v1/responses".to_string()),
|
||||
headers: None,
|
||||
|
||||
@@ -7,10 +7,12 @@ use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use codex_client::RetryOperation;
|
||||
use codex_features::Feature;
|
||||
use codex_http_client::RetryAfter;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
use tokio::time::Instant;
|
||||
use tracing::warn;
|
||||
|
||||
const INITIAL_CONNECTION_RETRY_DELAY: Duration = Duration::from_secs(5);
|
||||
@@ -64,6 +66,7 @@ pub(crate) async fn handle_response_stream_error(
|
||||
let Some(delay) = err.retry_delay(retry_count) else {
|
||||
return Err(err);
|
||||
};
|
||||
let retry_after = err.retry_after();
|
||||
|
||||
if turn_context
|
||||
.config
|
||||
@@ -129,8 +132,12 @@ pub(crate) async fn handle_response_stream_error(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let retry_at = retry_after
|
||||
.map(RetryAfter::deadline)
|
||||
.unwrap_or_else(|| Instant::now() + delay);
|
||||
let delay = retry_at.saturating_duration_since(Instant::now());
|
||||
codex_client::record_retry!(retry_count, delay, operation);
|
||||
tokio::time::sleep(delay).await;
|
||||
tokio::time::sleep_until(retry_at).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -138,9 +145,7 @@ pub(crate) async fn handle_response_stream_error(
|
||||
.thread_extension_data
|
||||
.insert(ExhaustedResponseRetry {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
retry_at: err
|
||||
.server_retry_delay()
|
||||
.and_then(|delay| tokio::time::Instant::now().checked_add(delay)),
|
||||
retry_at: retry_after.map(RetryAfter::deadline),
|
||||
});
|
||||
Err(err)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
use super::ResponsesStreamRequest;
|
||||
use super::ResponsesStreamRetryState;
|
||||
use super::handle_response_stream_error;
|
||||
use super::log_retry;
|
||||
use crate::realtime_history::RealtimeHistoryState;
|
||||
use crate::session::tests::make_session_and_context;
|
||||
use codex_http_client::RetryAfter;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::Instant;
|
||||
use tracing_test::internal::MockWriter;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -41,3 +47,51 @@ async fn sampling_retry_logs_stream_error_context() {
|
||||
"sampling_error=stream disconnected before completion: websocket closed by server before response.completed"
|
||||
));
|
||||
}
|
||||
|
||||
/// Time spent reporting a retry must count toward the server's original deadline.
|
||||
#[tokio::test]
|
||||
#[expect(
|
||||
clippy::await_holding_invalid_type,
|
||||
reason = "test holds the event-delivery lock to delay notification while virtual time advances"
|
||||
)]
|
||||
async fn stream_retry_preserves_deadline_across_delayed_notification() {
|
||||
let (mut session, turn_context) = make_session_and_context().await;
|
||||
session.realtime_history = Some(Mutex::new(RealtimeHistoryState::default()));
|
||||
let mut client_session = session.services.model_client.new_session();
|
||||
// The second retry emits a notification even when release builds hide the first.
|
||||
let mut retry_state = ResponsesStreamRetryState {
|
||||
retries: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
tokio::time::pause();
|
||||
let advice = RetryAfter::from_delay(Duration::from_secs(10)).expect("retry deadline");
|
||||
tokio::time::advance(Duration::from_secs(4)).await;
|
||||
|
||||
// Hold the event-delivery lock so reporting the error consumes part of the deadline.
|
||||
let history_guard = session.realtime_history.as_ref().unwrap().lock().await;
|
||||
let retry = handle_response_stream_error(
|
||||
&mut retry_state,
|
||||
/*max_retries*/ 2,
|
||||
CodexErr::InternalServerError.with_retry_after(advice),
|
||||
&mut client_session,
|
||||
&session,
|
||||
&turn_context,
|
||||
ResponsesStreamRequest::Sampling,
|
||||
);
|
||||
tokio::pin!(retry);
|
||||
assert!(futures::poll!(&mut retry).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(4)).await;
|
||||
drop(history_guard);
|
||||
|
||||
assert!(futures::poll!(&mut retry).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert!(futures::poll!(&mut retry).is_pending());
|
||||
retry.await.expect("retry should be allowed");
|
||||
// Tokio rounds timer deadlines up to the next millisecond.
|
||||
let resumed_at = Instant::now();
|
||||
assert!(
|
||||
(advice.deadline()..=advice.deadline() + Duration::from_millis(1)).contains(&resumed_at),
|
||||
"retry resumed at {resumed_at:?}, expected {advice:?}"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user