Retry transient exec-server connection failures (#62601)

## Why

A remote exec-server connection could fail during setup if the transport
closed before the client received the initialization response. Callers
waiting for readiness received the failure even if a new connection
could succeed moments later.

## What changed

Share retries among callers during startup, reconnect, and connection
refresh. After a retryable transport or initialization failure, use
backoff and allow new attempts for up to five seconds. Attempts already
underway can finish after that window.

Keep the registry's existing retry policy and reuse the provisioning
deadline across transport retries.

## Testing

Add regression coverage for shared startup recovery after a WebSocket
closes during initialization, registry retry limits and authentication
failures, and child-process cleanup when stdio initialization is
cancelled.

GitOrigin-RevId: 56004badb67f4e5df8d939f2bc6cdfa46246fdb0
This commit is contained in:
richardopenai
2026-11-19 21:44:07 +00:00
committed by copyberry
parent 665e0a9a65
commit 93000976e8
7 changed files with 345 additions and 36 deletions
+153 -13
View File
@@ -564,7 +564,7 @@ impl LazyRemoteExecServerClient {
} else {
self.startup
.result
.get_or_init(|| self.connect_once(&self.startup))
.get_or_init(|| self.connect_with_retry(&self.startup))
.await
.clone()
.map_err(ExecServerError::ConnectionAttempt)
@@ -599,7 +599,7 @@ impl LazyRemoteExecServerClient {
};
let result = attempt
.result
.get_or_init(|| self.connect_once(&attempt))
.get_or_init(|| self.connect_with_retry(&attempt))
.await;
let mut reconnect = self
.reconnect
@@ -2373,6 +2373,7 @@ mod tests {
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
/*started*/ &mut None,
/*provisioning_deadline*/ &mut None,
)
.await
.expect("stdio transport should connect");
@@ -2404,6 +2405,39 @@ mod tests {
assert_eq!(client.session_id().as_deref(), Some("stdio-test"));
}
#[cfg(unix)]
#[tokio::test]
async fn cancelling_stdio_initialization_terminates_spawned_process() {
let tempdir = tempfile::tempdir().expect("tempdir should be created");
let pid_file = tempdir.path().join("server.pid");
let script = format!(
"read _line; echo \"$$\" > {}; read _line",
shell_quote(pid_file.as_path()),
);
let client = LazyRemoteExecServerClient::new(
ExecServerTransportParams::StdioCommand {
command: StdioExecServerCommand {
program: "sh".to_string(),
args: vec!["-c".to_string(), script],
env: HashMap::new(),
cwd: None,
},
initialize_timeout: Duration::from_secs(60),
},
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
);
let initializing = tokio::spawn({
let client = client.clone();
async move { client.get().await }
});
let pid = read_pid_file(&pid_file).await;
assert!(process_exists(pid));
initializing.abort();
let _ = initializing.await;
wait_for_process_exit(pid).await;
assert!(!client.startup_finished());
}
#[cfg(unix)]
#[tokio::test]
async fn dropping_stdio_client_terminates_spawned_process() {
@@ -3403,14 +3437,21 @@ mod tests {
listener.local_addr().expect("listener should have address")
);
let (replacement_initialized_tx, replacement_initialized_rx) = oneshot::channel();
let (allow_replacement_tx, allow_replacement_rx) = watch::channel(false);
let server = tokio::spawn(async move {
let (mut failed_startup, _) = listener.accept().await.expect("startup should arrive");
failed_startup
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
let replacement = loop {
let (mut stream, _) = listener.accept().await.expect("startup should arrive");
if *allow_replacement_rx.borrow() {
break stream;
}
stream
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
.await
.expect("failed handshake response should write");
};
let mut replacement = accept_async(replacement)
.await
.expect("failed handshake response should write");
let mut replacement = accept_websocket(&listener).await;
.expect("replacement websocket");
complete_websocket_initialize(
&mut replacement,
"replacement-session",
@@ -3459,6 +3500,7 @@ mod tests {
EnvironmentObservedStatus::Disconnected { .. }
));
allow_replacement_tx.send(true).expect("allow replacement");
let (ready, first, second) =
tokio::join!(client.wait_until_ready(), client.get(), client.get());
ready.expect("later readiness check should retry startup");
@@ -3470,12 +3512,16 @@ mod tests {
.await
.expect("server should observe replacement initialization");
let outcomes = std::iter::from_fn(|| observed_rx.try_recv().ok()).collect::<Vec<_>>();
assert_eq!(
std::iter::from_fn(|| observed_rx.try_recv().ok()).collect::<Vec<_>>(),
[
crate::ConnectionAttemptOutcome::Failure,
crate::ConnectionAttemptOutcome::Success
],
outcomes.last(),
Some(&crate::ConnectionAttemptOutcome::Success)
);
assert!(outcomes.len() >= 2);
assert!(
outcomes[..outcomes.len() - 1]
.iter()
.all(|outcome| *outcome == crate::ConnectionAttemptOutcome::Failure)
);
drop(first);
@@ -3484,6 +3530,96 @@ mod tests {
server.await.expect("server task should finish");
}
#[tokio::test]
async fn startup_retry_is_shared_after_initialize_transport_closes() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("listener should bind");
let websocket_url = format!("ws://{}", listener.local_addr().expect("listener address"));
let (initialized_tx, initialized_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let mut failed = accept_websocket(&listener).await;
let initialize = read_jsonrpc_websocket(&mut failed).await;
assert!(
matches!(initialize, JSONRPCMessage::Request(request) if request.method == INITIALIZE_METHOD)
);
// The executor may have received initialize, but the caller never sees its response.
drop(failed);
let mut replacement = accept_websocket(&listener).await;
complete_websocket_initialize(
&mut replacement,
"replacement-session",
/*expected_resume_session_id*/ None,
)
.await;
initialized_tx.send(()).expect("replacement initialized");
timeout(Duration::from_secs(1), replacement.next())
.await
.expect("client should close");
});
let client = LazyRemoteExecServerClient::new(
ExecServerTransportParams::WebSocketUrl {
websocket_url,
connect_timeout: Duration::from_secs(1),
initialize_timeout: Duration::from_secs(1),
http_headers: HeaderMap::new(),
},
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
);
let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
assert!(
client
.connection_observer
.set(Arc::new(move |_, outcome| {
observed_tx
.send(outcome)
.expect("record connection outcome");
}))
.is_ok()
);
let background = client.start_connecting().expect("background startup task");
let initiator = tokio::spawn({
let client = client.clone();
async move { client.get().await }
});
assert_eq!(
observed_rx.recv().await,
Some(crate::ConnectionAttemptOutcome::Failure)
);
assert!(!client.startup_finished());
assert!(client.readiness_result().is_none());
assert!(matches!(
client.status().await,
EnvironmentObservedStatus::Pending
));
initiator.abort();
let _ = initiator.await;
timeout(Duration::from_secs(2), initialized_rx)
.await
.expect("startup should progress without any waiter")
.expect("server observed initialization");
let (ready, first, second) =
tokio::join!(client.wait_until_ready(), client.get(), client.get());
ready.expect("startup should recover");
let first = first.expect("first waiter should connect");
let second = second.expect("second waiter should connect");
assert_eq!(first.session_id().as_deref(), Some("replacement-session"));
assert!(Arc::ptr_eq(&first.inner, &second.inner));
assert!(client.startup_finished());
assert_eq!(
observed_rx.recv().await,
Some(crate::ConnectionAttemptOutcome::Success)
);
assert!(observed_rx.try_recv().is_err());
drop(first);
drop(second);
drop(background);
drop(client);
server.await.expect("server task should finish");
}
#[tokio::test]
async fn failed_reconnect_does_not_burn_environment() {
let listener = TcpListener::bind("127.0.0.1:0")
@@ -3553,6 +3689,10 @@ mod tests {
})
.await
.expect("client should observe disconnect");
timeout(Duration::from_millis(50), client.wait_until_ready())
.await
.expect("historical startup readiness must not reconnect")
.expect("initial startup succeeded");
let failed_reconnect = match client.get().await {
Ok(_) => panic!("first lazy reconnect should fail"),
Err(error) => error,
+54 -4
View File
@@ -19,11 +19,13 @@
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::time::Duration;
use futures::future::BoxFuture;
use tokio::sync::OnceCell;
use tokio::sync::watch;
use tokio::time::Instant;
use tokio::time::sleep;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
@@ -34,6 +36,9 @@ use super::ExecServerError;
use super::Inner;
use super::LazyRemoteExecServerClient;
use super::fail_all_in_flight_work;
use super::is_retryable_registry_error;
use super::is_transport_closed_error;
use super::registry_recovery_retry_delay;
use crate::ConnectionAttemptOutcome;
use crate::EnvironmentConnectionState;
use crate::NoiseChannelPublicKey;
@@ -43,6 +48,8 @@ use crate::client_api::NoiseRendezvousConnectBundle;
use crate::client_api::NoiseRendezvousConnectProvider;
use crate::client_transport::ExecServerReconnectStrategy;
const STARTUP_TRANSPORT_RETRY_TIMEOUT: Duration = Duration::from_secs(5);
/// Shared startup/reconnect result plus cancellation for work superseded by refresh.
/// The optional transport carries a refresh lookup's bundle into the normal connector.
#[derive(Default)]
@@ -201,7 +208,7 @@ impl LazyRemoteExecServerClient {
}
let result = attempt
.result
.get_or_init(|| self.connect_once(&attempt))
.get_or_init(|| self.connect_with_retry(&attempt))
.await
.clone();
let mut reconnect = self
@@ -224,10 +231,46 @@ impl LazyRemoteExecServerClient {
client.environment_status().await.map(drop)
}
#[tracing::instrument(name = "codex.exec_server.remote.connect", skip_all)]
pub(super) fn connect_once<'a>(
/// Run one shared setup operation. Registry failures have their own bounded retry policy;
/// only transient transport or initialization failures admit another physical attempt.
pub(super) fn connect_with_retry<'a>(
&'a self,
attempt: &'a ConnectionAttempt,
) -> BoxFuture<'a, ConnectionResult> {
Box::pin(async move {
let mut provisioning_deadline = None;
let mut result = self.connect_once(attempt, &mut provisioning_deadline).await;
if !self.can_reconnect() || !retry_startup_transport(&result) {
return result;
}
let deadline = Instant::now() + STARTUP_TRANSPORT_RETRY_TIMEOUT;
let retry_key = uuid::Uuid::new_v4().to_string();
let mut retry = 0;
loop {
let delay = registry_recovery_retry_delay(&retry_key, retry);
retry = retry.saturating_add(1);
tokio::select! {
biased;
_ = attempt.cancelled.cancelled() => return Err(Arc::new(ExecServerError::Disconnected("connection attempt was superseded".to_string()))),
_ = sleep(delay.min(deadline.saturating_duration_since(Instant::now()))) => {},
}
if Instant::now() >= deadline {
return result;
}
// Bound retry admission, not the duration of a registry or transport phase.
result = self.connect_once(attempt, &mut provisioning_deadline).await;
if !retry_startup_transport(&result) {
return result;
}
}
})
}
#[tracing::instrument(name = "codex.exec_server.remote.connect", skip_all)]
fn connect_once<'a>(
&'a self,
attempt: &'a ConnectionAttempt,
provisioning_deadline: &'a mut Option<Instant>,
) -> BoxFuture<'a, ConnectionResult> {
// Keep the transport future out of every caller's async layout, including
// the CLI entry point, which otherwise exceeds rustc's query-depth limit.
@@ -251,7 +294,7 @@ impl LazyRemoteExecServerClient {
let result = tokio::select! {
biased;
_ = attempt.cancelled.cancelled() => return Err(Arc::new(ExecServerError::Disconnected("connection attempt was superseded".to_string()))),
result = ExecServerClient::connect_for_transport(transport.clone(), self.http_client_factory.clone(), &mut observation.started) => result,
result = ExecServerClient::connect_for_transport(transport.clone(), self.http_client_factory.clone(), &mut observation.started, provisioning_deadline) => result,
};
let client = result.map_err(|error| {
observation.outcome = ConnectionAttemptOutcome::Failure;
@@ -281,6 +324,13 @@ impl LazyRemoteExecServerClient {
}
}
fn retry_startup_transport(result: &ConnectionResult) -> bool {
result.as_ref().is_err_and(|error| {
!is_retryable_registry_error(error)
&& (error.is_retryable_preparation_error() || is_transport_closed_error(error))
})
}
impl Inner {
async fn retire(self: &Arc<Self>) {
let message = "exec-server executor was replaced".to_string();
@@ -697,8 +697,13 @@ async fn failed_replacement_connection_keeps_old_handles_retired_and_get_retries
drop(unavailable.accept().await.unwrap());
}));
*registry.target.lock().unwrap() = target;
// The refresh now exhausts the transport retry window. Resume time to let it expire,
// then freeze it again for the following Noise handshake.
tokio::time::resume();
let refresh = client.refresh_connection().await;
tokio::time::pause();
assert!(matches!(
client.refresh_connection().await,
refresh,
Err(ExecServerError::ConnectionAttempt(_))
));
assert!(original.inner.retired.is_cancelled());
+21 -17
View File
@@ -361,6 +361,7 @@ impl ExecServerClient {
transport_params: ExecServerTransportParams,
http_client_factory: HttpClientFactory,
started: &mut Option<Instant>,
provisioning_deadline: &mut Option<Instant>,
) -> Result<Self, ExecServerError> {
let (transport_params, deferred_readiness) = match transport_params {
ExecServerTransportParams::Deferred(deferred) => {
@@ -369,28 +370,31 @@ impl ExecServerClient {
transport_params => (transport_params, None),
};
let provisioning_deadline = if let Some(mut readiness) = deferred_readiness {
let provisioning_result = readiness
.wait_for(Option::is_some)
.await
.map_err(|_| {
ExecServerError::Disconnected(
let effective_provisioning_deadline =
if let Some(mut readiness) = deferred_readiness {
let provisioning_result = readiness
.wait_for(Option::is_some)
.await
.map_err(|_| {
ExecServerError::Disconnected(
"environment unavailable: environment provisioning ended before completion"
.to_string(),
)
})?
.clone()
.ok_or_else(|| {
ExecServerError::Disconnected(
})?
.clone()
.ok_or_else(|| {
ExecServerError::Disconnected(
"environment unavailable: provisioning remained pending after completion"
.to_string(),
)
})?;
provisioning_result.map_err(ExecServerError::ProvisioningFailed)?;
Some(Instant::now() + PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT)
} else {
None
};
})?;
provisioning_result.map_err(ExecServerError::ProvisioningFailed)?;
Some(*provisioning_deadline.get_or_insert_with(|| {
Instant::now() + PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT
}))
} else {
None
};
*started = Some(Instant::now());
let websocket = match transport_params {
@@ -415,7 +419,7 @@ impl ExecServerClient {
&provider,
&identity,
http_client_factory.clone(),
provisioning_deadline,
effective_provisioning_deadline,
)
.await?;
let reconnect_strategy = ExecServerReconnectStrategy::NoiseRendezvous {
@@ -129,6 +129,7 @@ impl SequenceNoiseConnectProvider {
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
/*started*/ &mut None,
/*provisioning_deadline*/ &mut None,
)
.await
}
@@ -448,6 +449,113 @@ async fn deferred_initialize_timeout_reports_configured_budget() {
let _ = server.await;
}
#[tokio::test(start_paused = true)]
async fn shared_startup_does_not_restart_exhausted_registry_retries() -> Result<()> {
let provider = Arc::new(SequenceNoiseConnectProvider::default());
for _ in 0..=INITIAL_REGISTRY_MAX_RETRIES {
provider.push_error(registry_error(
http::StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
));
}
let client = crate::client::LazyRemoteExecServerClient::new(
ExecServerTransportParams::NoiseRendezvous {
provider: provider.clone(),
identity: NoiseChannelIdentity::generate()?,
},
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
);
let started = tokio::time::Instant::now();
let (ready, first, second) =
tokio::join!(client.wait_until_ready(), client.get(), client.get());
assert!(ready.is_err());
assert!(first.is_err());
assert!(second.is_err());
assert_eq!(
provider.requested_keys().len(),
INITIAL_REGISTRY_MAX_RETRIES as usize + 1
);
assert!(started.elapsed() <= INITIAL_REGISTRY_OPERATION_TIMEOUT);
Ok(())
}
#[tokio::test]
async fn startup_retry_allows_registry_phase_to_finish_after_grace() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let provider = Arc::new(SequenceNoiseConnectProvider::default());
let bundle = test_bundle(format!("ws://{}", listener.local_addr()?))?;
provider.push_response(async move { Ok(bundle) });
provider.push_response(async {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
Err(ExecServerError::Protocol(
"registry phase finished".to_string(),
))
});
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
socket
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
.await
.unwrap();
});
let client = crate::client::LazyRemoteExecServerClient::new(
ExecServerTransportParams::NoiseRendezvous {
provider: provider.clone(),
identity: NoiseChannelIdentity::generate()?,
},
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
);
let error = client.get().await.err().expect("registry error");
assert!(
error.to_string().contains("registry phase finished"),
"{error}"
);
assert_eq!(provider.requested_keys().len(), 2);
server.await?;
Ok(())
}
#[tokio::test]
async fn initial_registry_auth_failure_remains_shared_by_readiness_and_get() -> Result<()> {
let provider = Arc::new(SequenceNoiseConnectProvider::default());
provider.push_error(ExecServerError::EnvironmentRegistryAuth(
"expired".to_string(),
));
let client = crate::client::LazyRemoteExecServerClient::new(
ExecServerTransportParams::NoiseRendezvous {
provider: provider.clone(),
identity: NoiseChannelIdentity::generate()?,
},
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
);
assert!(
client
.wait_until_ready()
.await
.err()
.unwrap()
.to_string()
.contains("expired")
);
assert!(
client
.get()
.await
.err()
.unwrap()
.to_string()
.contains("expired")
);
assert_eq!(provider.requested_keys().len(), 1);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn initial_noise_connection_bounds_offline_retries() -> Result<()> {
let sequence = Arc::new(SequenceNoiseConnectProvider::default());
+2
View File
@@ -1883,6 +1883,7 @@ mod tests {
let environment = manager
.get_environment("executor-a")
.expect("first remote environment");
let retained_filesystem = environment.get_filesystem();
let (observed_tx, mut observed_rx) = tokio::sync::mpsc::unbounded_channel();
environment.observe_connection_attempts(move |_, outcome| {
observed_tx
@@ -1923,6 +1924,7 @@ mod tests {
})
.await
.expect("replacing the environment should cancel its startup task");
drop(retained_filesystem);
assert_eq!(
observed_rx.try_recv(),
Ok(super::ConnectionAttemptOutcome::Cancelled)