From 1c2e4cd45f15ea7d9576d638f6a1112b8fa58f77 Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Wed, 18 Nov 2026 01:21:42 +0000 Subject: [PATCH] Limit recovery after repeated cloud reply stream disconnects (#62151) ## Why A cloud reply stream can drop again shortly after reconnecting, leaving users in a cycle of disconnects and recovery. ## What changed Delay recovery by 500 ms for each replacement stream already installed in the current recovery sequence. After three replacements, return an error that tells users their transcript and draft are still available. Clear the count when recovery state is reset or when a stream disconnects after staying connected for at least 15 seconds. GitOrigin-RevId: 6ae2e88023464fc41d8e26d47dd94af624660d5f --- .../app-server-client/src/cloud_backend/idle.rs | 15 ++++++++++++++- .../app-server-client/src/cloud_backend/mod.rs | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/codex-rs/app-server-client/src/cloud_backend/idle.rs b/codex-rs/app-server-client/src/cloud_backend/idle.rs index 5fc100e629..3ae171dc07 100644 --- a/codex-rs/app-server-client/src/cloud_backend/idle.rs +++ b/codex-rs/app-server-client/src/cloud_backend/idle.rs @@ -129,6 +129,7 @@ pub(super) struct Recovery { pub(super) pending: Option<(Arc, Open)>, pub(super) installed: Option, pub(super) held: Option<(String, Instant, BoxFuture<'static, Option>)>, + pub(super) attempts: u8, // None until either the initial Attach event or the activation snapshot establishes status. pub(super) observed_idle: Option, pub(super) client: Arc, @@ -176,12 +177,22 @@ impl Recovery { )))); return; } + let attempts = self.attempts; + if attempts >= 3 { + let _ = response.send(Ok(Err(recovery_error( + "The cloud reply stream keeps disconnecting. Your transcript and draft are still here.", + )))); + return; + } self.cancel(); let client = Arc::clone(&self.client); let login = Arc::clone(&self.login); let root = thread_id.clone(); let open = async move { - let attempt = login.permit.run(login.attach(&client, &root)); + let attempt = login.permit.run(async { + tokio::time::sleep(Duration::from_millis(500 * u64::from(attempts))).await; + login.attach(&client, &root).await + }); let opened = tokio::select! { biased; _ = response.closed() => return None, @@ -246,6 +257,7 @@ impl Recovery { pub(super) fn reset(&mut self) { self.held = None; + self.attempts = 0; drop(self.pending.take()); self.state.send_replace(IdleReconnect::Inactive); self.installed = None; @@ -294,6 +306,7 @@ impl Recovery { pending: None, installed: None, held: None, + attempts: 0, observed_idle: None, client, login: Arc::new(login), diff --git a/codex-rs/app-server-client/src/cloud_backend/mod.rs b/codex-rs/app-server-client/src/cloud_backend/mod.rs index 75b2108f87..61673acd61 100644 --- a/codex-rs/app-server-client/src/cloud_backend/mod.rs +++ b/codex-rs/app-server-client/src/cloud_backend/mod.rs @@ -101,6 +101,7 @@ impl Replies { if opened.ack.send(Ok(())).is_err() { return; } + recovery.attempts += 1; recovery.installed = Some(tokio::time::Instant::now()); recovery.observed_idle = None; self.stream_losses.supersede(&root); @@ -165,6 +166,9 @@ impl Replies { let stable = recovery .installed .is_none_or(|start| start.elapsed() >= std::time::Duration::from_secs(15)); + if stable { + recovery.attempts = 0; + } if recovery.observed_idle == Some(true) && stable && idle::transient(error.as_ref())