Limit recovery after repeated cloud reply stream disconnects (#62151)

## Why

A cloud reply stream can drop again shortly after reconnecting, leaving
users in a cycle of disconnects and recovery.

## What changed

Delay recovery by 500 ms for each replacement stream already installed
in the current recovery sequence. After three replacements, return an
error that tells users their transcript and draft are still available.

Clear the count when recovery state is reset or when a stream disconnects
after staying connected for at least 15 seconds.

GitOrigin-RevId: 6ae2e88023464fc41d8e26d47dd94af624660d5f
This commit is contained in:
Benjamin Carlsson
2026-11-18 01:36:57 +00:00
committed by copyberry
parent 2f7c5a8588
commit 1c2e4cd45f
2 changed files with 18 additions and 1 deletions
@@ -129,6 +129,7 @@ pub(super) struct Recovery {
pub(super) pending: Option<(Arc<str>, Open)>,
pub(super) installed: Option<Instant>,
pub(super) held: Option<(String, Instant, BoxFuture<'static, Option<Opened>>)>,
pub(super) attempts: u8,
// None until either the initial Attach event or the activation snapshot establishes status.
pub(super) observed_idle: Option<bool>,
pub(super) client: Arc<cloud::Client>,
@@ -176,12 +177,22 @@ impl Recovery {
))));
return;
}
let attempts = self.attempts;
if attempts >= 3 {
let _ = response.send(Ok(Err(recovery_error(
"The cloud reply stream keeps disconnecting. Your transcript and draft are still here.",
))));
return;
}
self.cancel();
let client = Arc::clone(&self.client);
let login = Arc::clone(&self.login);
let root = thread_id.clone();
let open = async move {
let attempt = login.permit.run(login.attach(&client, &root));
let attempt = login.permit.run(async {
tokio::time::sleep(Duration::from_millis(500 * u64::from(attempts))).await;
login.attach(&client, &root).await
});
let opened = tokio::select! {
biased;
_ = response.closed() => return None,
@@ -246,6 +257,7 @@ impl Recovery {
pub(super) fn reset(&mut self) {
self.held = None;
self.attempts = 0;
drop(self.pending.take());
self.state.send_replace(IdleReconnect::Inactive);
self.installed = None;
@@ -294,6 +306,7 @@ impl Recovery {
pending: None,
installed: None,
held: None,
attempts: 0,
observed_idle: None,
client,
login: Arc::new(login),
@@ -101,6 +101,7 @@ impl Replies {
if opened.ack.send(Ok(())).is_err() {
return;
}
recovery.attempts += 1;
recovery.installed = Some(tokio::time::Instant::now());
recovery.observed_idle = None;
self.stream_losses.supersede(&root);
@@ -165,6 +166,9 @@ impl Replies {
let stable = recovery
.installed
.is_none_or(|start| start.elapsed() >= std::time::Duration::from_secs(15));
if stable {
recovery.attempts = 0;
}
if recovery.observed_idle == Some(true)
&& stable
&& idle::transient(error.as_ref())