Report Cloud stream recovery failures and cancel abandoned reads (#61599)

## Why

Cloud stream failures previously used a generic reopen message even when the conversation was missing, access was denied, or sign-in had changed. Abandoned turn and item list requests could also keep the native worker occupied.

## What changed

- Report specific disconnect reasons for missing conversations, lost access, and changed or expired sign-in, including failures during idle reconnection.
- Report exhausted reconnect attempts and other nontransient recovery failures with dedicated messages.
- Cancel in-flight `ThreadTurnsList` and `ThreadItemsList` requests when their callers drop, matching `ThreadList` behavior.

## Testing

Add regression coverage for abandoned history reads, changed sign-in during accepted idle recovery, and access-denied stream errors.

GitOrigin-RevId: 29923b46c2d377f2247d699757553777a31d08ca
This commit is contained in:
Benjamin Carlsson
2026-11-15 03:15:06 +00:00
committed by copyberry
parent b7056f3408
commit 8e30237f64
3 changed files with 240 additions and 91 deletions
@@ -226,15 +226,37 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R
}
#[tokio::test]
async fn dropping_an_in_flight_list_unblocks_the_native_worker() -> Result<()> {
assert_abandoned_request_unblocks_worker(
"ListThreads",
CloudOperation::Request(Box::new(protocol::ClientRequest::ThreadList {
request_id: protocol::RequestId::Integer(1),
params: serde_json::from_value(serde_json::json!({"limit": 10}))?,
})),
)
.await
async fn dropping_an_in_flight_read_unblocks_the_native_worker() -> Result<()> {
for (rpc, request) in [
(
"ListThreads",
protocol::ClientRequest::ThreadList {
request_id: protocol::RequestId::Integer(1),
params: serde_json::from_value(serde_json::json!({"limit": 10}))?,
},
),
(
"ListTurns",
protocol::ClientRequest::ThreadTurnsList {
request_id: protocol::RequestId::Integer(1),
params: serde_json::from_value(serde_json::json!({
"threadId": "t",
"itemsView": "notLoaded",
}))?,
},
),
(
"ListItems",
protocol::ClientRequest::ThreadItemsList {
request_id: protocol::RequestId::Integer(1),
params: serde_json::from_value(serde_json::json!({"threadId": "t"}))?,
},
),
] {
assert_abandoned_request_unblocks_worker(rpc, CloudOperation::Request(Box::new(request)))
.await?;
}
Ok(())
}
#[tokio::test]
@@ -608,17 +630,85 @@ async fn successful_reopen_supersedes_only_earlier_losses_for_that_thread() -> R
Ok(())
}
#[tokio::test]
async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> {
let server = MockServer::start().await;
let home = tempfile::tempdir()?;
let config = crate::legacy_core::config::ConfigBuilder::default()
.codex_home(home.path().to_owned())
.fallback_cwd(Some(home.path().to_owned()))
.without_project_context()
.build()
.await?;
let factory = config.http_client_factory();
let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1);
let handle = client.request_handle();
let mut replies = Replies {
idle: Some(idle::Recovery {
state: handle.reconnect.clone(),
pending: None,
installed: None,
observed_idle: Some(true),
client: Arc::new(cloud::Client::new(
&factory,
&server.uri(),
cloud::Credentials::new("synthetic-bearer", "synthetic-account")?,
)?),
login: Arc::new(idle::Login {
storage: config.auth_config(),
mode: codex_protocol::auth::AuthMode::Chatgpt,
token: "synthetic-bearer".into(),
account: "synthetic-account".into(),
permit: factory.network_policy().acquire(&server.uri().parse()?)?,
}),
}),
active: Some(Attachment {
thread_id: "t".into(),
stream: stream::empty().boxed(),
}),
stream_losses: client.stream_losses.clone(),
..Default::default()
};
replies.forward_next(&output).await;
assert!(matches!(
client.next_event().await,
Some(AppServerEvent::ThreadStreamDisconnected(_))
));
assert!(handle.reconnect_idle_thread("t"));
replies.forward_next(&output).await;
let Some(AppServerEvent::ThreadStreamDisconnected(error)) = client.next_event().await else {
panic!("expected a disconnect notification after the accepted reconnect failed");
};
assert_eq!(
*error,
cloud_error(
"t",
"Your cloud sign-in changed or expired. Sign in again to access this conversation."
)
);
Ok(())
}
#[tokio::test]
async fn attachment_status_follows_its_server_notifications() -> Result<()> {
let (output, _notifications) = mpsc::unbounded_channel();
let (output, mut events) = mpsc::unbounded_channel();
let mut replies = Replies::default();
let notifications: [&[u8]; 3] = [
b"\x1a\x0d\x12\x0b\x0a\x06\x22\x04\x0a\x02\x08\x01\x12\x01t", // t is waiting
b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01x", // x is idle
b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t", // t is idle
];
let stream =
stream::iter(notifications.map(|bytes| Ok(cloud::Event::Notification(bytes.into()))));
let stream = stream::iter(
notifications
.map(|bytes| Ok(cloud::Event::Notification(bytes.into())))
.into_iter()
.chain([Err(cloud::Error::Rpc(cloud::RpcStatus {
code: 7,
message: "private backend details".into(),
details: Default::default(),
}))]),
);
replies
.replace("t", async { Ok(stream.boxed()) }, &output)
.await?;
@@ -634,6 +724,16 @@ async fn attachment_status_follows_its_server_notifications() -> Result<()> {
assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle));
replies.forward_next(&output).await;
assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle));
let AppServerEvent::ThreadStreamDisconnected(error) = events.try_recv()? else {
panic!("expected a Cloud stream loss");
};
assert_eq!(
*error,
cloud_error(
"t",
"Your account no longer has access to this cloud conversation."
)
);
replies
.replace("t", async { Ok(stream::pending().boxed()) }, &output)
@@ -1,6 +1,7 @@
//! Reopens only the lost reply stream of a displayed, idle Cloud conversation.
use super::ReplyStream;
use super::check_reply_stream;
use crate::cloud::IdleReconnect;
use codex_cloud_client as cloud;
use codex_http_client::NetworkPermit;
@@ -8,7 +9,6 @@ use codex_login::AuthConfig;
use codex_login::token_data::parse_jwt_expiration;
use codex_protocol::auth::AuthMode;
use futures::FutureExt;
use futures::StreamExt;
use futures::future::BoxFuture;
use std::sync::Arc;
use std::time::Duration;
@@ -16,6 +16,23 @@ use std::time::SystemTime;
use tokio::sync::watch;
use tokio::time::Instant;
const SIGN_IN_AGAIN: &str =
"Your cloud sign-in changed or expired. Sign in again to access this conversation.";
pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> {
let cloud::Error::Rpc(status) = error else {
return None;
};
match status.code {
5 => Some("Cloud could not find this conversation."),
7 => Some("Your account no longer has access to this cloud conversation."),
16 => Some(SIGN_IN_AGAIN),
_ => None,
}
}
const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation.";
pub(super) struct Login {
pub(super) storage: AuthConfig,
pub(super) mode: AuthMode,
@@ -26,6 +43,9 @@ pub(super) struct Login {
impl Login {
async fn current(&self) -> bool {
if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) {
return false;
}
let Ok(Some(auth)) = self
.storage
.load_auth_with_modes(/*enable_codex_api_key_env*/ false, Some(&[self.mode]))
@@ -44,9 +64,48 @@ impl Login {
expiry.is_none_or(|expiry| SystemTime::from(expiry) > SystemTime::now())
})
}
async fn attach(
&self,
client: &cloud::Client,
thread_id: &str,
) -> Result<(ReplyStream, Option<cloud::Event>), &'static str> {
for delay in [
Duration::ZERO,
Duration::from_millis(500),
Duration::from_secs(1),
] {
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
if !self.current().await {
return Err(SIGN_IN_AGAIN);
}
match client
.attach(thread_id)
.await
.map_err(Some)
.and_then(|mut stream| {
let first = check_reply_stream(&mut stream)?;
Ok((stream, first))
}) {
Ok(attachment) => {
if !self.current().await {
return Err(SIGN_IN_AGAIN);
}
return Ok(attachment);
}
Err(Some(error)) if !transient(Some(&error)) => {
return Err(blocked_recovery(&error).unwrap_or(RECOVERY_FAILED));
}
Err(_) => {}
}
}
Err("Cloud did not restore the reply stream after several attempts.")
}
}
type Open = BoxFuture<'static, Option<(ReplyStream, Option<cloud::Event>)>>;
type Open = BoxFuture<'static, Result<(ReplyStream, Option<cloud::Event>), Option<&'static str>>>;
pub(super) struct Recovery {
pub(super) state: watch::Sender<IdleReconnect>,
@@ -138,49 +197,14 @@ impl Recovery {
|state| matches!(state, IdleReconnect::Opening(id) if Arc::ptr_eq(id, &selected)),
)
.await
.ok()?;
.map_err(|_| None)?;
// Use the policy bound to the frozen serving credential when this client opened.
let attempts = login
.permit
// A revoke-then-allow cannot restart this episode or release a buffered first event.
.run(async {
for delay in [
Duration::ZERO,
Duration::from_millis(500),
Duration::from_secs(1),
] {
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
if !login.current().await {
return None;
}
match client.attach(&selected).await {
Ok(mut stream) => {
let first =
tokio::task::unconstrained(stream.next()).now_or_never();
let event = match first {
None => None,
Some(Some(Ok(event))) => Some(event),
Some(Some(Err(error))) if !transient(Some(&error)) => {
return None;
}
Some(_) => continue,
};
return login.current().await.then_some((stream, event));
}
Err(error) if transient(Some(&error)) => {}
Err(_) => return None,
}
}
None
});
// Bound this accepted recovery episode, including credential checks and retry backoff.
tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempts)
let attempt = login.permit.run(login.attach(&client, &selected));
tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempt)
.await
.ok()?
.ok()
.flatten()
.map_err(|_| None)?
.map_err(|_| None)?
.map_err(Some)
};
self.pending = Some((root, open.boxed()));
}
@@ -16,6 +16,7 @@ use codex_app_server_protocol as protocol;
use codex_cloud_client as cloud;
use color_eyre::eyre::Result;
use color_eyre::eyre::bail;
use color_eyre::eyre::eyre;
use futures::FutureExt;
use futures::StreamExt;
@@ -30,6 +31,18 @@ pub use auth::connect;
type ReplyStream = BoxStream<'static, Result<cloud::Event, cloud::Error>>;
type OpenReplyStream = (ReplyStream, Option<cloud::Event>);
/// Separate an immediate EOF from a quiet, successfully attached stream.
fn check_reply_stream(
stream: &mut ReplyStream,
) -> Result<Option<cloud::Event>, Option<cloud::Error>> {
match tokio::task::unconstrained(stream.next()).now_or_never() {
Some(None) => Err(None),
Some(Some(Err(error))) => Err(Some(error)),
Some(Some(Ok(event))) => Ok(Some(event)),
None => Ok(None),
}
}
/// Rejected locally before an admission request can be sent.
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
@@ -63,22 +76,27 @@ impl Replies {
let opened = open.await;
let root = Arc::clone(root);
recovery.pending = None;
if let Some((stream, first)) = opened.filter(|_| recovery.login.permit.check().is_ok())
{
recovery.installed = Some(tokio::time::Instant::now());
// A same-thread Attach can stay quiet. Keep the last observation until an
// event or a dispatched mutation changes recovery eligibility.
self.install(&root, stream, first, output);
if let Some(recovery) = &self.idle
&& recovery.observed_idle != Some(false)
{
recovery.notify_connected(&root, output);
match opened.and_then(|attachment| {
recovery.login.permit.check().map_err(|_| None)?;
Ok(attachment)
}) {
Ok((stream, first)) => {
recovery.installed = Some(tokio::time::Instant::now());
// A same-thread Attach can stay quiet. Keep the last observation until an
// event or a dispatched mutation changes recovery eligibility.
self.install(&root, stream, first, output);
if let Some(recovery) = &self.idle
&& recovery.observed_idle != Some(false)
{
recovery.notify_connected(&root, output);
}
}
Err(reason) => {
recovery
.state
.send_replace(crate::cloud::IdleReconnect::Inactive);
self.report_loss(&root, reason, output);
}
} else {
recovery
.state
.send_replace(crate::cloud::IdleReconnect::Inactive);
self.report_loss(&root, output);
}
return;
}
@@ -95,6 +113,7 @@ impl Replies {
),
ended => {
let id = active.thread_id.clone();
let error = ended.and_then(Result::err);
self.active = None;
if let Some(recovery) = &mut self.idle {
recovery
@@ -105,22 +124,27 @@ impl Replies {
.is_none_or(|start| start.elapsed() >= std::time::Duration::from_secs(15));
if recovery.observed_idle == Some(true)
&& stable
&& idle::transient(ended.and_then(Result::err).as_ref())
&& idle::transient(error.as_ref())
{
recovery.offer(&id);
}
}
self.report_loss(&id, output);
self.report_loss(&id, error.as_ref().and_then(idle::blocked_recovery), output);
}
}
}
fn report_loss(&self, id: &str, output: &mpsc::UnboundedSender<AppServerEvent>) {
fn report_loss(
&self,
id: &str,
reason: Option<&str>,
output: &mpsc::UnboundedSender<AppServerEvent>,
) {
self.stream_losses.record(id);
let _ = output.send(AppServerEvent::ThreadStreamDisconnected(Box::new(
cloud_error(
id,
"Cloud stream ended. Reopen the conversation to reconnect.",
reason.unwrap_or("Cloud stream ended. Reopen the conversation to reconnect."),
),
)));
}
@@ -255,16 +279,11 @@ impl Replies {
}
fn probe_reply_stream(stream: &mut ReplyStream) -> Result<Option<cloud::Event>> {
// Attach can close after successful headers. A healthy idle stream need not emit an event.
match tokio::task::unconstrained(stream.next()).now_or_never() {
Some(None | Some(Err(_))) => {
bail!(
"Cloud conversation activated but its stream did not connect. Reopen it before sending."
);
}
Some(Some(Ok(event))) => Ok(Some(event)),
None => Ok(None),
}
check_reply_stream(stream).map_err(|_| {
eyre!(
"Cloud conversation activated but its stream did not connect. Reopen it before sending."
)
})
}
/// Opens an app-server client backed by the authenticated native Cloud client.
@@ -385,7 +404,7 @@ async fn run_worker_with_prepared(
if let Some(candidate) = &mut prepared {
candidate.canceled_recovery = canceled;
} else if let Some(root) = canceled {
replies.report_loss(&root, output);
replies.report_loss(&root, /*reason*/ None, output);
}
continue;
}
@@ -434,15 +453,21 @@ async fn run_worker_with_prepared(
candidate.settings.thread.id == *id
|| loaded.is_some_and(|loaded| loaded.thread.id == *id)
});
let is_thread_list = matches!(&request, protocol::ClientRequest::ThreadList { .. });
// Inventory and history reads have no conversation state to commit.
let is_read = matches!(
&request,
protocol::ClientRequest::ThreadList { .. }
| protocol::ClientRequest::ThreadTurnsList { .. }
| protocol::ClientRequest::ThreadItemsList { .. }
);
// When closure and the handshake are both ready, cancel before
// the handshake can replace the active attachment.
let result = tokio::select! {
biased;
_ = output.closed() => return,
_ = command.response.closed(), if is_thread_list || is_thread_start => {
_ = command.response.closed(), if is_read || is_thread_start => {
if let Some(root) = canceled {
replies.report_loss(&root, output);
replies.report_loss(&root, /*reason*/ None, output);
}
continue;
}
@@ -455,7 +480,7 @@ async fn run_worker_with_prepared(
.and_then(|candidate| candidate.canceled_recovery)
&& requests.loaded.is_some()
{
replies.report_loss(&root, output);
replies.report_loss(&root, /*reason*/ None, output);
}
result
}
@@ -464,7 +489,7 @@ async fn run_worker_with_prepared(
if result.is_err()
&& let Some(root) = canceled
{
replies.report_loss(&root, output);
replies.report_loss(&root, /*reason*/ None, output);
}
let _ = command.response.send(Ok(request_result(result)));
}
@@ -647,7 +672,7 @@ async fn advance_prepared_resume(
}
}
if let Some(root) = candidate.canceled_recovery {
replies.report_loss(&root, output);
replies.report_loss(&root, /*reason*/ None, output);
}
None
}