diff --git a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs index 4fea016f4b..e3d45ab24a 100644 --- a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs +++ b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs @@ -226,15 +226,37 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R } #[tokio::test] -async fn dropping_an_in_flight_list_unblocks_the_native_worker() -> Result<()> { - assert_abandoned_request_unblocks_worker( - "ListThreads", - CloudOperation::Request(Box::new(protocol::ClientRequest::ThreadList { - request_id: protocol::RequestId::Integer(1), - params: serde_json::from_value(serde_json::json!({"limit": 10}))?, - })), - ) - .await +async fn dropping_an_in_flight_read_unblocks_the_native_worker() -> Result<()> { + for (rpc, request) in [ + ( + "ListThreads", + protocol::ClientRequest::ThreadList { + request_id: protocol::RequestId::Integer(1), + params: serde_json::from_value(serde_json::json!({"limit": 10}))?, + }, + ), + ( + "ListTurns", + protocol::ClientRequest::ThreadTurnsList { + request_id: protocol::RequestId::Integer(1), + params: serde_json::from_value(serde_json::json!({ + "threadId": "t", + "itemsView": "notLoaded", + }))?, + }, + ), + ( + "ListItems", + protocol::ClientRequest::ThreadItemsList { + request_id: protocol::RequestId::Integer(1), + params: serde_json::from_value(serde_json::json!({"threadId": "t"}))?, + }, + ), + ] { + assert_abandoned_request_unblocks_worker(rpc, CloudOperation::Request(Box::new(request))) + .await?; + } + Ok(()) } #[tokio::test] @@ -608,17 +630,85 @@ async fn successful_reopen_supersedes_only_earlier_losses_for_that_thread() -> R Ok(()) } +#[tokio::test] +async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> { + let server = MockServer::start().await; + let home = tempfile::tempdir()?; + let config = crate::legacy_core::config::ConfigBuilder::default() + .codex_home(home.path().to_owned()) + .fallback_cwd(Some(home.path().to_owned())) + .without_project_context() + .build() + .await?; + let factory = config.http_client_factory(); + let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1); + let handle = client.request_handle(); + let mut replies = Replies { + idle: Some(idle::Recovery { + state: handle.reconnect.clone(), + pending: None, + installed: None, + observed_idle: Some(true), + client: Arc::new(cloud::Client::new( + &factory, + &server.uri(), + cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, + )?), + login: Arc::new(idle::Login { + storage: config.auth_config(), + mode: codex_protocol::auth::AuthMode::Chatgpt, + token: "synthetic-bearer".into(), + account: "synthetic-account".into(), + permit: factory.network_policy().acquire(&server.uri().parse()?)?, + }), + }), + active: Some(Attachment { + thread_id: "t".into(), + stream: stream::empty().boxed(), + }), + stream_losses: client.stream_losses.clone(), + ..Default::default() + }; + replies.forward_next(&output).await; + assert!(matches!( + client.next_event().await, + Some(AppServerEvent::ThreadStreamDisconnected(_)) + )); + assert!(handle.reconnect_idle_thread("t")); + + replies.forward_next(&output).await; + let Some(AppServerEvent::ThreadStreamDisconnected(error)) = client.next_event().await else { + panic!("expected a disconnect notification after the accepted reconnect failed"); + }; + assert_eq!( + *error, + cloud_error( + "t", + "Your cloud sign-in changed or expired. Sign in again to access this conversation." + ) + ); + Ok(()) +} + #[tokio::test] async fn attachment_status_follows_its_server_notifications() -> Result<()> { - let (output, _notifications) = mpsc::unbounded_channel(); + let (output, mut events) = mpsc::unbounded_channel(); let mut replies = Replies::default(); let notifications: [&[u8]; 3] = [ b"\x1a\x0d\x12\x0b\x0a\x06\x22\x04\x0a\x02\x08\x01\x12\x01t", // t is waiting b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01x", // x is idle b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t", // t is idle ]; - let stream = - stream::iter(notifications.map(|bytes| Ok(cloud::Event::Notification(bytes.into())))); + let stream = stream::iter( + notifications + .map(|bytes| Ok(cloud::Event::Notification(bytes.into()))) + .into_iter() + .chain([Err(cloud::Error::Rpc(cloud::RpcStatus { + code: 7, + message: "private backend details".into(), + details: Default::default(), + }))]), + ); replies .replace("t", async { Ok(stream.boxed()) }, &output) .await?; @@ -634,6 +724,16 @@ async fn attachment_status_follows_its_server_notifications() -> Result<()> { assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle)); replies.forward_next(&output).await; assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle)); + let AppServerEvent::ThreadStreamDisconnected(error) = events.try_recv()? else { + panic!("expected a Cloud stream loss"); + }; + assert_eq!( + *error, + cloud_error( + "t", + "Your account no longer has access to this cloud conversation." + ) + ); replies .replace("t", async { Ok(stream::pending().boxed()) }, &output) diff --git a/codex-rs/app-server-client/src/cloud_backend/idle.rs b/codex-rs/app-server-client/src/cloud_backend/idle.rs index 1fbf67ca6e..3d6c4303fb 100644 --- a/codex-rs/app-server-client/src/cloud_backend/idle.rs +++ b/codex-rs/app-server-client/src/cloud_backend/idle.rs @@ -1,6 +1,7 @@ //! Reopens only the lost reply stream of a displayed, idle Cloud conversation. use super::ReplyStream; +use super::check_reply_stream; use crate::cloud::IdleReconnect; use codex_cloud_client as cloud; use codex_http_client::NetworkPermit; @@ -8,7 +9,6 @@ use codex_login::AuthConfig; use codex_login::token_data::parse_jwt_expiration; use codex_protocol::auth::AuthMode; use futures::FutureExt; -use futures::StreamExt; use futures::future::BoxFuture; use std::sync::Arc; use std::time::Duration; @@ -16,6 +16,23 @@ use std::time::SystemTime; use tokio::sync::watch; use tokio::time::Instant; +const SIGN_IN_AGAIN: &str = + "Your cloud sign-in changed or expired. Sign in again to access this conversation."; + +pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> { + let cloud::Error::Rpc(status) = error else { + return None; + }; + match status.code { + 5 => Some("Cloud could not find this conversation."), + 7 => Some("Your account no longer has access to this cloud conversation."), + 16 => Some(SIGN_IN_AGAIN), + _ => None, + } +} + +const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation."; + pub(super) struct Login { pub(super) storage: AuthConfig, pub(super) mode: AuthMode, @@ -26,6 +43,9 @@ pub(super) struct Login { impl Login { async fn current(&self) -> bool { + if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) { + return false; + } let Ok(Some(auth)) = self .storage .load_auth_with_modes(/*enable_codex_api_key_env*/ false, Some(&[self.mode])) @@ -44,9 +64,48 @@ impl Login { expiry.is_none_or(|expiry| SystemTime::from(expiry) > SystemTime::now()) }) } + + async fn attach( + &self, + client: &cloud::Client, + thread_id: &str, + ) -> Result<(ReplyStream, Option), &'static str> { + for delay in [ + Duration::ZERO, + Duration::from_millis(500), + Duration::from_secs(1), + ] { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + if !self.current().await { + return Err(SIGN_IN_AGAIN); + } + match client + .attach(thread_id) + .await + .map_err(Some) + .and_then(|mut stream| { + let first = check_reply_stream(&mut stream)?; + Ok((stream, first)) + }) { + Ok(attachment) => { + if !self.current().await { + return Err(SIGN_IN_AGAIN); + } + return Ok(attachment); + } + Err(Some(error)) if !transient(Some(&error)) => { + return Err(blocked_recovery(&error).unwrap_or(RECOVERY_FAILED)); + } + Err(_) => {} + } + } + Err("Cloud did not restore the reply stream after several attempts.") + } } -type Open = BoxFuture<'static, Option<(ReplyStream, Option)>>; +type Open = BoxFuture<'static, Result<(ReplyStream, Option), Option<&'static str>>>; pub(super) struct Recovery { pub(super) state: watch::Sender, @@ -138,49 +197,14 @@ impl Recovery { |state| matches!(state, IdleReconnect::Opening(id) if Arc::ptr_eq(id, &selected)), ) .await - .ok()?; + .map_err(|_| None)?; // Use the policy bound to the frozen serving credential when this client opened. - let attempts = login - .permit - // A revoke-then-allow cannot restart this episode or release a buffered first event. - .run(async { - for delay in [ - Duration::ZERO, - Duration::from_millis(500), - Duration::from_secs(1), - ] { - if !delay.is_zero() { - tokio::time::sleep(delay).await; - } - if !login.current().await { - return None; - } - match client.attach(&selected).await { - Ok(mut stream) => { - let first = - tokio::task::unconstrained(stream.next()).now_or_never(); - let event = match first { - None => None, - Some(Some(Ok(event))) => Some(event), - Some(Some(Err(error))) if !transient(Some(&error)) => { - return None; - } - Some(_) => continue, - }; - return login.current().await.then_some((stream, event)); - } - Err(error) if transient(Some(&error)) => {} - Err(_) => return None, - } - } - None - }); - // Bound this accepted recovery episode, including credential checks and retry backoff. - tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempts) + let attempt = login.permit.run(login.attach(&client, &selected)); + tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempt) .await - .ok()? - .ok() - .flatten() + .map_err(|_| None)? + .map_err(|_| None)? + .map_err(Some) }; self.pending = Some((root, open.boxed())); } diff --git a/codex-rs/app-server-client/src/cloud_backend/mod.rs b/codex-rs/app-server-client/src/cloud_backend/mod.rs index 0b2741b36a..a59b91f6dc 100644 --- a/codex-rs/app-server-client/src/cloud_backend/mod.rs +++ b/codex-rs/app-server-client/src/cloud_backend/mod.rs @@ -16,6 +16,7 @@ use codex_app_server_protocol as protocol; use codex_cloud_client as cloud; use color_eyre::eyre::Result; use color_eyre::eyre::bail; + use color_eyre::eyre::eyre; use futures::FutureExt; use futures::StreamExt; @@ -30,6 +31,18 @@ pub use auth::connect; type ReplyStream = BoxStream<'static, Result>; type OpenReplyStream = (ReplyStream, Option); +/// Separate an immediate EOF from a quiet, successfully attached stream. +fn check_reply_stream( + stream: &mut ReplyStream, +) -> Result, Option> { + match tokio::task::unconstrained(stream.next()).now_or_never() { + Some(None) => Err(None), + Some(Some(Err(error))) => Err(Some(error)), + Some(Some(Ok(event))) => Ok(Some(event)), + None => Ok(None), + } +} + /// Rejected locally before an admission request can be sent. #[derive(Debug, thiserror::Error)] #[error("{0}")] @@ -63,22 +76,27 @@ impl Replies { let opened = open.await; let root = Arc::clone(root); recovery.pending = None; - if let Some((stream, first)) = opened.filter(|_| recovery.login.permit.check().is_ok()) - { - recovery.installed = Some(tokio::time::Instant::now()); - // A same-thread Attach can stay quiet. Keep the last observation until an - // event or a dispatched mutation changes recovery eligibility. - self.install(&root, stream, first, output); - if let Some(recovery) = &self.idle - && recovery.observed_idle != Some(false) - { - recovery.notify_connected(&root, output); + match opened.and_then(|attachment| { + recovery.login.permit.check().map_err(|_| None)?; + Ok(attachment) + }) { + Ok((stream, first)) => { + recovery.installed = Some(tokio::time::Instant::now()); + // A same-thread Attach can stay quiet. Keep the last observation until an + // event or a dispatched mutation changes recovery eligibility. + self.install(&root, stream, first, output); + if let Some(recovery) = &self.idle + && recovery.observed_idle != Some(false) + { + recovery.notify_connected(&root, output); + } + } + Err(reason) => { + recovery + .state + .send_replace(crate::cloud::IdleReconnect::Inactive); + self.report_loss(&root, reason, output); } - } else { - recovery - .state - .send_replace(crate::cloud::IdleReconnect::Inactive); - self.report_loss(&root, output); } return; } @@ -95,6 +113,7 @@ impl Replies { ), ended => { let id = active.thread_id.clone(); + let error = ended.and_then(Result::err); self.active = None; if let Some(recovery) = &mut self.idle { recovery @@ -105,22 +124,27 @@ impl Replies { .is_none_or(|start| start.elapsed() >= std::time::Duration::from_secs(15)); if recovery.observed_idle == Some(true) && stable - && idle::transient(ended.and_then(Result::err).as_ref()) + && idle::transient(error.as_ref()) { recovery.offer(&id); } } - self.report_loss(&id, output); + self.report_loss(&id, error.as_ref().and_then(idle::blocked_recovery), output); } } } - fn report_loss(&self, id: &str, output: &mpsc::UnboundedSender) { + fn report_loss( + &self, + id: &str, + reason: Option<&str>, + output: &mpsc::UnboundedSender, + ) { self.stream_losses.record(id); let _ = output.send(AppServerEvent::ThreadStreamDisconnected(Box::new( cloud_error( id, - "Cloud stream ended. Reopen the conversation to reconnect.", + reason.unwrap_or("Cloud stream ended. Reopen the conversation to reconnect."), ), ))); } @@ -255,16 +279,11 @@ impl Replies { } fn probe_reply_stream(stream: &mut ReplyStream) -> Result> { - // Attach can close after successful headers. A healthy idle stream need not emit an event. - match tokio::task::unconstrained(stream.next()).now_or_never() { - Some(None | Some(Err(_))) => { - bail!( - "Cloud conversation activated but its stream did not connect. Reopen it before sending." - ); - } - Some(Some(Ok(event))) => Ok(Some(event)), - None => Ok(None), - } + check_reply_stream(stream).map_err(|_| { + eyre!( + "Cloud conversation activated but its stream did not connect. Reopen it before sending." + ) + }) } /// Opens an app-server client backed by the authenticated native Cloud client. @@ -385,7 +404,7 @@ async fn run_worker_with_prepared( if let Some(candidate) = &mut prepared { candidate.canceled_recovery = canceled; } else if let Some(root) = canceled { - replies.report_loss(&root, output); + replies.report_loss(&root, /*reason*/ None, output); } continue; } @@ -434,15 +453,21 @@ async fn run_worker_with_prepared( candidate.settings.thread.id == *id || loaded.is_some_and(|loaded| loaded.thread.id == *id) }); - let is_thread_list = matches!(&request, protocol::ClientRequest::ThreadList { .. }); + // Inventory and history reads have no conversation state to commit. + let is_read = matches!( + &request, + protocol::ClientRequest::ThreadList { .. } + | protocol::ClientRequest::ThreadTurnsList { .. } + | protocol::ClientRequest::ThreadItemsList { .. } + ); // When closure and the handshake are both ready, cancel before // the handshake can replace the active attachment. let result = tokio::select! { biased; _ = output.closed() => return, - _ = command.response.closed(), if is_thread_list || is_thread_start => { + _ = command.response.closed(), if is_read || is_thread_start => { if let Some(root) = canceled { - replies.report_loss(&root, output); + replies.report_loss(&root, /*reason*/ None, output); } continue; } @@ -455,7 +480,7 @@ async fn run_worker_with_prepared( .and_then(|candidate| candidate.canceled_recovery) && requests.loaded.is_some() { - replies.report_loss(&root, output); + replies.report_loss(&root, /*reason*/ None, output); } result } @@ -464,7 +489,7 @@ async fn run_worker_with_prepared( if result.is_err() && let Some(root) = canceled { - replies.report_loss(&root, output); + replies.report_loss(&root, /*reason*/ None, output); } let _ = command.response.send(Ok(request_result(result))); } @@ -647,7 +672,7 @@ async fn advance_prepared_resume( } } if let Some(root) = candidate.canceled_recovery { - replies.report_loss(&root, output); + replies.report_loss(&root, /*reason*/ None, output); } None }