From f39fcc1bfbcb04afbe7df8793545bf4a00f5f266 Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Tue, 17 Nov 2026 23:14:01 +0000 Subject: [PATCH] Hold Cloud reply streams until history can be refreshed (#62140) ## Why A Cloud conversation can miss an entire turn while disconnected. A successful reattach, even with an idle thread, cannot establish that the client's history is complete. ## What changed Add an app-server client operation that reattaches a disconnected, loaded Cloud thread and returns its cached settings. The worker holds the replacement stream so the caller can refresh turn and item history before explicitly installing the stream and receiving buffered events. Reattachment, recovery history reads, and stream installation share a deadline and use the original Cloud sign-in and network permit. Dropping the release handle cancels the replacement. Personal access token sessions still reconnect by reopening the conversation. ## Testing Extend Cloud connection tests to cover quiet and busy reattachments, buffered event delivery, cancellation and retry, slow history reads, and sign-in changes during recovery. GitOrigin-RevId: a4e5cb967e097021596bc83aac2f8d1a95373712 --- codex-rs/app-server-client/src/cloud.rs | 61 ++++ .../src/cloud_backend/connection_tests.rs | 331 ++++++++++++++++-- .../src/cloud_backend/idle.rs | 139 +++++++- .../src/cloud_backend/mod.rs | 149 ++++++-- codex-rs/app-server-client/src/lib.rs | 3 + 5 files changed, 639 insertions(+), 44 deletions(-) diff --git a/codex-rs/app-server-client/src/cloud.rs b/codex-rs/app-server-client/src/cloud.rs index d575c13170..7de4358d20 100644 --- a/codex-rs/app-server-client/src/cloud.rs +++ b/codex-rs/app-server-client/src/cloud.rs @@ -11,6 +11,7 @@ use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::Result as JsonRpcResult; +use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadSetNameParams; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnStartParams; @@ -22,6 +23,12 @@ use std::sync::MutexGuard; use std::sync::PoisonError; use tokio::sync::mpsc; use tokio::sync::oneshot; +use tokio::time::Instant; +use tokio::time::timeout_at; + +/// Rejection from the original network policy during Cloud recovery. +pub const CLOUD_RECOVERY_NETWORK_BLOCKED: &str = + "The current network policy blocks cloud reconnection."; /// Operations the cloud worker must handle without forwarding to another backend. /// Unsupported operations must return an explicit error; they must not claim success. @@ -44,6 +51,12 @@ pub enum CloudOperation { SubmitUserMessage(Box), /// Apply the result of a name write that already succeeded outside the command queue. ThreadNameSaved(ThreadSetNameParams), + ResumeDisconnectedThread { + thread_id: String, + require_history: bool, + deadline: Instant, + release: oneshot::Receiver>>, + }, Notify(ClientNotification), Resolve { request_id: RequestId, @@ -72,6 +85,30 @@ pub struct CloudCommand { pub response: oneshot::Sender>, } +/// Allows the worker to install a held stream after the caller reads its history. +/// Dropping the token cancels that replacement. +pub struct CloudRecoveryRelease { + release: oneshot::Sender>>, + deadline: Instant, +} + +impl CloudRecoveryRelease { + pub async fn install(self) -> Result<(), &'static str> { + let (ack, mut installed) = oneshot::channel(); + self.release + .send(ack) + .map_err(|_| "Cloud reconnection expired.")?; + let response = match timeout_at(self.deadline, &mut installed).await { + Ok(result) => result.ok(), + Err(_) => { + installed.close(); + installed.try_recv().ok() + } + }; + response.unwrap_or(Err("Cloud reconnection expired.")) + } +} + #[derive(Clone, Default)] pub(super) enum IdleReconnect { #[default] @@ -298,6 +335,30 @@ impl CloudRequestHandle { }) } + pub async fn resume_disconnected_thread( + &self, + thread_id: String, + require_history: bool, + deadline: Instant, + ) -> Result<(Option, CloudRecoveryRelease), TypedRequestError> { + let (release, held) = oneshot::channel(); + let response = timeout_at( + deadline, + self.send(CloudOperation::ResumeDisconnectedThread { + thread_id, + require_history, + deadline, + release: held, + }), + ) + .await + .map_err(io::Error::from) + .and_then(std::convert::identity); + let settings: Option = + crate::decode_typed_response("thread/resume", response)?; + Ok((settings, CloudRecoveryRelease { release, deadline })) + } + /// Whether a completed model read still belongs to the bound Cloud account. pub fn model_catalog_is_current(&self) -> bool { self.native diff --git a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs index 8f72bc5b48..0b1dc1c539 100644 --- a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs +++ b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs @@ -3,6 +3,7 @@ //! Exercises native worker progress and reply attachment. use super::*; +use crate::AppServerRequestHandle; use futures::stream; use pretty_assertions::assert_eq; use std::sync::Mutex; @@ -93,12 +94,30 @@ async fn rejected_rename_preserves_loaded_cloud_thread_name() -> Result<()> { async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> Result<()> { let server = MockServer::start().await; let home = tempfile::tempdir()?; - let config = crate::legacy_core::config::ConfigBuilder::default() + let mut config = crate::legacy_core::config::ConfigBuilder::default() .codex_home(home.path().to_owned()) .fallback_cwd(Some(home.path().to_owned())) .without_project_context() .build() .await?; + config.cli_auth_credentials_store_mode = codex_login::AuthCredentialsStoreMode::File; + let storage = config.auth_config(); + let token = "e30.e30.sig"; + let account = "synthetic-account"; + let auth = serde_json::from_value(serde_json::json!({ + "auth_mode": "chatgpt", + "tokens": { + "id_token": token, "access_token": token, "refresh_token": "test", "account_id": account, + }, + "last_refresh": "2026-01-01T00:00:00Z", + }))?; + codex_login::save_auth( + home.path(), + &auth, + storage.auth_credentials_store_mode, + storage.keyring_backend_kind, + )?; + let cached = Box::new(serde_json::from_value::( serde_json::json!({ "thread": { @@ -116,7 +135,7 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R client: std::sync::Arc::new(cloud::Client::new( &config.http_client_factory(), &server.uri(), - cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, + cloud::Credentials::new(token, account)?, )?), account: protocol::GetAccountResponse { user_id: None, @@ -222,6 +241,279 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R ) ); assert!(notifications.try_recv().is_err()); + let login = idle::Login { + storage, + mode: codex_protocol::auth::AuthMode::Chatgpt, + token: token.into(), + account: account.into(), + permit: config + .http_client_factory() + .network_policy() + .acquire(&server.uri().parse()?)?, + }; + // A live replacement installs and forwards held and buffered events. + let (ack, installed) = oneshot::channel(); + let (state, _) = tokio::sync::watch::channel(crate::cloud::IdleReconnect::Inactive); + let mut recovery = idle::Recovery::new(state, Arc::clone(&requests.client), login.clone()); + recovery.held = Some(( + "t".into(), + tokio::time::Instant::now() + std::time::Duration::from_secs(8), + Box::pin(async move { + let idle: &[u8] = b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t"; + Some(idle::Opened { + stream: stream::iter([Ok(cloud::Event::Notification(idle.into()))]) + .chain(stream::pending()) + .boxed(), + first: Some(cloud::Event::ServerRequest(Default::default())), + ack, + }) + }), + )); + replies.idle = Some(recovery); + requests.warned_partial_history = Some("t".into()); + replies.next_event(&output).await; + assert_eq!(installed.await?, Ok(())); + let AppServerEvent::ServerNotification(notification) = notifications.try_recv()? else { + panic!("expected held first event"); + }; + assert!(matches!( + *notification, + protocol::ServerNotification::Warning(_) + )); + replies.next_event(&output).await; + assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle)); + // Installation must not erase a warning already emitted by the refreshed history. + let (commands, receiver) = mpsc::channel(1); + drop(commands); + run_worker(receiver, &mut requests, &mut replies, &output).await; + assert_eq!(requests.warned_partial_history.as_deref(), Some("t")); + + // A quiet Attach cannot prove that no whole turn was missed while disconnected. + let attach_path = "/oaiproto.codex_app_server_backend.ThreadService/Attach"; + for (idle, statuses, outcome) in [ + (true, &[][..], "pat"), + (true, &["quiet"][..], "install"), + (true, &["7"][..], "install"), + (false, &["14", "7"][..], "install"), + (false, &["14", "busy"][..], "install"), + (true, &["busy"][..], "install"), + (true, &["busy"][..], "cancel"), + (true, &["busy", "busy"][..], "retry"), + (true, &["busy"][..], "slow-history"), + (true, &["busy"][..], "auth-change"), + ] { + server.reset().await; + let attempt = std::sync::atomic::AtomicUsize::new(0); + Mock::given(method("POST")) + .and(path(attach_path)) + .and(header("authorization", format!("Bearer {token}"))) + .respond_with(move |_: &wiremock::Request| { + let index = attempt.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let response = + ResponseTemplate::new(200).insert_header("content-type", "application/grpc"); + match statuses[index] { + "quiet" => response.set_body_bytes(b"\x00\x00\x00\x00\x02\x0a\x00".to_vec()), + "busy" => response.set_body_bytes(vec![0, 0, 0, 0, 2, 18, 0]), + status => response + .insert_header("grpc-status", status) + .insert_header("grpc-message", "private backend details"), + } + }) + .expect(statuses.len() as u64) + .mount(&server) + .await; + let (client, commands, events) = CloudClient::channel(/*capacity*/ 4); + let mut client = AppServerClient::Cloud(client); + let requests_handle = client.request_handle(); + let AppServerRequestHandle::Cloud(handle) = &requests_handle else { + unreachable!(); + }; + let mut recovery = idle::Recovery::new( + handle.reconnect.clone(), + Arc::clone(&requests.client), + if outcome == "pat" { + idle::Login { + mode: codex_protocol::auth::AuthMode::PersonalAccessToken, + ..login.clone() + } + } else { + login.clone() + }, + ); + recovery.observed_idle = Some(idle); + let mut worker_replies = Replies { + idle: Some(recovery), + ..Default::default() + }; + let mut worker_requests = requests::Requests { + client: Arc::clone(&requests.client), + account: requests.account.clone(), + loaded: Some(cached.clone()), + warned_partial_history: Some("t".into()), + }; + let worker = tokio::spawn(async move { + run_worker(commands, &mut worker_requests, &mut worker_replies, &events).await; + }); + let budget = std::time::Duration::from_secs(if outcome == "slow-history" { 2 } else { 8 }); + let result = handle + .resume_disconnected_thread( + "t".into(), + /*require_history*/ false, + tokio::time::Instant::now() + budget, + ) + .await; + if outcome == "pat" || statuses.last() == Some(&"7") { + let Err(crate::TypedRequestError::Server { source, .. }) = result else { + panic!("expected recovery to reject this authentication mode or access denial"); + }; + assert_eq!( + source, + idle::recovery_error(if outcome == "pat" { + "Reopen this cloud conversation to reconnect with your personal access token." + } else { + "Your account no longer has access to this cloud conversation." + }) + ); + } else { + let (settings, release) = result?; + assert_eq!(settings.as_ref(), Some(cached.as_ref())); + assert!(client.next_event().now_or_never().is_none()); + let loaded: protocol::ThreadLoadedListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadLoadedList { + request_id: protocol::RequestId::Integer(4), + params: Default::default(), + }) + .await?; + assert!(loaded.data.is_empty()); + + let auth_path = home.path().join("auth.json"); + Mock::given(method("POST")) + .and(wiremock::matchers::path_regex( + "/oaiproto.codex_app_server_backend.ThreadService/List(Turns|Items)", + )) + .respond_with(move |request: &wiremock::Request| { + if outcome == "auth-change" { + std::fs::remove_file(&auth_path).unwrap(); + } + let body: &[u8] = if request.url.path().ends_with("ListItems") { + b"\0\0\0\0\x0f\x0a\x0a\x0a\x08\x0a\x01u\x12\x03\xa2\x01\0\x12\x01x" + } else { + b"\x00\x00\x00\x00\x02\x0a\x00" + }; + let response = ResponseTemplate::new(200) + .insert_header("content-type", "application/grpc") + .set_body_bytes(body); + if outcome == "slow-history" { + response.set_delay(std::time::Duration::from_secs(10)) + } else { + response + } + }) + .expect(if matches!(outcome, "auth-change" | "slow-history") { + 1 + } else { + 3 + }) + .mount(&server) + .await; + let history: Result = tokio::time::timeout( + std::time::Duration::from_secs(if outcome == "slow-history" { 3 } else { 2 }), + requests_handle.request_typed(protocol::ClientRequest::ThreadTurnsList { + request_id: protocol::RequestId::Integer(5), + params: protocol::ThreadTurnsListParams { + thread_id: "t".into(), + cursor: None, + limit: Some(5), + sort_direction: Some(protocol::SortDirection::Desc), + items_view: Some(protocol::TurnItemsView::NotLoaded), + }, + }), + ) + .await?; + if outcome == "auth-change" { + let Err(crate::TypedRequestError::Transport { source, .. }) = history else { + panic!("expected changed sign-in to withhold the fetched page"); + }; + assert!(source.get_ref().unwrap().is::()); + assert_eq!(source.to_string(), idle::SIGN_IN_AGAIN); + let items = requests_handle + .request(protocol::ClientRequest::ThreadItemsList { + request_id: protocol::RequestId::Integer(7), + params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?, + }) + .await + .expect_err("the changed sign-in must reject the next page before RPC"); + assert!(items.get_ref().unwrap().is::()); + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|request| !request.url.path().ends_with("/ListItems")) + ); + } else if outcome == "slow-history" { + let Err(crate::TypedRequestError::Server { source, .. }) = history else { + panic!("expected the recovery deadline to withhold the page"); + }; + assert_eq!(source, idle::recovery_error(idle::RECOVERY_TIMEOUT)); + } else { + assert_eq!( + history?, + protocol::ThreadTurnsListResponse { + data: Vec::new(), + next_cursor: None, + backwards_cursor: None, + } + ); + for page in 0..2 { + let _: protocol::ThreadItemsListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadItemsList { + request_id: protocol::RequestId::Integer(7 + page), + params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?, + }) + .await?; + if page == 0 { + assert!(matches!(client.next_event().now_or_never().flatten(), + Some(AppServerEvent::ServerNotification(notification)) + if matches!(*notification, protocol::ServerNotification::Warning(_)))); + } else { + assert!(client.next_event().now_or_never().is_none()); + } + } + } + if outcome != "install" { + drop(release); + if outcome == "retry" { + // No inventory read or yield may retire the canceled hold before this retry. + let (settings, release) = handle + .resume_disconnected_thread( + "t".into(), + /*require_history*/ true, + tokio::time::Instant::now() + std::time::Duration::from_secs(8), + ) + .await?; + assert_eq!(settings.as_ref(), Some(cached.as_ref())); + drop(release); + } + let loaded: protocol::ThreadLoadedListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadLoadedList { + request_id: protocol::RequestId::Integer(6), + params: Default::default(), + }) + .await?; + assert!(loaded.data.is_empty()); + assert!(client.next_event().now_or_never().is_none()); + } else { + // The wiremock response is finite: it must never ACK a live installation. + assert!(release.install().await.is_err()); + assert!(client.next_event().now_or_never().is_none()); + } + } + client.shutdown().await?; + worker.await?; + server.verify().await; + } Ok(()) } @@ -643,25 +935,24 @@ async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> { let factory = config.http_client_factory(); let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1); let handle = client.request_handle(); + let mut recovery = idle::Recovery::new( + handle.reconnect.clone(), + Arc::new(cloud::Client::new( + &factory, + &server.uri(), + cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, + )?), + idle::Login { + storage: config.auth_config(), + mode: codex_protocol::auth::AuthMode::Chatgpt, + token: "synthetic-bearer".into(), + account: "synthetic-account".into(), + permit: factory.network_policy().acquire(&server.uri().parse()?)?, + }, + ); + recovery.observed_idle = Some(true); let mut replies = Replies { - idle: Some(idle::Recovery { - state: handle.reconnect.clone(), - pending: None, - installed: None, - observed_idle: Some(true), - client: Arc::new(cloud::Client::new( - &factory, - &server.uri(), - cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, - )?), - login: Arc::new(idle::Login { - storage: config.auth_config(), - mode: codex_protocol::auth::AuthMode::Chatgpt, - token: "synthetic-bearer".into(), - account: "synthetic-account".into(), - permit: factory.network_policy().acquire(&server.uri().parse()?)?, - }), - }), + idle: Some(recovery), active: Some(Attachment { thread_id: "t".into(), stream: stream::empty().boxed(), diff --git a/codex-rs/app-server-client/src/cloud_backend/idle.rs b/codex-rs/app-server-client/src/cloud_backend/idle.rs index 3d6c4303fb..5fc100e629 100644 --- a/codex-rs/app-server-client/src/cloud_backend/idle.rs +++ b/codex-rs/app-server-client/src/cloud_backend/idle.rs @@ -1,8 +1,13 @@ -//! Reopens only the lost reply stream of a displayed, idle Cloud conversation. +//! Restores Cloud reply streams under the original login and network permit. +//! An active recovery holds its stream until the caller has read history. use super::ReplyStream; use super::check_reply_stream; +use crate::CLOUD_RECOVERY_NETWORK_BLOCKED; +use crate::RequestResult; use crate::cloud::IdleReconnect; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::ThreadResumeResponse; use codex_cloud_client as cloud; use codex_http_client::NetworkPermit; use codex_login::AuthConfig; @@ -10,13 +15,15 @@ use codex_login::token_data::parse_jwt_expiration; use codex_protocol::auth::AuthMode; use futures::FutureExt; use futures::future::BoxFuture; +use std::io; use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; +use tokio::sync::oneshot; use tokio::sync::watch; use tokio::time::Instant; -const SIGN_IN_AGAIN: &str = +pub(super) const SIGN_IN_AGAIN: &str = "Your cloud sign-in changed or expired. Sign in again to access this conversation."; pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> { @@ -33,6 +40,7 @@ pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> { const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation."; +#[derive(Clone)] pub(super) struct Login { pub(super) storage: AuthConfig, pub(super) mode: AuthMode, @@ -42,7 +50,7 @@ pub(super) struct Login { } impl Login { - async fn current(&self) -> bool { + pub(super) async fn current(&self) -> bool { if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) { return false; } @@ -105,12 +113,22 @@ impl Login { } } +pub(super) const RECOVERY_TIMEOUT: &str = "Cloud could not restore the reply stream in time."; +type Release = oneshot::Receiver>>; + +pub(super) struct Opened { + pub(super) stream: ReplyStream, + pub(super) first: Option, + pub(super) ack: oneshot::Sender>, +} + type Open = BoxFuture<'static, Result<(ReplyStream, Option), Option<&'static str>>>; pub(super) struct Recovery { pub(super) state: watch::Sender, pub(super) pending: Option<(Arc, Open)>, pub(super) installed: Option, + pub(super) held: Option<(String, Instant, BoxFuture<'static, Option>)>, // None until either the initial Attach event or the activation snapshot establishes status. pub(super) observed_idle: Option, pub(super) client: Arc, @@ -126,7 +144,97 @@ pub(super) fn transient(error: Option<&cloud::Error>) -> bool { } impl Recovery { + pub(super) fn check_permit(&self) -> Result<(), &'static str> { + self.login + .permit + .check() + .map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED) + } + + pub(super) fn history_login(&self, thread_id: &str) -> Option<(Arc, Instant)> { + self.held + .as_ref() + .filter(|(root, _, _)| root == thread_id) + .map(|(_, deadline, _)| (Arc::clone(&self.login), *deadline)) + } + + pub(super) fn start( + &mut self, + thread_id: String, + cached: Box, + _require_history: bool, + deadline: Instant, + release: Release, + mut response: oneshot::Sender>, + ) { + if response.is_closed() { + return; + } + if self.login.mode == AuthMode::PersonalAccessToken { + let _ = response.send(Ok(Err(recovery_error( + "Reopen this cloud conversation to reconnect with your personal access token.", + )))); + return; + } + self.cancel(); + let client = Arc::clone(&self.client); + let login = Arc::clone(&self.login); + let root = thread_id.clone(); + let open = async move { + let attempt = login.permit.run(login.attach(&client, &root)); + let opened = tokio::select! { + biased; + _ = response.closed() => return None, + result = tokio::time::timeout_at(deadline, attempt) => match result { + Ok(Ok(result)) => result, + Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED), + Err(_) => Err(RECOVERY_TIMEOUT), + }, + }; + let (stream, first) = match opened { + Ok(opened) => opened, + Err(reason) => { + let _ = response.send(Ok(Err(recovery_error(reason)))); + return None; + } + }; + // Attach cannot prove that no complete turn was missed during the outage. + // Attach resumes the conversation without configuration overrides. Keep + // the settings already loaded for this thread and refresh its history. + let settings = Some(cached); + let value = match serde_json::to_value(&settings) { + Ok(value) => value, + Err(error) => { + let _ = response.send(Err(io::Error::other(error))); + return None; + } + }; + response.send(Ok(Ok(value))).ok()?; + let ack = tokio::time::timeout_at(deadline, release) + .await + .ok()? + .ok()?; + let checked = login.permit.run(login.current()); + let authorized = match tokio::time::timeout_at(deadline, checked).await { + Ok(Ok(true)) => login + .permit + .check() + .map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED), + Ok(Ok(false)) => Err(SIGN_IN_AGAIN), + Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED), + Err(_) => Err(RECOVERY_TIMEOUT), + }; + if let Err(reason) = authorized { + let _ = ack.send(Err(reason)); + return None; + } + Some(Opened { stream, first, ack }) + }; + self.held = Some((thread_id, deadline, open.boxed())); + } + pub(super) fn cancel(&mut self) -> Option> { + self.held = None; drop(self.pending.take()?); let state = self.state.send_replace(IdleReconnect::Inactive); if let IdleReconnect::Opening(root) = state { @@ -137,6 +245,7 @@ impl Recovery { } pub(super) fn reset(&mut self) { + self.held = None; drop(self.pending.take()); self.state.send_replace(IdleReconnect::Inactive); self.installed = None; @@ -175,6 +284,22 @@ impl Recovery { } } + pub(super) fn new( + state: watch::Sender, + client: Arc, + login: Login, + ) -> Self { + Self { + state, + pending: None, + installed: None, + held: None, + observed_idle: None, + client, + login: Arc::new(login), + } + } + pub(super) fn offer(&mut self, id: &str) { // Only OAuth credentials can be rechecked locally. PATs use ordinary explicit reopen. if !matches!( @@ -209,3 +334,11 @@ impl Recovery { self.pending = Some((root, open.boxed())); } } + +pub(super) fn recovery_error(reason: &'static str) -> JSONRPCErrorError { + JSONRPCErrorError { + code: -32000, + message: reason.to_owned(), + data: None, + } +} diff --git a/codex-rs/app-server-client/src/cloud_backend/mod.rs b/codex-rs/app-server-client/src/cloud_backend/mod.rs index 355a77ba59..75b2108f87 100644 --- a/codex-rs/app-server-client/src/cloud_backend/mod.rs +++ b/codex-rs/app-server-client/src/cloud_backend/mod.rs @@ -10,6 +10,7 @@ mod thread_title; use crate::AppServerClient; use crate::AppServerEvent; +use crate::CLOUD_RECOVERY_NETWORK_BLOCKED; use crate::CloudClient; use crate::CloudOperation; use crate::cloud::CloudStreamLosses; @@ -54,6 +55,12 @@ struct PreflightRejection(&'static str); #[error("Cloud stream is not connected. Reopen the conversation before sending.")] struct MissingReplyAttachment; +/// Local rejection when recovery history no longer belongs to the bound sign-in. +/// This carries no credentials and never enters the app-server wire envelope. +#[derive(Debug, thiserror::Error)] +#[error("{}", idle::SIGN_IN_AGAIN)] +pub struct CloudRecoverySignInChanged; + struct Attachment { thread_id: String, stream: ReplyStream, @@ -70,6 +77,40 @@ struct Replies { } impl Replies { + async fn next_event(&mut self, output: &mpsc::UnboundedSender) { + if let Some(recovery) = &mut self.idle + && let Some((root, _, open)) = &mut recovery.held + { + let opened = open.await; + let root = std::mem::take(root); + recovery.held = None; + if let Some(opened) = opened { + if let Err(reason) = recovery.check_permit() { + let _ = opened.ack.send(Err(reason)); + return; + } + let Ok((stream, first)) = check_prepared_stream((opened.stream, opened.first)) + else { + let _ = opened + .ack + .send(Err("Cloud reply stream disconnected again.")); + return; + }; + // Commit through the acknowledgement. If the caller timed out + // and closed it first, leave the stream held and drop it here. + if opened.ack.send(Ok(())).is_err() { + return; + } + recovery.installed = Some(tokio::time::Instant::now()); + recovery.observed_idle = None; + self.stream_losses.supersede(&root); + self.install(&root, stream, first, output); + } + } else { + self.forward_next(output).await; + } + } + async fn forward_next(&mut self, output: &mpsc::UnboundedSender) { if self.active.is_none() && let Some(recovery) = &mut self.idle @@ -161,7 +202,7 @@ impl Replies { tokio::select! { biased; result = &mut future => return result, - () = self.forward_next(output) => {} + () = self.next_event(output) => {} } } } @@ -238,10 +279,11 @@ impl Replies { fn detach(&mut self, id: &str) { if let Some(idle) = &mut self.idle - && (idle - .pending - .as_ref() - .is_some_and(|(root, _)| root.as_ref() == id) + && (idle.held.as_ref().is_some_and(|(root, _, _)| root == id) + || idle + .pending + .as_ref() + .is_some_and(|(root, _)| root.as_ref() == id) || self .active .as_ref() @@ -305,14 +347,11 @@ fn start_worker( warned_partial_history: None, }; let mut replies = Replies { - idle: Some(idle::Recovery { - state: app_client.request_handle().reconnect, - pending: None, - installed: None, - observed_idle: None, + idle: Some(idle::Recovery::new( + app_client.request_handle().reconnect, client, - login: Arc::new(login), - }), + login, + )), stream_losses: app_client.stream_losses.clone(), ..Default::default() }; @@ -341,7 +380,7 @@ async fn run_worker_with_prepared( loop { let mut command = tokio::select! { _ = output.closed() => return, - () = replies.forward_next(output) => continue, + () = replies.next_event(output) => continue, command = commands.recv() => match command { Some(command) => command, None => return }, command = async { match &mut prepared { @@ -440,6 +479,45 @@ async fn run_worker_with_prepared( cloud::SubmissionMode::Steer, None, )), + CloudOperation::ResumeDisconnectedThread { + thread_id, + require_history, + deadline, + release, + } => { + // Retire canceled holds, but commit a ready handoff through its normal ACK. + if replies + .idle + .as_ref() + .is_some_and(|recovery| recovery.held.is_some()) + { + let _ = replies.next_event(output).now_or_never(); + } + if let Some(recovery) = &mut replies.idle + && let Some(cached) = requests + .loaded + .as_ref() + .filter(|settings| settings.thread.id == thread_id) + && replies.active.is_none() + && recovery.held.is_none() + { + // Start a fresh warning epoch before any recovery history is read. + requests.warned_partial_history = None; + recovery.start( + thread_id, + cached.clone(), + require_history, + deadline, + release, + command.response, + ); + } else { + let _ = command.response.send(Ok(Err(idle::recovery_error( + "This cloud conversation no longer has a reply attachment to restore.", + )))); + } + continue; + } CloudOperation::Notify(_) => { Err(eyre!("Cloud client notifications are not supported yet.")) } @@ -462,12 +540,18 @@ async fn run_worker_with_prepared( || loaded.is_some_and(|loaded| loaded.thread.id == *id) }); // Inventory and history reads have no conversation state to commit. - let is_read = matches!( - &request, - protocol::ClientRequest::ThreadList { .. } - | protocol::ClientRequest::ThreadTurnsList { .. } - | protocol::ClientRequest::ThreadItemsList { .. } - ); + let history_thread = match &request { + protocol::ClientRequest::ThreadTurnsList { params, .. } => { + Some(¶ms.thread_id) + } + protocol::ClientRequest::ThreadItemsList { params, .. } => { + Some(¶ms.thread_id) + } + _ => None, + }; + let is_read = history_thread.is_some() + || matches!(&request, protocol::ClientRequest::ThreadList { .. }); + let login = history_thread.and_then(|id| replies.idle.as_ref()?.history_login(id)); // When closure and the handshake are both ready, cancel before // the handshake can replace the active attachment. let result = tokio::select! { @@ -479,7 +563,24 @@ async fn run_worker_with_prepared( } continue; } - result = handle_request(request, mode, environment, requests, replies, output) => result, + result = async { + let read = handle_request(request, mode, environment, requests, replies, output); + let Some((login, deadline)) = login else { + return read.await; + }; + tokio::time::timeout_at(deadline, login.permit.run(async { + if !login.current().await { + return Err(eyre!(CloudRecoverySignInChanged)); + } + let result = read.await; + if !login.current().await { + return Err(eyre!(CloudRecoverySignInChanged)); + } + login.permit.check().map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))?; + result + })).await.map_err(|_| eyre!(idle::RECOVERY_TIMEOUT))? + .map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))? + } => result, }; if unloads && result.is_ok() @@ -499,7 +600,13 @@ async fn run_worker_with_prepared( { replies.report_loss(&root, /*reason*/ None, output); } - let _ = command.response.send(Ok(request_result(result))); + let response = match result { + Err(error) if error.is::() => { + Err(std::io::Error::other(CloudRecoverySignInChanged)) + } + result => Ok(request_result(result)), + }; + let _ = command.response.send(response); } } diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index d17c86cb94..14dbcbea0b 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -29,11 +29,14 @@ mod path; mod remote; mod remote_addr; mod remote_websocket; +pub use cloud::CLOUD_RECOVERY_NETWORK_BLOCKED; pub use cloud::CloudClient; pub use cloud::CloudCommand; pub use cloud::CloudOperation; +pub use cloud::CloudRecoveryRelease; pub use cloud::CloudRequestHandle; pub use cloud::CloudUserMessageReceipt; +pub use cloud_backend::CloudRecoverySignInChanged; mod server_request; pub use server_request::AppServerReplyHandle;