diff --git a/codex-rs/app-server-client/src/cloud.rs b/codex-rs/app-server-client/src/cloud.rs index d575c13170..7de4358d20 100644 --- a/codex-rs/app-server-client/src/cloud.rs +++ b/codex-rs/app-server-client/src/cloud.rs @@ -11,6 +11,7 @@ use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::Result as JsonRpcResult; +use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadSetNameParams; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnStartParams; @@ -22,6 +23,12 @@ use std::sync::MutexGuard; use std::sync::PoisonError; use tokio::sync::mpsc; use tokio::sync::oneshot; +use tokio::time::Instant; +use tokio::time::timeout_at; + +/// Rejection from the original network policy during Cloud recovery. +pub const CLOUD_RECOVERY_NETWORK_BLOCKED: &str = + "The current network policy blocks cloud reconnection."; /// Operations the cloud worker must handle without forwarding to another backend. /// Unsupported operations must return an explicit error; they must not claim success. @@ -44,6 +51,12 @@ pub enum CloudOperation { SubmitUserMessage(Box), /// Apply the result of a name write that already succeeded outside the command queue. ThreadNameSaved(ThreadSetNameParams), + ResumeDisconnectedThread { + thread_id: String, + require_history: bool, + deadline: Instant, + release: oneshot::Receiver>>, + }, Notify(ClientNotification), Resolve { request_id: RequestId, @@ -72,6 +85,30 @@ pub struct CloudCommand { pub response: oneshot::Sender>, } +/// Allows the worker to install a held stream after the caller reads its history. +/// Dropping the token cancels that replacement. +pub struct CloudRecoveryRelease { + release: oneshot::Sender>>, + deadline: Instant, +} + +impl CloudRecoveryRelease { + pub async fn install(self) -> Result<(), &'static str> { + let (ack, mut installed) = oneshot::channel(); + self.release + .send(ack) + .map_err(|_| "Cloud reconnection expired.")?; + let response = match timeout_at(self.deadline, &mut installed).await { + Ok(result) => result.ok(), + Err(_) => { + installed.close(); + installed.try_recv().ok() + } + }; + response.unwrap_or(Err("Cloud reconnection expired.")) + } +} + #[derive(Clone, Default)] pub(super) enum IdleReconnect { #[default] @@ -298,6 +335,30 @@ impl CloudRequestHandle { }) } + pub async fn resume_disconnected_thread( + &self, + thread_id: String, + require_history: bool, + deadline: Instant, + ) -> Result<(Option, CloudRecoveryRelease), TypedRequestError> { + let (release, held) = oneshot::channel(); + let response = timeout_at( + deadline, + self.send(CloudOperation::ResumeDisconnectedThread { + thread_id, + require_history, + deadline, + release: held, + }), + ) + .await + .map_err(io::Error::from) + .and_then(std::convert::identity); + let settings: Option = + crate::decode_typed_response("thread/resume", response)?; + Ok((settings, CloudRecoveryRelease { release, deadline })) + } + /// Whether a completed model read still belongs to the bound Cloud account. pub fn model_catalog_is_current(&self) -> bool { self.native diff --git a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs index 8f72bc5b48..0b1dc1c539 100644 --- a/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs +++ b/codex-rs/app-server-client/src/cloud_backend/connection_tests.rs @@ -3,6 +3,7 @@ //! Exercises native worker progress and reply attachment. use super::*; +use crate::AppServerRequestHandle; use futures::stream; use pretty_assertions::assert_eq; use std::sync::Mutex; @@ -93,12 +94,30 @@ async fn rejected_rename_preserves_loaded_cloud_thread_name() -> Result<()> { async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> Result<()> { let server = MockServer::start().await; let home = tempfile::tempdir()?; - let config = crate::legacy_core::config::ConfigBuilder::default() + let mut config = crate::legacy_core::config::ConfigBuilder::default() .codex_home(home.path().to_owned()) .fallback_cwd(Some(home.path().to_owned())) .without_project_context() .build() .await?; + config.cli_auth_credentials_store_mode = codex_login::AuthCredentialsStoreMode::File; + let storage = config.auth_config(); + let token = "e30.e30.sig"; + let account = "synthetic-account"; + let auth = serde_json::from_value(serde_json::json!({ + "auth_mode": "chatgpt", + "tokens": { + "id_token": token, "access_token": token, "refresh_token": "test", "account_id": account, + }, + "last_refresh": "2026-01-01T00:00:00Z", + }))?; + codex_login::save_auth( + home.path(), + &auth, + storage.auth_credentials_store_mode, + storage.keyring_backend_kind, + )?; + let cached = Box::new(serde_json::from_value::( serde_json::json!({ "thread": { @@ -116,7 +135,7 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R client: std::sync::Arc::new(cloud::Client::new( &config.http_client_factory(), &server.uri(), - cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, + cloud::Credentials::new(token, account)?, )?), account: protocol::GetAccountResponse { user_id: None, @@ -222,6 +241,279 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R ) ); assert!(notifications.try_recv().is_err()); + let login = idle::Login { + storage, + mode: codex_protocol::auth::AuthMode::Chatgpt, + token: token.into(), + account: account.into(), + permit: config + .http_client_factory() + .network_policy() + .acquire(&server.uri().parse()?)?, + }; + // A live replacement installs and forwards held and buffered events. + let (ack, installed) = oneshot::channel(); + let (state, _) = tokio::sync::watch::channel(crate::cloud::IdleReconnect::Inactive); + let mut recovery = idle::Recovery::new(state, Arc::clone(&requests.client), login.clone()); + recovery.held = Some(( + "t".into(), + tokio::time::Instant::now() + std::time::Duration::from_secs(8), + Box::pin(async move { + let idle: &[u8] = b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t"; + Some(idle::Opened { + stream: stream::iter([Ok(cloud::Event::Notification(idle.into()))]) + .chain(stream::pending()) + .boxed(), + first: Some(cloud::Event::ServerRequest(Default::default())), + ack, + }) + }), + )); + replies.idle = Some(recovery); + requests.warned_partial_history = Some("t".into()); + replies.next_event(&output).await; + assert_eq!(installed.await?, Ok(())); + let AppServerEvent::ServerNotification(notification) = notifications.try_recv()? else { + panic!("expected held first event"); + }; + assert!(matches!( + *notification, + protocol::ServerNotification::Warning(_) + )); + replies.next_event(&output).await; + assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle)); + // Installation must not erase a warning already emitted by the refreshed history. + let (commands, receiver) = mpsc::channel(1); + drop(commands); + run_worker(receiver, &mut requests, &mut replies, &output).await; + assert_eq!(requests.warned_partial_history.as_deref(), Some("t")); + + // A quiet Attach cannot prove that no whole turn was missed while disconnected. + let attach_path = "/oaiproto.codex_app_server_backend.ThreadService/Attach"; + for (idle, statuses, outcome) in [ + (true, &[][..], "pat"), + (true, &["quiet"][..], "install"), + (true, &["7"][..], "install"), + (false, &["14", "7"][..], "install"), + (false, &["14", "busy"][..], "install"), + (true, &["busy"][..], "install"), + (true, &["busy"][..], "cancel"), + (true, &["busy", "busy"][..], "retry"), + (true, &["busy"][..], "slow-history"), + (true, &["busy"][..], "auth-change"), + ] { + server.reset().await; + let attempt = std::sync::atomic::AtomicUsize::new(0); + Mock::given(method("POST")) + .and(path(attach_path)) + .and(header("authorization", format!("Bearer {token}"))) + .respond_with(move |_: &wiremock::Request| { + let index = attempt.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let response = + ResponseTemplate::new(200).insert_header("content-type", "application/grpc"); + match statuses[index] { + "quiet" => response.set_body_bytes(b"\x00\x00\x00\x00\x02\x0a\x00".to_vec()), + "busy" => response.set_body_bytes(vec![0, 0, 0, 0, 2, 18, 0]), + status => response + .insert_header("grpc-status", status) + .insert_header("grpc-message", "private backend details"), + } + }) + .expect(statuses.len() as u64) + .mount(&server) + .await; + let (client, commands, events) = CloudClient::channel(/*capacity*/ 4); + let mut client = AppServerClient::Cloud(client); + let requests_handle = client.request_handle(); + let AppServerRequestHandle::Cloud(handle) = &requests_handle else { + unreachable!(); + }; + let mut recovery = idle::Recovery::new( + handle.reconnect.clone(), + Arc::clone(&requests.client), + if outcome == "pat" { + idle::Login { + mode: codex_protocol::auth::AuthMode::PersonalAccessToken, + ..login.clone() + } + } else { + login.clone() + }, + ); + recovery.observed_idle = Some(idle); + let mut worker_replies = Replies { + idle: Some(recovery), + ..Default::default() + }; + let mut worker_requests = requests::Requests { + client: Arc::clone(&requests.client), + account: requests.account.clone(), + loaded: Some(cached.clone()), + warned_partial_history: Some("t".into()), + }; + let worker = tokio::spawn(async move { + run_worker(commands, &mut worker_requests, &mut worker_replies, &events).await; + }); + let budget = std::time::Duration::from_secs(if outcome == "slow-history" { 2 } else { 8 }); + let result = handle + .resume_disconnected_thread( + "t".into(), + /*require_history*/ false, + tokio::time::Instant::now() + budget, + ) + .await; + if outcome == "pat" || statuses.last() == Some(&"7") { + let Err(crate::TypedRequestError::Server { source, .. }) = result else { + panic!("expected recovery to reject this authentication mode or access denial"); + }; + assert_eq!( + source, + idle::recovery_error(if outcome == "pat" { + "Reopen this cloud conversation to reconnect with your personal access token." + } else { + "Your account no longer has access to this cloud conversation." + }) + ); + } else { + let (settings, release) = result?; + assert_eq!(settings.as_ref(), Some(cached.as_ref())); + assert!(client.next_event().now_or_never().is_none()); + let loaded: protocol::ThreadLoadedListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadLoadedList { + request_id: protocol::RequestId::Integer(4), + params: Default::default(), + }) + .await?; + assert!(loaded.data.is_empty()); + + let auth_path = home.path().join("auth.json"); + Mock::given(method("POST")) + .and(wiremock::matchers::path_regex( + "/oaiproto.codex_app_server_backend.ThreadService/List(Turns|Items)", + )) + .respond_with(move |request: &wiremock::Request| { + if outcome == "auth-change" { + std::fs::remove_file(&auth_path).unwrap(); + } + let body: &[u8] = if request.url.path().ends_with("ListItems") { + b"\0\0\0\0\x0f\x0a\x0a\x0a\x08\x0a\x01u\x12\x03\xa2\x01\0\x12\x01x" + } else { + b"\x00\x00\x00\x00\x02\x0a\x00" + }; + let response = ResponseTemplate::new(200) + .insert_header("content-type", "application/grpc") + .set_body_bytes(body); + if outcome == "slow-history" { + response.set_delay(std::time::Duration::from_secs(10)) + } else { + response + } + }) + .expect(if matches!(outcome, "auth-change" | "slow-history") { + 1 + } else { + 3 + }) + .mount(&server) + .await; + let history: Result = tokio::time::timeout( + std::time::Duration::from_secs(if outcome == "slow-history" { 3 } else { 2 }), + requests_handle.request_typed(protocol::ClientRequest::ThreadTurnsList { + request_id: protocol::RequestId::Integer(5), + params: protocol::ThreadTurnsListParams { + thread_id: "t".into(), + cursor: None, + limit: Some(5), + sort_direction: Some(protocol::SortDirection::Desc), + items_view: Some(protocol::TurnItemsView::NotLoaded), + }, + }), + ) + .await?; + if outcome == "auth-change" { + let Err(crate::TypedRequestError::Transport { source, .. }) = history else { + panic!("expected changed sign-in to withhold the fetched page"); + }; + assert!(source.get_ref().unwrap().is::()); + assert_eq!(source.to_string(), idle::SIGN_IN_AGAIN); + let items = requests_handle + .request(protocol::ClientRequest::ThreadItemsList { + request_id: protocol::RequestId::Integer(7), + params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?, + }) + .await + .expect_err("the changed sign-in must reject the next page before RPC"); + assert!(items.get_ref().unwrap().is::()); + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|request| !request.url.path().ends_with("/ListItems")) + ); + } else if outcome == "slow-history" { + let Err(crate::TypedRequestError::Server { source, .. }) = history else { + panic!("expected the recovery deadline to withhold the page"); + }; + assert_eq!(source, idle::recovery_error(idle::RECOVERY_TIMEOUT)); + } else { + assert_eq!( + history?, + protocol::ThreadTurnsListResponse { + data: Vec::new(), + next_cursor: None, + backwards_cursor: None, + } + ); + for page in 0..2 { + let _: protocol::ThreadItemsListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadItemsList { + request_id: protocol::RequestId::Integer(7 + page), + params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?, + }) + .await?; + if page == 0 { + assert!(matches!(client.next_event().now_or_never().flatten(), + Some(AppServerEvent::ServerNotification(notification)) + if matches!(*notification, protocol::ServerNotification::Warning(_)))); + } else { + assert!(client.next_event().now_or_never().is_none()); + } + } + } + if outcome != "install" { + drop(release); + if outcome == "retry" { + // No inventory read or yield may retire the canceled hold before this retry. + let (settings, release) = handle + .resume_disconnected_thread( + "t".into(), + /*require_history*/ true, + tokio::time::Instant::now() + std::time::Duration::from_secs(8), + ) + .await?; + assert_eq!(settings.as_ref(), Some(cached.as_ref())); + drop(release); + } + let loaded: protocol::ThreadLoadedListResponse = requests_handle + .request_typed(protocol::ClientRequest::ThreadLoadedList { + request_id: protocol::RequestId::Integer(6), + params: Default::default(), + }) + .await?; + assert!(loaded.data.is_empty()); + assert!(client.next_event().now_or_never().is_none()); + } else { + // The wiremock response is finite: it must never ACK a live installation. + assert!(release.install().await.is_err()); + assert!(client.next_event().now_or_never().is_none()); + } + } + client.shutdown().await?; + worker.await?; + server.verify().await; + } Ok(()) } @@ -643,25 +935,24 @@ async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> { let factory = config.http_client_factory(); let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1); let handle = client.request_handle(); + let mut recovery = idle::Recovery::new( + handle.reconnect.clone(), + Arc::new(cloud::Client::new( + &factory, + &server.uri(), + cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, + )?), + idle::Login { + storage: config.auth_config(), + mode: codex_protocol::auth::AuthMode::Chatgpt, + token: "synthetic-bearer".into(), + account: "synthetic-account".into(), + permit: factory.network_policy().acquire(&server.uri().parse()?)?, + }, + ); + recovery.observed_idle = Some(true); let mut replies = Replies { - idle: Some(idle::Recovery { - state: handle.reconnect.clone(), - pending: None, - installed: None, - observed_idle: Some(true), - client: Arc::new(cloud::Client::new( - &factory, - &server.uri(), - cloud::Credentials::new("synthetic-bearer", "synthetic-account")?, - )?), - login: Arc::new(idle::Login { - storage: config.auth_config(), - mode: codex_protocol::auth::AuthMode::Chatgpt, - token: "synthetic-bearer".into(), - account: "synthetic-account".into(), - permit: factory.network_policy().acquire(&server.uri().parse()?)?, - }), - }), + idle: Some(recovery), active: Some(Attachment { thread_id: "t".into(), stream: stream::empty().boxed(), diff --git a/codex-rs/app-server-client/src/cloud_backend/idle.rs b/codex-rs/app-server-client/src/cloud_backend/idle.rs index 3d6c4303fb..5fc100e629 100644 --- a/codex-rs/app-server-client/src/cloud_backend/idle.rs +++ b/codex-rs/app-server-client/src/cloud_backend/idle.rs @@ -1,8 +1,13 @@ -//! Reopens only the lost reply stream of a displayed, idle Cloud conversation. +//! Restores Cloud reply streams under the original login and network permit. +//! An active recovery holds its stream until the caller has read history. use super::ReplyStream; use super::check_reply_stream; +use crate::CLOUD_RECOVERY_NETWORK_BLOCKED; +use crate::RequestResult; use crate::cloud::IdleReconnect; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::ThreadResumeResponse; use codex_cloud_client as cloud; use codex_http_client::NetworkPermit; use codex_login::AuthConfig; @@ -10,13 +15,15 @@ use codex_login::token_data::parse_jwt_expiration; use codex_protocol::auth::AuthMode; use futures::FutureExt; use futures::future::BoxFuture; +use std::io; use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; +use tokio::sync::oneshot; use tokio::sync::watch; use tokio::time::Instant; -const SIGN_IN_AGAIN: &str = +pub(super) const SIGN_IN_AGAIN: &str = "Your cloud sign-in changed or expired. Sign in again to access this conversation."; pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> { @@ -33,6 +40,7 @@ pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> { const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation."; +#[derive(Clone)] pub(super) struct Login { pub(super) storage: AuthConfig, pub(super) mode: AuthMode, @@ -42,7 +50,7 @@ pub(super) struct Login { } impl Login { - async fn current(&self) -> bool { + pub(super) async fn current(&self) -> bool { if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) { return false; } @@ -105,12 +113,22 @@ impl Login { } } +pub(super) const RECOVERY_TIMEOUT: &str = "Cloud could not restore the reply stream in time."; +type Release = oneshot::Receiver>>; + +pub(super) struct Opened { + pub(super) stream: ReplyStream, + pub(super) first: Option, + pub(super) ack: oneshot::Sender>, +} + type Open = BoxFuture<'static, Result<(ReplyStream, Option), Option<&'static str>>>; pub(super) struct Recovery { pub(super) state: watch::Sender, pub(super) pending: Option<(Arc, Open)>, pub(super) installed: Option, + pub(super) held: Option<(String, Instant, BoxFuture<'static, Option>)>, // None until either the initial Attach event or the activation snapshot establishes status. pub(super) observed_idle: Option, pub(super) client: Arc, @@ -126,7 +144,97 @@ pub(super) fn transient(error: Option<&cloud::Error>) -> bool { } impl Recovery { + pub(super) fn check_permit(&self) -> Result<(), &'static str> { + self.login + .permit + .check() + .map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED) + } + + pub(super) fn history_login(&self, thread_id: &str) -> Option<(Arc, Instant)> { + self.held + .as_ref() + .filter(|(root, _, _)| root == thread_id) + .map(|(_, deadline, _)| (Arc::clone(&self.login), *deadline)) + } + + pub(super) fn start( + &mut self, + thread_id: String, + cached: Box, + _require_history: bool, + deadline: Instant, + release: Release, + mut response: oneshot::Sender>, + ) { + if response.is_closed() { + return; + } + if self.login.mode == AuthMode::PersonalAccessToken { + let _ = response.send(Ok(Err(recovery_error( + "Reopen this cloud conversation to reconnect with your personal access token.", + )))); + return; + } + self.cancel(); + let client = Arc::clone(&self.client); + let login = Arc::clone(&self.login); + let root = thread_id.clone(); + let open = async move { + let attempt = login.permit.run(login.attach(&client, &root)); + let opened = tokio::select! { + biased; + _ = response.closed() => return None, + result = tokio::time::timeout_at(deadline, attempt) => match result { + Ok(Ok(result)) => result, + Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED), + Err(_) => Err(RECOVERY_TIMEOUT), + }, + }; + let (stream, first) = match opened { + Ok(opened) => opened, + Err(reason) => { + let _ = response.send(Ok(Err(recovery_error(reason)))); + return None; + } + }; + // Attach cannot prove that no complete turn was missed during the outage. + // Attach resumes the conversation without configuration overrides. Keep + // the settings already loaded for this thread and refresh its history. + let settings = Some(cached); + let value = match serde_json::to_value(&settings) { + Ok(value) => value, + Err(error) => { + let _ = response.send(Err(io::Error::other(error))); + return None; + } + }; + response.send(Ok(Ok(value))).ok()?; + let ack = tokio::time::timeout_at(deadline, release) + .await + .ok()? + .ok()?; + let checked = login.permit.run(login.current()); + let authorized = match tokio::time::timeout_at(deadline, checked).await { + Ok(Ok(true)) => login + .permit + .check() + .map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED), + Ok(Ok(false)) => Err(SIGN_IN_AGAIN), + Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED), + Err(_) => Err(RECOVERY_TIMEOUT), + }; + if let Err(reason) = authorized { + let _ = ack.send(Err(reason)); + return None; + } + Some(Opened { stream, first, ack }) + }; + self.held = Some((thread_id, deadline, open.boxed())); + } + pub(super) fn cancel(&mut self) -> Option> { + self.held = None; drop(self.pending.take()?); let state = self.state.send_replace(IdleReconnect::Inactive); if let IdleReconnect::Opening(root) = state { @@ -137,6 +245,7 @@ impl Recovery { } pub(super) fn reset(&mut self) { + self.held = None; drop(self.pending.take()); self.state.send_replace(IdleReconnect::Inactive); self.installed = None; @@ -175,6 +284,22 @@ impl Recovery { } } + pub(super) fn new( + state: watch::Sender, + client: Arc, + login: Login, + ) -> Self { + Self { + state, + pending: None, + installed: None, + held: None, + observed_idle: None, + client, + login: Arc::new(login), + } + } + pub(super) fn offer(&mut self, id: &str) { // Only OAuth credentials can be rechecked locally. PATs use ordinary explicit reopen. if !matches!( @@ -209,3 +334,11 @@ impl Recovery { self.pending = Some((root, open.boxed())); } } + +pub(super) fn recovery_error(reason: &'static str) -> JSONRPCErrorError { + JSONRPCErrorError { + code: -32000, + message: reason.to_owned(), + data: None, + } +} diff --git a/codex-rs/app-server-client/src/cloud_backend/mod.rs b/codex-rs/app-server-client/src/cloud_backend/mod.rs index 355a77ba59..75b2108f87 100644 --- a/codex-rs/app-server-client/src/cloud_backend/mod.rs +++ b/codex-rs/app-server-client/src/cloud_backend/mod.rs @@ -10,6 +10,7 @@ mod thread_title; use crate::AppServerClient; use crate::AppServerEvent; +use crate::CLOUD_RECOVERY_NETWORK_BLOCKED; use crate::CloudClient; use crate::CloudOperation; use crate::cloud::CloudStreamLosses; @@ -54,6 +55,12 @@ struct PreflightRejection(&'static str); #[error("Cloud stream is not connected. Reopen the conversation before sending.")] struct MissingReplyAttachment; +/// Local rejection when recovery history no longer belongs to the bound sign-in. +/// This carries no credentials and never enters the app-server wire envelope. +#[derive(Debug, thiserror::Error)] +#[error("{}", idle::SIGN_IN_AGAIN)] +pub struct CloudRecoverySignInChanged; + struct Attachment { thread_id: String, stream: ReplyStream, @@ -70,6 +77,40 @@ struct Replies { } impl Replies { + async fn next_event(&mut self, output: &mpsc::UnboundedSender) { + if let Some(recovery) = &mut self.idle + && let Some((root, _, open)) = &mut recovery.held + { + let opened = open.await; + let root = std::mem::take(root); + recovery.held = None; + if let Some(opened) = opened { + if let Err(reason) = recovery.check_permit() { + let _ = opened.ack.send(Err(reason)); + return; + } + let Ok((stream, first)) = check_prepared_stream((opened.stream, opened.first)) + else { + let _ = opened + .ack + .send(Err("Cloud reply stream disconnected again.")); + return; + }; + // Commit through the acknowledgement. If the caller timed out + // and closed it first, leave the stream held and drop it here. + if opened.ack.send(Ok(())).is_err() { + return; + } + recovery.installed = Some(tokio::time::Instant::now()); + recovery.observed_idle = None; + self.stream_losses.supersede(&root); + self.install(&root, stream, first, output); + } + } else { + self.forward_next(output).await; + } + } + async fn forward_next(&mut self, output: &mpsc::UnboundedSender) { if self.active.is_none() && let Some(recovery) = &mut self.idle @@ -161,7 +202,7 @@ impl Replies { tokio::select! { biased; result = &mut future => return result, - () = self.forward_next(output) => {} + () = self.next_event(output) => {} } } } @@ -238,10 +279,11 @@ impl Replies { fn detach(&mut self, id: &str) { if let Some(idle) = &mut self.idle - && (idle - .pending - .as_ref() - .is_some_and(|(root, _)| root.as_ref() == id) + && (idle.held.as_ref().is_some_and(|(root, _, _)| root == id) + || idle + .pending + .as_ref() + .is_some_and(|(root, _)| root.as_ref() == id) || self .active .as_ref() @@ -305,14 +347,11 @@ fn start_worker( warned_partial_history: None, }; let mut replies = Replies { - idle: Some(idle::Recovery { - state: app_client.request_handle().reconnect, - pending: None, - installed: None, - observed_idle: None, + idle: Some(idle::Recovery::new( + app_client.request_handle().reconnect, client, - login: Arc::new(login), - }), + login, + )), stream_losses: app_client.stream_losses.clone(), ..Default::default() }; @@ -341,7 +380,7 @@ async fn run_worker_with_prepared( loop { let mut command = tokio::select! { _ = output.closed() => return, - () = replies.forward_next(output) => continue, + () = replies.next_event(output) => continue, command = commands.recv() => match command { Some(command) => command, None => return }, command = async { match &mut prepared { @@ -440,6 +479,45 @@ async fn run_worker_with_prepared( cloud::SubmissionMode::Steer, None, )), + CloudOperation::ResumeDisconnectedThread { + thread_id, + require_history, + deadline, + release, + } => { + // Retire canceled holds, but commit a ready handoff through its normal ACK. + if replies + .idle + .as_ref() + .is_some_and(|recovery| recovery.held.is_some()) + { + let _ = replies.next_event(output).now_or_never(); + } + if let Some(recovery) = &mut replies.idle + && let Some(cached) = requests + .loaded + .as_ref() + .filter(|settings| settings.thread.id == thread_id) + && replies.active.is_none() + && recovery.held.is_none() + { + // Start a fresh warning epoch before any recovery history is read. + requests.warned_partial_history = None; + recovery.start( + thread_id, + cached.clone(), + require_history, + deadline, + release, + command.response, + ); + } else { + let _ = command.response.send(Ok(Err(idle::recovery_error( + "This cloud conversation no longer has a reply attachment to restore.", + )))); + } + continue; + } CloudOperation::Notify(_) => { Err(eyre!("Cloud client notifications are not supported yet.")) } @@ -462,12 +540,18 @@ async fn run_worker_with_prepared( || loaded.is_some_and(|loaded| loaded.thread.id == *id) }); // Inventory and history reads have no conversation state to commit. - let is_read = matches!( - &request, - protocol::ClientRequest::ThreadList { .. } - | protocol::ClientRequest::ThreadTurnsList { .. } - | protocol::ClientRequest::ThreadItemsList { .. } - ); + let history_thread = match &request { + protocol::ClientRequest::ThreadTurnsList { params, .. } => { + Some(¶ms.thread_id) + } + protocol::ClientRequest::ThreadItemsList { params, .. } => { + Some(¶ms.thread_id) + } + _ => None, + }; + let is_read = history_thread.is_some() + || matches!(&request, protocol::ClientRequest::ThreadList { .. }); + let login = history_thread.and_then(|id| replies.idle.as_ref()?.history_login(id)); // When closure and the handshake are both ready, cancel before // the handshake can replace the active attachment. let result = tokio::select! { @@ -479,7 +563,24 @@ async fn run_worker_with_prepared( } continue; } - result = handle_request(request, mode, environment, requests, replies, output) => result, + result = async { + let read = handle_request(request, mode, environment, requests, replies, output); + let Some((login, deadline)) = login else { + return read.await; + }; + tokio::time::timeout_at(deadline, login.permit.run(async { + if !login.current().await { + return Err(eyre!(CloudRecoverySignInChanged)); + } + let result = read.await; + if !login.current().await { + return Err(eyre!(CloudRecoverySignInChanged)); + } + login.permit.check().map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))?; + result + })).await.map_err(|_| eyre!(idle::RECOVERY_TIMEOUT))? + .map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))? + } => result, }; if unloads && result.is_ok() @@ -499,7 +600,13 @@ async fn run_worker_with_prepared( { replies.report_loss(&root, /*reason*/ None, output); } - let _ = command.response.send(Ok(request_result(result))); + let response = match result { + Err(error) if error.is::() => { + Err(std::io::Error::other(CloudRecoverySignInChanged)) + } + result => Ok(request_result(result)), + }; + let _ = command.response.send(response); } } diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index d17c86cb94..14dbcbea0b 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -29,11 +29,14 @@ mod path; mod remote; mod remote_addr; mod remote_websocket; +pub use cloud::CLOUD_RECOVERY_NETWORK_BLOCKED; pub use cloud::CloudClient; pub use cloud::CloudCommand; pub use cloud::CloudOperation; +pub use cloud::CloudRecoveryRelease; pub use cloud::CloudRequestHandle; pub use cloud::CloudUserMessageReceipt; +pub use cloud_backend::CloudRecoverySignInChanged; mod server_request; pub use server_request::AppServerReplyHandle;