mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Hold Cloud reply streams until history can be refreshed (#62140)
## Why A Cloud conversation can miss an entire turn while disconnected. A successful reattach, even with an idle thread, cannot establish that the client's history is complete. ## What changed Add an app-server client operation that reattaches a disconnected, loaded Cloud thread and returns its cached settings. The worker holds the replacement stream so the caller can refresh turn and item history before explicitly installing the stream and receiving buffered events. Reattachment, recovery history reads, and stream installation share a deadline and use the original Cloud sign-in and network permit. Dropping the release handle cancels the replacement. Personal access token sessions still reconnect by reopening the conversation. ## Testing Extend Cloud connection tests to cover quiet and busy reattachments, buffered event delivery, cancellation and retry, slow history reads, and sign-in changes during recovery. GitOrigin-RevId: a4e5cb967e097021596bc83aac2f8d1a95373712
This commit is contained in:
committed by
copyberry
parent
1283c91ad7
commit
f39fcc1bfb
@@ -3,6 +3,7 @@
|
||||
//! Exercises native worker progress and reply attachment.
|
||||
|
||||
use super::*;
|
||||
use crate::AppServerRequestHandle;
|
||||
use futures::stream;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::sync::Mutex;
|
||||
@@ -93,12 +94,30 @@ async fn rejected_rename_preserves_loaded_cloud_thread_name() -> Result<()> {
|
||||
async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> Result<()> {
|
||||
let server = MockServer::start().await;
|
||||
let home = tempfile::tempdir()?;
|
||||
let config = crate::legacy_core::config::ConfigBuilder::default()
|
||||
let mut config = crate::legacy_core::config::ConfigBuilder::default()
|
||||
.codex_home(home.path().to_owned())
|
||||
.fallback_cwd(Some(home.path().to_owned()))
|
||||
.without_project_context()
|
||||
.build()
|
||||
.await?;
|
||||
config.cli_auth_credentials_store_mode = codex_login::AuthCredentialsStoreMode::File;
|
||||
let storage = config.auth_config();
|
||||
let token = "e30.e30.sig";
|
||||
let account = "synthetic-account";
|
||||
let auth = serde_json::from_value(serde_json::json!({
|
||||
"auth_mode": "chatgpt",
|
||||
"tokens": {
|
||||
"id_token": token, "access_token": token, "refresh_token": "test", "account_id": account,
|
||||
},
|
||||
"last_refresh": "2026-01-01T00:00:00Z",
|
||||
}))?;
|
||||
codex_login::save_auth(
|
||||
home.path(),
|
||||
&auth,
|
||||
storage.auth_credentials_store_mode,
|
||||
storage.keyring_backend_kind,
|
||||
)?;
|
||||
|
||||
let cached = Box::new(serde_json::from_value::<protocol::ThreadResumeResponse>(
|
||||
serde_json::json!({
|
||||
"thread": {
|
||||
@@ -116,7 +135,7 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R
|
||||
client: std::sync::Arc::new(cloud::Client::new(
|
||||
&config.http_client_factory(),
|
||||
&server.uri(),
|
||||
cloud::Credentials::new("synthetic-bearer", "synthetic-account")?,
|
||||
cloud::Credentials::new(token, account)?,
|
||||
)?),
|
||||
account: protocol::GetAccountResponse {
|
||||
user_id: None,
|
||||
@@ -222,6 +241,279 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R
|
||||
)
|
||||
);
|
||||
assert!(notifications.try_recv().is_err());
|
||||
let login = idle::Login {
|
||||
storage,
|
||||
mode: codex_protocol::auth::AuthMode::Chatgpt,
|
||||
token: token.into(),
|
||||
account: account.into(),
|
||||
permit: config
|
||||
.http_client_factory()
|
||||
.network_policy()
|
||||
.acquire(&server.uri().parse()?)?,
|
||||
};
|
||||
// A live replacement installs and forwards held and buffered events.
|
||||
let (ack, installed) = oneshot::channel();
|
||||
let (state, _) = tokio::sync::watch::channel(crate::cloud::IdleReconnect::Inactive);
|
||||
let mut recovery = idle::Recovery::new(state, Arc::clone(&requests.client), login.clone());
|
||||
recovery.held = Some((
|
||||
"t".into(),
|
||||
tokio::time::Instant::now() + std::time::Duration::from_secs(8),
|
||||
Box::pin(async move {
|
||||
let idle: &[u8] = b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t";
|
||||
Some(idle::Opened {
|
||||
stream: stream::iter([Ok(cloud::Event::Notification(idle.into()))])
|
||||
.chain(stream::pending())
|
||||
.boxed(),
|
||||
first: Some(cloud::Event::ServerRequest(Default::default())),
|
||||
ack,
|
||||
})
|
||||
}),
|
||||
));
|
||||
replies.idle = Some(recovery);
|
||||
requests.warned_partial_history = Some("t".into());
|
||||
replies.next_event(&output).await;
|
||||
assert_eq!(installed.await?, Ok(()));
|
||||
let AppServerEvent::ServerNotification(notification) = notifications.try_recv()? else {
|
||||
panic!("expected held first event");
|
||||
};
|
||||
assert!(matches!(
|
||||
*notification,
|
||||
protocol::ServerNotification::Warning(_)
|
||||
));
|
||||
replies.next_event(&output).await;
|
||||
assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle));
|
||||
// Installation must not erase a warning already emitted by the refreshed history.
|
||||
let (commands, receiver) = mpsc::channel(1);
|
||||
drop(commands);
|
||||
run_worker(receiver, &mut requests, &mut replies, &output).await;
|
||||
assert_eq!(requests.warned_partial_history.as_deref(), Some("t"));
|
||||
|
||||
// A quiet Attach cannot prove that no whole turn was missed while disconnected.
|
||||
let attach_path = "/oaiproto.codex_app_server_backend.ThreadService/Attach";
|
||||
for (idle, statuses, outcome) in [
|
||||
(true, &[][..], "pat"),
|
||||
(true, &["quiet"][..], "install"),
|
||||
(true, &["7"][..], "install"),
|
||||
(false, &["14", "7"][..], "install"),
|
||||
(false, &["14", "busy"][..], "install"),
|
||||
(true, &["busy"][..], "install"),
|
||||
(true, &["busy"][..], "cancel"),
|
||||
(true, &["busy", "busy"][..], "retry"),
|
||||
(true, &["busy"][..], "slow-history"),
|
||||
(true, &["busy"][..], "auth-change"),
|
||||
] {
|
||||
server.reset().await;
|
||||
let attempt = std::sync::atomic::AtomicUsize::new(0);
|
||||
Mock::given(method("POST"))
|
||||
.and(path(attach_path))
|
||||
.and(header("authorization", format!("Bearer {token}")))
|
||||
.respond_with(move |_: &wiremock::Request| {
|
||||
let index = attempt.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let response =
|
||||
ResponseTemplate::new(200).insert_header("content-type", "application/grpc");
|
||||
match statuses[index] {
|
||||
"quiet" => response.set_body_bytes(b"\x00\x00\x00\x00\x02\x0a\x00".to_vec()),
|
||||
"busy" => response.set_body_bytes(vec![0, 0, 0, 0, 2, 18, 0]),
|
||||
status => response
|
||||
.insert_header("grpc-status", status)
|
||||
.insert_header("grpc-message", "private backend details"),
|
||||
}
|
||||
})
|
||||
.expect(statuses.len() as u64)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (client, commands, events) = CloudClient::channel(/*capacity*/ 4);
|
||||
let mut client = AppServerClient::Cloud(client);
|
||||
let requests_handle = client.request_handle();
|
||||
let AppServerRequestHandle::Cloud(handle) = &requests_handle else {
|
||||
unreachable!();
|
||||
};
|
||||
let mut recovery = idle::Recovery::new(
|
||||
handle.reconnect.clone(),
|
||||
Arc::clone(&requests.client),
|
||||
if outcome == "pat" {
|
||||
idle::Login {
|
||||
mode: codex_protocol::auth::AuthMode::PersonalAccessToken,
|
||||
..login.clone()
|
||||
}
|
||||
} else {
|
||||
login.clone()
|
||||
},
|
||||
);
|
||||
recovery.observed_idle = Some(idle);
|
||||
let mut worker_replies = Replies {
|
||||
idle: Some(recovery),
|
||||
..Default::default()
|
||||
};
|
||||
let mut worker_requests = requests::Requests {
|
||||
client: Arc::clone(&requests.client),
|
||||
account: requests.account.clone(),
|
||||
loaded: Some(cached.clone()),
|
||||
warned_partial_history: Some("t".into()),
|
||||
};
|
||||
let worker = tokio::spawn(async move {
|
||||
run_worker(commands, &mut worker_requests, &mut worker_replies, &events).await;
|
||||
});
|
||||
let budget = std::time::Duration::from_secs(if outcome == "slow-history" { 2 } else { 8 });
|
||||
let result = handle
|
||||
.resume_disconnected_thread(
|
||||
"t".into(),
|
||||
/*require_history*/ false,
|
||||
tokio::time::Instant::now() + budget,
|
||||
)
|
||||
.await;
|
||||
if outcome == "pat" || statuses.last() == Some(&"7") {
|
||||
let Err(crate::TypedRequestError::Server { source, .. }) = result else {
|
||||
panic!("expected recovery to reject this authentication mode or access denial");
|
||||
};
|
||||
assert_eq!(
|
||||
source,
|
||||
idle::recovery_error(if outcome == "pat" {
|
||||
"Reopen this cloud conversation to reconnect with your personal access token."
|
||||
} else {
|
||||
"Your account no longer has access to this cloud conversation."
|
||||
})
|
||||
);
|
||||
} else {
|
||||
let (settings, release) = result?;
|
||||
assert_eq!(settings.as_ref(), Some(cached.as_ref()));
|
||||
assert!(client.next_event().now_or_never().is_none());
|
||||
let loaded: protocol::ThreadLoadedListResponse = requests_handle
|
||||
.request_typed(protocol::ClientRequest::ThreadLoadedList {
|
||||
request_id: protocol::RequestId::Integer(4),
|
||||
params: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
assert!(loaded.data.is_empty());
|
||||
|
||||
let auth_path = home.path().join("auth.json");
|
||||
Mock::given(method("POST"))
|
||||
.and(wiremock::matchers::path_regex(
|
||||
"/oaiproto.codex_app_server_backend.ThreadService/List(Turns|Items)",
|
||||
))
|
||||
.respond_with(move |request: &wiremock::Request| {
|
||||
if outcome == "auth-change" {
|
||||
std::fs::remove_file(&auth_path).unwrap();
|
||||
}
|
||||
let body: &[u8] = if request.url.path().ends_with("ListItems") {
|
||||
b"\0\0\0\0\x0f\x0a\x0a\x0a\x08\x0a\x01u\x12\x03\xa2\x01\0\x12\x01x"
|
||||
} else {
|
||||
b"\x00\x00\x00\x00\x02\x0a\x00"
|
||||
};
|
||||
let response = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "application/grpc")
|
||||
.set_body_bytes(body);
|
||||
if outcome == "slow-history" {
|
||||
response.set_delay(std::time::Duration::from_secs(10))
|
||||
} else {
|
||||
response
|
||||
}
|
||||
})
|
||||
.expect(if matches!(outcome, "auth-change" | "slow-history") {
|
||||
1
|
||||
} else {
|
||||
3
|
||||
})
|
||||
.mount(&server)
|
||||
.await;
|
||||
let history: Result<protocol::ThreadTurnsListResponse, _> = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(if outcome == "slow-history" { 3 } else { 2 }),
|
||||
requests_handle.request_typed(protocol::ClientRequest::ThreadTurnsList {
|
||||
request_id: protocol::RequestId::Integer(5),
|
||||
params: protocol::ThreadTurnsListParams {
|
||||
thread_id: "t".into(),
|
||||
cursor: None,
|
||||
limit: Some(5),
|
||||
sort_direction: Some(protocol::SortDirection::Desc),
|
||||
items_view: Some(protocol::TurnItemsView::NotLoaded),
|
||||
},
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
if outcome == "auth-change" {
|
||||
let Err(crate::TypedRequestError::Transport { source, .. }) = history else {
|
||||
panic!("expected changed sign-in to withhold the fetched page");
|
||||
};
|
||||
assert!(source.get_ref().unwrap().is::<CloudRecoverySignInChanged>());
|
||||
assert_eq!(source.to_string(), idle::SIGN_IN_AGAIN);
|
||||
let items = requests_handle
|
||||
.request(protocol::ClientRequest::ThreadItemsList {
|
||||
request_id: protocol::RequestId::Integer(7),
|
||||
params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?,
|
||||
})
|
||||
.await
|
||||
.expect_err("the changed sign-in must reject the next page before RPC");
|
||||
assert!(items.get_ref().unwrap().is::<CloudRecoverySignInChanged>());
|
||||
assert!(
|
||||
server
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|request| !request.url.path().ends_with("/ListItems"))
|
||||
);
|
||||
} else if outcome == "slow-history" {
|
||||
let Err(crate::TypedRequestError::Server { source, .. }) = history else {
|
||||
panic!("expected the recovery deadline to withhold the page");
|
||||
};
|
||||
assert_eq!(source, idle::recovery_error(idle::RECOVERY_TIMEOUT));
|
||||
} else {
|
||||
assert_eq!(
|
||||
history?,
|
||||
protocol::ThreadTurnsListResponse {
|
||||
data: Vec::new(),
|
||||
next_cursor: None,
|
||||
backwards_cursor: None,
|
||||
}
|
||||
);
|
||||
for page in 0..2 {
|
||||
let _: protocol::ThreadItemsListResponse = requests_handle
|
||||
.request_typed(protocol::ClientRequest::ThreadItemsList {
|
||||
request_id: protocol::RequestId::Integer(7 + page),
|
||||
params: serde_json::from_value(serde_json::json!({"threadId":"t"}))?,
|
||||
})
|
||||
.await?;
|
||||
if page == 0 {
|
||||
assert!(matches!(client.next_event().now_or_never().flatten(),
|
||||
Some(AppServerEvent::ServerNotification(notification))
|
||||
if matches!(*notification, protocol::ServerNotification::Warning(_))));
|
||||
} else {
|
||||
assert!(client.next_event().now_or_never().is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
if outcome != "install" {
|
||||
drop(release);
|
||||
if outcome == "retry" {
|
||||
// No inventory read or yield may retire the canceled hold before this retry.
|
||||
let (settings, release) = handle
|
||||
.resume_disconnected_thread(
|
||||
"t".into(),
|
||||
/*require_history*/ true,
|
||||
tokio::time::Instant::now() + std::time::Duration::from_secs(8),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(settings.as_ref(), Some(cached.as_ref()));
|
||||
drop(release);
|
||||
}
|
||||
let loaded: protocol::ThreadLoadedListResponse = requests_handle
|
||||
.request_typed(protocol::ClientRequest::ThreadLoadedList {
|
||||
request_id: protocol::RequestId::Integer(6),
|
||||
params: Default::default(),
|
||||
})
|
||||
.await?;
|
||||
assert!(loaded.data.is_empty());
|
||||
assert!(client.next_event().now_or_never().is_none());
|
||||
} else {
|
||||
// The wiremock response is finite: it must never ACK a live installation.
|
||||
assert!(release.install().await.is_err());
|
||||
assert!(client.next_event().now_or_never().is_none());
|
||||
}
|
||||
}
|
||||
client.shutdown().await?;
|
||||
worker.await?;
|
||||
server.verify().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -643,25 +935,24 @@ async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> {
|
||||
let factory = config.http_client_factory();
|
||||
let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1);
|
||||
let handle = client.request_handle();
|
||||
let mut recovery = idle::Recovery::new(
|
||||
handle.reconnect.clone(),
|
||||
Arc::new(cloud::Client::new(
|
||||
&factory,
|
||||
&server.uri(),
|
||||
cloud::Credentials::new("synthetic-bearer", "synthetic-account")?,
|
||||
)?),
|
||||
idle::Login {
|
||||
storage: config.auth_config(),
|
||||
mode: codex_protocol::auth::AuthMode::Chatgpt,
|
||||
token: "synthetic-bearer".into(),
|
||||
account: "synthetic-account".into(),
|
||||
permit: factory.network_policy().acquire(&server.uri().parse()?)?,
|
||||
},
|
||||
);
|
||||
recovery.observed_idle = Some(true);
|
||||
let mut replies = Replies {
|
||||
idle: Some(idle::Recovery {
|
||||
state: handle.reconnect.clone(),
|
||||
pending: None,
|
||||
installed: None,
|
||||
observed_idle: Some(true),
|
||||
client: Arc::new(cloud::Client::new(
|
||||
&factory,
|
||||
&server.uri(),
|
||||
cloud::Credentials::new("synthetic-bearer", "synthetic-account")?,
|
||||
)?),
|
||||
login: Arc::new(idle::Login {
|
||||
storage: config.auth_config(),
|
||||
mode: codex_protocol::auth::AuthMode::Chatgpt,
|
||||
token: "synthetic-bearer".into(),
|
||||
account: "synthetic-account".into(),
|
||||
permit: factory.network_policy().acquire(&server.uri().parse()?)?,
|
||||
}),
|
||||
}),
|
||||
idle: Some(recovery),
|
||||
active: Some(Attachment {
|
||||
thread_id: "t".into(),
|
||||
stream: stream::empty().boxed(),
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
//! Reopens only the lost reply stream of a displayed, idle Cloud conversation.
|
||||
//! Restores Cloud reply streams under the original login and network permit.
|
||||
//! An active recovery holds its stream until the caller has read history.
|
||||
|
||||
use super::ReplyStream;
|
||||
use super::check_reply_stream;
|
||||
use crate::CLOUD_RECOVERY_NETWORK_BLOCKED;
|
||||
use crate::RequestResult;
|
||||
use crate::cloud::IdleReconnect;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_cloud_client as cloud;
|
||||
use codex_http_client::NetworkPermit;
|
||||
use codex_login::AuthConfig;
|
||||
@@ -10,13 +15,15 @@ use codex_login::token_data::parse_jwt_expiration;
|
||||
use codex_protocol::auth::AuthMode;
|
||||
use futures::FutureExt;
|
||||
use futures::future::BoxFuture;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::sync::watch;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const SIGN_IN_AGAIN: &str =
|
||||
pub(super) const SIGN_IN_AGAIN: &str =
|
||||
"Your cloud sign-in changed or expired. Sign in again to access this conversation.";
|
||||
|
||||
pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> {
|
||||
@@ -33,6 +40,7 @@ pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> {
|
||||
|
||||
const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation.";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct Login {
|
||||
pub(super) storage: AuthConfig,
|
||||
pub(super) mode: AuthMode,
|
||||
@@ -42,7 +50,7 @@ pub(super) struct Login {
|
||||
}
|
||||
|
||||
impl Login {
|
||||
async fn current(&self) -> bool {
|
||||
pub(super) async fn current(&self) -> bool {
|
||||
if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) {
|
||||
return false;
|
||||
}
|
||||
@@ -105,12 +113,22 @@ impl Login {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const RECOVERY_TIMEOUT: &str = "Cloud could not restore the reply stream in time.";
|
||||
type Release = oneshot::Receiver<oneshot::Sender<Result<(), &'static str>>>;
|
||||
|
||||
pub(super) struct Opened {
|
||||
pub(super) stream: ReplyStream,
|
||||
pub(super) first: Option<cloud::Event>,
|
||||
pub(super) ack: oneshot::Sender<Result<(), &'static str>>,
|
||||
}
|
||||
|
||||
type Open = BoxFuture<'static, Result<(ReplyStream, Option<cloud::Event>), Option<&'static str>>>;
|
||||
|
||||
pub(super) struct Recovery {
|
||||
pub(super) state: watch::Sender<IdleReconnect>,
|
||||
pub(super) pending: Option<(Arc<str>, Open)>,
|
||||
pub(super) installed: Option<Instant>,
|
||||
pub(super) held: Option<(String, Instant, BoxFuture<'static, Option<Opened>>)>,
|
||||
// None until either the initial Attach event or the activation snapshot establishes status.
|
||||
pub(super) observed_idle: Option<bool>,
|
||||
pub(super) client: Arc<cloud::Client>,
|
||||
@@ -126,7 +144,97 @@ pub(super) fn transient(error: Option<&cloud::Error>) -> bool {
|
||||
}
|
||||
|
||||
impl Recovery {
|
||||
pub(super) fn check_permit(&self) -> Result<(), &'static str> {
|
||||
self.login
|
||||
.permit
|
||||
.check()
|
||||
.map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED)
|
||||
}
|
||||
|
||||
pub(super) fn history_login(&self, thread_id: &str) -> Option<(Arc<Login>, Instant)> {
|
||||
self.held
|
||||
.as_ref()
|
||||
.filter(|(root, _, _)| root == thread_id)
|
||||
.map(|(_, deadline, _)| (Arc::clone(&self.login), *deadline))
|
||||
}
|
||||
|
||||
pub(super) fn start(
|
||||
&mut self,
|
||||
thread_id: String,
|
||||
cached: Box<ThreadResumeResponse>,
|
||||
_require_history: bool,
|
||||
deadline: Instant,
|
||||
release: Release,
|
||||
mut response: oneshot::Sender<io::Result<RequestResult>>,
|
||||
) {
|
||||
if response.is_closed() {
|
||||
return;
|
||||
}
|
||||
if self.login.mode == AuthMode::PersonalAccessToken {
|
||||
let _ = response.send(Ok(Err(recovery_error(
|
||||
"Reopen this cloud conversation to reconnect with your personal access token.",
|
||||
))));
|
||||
return;
|
||||
}
|
||||
self.cancel();
|
||||
let client = Arc::clone(&self.client);
|
||||
let login = Arc::clone(&self.login);
|
||||
let root = thread_id.clone();
|
||||
let open = async move {
|
||||
let attempt = login.permit.run(login.attach(&client, &root));
|
||||
let opened = tokio::select! {
|
||||
biased;
|
||||
_ = response.closed() => return None,
|
||||
result = tokio::time::timeout_at(deadline, attempt) => match result {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED),
|
||||
Err(_) => Err(RECOVERY_TIMEOUT),
|
||||
},
|
||||
};
|
||||
let (stream, first) = match opened {
|
||||
Ok(opened) => opened,
|
||||
Err(reason) => {
|
||||
let _ = response.send(Ok(Err(recovery_error(reason))));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// Attach cannot prove that no complete turn was missed during the outage.
|
||||
// Attach resumes the conversation without configuration overrides. Keep
|
||||
// the settings already loaded for this thread and refresh its history.
|
||||
let settings = Some(cached);
|
||||
let value = match serde_json::to_value(&settings) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
let _ = response.send(Err(io::Error::other(error)));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
response.send(Ok(Ok(value))).ok()?;
|
||||
let ack = tokio::time::timeout_at(deadline, release)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()?;
|
||||
let checked = login.permit.run(login.current());
|
||||
let authorized = match tokio::time::timeout_at(deadline, checked).await {
|
||||
Ok(Ok(true)) => login
|
||||
.permit
|
||||
.check()
|
||||
.map_err(|_| CLOUD_RECOVERY_NETWORK_BLOCKED),
|
||||
Ok(Ok(false)) => Err(SIGN_IN_AGAIN),
|
||||
Ok(Err(_)) => Err(CLOUD_RECOVERY_NETWORK_BLOCKED),
|
||||
Err(_) => Err(RECOVERY_TIMEOUT),
|
||||
};
|
||||
if let Err(reason) = authorized {
|
||||
let _ = ack.send(Err(reason));
|
||||
return None;
|
||||
}
|
||||
Some(Opened { stream, first, ack })
|
||||
};
|
||||
self.held = Some((thread_id, deadline, open.boxed()));
|
||||
}
|
||||
|
||||
pub(super) fn cancel(&mut self) -> Option<Arc<str>> {
|
||||
self.held = None;
|
||||
drop(self.pending.take()?);
|
||||
let state = self.state.send_replace(IdleReconnect::Inactive);
|
||||
if let IdleReconnect::Opening(root) = state {
|
||||
@@ -137,6 +245,7 @@ impl Recovery {
|
||||
}
|
||||
|
||||
pub(super) fn reset(&mut self) {
|
||||
self.held = None;
|
||||
drop(self.pending.take());
|
||||
self.state.send_replace(IdleReconnect::Inactive);
|
||||
self.installed = None;
|
||||
@@ -175,6 +284,22 @@ impl Recovery {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn new(
|
||||
state: watch::Sender<IdleReconnect>,
|
||||
client: Arc<cloud::Client>,
|
||||
login: Login,
|
||||
) -> Self {
|
||||
Self {
|
||||
state,
|
||||
pending: None,
|
||||
installed: None,
|
||||
held: None,
|
||||
observed_idle: None,
|
||||
client,
|
||||
login: Arc::new(login),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn offer(&mut self, id: &str) {
|
||||
// Only OAuth credentials can be rechecked locally. PATs use ordinary explicit reopen.
|
||||
if !matches!(
|
||||
@@ -209,3 +334,11 @@ impl Recovery {
|
||||
self.pending = Some((root, open.boxed()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn recovery_error(reason: &'static str) -> JSONRPCErrorError {
|
||||
JSONRPCErrorError {
|
||||
code: -32000,
|
||||
message: reason.to_owned(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ mod thread_title;
|
||||
|
||||
use crate::AppServerClient;
|
||||
use crate::AppServerEvent;
|
||||
use crate::CLOUD_RECOVERY_NETWORK_BLOCKED;
|
||||
use crate::CloudClient;
|
||||
use crate::CloudOperation;
|
||||
use crate::cloud::CloudStreamLosses;
|
||||
@@ -54,6 +55,12 @@ struct PreflightRejection(&'static str);
|
||||
#[error("Cloud stream is not connected. Reopen the conversation before sending.")]
|
||||
struct MissingReplyAttachment;
|
||||
|
||||
/// Local rejection when recovery history no longer belongs to the bound sign-in.
|
||||
/// This carries no credentials and never enters the app-server wire envelope.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{}", idle::SIGN_IN_AGAIN)]
|
||||
pub struct CloudRecoverySignInChanged;
|
||||
|
||||
struct Attachment {
|
||||
thread_id: String,
|
||||
stream: ReplyStream,
|
||||
@@ -70,6 +77,40 @@ struct Replies {
|
||||
}
|
||||
|
||||
impl Replies {
|
||||
async fn next_event(&mut self, output: &mpsc::UnboundedSender<AppServerEvent>) {
|
||||
if let Some(recovery) = &mut self.idle
|
||||
&& let Some((root, _, open)) = &mut recovery.held
|
||||
{
|
||||
let opened = open.await;
|
||||
let root = std::mem::take(root);
|
||||
recovery.held = None;
|
||||
if let Some(opened) = opened {
|
||||
if let Err(reason) = recovery.check_permit() {
|
||||
let _ = opened.ack.send(Err(reason));
|
||||
return;
|
||||
}
|
||||
let Ok((stream, first)) = check_prepared_stream((opened.stream, opened.first))
|
||||
else {
|
||||
let _ = opened
|
||||
.ack
|
||||
.send(Err("Cloud reply stream disconnected again."));
|
||||
return;
|
||||
};
|
||||
// Commit through the acknowledgement. If the caller timed out
|
||||
// and closed it first, leave the stream held and drop it here.
|
||||
if opened.ack.send(Ok(())).is_err() {
|
||||
return;
|
||||
}
|
||||
recovery.installed = Some(tokio::time::Instant::now());
|
||||
recovery.observed_idle = None;
|
||||
self.stream_losses.supersede(&root);
|
||||
self.install(&root, stream, first, output);
|
||||
}
|
||||
} else {
|
||||
self.forward_next(output).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn forward_next(&mut self, output: &mpsc::UnboundedSender<AppServerEvent>) {
|
||||
if self.active.is_none()
|
||||
&& let Some(recovery) = &mut self.idle
|
||||
@@ -161,7 +202,7 @@ impl Replies {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut future => return result,
|
||||
() = self.forward_next(output) => {}
|
||||
() = self.next_event(output) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -238,10 +279,11 @@ impl Replies {
|
||||
|
||||
fn detach(&mut self, id: &str) {
|
||||
if let Some(idle) = &mut self.idle
|
||||
&& (idle
|
||||
.pending
|
||||
.as_ref()
|
||||
.is_some_and(|(root, _)| root.as_ref() == id)
|
||||
&& (idle.held.as_ref().is_some_and(|(root, _, _)| root == id)
|
||||
|| idle
|
||||
.pending
|
||||
.as_ref()
|
||||
.is_some_and(|(root, _)| root.as_ref() == id)
|
||||
|| self
|
||||
.active
|
||||
.as_ref()
|
||||
@@ -305,14 +347,11 @@ fn start_worker(
|
||||
warned_partial_history: None,
|
||||
};
|
||||
let mut replies = Replies {
|
||||
idle: Some(idle::Recovery {
|
||||
state: app_client.request_handle().reconnect,
|
||||
pending: None,
|
||||
installed: None,
|
||||
observed_idle: None,
|
||||
idle: Some(idle::Recovery::new(
|
||||
app_client.request_handle().reconnect,
|
||||
client,
|
||||
login: Arc::new(login),
|
||||
}),
|
||||
login,
|
||||
)),
|
||||
stream_losses: app_client.stream_losses.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -341,7 +380,7 @@ async fn run_worker_with_prepared(
|
||||
loop {
|
||||
let mut command = tokio::select! {
|
||||
_ = output.closed() => return,
|
||||
() = replies.forward_next(output) => continue,
|
||||
() = replies.next_event(output) => continue,
|
||||
command = commands.recv() => match command { Some(command) => command, None => return },
|
||||
command = async {
|
||||
match &mut prepared {
|
||||
@@ -440,6 +479,45 @@ async fn run_worker_with_prepared(
|
||||
cloud::SubmissionMode::Steer,
|
||||
None,
|
||||
)),
|
||||
CloudOperation::ResumeDisconnectedThread {
|
||||
thread_id,
|
||||
require_history,
|
||||
deadline,
|
||||
release,
|
||||
} => {
|
||||
// Retire canceled holds, but commit a ready handoff through its normal ACK.
|
||||
if replies
|
||||
.idle
|
||||
.as_ref()
|
||||
.is_some_and(|recovery| recovery.held.is_some())
|
||||
{
|
||||
let _ = replies.next_event(output).now_or_never();
|
||||
}
|
||||
if let Some(recovery) = &mut replies.idle
|
||||
&& let Some(cached) = requests
|
||||
.loaded
|
||||
.as_ref()
|
||||
.filter(|settings| settings.thread.id == thread_id)
|
||||
&& replies.active.is_none()
|
||||
&& recovery.held.is_none()
|
||||
{
|
||||
// Start a fresh warning epoch before any recovery history is read.
|
||||
requests.warned_partial_history = None;
|
||||
recovery.start(
|
||||
thread_id,
|
||||
cached.clone(),
|
||||
require_history,
|
||||
deadline,
|
||||
release,
|
||||
command.response,
|
||||
);
|
||||
} else {
|
||||
let _ = command.response.send(Ok(Err(idle::recovery_error(
|
||||
"This cloud conversation no longer has a reply attachment to restore.",
|
||||
))));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
CloudOperation::Notify(_) => {
|
||||
Err(eyre!("Cloud client notifications are not supported yet."))
|
||||
}
|
||||
@@ -462,12 +540,18 @@ async fn run_worker_with_prepared(
|
||||
|| loaded.is_some_and(|loaded| loaded.thread.id == *id)
|
||||
});
|
||||
// Inventory and history reads have no conversation state to commit.
|
||||
let is_read = matches!(
|
||||
&request,
|
||||
protocol::ClientRequest::ThreadList { .. }
|
||||
| protocol::ClientRequest::ThreadTurnsList { .. }
|
||||
| protocol::ClientRequest::ThreadItemsList { .. }
|
||||
);
|
||||
let history_thread = match &request {
|
||||
protocol::ClientRequest::ThreadTurnsList { params, .. } => {
|
||||
Some(¶ms.thread_id)
|
||||
}
|
||||
protocol::ClientRequest::ThreadItemsList { params, .. } => {
|
||||
Some(¶ms.thread_id)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let is_read = history_thread.is_some()
|
||||
|| matches!(&request, protocol::ClientRequest::ThreadList { .. });
|
||||
let login = history_thread.and_then(|id| replies.idle.as_ref()?.history_login(id));
|
||||
// When closure and the handshake are both ready, cancel before
|
||||
// the handshake can replace the active attachment.
|
||||
let result = tokio::select! {
|
||||
@@ -479,7 +563,24 @@ async fn run_worker_with_prepared(
|
||||
}
|
||||
continue;
|
||||
}
|
||||
result = handle_request(request, mode, environment, requests, replies, output) => result,
|
||||
result = async {
|
||||
let read = handle_request(request, mode, environment, requests, replies, output);
|
||||
let Some((login, deadline)) = login else {
|
||||
return read.await;
|
||||
};
|
||||
tokio::time::timeout_at(deadline, login.permit.run(async {
|
||||
if !login.current().await {
|
||||
return Err(eyre!(CloudRecoverySignInChanged));
|
||||
}
|
||||
let result = read.await;
|
||||
if !login.current().await {
|
||||
return Err(eyre!(CloudRecoverySignInChanged));
|
||||
}
|
||||
login.permit.check().map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))?;
|
||||
result
|
||||
})).await.map_err(|_| eyre!(idle::RECOVERY_TIMEOUT))?
|
||||
.map_err(|_| eyre!(CLOUD_RECOVERY_NETWORK_BLOCKED))?
|
||||
} => result,
|
||||
};
|
||||
if unloads
|
||||
&& result.is_ok()
|
||||
@@ -499,7 +600,13 @@ async fn run_worker_with_prepared(
|
||||
{
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
let _ = command.response.send(Ok(request_result(result)));
|
||||
let response = match result {
|
||||
Err(error) if error.is::<CloudRecoverySignInChanged>() => {
|
||||
Err(std::io::Error::other(CloudRecoverySignInChanged))
|
||||
}
|
||||
result => Ok(request_result(result)),
|
||||
};
|
||||
let _ = command.response.send(response);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user