mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Report Cloud stream recovery failures and cancel abandoned reads (#61599)
## Why Cloud stream failures previously used a generic reopen message even when the conversation was missing, access was denied, or sign-in had changed. Abandoned turn and item list requests could also keep the native worker occupied. ## What changed - Report specific disconnect reasons for missing conversations, lost access, and changed or expired sign-in, including failures during idle reconnection. - Report exhausted reconnect attempts and other nontransient recovery failures with dedicated messages. - Cancel in-flight `ThreadTurnsList` and `ThreadItemsList` requests when their callers drop, matching `ThreadList` behavior. ## Testing Add regression coverage for abandoned history reads, changed sign-in during accepted idle recovery, and access-denied stream errors. GitOrigin-RevId: 29923b46c2d377f2247d699757553777a31d08ca
This commit is contained in:
committed by
copyberry
parent
b7056f3408
commit
8e30237f64
@@ -226,15 +226,37 @@ async fn failed_archive_loaded_list_tracks_attachment_not_cached_metadata() -> R
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_an_in_flight_list_unblocks_the_native_worker() -> Result<()> {
|
||||
assert_abandoned_request_unblocks_worker(
|
||||
"ListThreads",
|
||||
CloudOperation::Request(Box::new(protocol::ClientRequest::ThreadList {
|
||||
request_id: protocol::RequestId::Integer(1),
|
||||
params: serde_json::from_value(serde_json::json!({"limit": 10}))?,
|
||||
})),
|
||||
)
|
||||
.await
|
||||
async fn dropping_an_in_flight_read_unblocks_the_native_worker() -> Result<()> {
|
||||
for (rpc, request) in [
|
||||
(
|
||||
"ListThreads",
|
||||
protocol::ClientRequest::ThreadList {
|
||||
request_id: protocol::RequestId::Integer(1),
|
||||
params: serde_json::from_value(serde_json::json!({"limit": 10}))?,
|
||||
},
|
||||
),
|
||||
(
|
||||
"ListTurns",
|
||||
protocol::ClientRequest::ThreadTurnsList {
|
||||
request_id: protocol::RequestId::Integer(1),
|
||||
params: serde_json::from_value(serde_json::json!({
|
||||
"threadId": "t",
|
||||
"itemsView": "notLoaded",
|
||||
}))?,
|
||||
},
|
||||
),
|
||||
(
|
||||
"ListItems",
|
||||
protocol::ClientRequest::ThreadItemsList {
|
||||
request_id: protocol::RequestId::Integer(1),
|
||||
params: serde_json::from_value(serde_json::json!({"threadId": "t"}))?,
|
||||
},
|
||||
),
|
||||
] {
|
||||
assert_abandoned_request_unblocks_worker(rpc, CloudOperation::Request(Box::new(request)))
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -608,17 +630,85 @@ async fn successful_reopen_supersedes_only_earlier_losses_for_that_thread() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_idle_reconnect_reports_changed_login() -> Result<()> {
|
||||
let server = MockServer::start().await;
|
||||
let home = tempfile::tempdir()?;
|
||||
let config = crate::legacy_core::config::ConfigBuilder::default()
|
||||
.codex_home(home.path().to_owned())
|
||||
.fallback_cwd(Some(home.path().to_owned()))
|
||||
.without_project_context()
|
||||
.build()
|
||||
.await?;
|
||||
let factory = config.http_client_factory();
|
||||
let (mut client, _commands, output) = CloudClient::channel(/*capacity*/ 1);
|
||||
let handle = client.request_handle();
|
||||
let mut replies = Replies {
|
||||
idle: Some(idle::Recovery {
|
||||
state: handle.reconnect.clone(),
|
||||
pending: None,
|
||||
installed: None,
|
||||
observed_idle: Some(true),
|
||||
client: Arc::new(cloud::Client::new(
|
||||
&factory,
|
||||
&server.uri(),
|
||||
cloud::Credentials::new("synthetic-bearer", "synthetic-account")?,
|
||||
)?),
|
||||
login: Arc::new(idle::Login {
|
||||
storage: config.auth_config(),
|
||||
mode: codex_protocol::auth::AuthMode::Chatgpt,
|
||||
token: "synthetic-bearer".into(),
|
||||
account: "synthetic-account".into(),
|
||||
permit: factory.network_policy().acquire(&server.uri().parse()?)?,
|
||||
}),
|
||||
}),
|
||||
active: Some(Attachment {
|
||||
thread_id: "t".into(),
|
||||
stream: stream::empty().boxed(),
|
||||
}),
|
||||
stream_losses: client.stream_losses.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
replies.forward_next(&output).await;
|
||||
assert!(matches!(
|
||||
client.next_event().await,
|
||||
Some(AppServerEvent::ThreadStreamDisconnected(_))
|
||||
));
|
||||
assert!(handle.reconnect_idle_thread("t"));
|
||||
|
||||
replies.forward_next(&output).await;
|
||||
let Some(AppServerEvent::ThreadStreamDisconnected(error)) = client.next_event().await else {
|
||||
panic!("expected a disconnect notification after the accepted reconnect failed");
|
||||
};
|
||||
assert_eq!(
|
||||
*error,
|
||||
cloud_error(
|
||||
"t",
|
||||
"Your cloud sign-in changed or expired. Sign in again to access this conversation."
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attachment_status_follows_its_server_notifications() -> Result<()> {
|
||||
let (output, _notifications) = mpsc::unbounded_channel();
|
||||
let (output, mut events) = mpsc::unbounded_channel();
|
||||
let mut replies = Replies::default();
|
||||
let notifications: [&[u8]; 3] = [
|
||||
b"\x1a\x0d\x12\x0b\x0a\x06\x22\x04\x0a\x02\x08\x01\x12\x01t", // t is waiting
|
||||
b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01x", // x is idle
|
||||
b"\x1a\x09\x12\x07\x0a\x02\x12\x00\x12\x01t", // t is idle
|
||||
];
|
||||
let stream =
|
||||
stream::iter(notifications.map(|bytes| Ok(cloud::Event::Notification(bytes.into()))));
|
||||
let stream = stream::iter(
|
||||
notifications
|
||||
.map(|bytes| Ok(cloud::Event::Notification(bytes.into())))
|
||||
.into_iter()
|
||||
.chain([Err(cloud::Error::Rpc(cloud::RpcStatus {
|
||||
code: 7,
|
||||
message: "private backend details".into(),
|
||||
details: Default::default(),
|
||||
}))]),
|
||||
);
|
||||
replies
|
||||
.replace("t", async { Ok(stream.boxed()) }, &output)
|
||||
.await?;
|
||||
@@ -634,6 +724,16 @@ async fn attachment_status_follows_its_server_notifications() -> Result<()> {
|
||||
assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle));
|
||||
replies.forward_next(&output).await;
|
||||
assert_eq!(replies.thread_status("t"), Some(&cloud::ThreadStatus::Idle));
|
||||
let AppServerEvent::ThreadStreamDisconnected(error) = events.try_recv()? else {
|
||||
panic!("expected a Cloud stream loss");
|
||||
};
|
||||
assert_eq!(
|
||||
*error,
|
||||
cloud_error(
|
||||
"t",
|
||||
"Your account no longer has access to this cloud conversation."
|
||||
)
|
||||
);
|
||||
|
||||
replies
|
||||
.replace("t", async { Ok(stream::pending().boxed()) }, &output)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Reopens only the lost reply stream of a displayed, idle Cloud conversation.
|
||||
|
||||
use super::ReplyStream;
|
||||
use super::check_reply_stream;
|
||||
use crate::cloud::IdleReconnect;
|
||||
use codex_cloud_client as cloud;
|
||||
use codex_http_client::NetworkPermit;
|
||||
@@ -8,7 +9,6 @@ use codex_login::AuthConfig;
|
||||
use codex_login::token_data::parse_jwt_expiration;
|
||||
use codex_protocol::auth::AuthMode;
|
||||
use futures::FutureExt;
|
||||
use futures::StreamExt;
|
||||
use futures::future::BoxFuture;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -16,6 +16,23 @@ use std::time::SystemTime;
|
||||
use tokio::sync::watch;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const SIGN_IN_AGAIN: &str =
|
||||
"Your cloud sign-in changed or expired. Sign in again to access this conversation.";
|
||||
|
||||
pub(super) fn blocked_recovery(error: &cloud::Error) -> Option<&'static str> {
|
||||
let cloud::Error::Rpc(status) = error else {
|
||||
return None;
|
||||
};
|
||||
match status.code {
|
||||
5 => Some("Cloud could not find this conversation."),
|
||||
7 => Some("Your account no longer has access to this cloud conversation."),
|
||||
16 => Some(SIGN_IN_AGAIN),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const RECOVERY_FAILED: &str = "Cloud could not restore the reply stream for this conversation.";
|
||||
|
||||
pub(super) struct Login {
|
||||
pub(super) storage: AuthConfig,
|
||||
pub(super) mode: AuthMode,
|
||||
@@ -26,6 +43,9 @@ pub(super) struct Login {
|
||||
|
||||
impl Login {
|
||||
async fn current(&self) -> bool {
|
||||
if !matches!(self.mode, AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens) {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(auth)) = self
|
||||
.storage
|
||||
.load_auth_with_modes(/*enable_codex_api_key_env*/ false, Some(&[self.mode]))
|
||||
@@ -44,9 +64,48 @@ impl Login {
|
||||
expiry.is_none_or(|expiry| SystemTime::from(expiry) > SystemTime::now())
|
||||
})
|
||||
}
|
||||
|
||||
async fn attach(
|
||||
&self,
|
||||
client: &cloud::Client,
|
||||
thread_id: &str,
|
||||
) -> Result<(ReplyStream, Option<cloud::Event>), &'static str> {
|
||||
for delay in [
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(500),
|
||||
Duration::from_secs(1),
|
||||
] {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if !self.current().await {
|
||||
return Err(SIGN_IN_AGAIN);
|
||||
}
|
||||
match client
|
||||
.attach(thread_id)
|
||||
.await
|
||||
.map_err(Some)
|
||||
.and_then(|mut stream| {
|
||||
let first = check_reply_stream(&mut stream)?;
|
||||
Ok((stream, first))
|
||||
}) {
|
||||
Ok(attachment) => {
|
||||
if !self.current().await {
|
||||
return Err(SIGN_IN_AGAIN);
|
||||
}
|
||||
return Ok(attachment);
|
||||
}
|
||||
Err(Some(error)) if !transient(Some(&error)) => {
|
||||
return Err(blocked_recovery(&error).unwrap_or(RECOVERY_FAILED));
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Err("Cloud did not restore the reply stream after several attempts.")
|
||||
}
|
||||
}
|
||||
|
||||
type Open = BoxFuture<'static, Option<(ReplyStream, Option<cloud::Event>)>>;
|
||||
type Open = BoxFuture<'static, Result<(ReplyStream, Option<cloud::Event>), Option<&'static str>>>;
|
||||
|
||||
pub(super) struct Recovery {
|
||||
pub(super) state: watch::Sender<IdleReconnect>,
|
||||
@@ -138,49 +197,14 @@ impl Recovery {
|
||||
|state| matches!(state, IdleReconnect::Opening(id) if Arc::ptr_eq(id, &selected)),
|
||||
)
|
||||
.await
|
||||
.ok()?;
|
||||
.map_err(|_| None)?;
|
||||
// Use the policy bound to the frozen serving credential when this client opened.
|
||||
let attempts = login
|
||||
.permit
|
||||
// A revoke-then-allow cannot restart this episode or release a buffered first event.
|
||||
.run(async {
|
||||
for delay in [
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(500),
|
||||
Duration::from_secs(1),
|
||||
] {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if !login.current().await {
|
||||
return None;
|
||||
}
|
||||
match client.attach(&selected).await {
|
||||
Ok(mut stream) => {
|
||||
let first =
|
||||
tokio::task::unconstrained(stream.next()).now_or_never();
|
||||
let event = match first {
|
||||
None => None,
|
||||
Some(Some(Ok(event))) => Some(event),
|
||||
Some(Some(Err(error))) if !transient(Some(&error)) => {
|
||||
return None;
|
||||
}
|
||||
Some(_) => continue,
|
||||
};
|
||||
return login.current().await.then_some((stream, event));
|
||||
}
|
||||
Err(error) if transient(Some(&error)) => {}
|
||||
Err(_) => return None,
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
// Bound this accepted recovery episode, including credential checks and retry backoff.
|
||||
tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempts)
|
||||
let attempt = login.permit.run(login.attach(&client, &selected));
|
||||
tokio::time::timeout(Duration::from_secs(/*secs*/ 8), attempt)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_err(|_| None)?
|
||||
.map_err(|_| None)?
|
||||
.map_err(Some)
|
||||
};
|
||||
self.pending = Some((root, open.boxed()));
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ use codex_app_server_protocol as protocol;
|
||||
use codex_cloud_client as cloud;
|
||||
use color_eyre::eyre::Result;
|
||||
use color_eyre::eyre::bail;
|
||||
|
||||
use color_eyre::eyre::eyre;
|
||||
use futures::FutureExt;
|
||||
use futures::StreamExt;
|
||||
@@ -30,6 +31,18 @@ pub use auth::connect;
|
||||
type ReplyStream = BoxStream<'static, Result<cloud::Event, cloud::Error>>;
|
||||
type OpenReplyStream = (ReplyStream, Option<cloud::Event>);
|
||||
|
||||
/// Separate an immediate EOF from a quiet, successfully attached stream.
|
||||
fn check_reply_stream(
|
||||
stream: &mut ReplyStream,
|
||||
) -> Result<Option<cloud::Event>, Option<cloud::Error>> {
|
||||
match tokio::task::unconstrained(stream.next()).now_or_never() {
|
||||
Some(None) => Err(None),
|
||||
Some(Some(Err(error))) => Err(Some(error)),
|
||||
Some(Some(Ok(event))) => Ok(Some(event)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Rejected locally before an admission request can be sent.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
@@ -63,22 +76,27 @@ impl Replies {
|
||||
let opened = open.await;
|
||||
let root = Arc::clone(root);
|
||||
recovery.pending = None;
|
||||
if let Some((stream, first)) = opened.filter(|_| recovery.login.permit.check().is_ok())
|
||||
{
|
||||
recovery.installed = Some(tokio::time::Instant::now());
|
||||
// A same-thread Attach can stay quiet. Keep the last observation until an
|
||||
// event or a dispatched mutation changes recovery eligibility.
|
||||
self.install(&root, stream, first, output);
|
||||
if let Some(recovery) = &self.idle
|
||||
&& recovery.observed_idle != Some(false)
|
||||
{
|
||||
recovery.notify_connected(&root, output);
|
||||
match opened.and_then(|attachment| {
|
||||
recovery.login.permit.check().map_err(|_| None)?;
|
||||
Ok(attachment)
|
||||
}) {
|
||||
Ok((stream, first)) => {
|
||||
recovery.installed = Some(tokio::time::Instant::now());
|
||||
// A same-thread Attach can stay quiet. Keep the last observation until an
|
||||
// event or a dispatched mutation changes recovery eligibility.
|
||||
self.install(&root, stream, first, output);
|
||||
if let Some(recovery) = &self.idle
|
||||
&& recovery.observed_idle != Some(false)
|
||||
{
|
||||
recovery.notify_connected(&root, output);
|
||||
}
|
||||
}
|
||||
Err(reason) => {
|
||||
recovery
|
||||
.state
|
||||
.send_replace(crate::cloud::IdleReconnect::Inactive);
|
||||
self.report_loss(&root, reason, output);
|
||||
}
|
||||
} else {
|
||||
recovery
|
||||
.state
|
||||
.send_replace(crate::cloud::IdleReconnect::Inactive);
|
||||
self.report_loss(&root, output);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -95,6 +113,7 @@ impl Replies {
|
||||
),
|
||||
ended => {
|
||||
let id = active.thread_id.clone();
|
||||
let error = ended.and_then(Result::err);
|
||||
self.active = None;
|
||||
if let Some(recovery) = &mut self.idle {
|
||||
recovery
|
||||
@@ -105,22 +124,27 @@ impl Replies {
|
||||
.is_none_or(|start| start.elapsed() >= std::time::Duration::from_secs(15));
|
||||
if recovery.observed_idle == Some(true)
|
||||
&& stable
|
||||
&& idle::transient(ended.and_then(Result::err).as_ref())
|
||||
&& idle::transient(error.as_ref())
|
||||
{
|
||||
recovery.offer(&id);
|
||||
}
|
||||
}
|
||||
self.report_loss(&id, output);
|
||||
self.report_loss(&id, error.as_ref().and_then(idle::blocked_recovery), output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn report_loss(&self, id: &str, output: &mpsc::UnboundedSender<AppServerEvent>) {
|
||||
fn report_loss(
|
||||
&self,
|
||||
id: &str,
|
||||
reason: Option<&str>,
|
||||
output: &mpsc::UnboundedSender<AppServerEvent>,
|
||||
) {
|
||||
self.stream_losses.record(id);
|
||||
let _ = output.send(AppServerEvent::ThreadStreamDisconnected(Box::new(
|
||||
cloud_error(
|
||||
id,
|
||||
"Cloud stream ended. Reopen the conversation to reconnect.",
|
||||
reason.unwrap_or("Cloud stream ended. Reopen the conversation to reconnect."),
|
||||
),
|
||||
)));
|
||||
}
|
||||
@@ -255,16 +279,11 @@ impl Replies {
|
||||
}
|
||||
|
||||
fn probe_reply_stream(stream: &mut ReplyStream) -> Result<Option<cloud::Event>> {
|
||||
// Attach can close after successful headers. A healthy idle stream need not emit an event.
|
||||
match tokio::task::unconstrained(stream.next()).now_or_never() {
|
||||
Some(None | Some(Err(_))) => {
|
||||
bail!(
|
||||
"Cloud conversation activated but its stream did not connect. Reopen it before sending."
|
||||
);
|
||||
}
|
||||
Some(Some(Ok(event))) => Ok(Some(event)),
|
||||
None => Ok(None),
|
||||
}
|
||||
check_reply_stream(stream).map_err(|_| {
|
||||
eyre!(
|
||||
"Cloud conversation activated but its stream did not connect. Reopen it before sending."
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Opens an app-server client backed by the authenticated native Cloud client.
|
||||
@@ -385,7 +404,7 @@ async fn run_worker_with_prepared(
|
||||
if let Some(candidate) = &mut prepared {
|
||||
candidate.canceled_recovery = canceled;
|
||||
} else if let Some(root) = canceled {
|
||||
replies.report_loss(&root, output);
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -434,15 +453,21 @@ async fn run_worker_with_prepared(
|
||||
candidate.settings.thread.id == *id
|
||||
|| loaded.is_some_and(|loaded| loaded.thread.id == *id)
|
||||
});
|
||||
let is_thread_list = matches!(&request, protocol::ClientRequest::ThreadList { .. });
|
||||
// Inventory and history reads have no conversation state to commit.
|
||||
let is_read = matches!(
|
||||
&request,
|
||||
protocol::ClientRequest::ThreadList { .. }
|
||||
| protocol::ClientRequest::ThreadTurnsList { .. }
|
||||
| protocol::ClientRequest::ThreadItemsList { .. }
|
||||
);
|
||||
// When closure and the handshake are both ready, cancel before
|
||||
// the handshake can replace the active attachment.
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = output.closed() => return,
|
||||
_ = command.response.closed(), if is_thread_list || is_thread_start => {
|
||||
_ = command.response.closed(), if is_read || is_thread_start => {
|
||||
if let Some(root) = canceled {
|
||||
replies.report_loss(&root, output);
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -455,7 +480,7 @@ async fn run_worker_with_prepared(
|
||||
.and_then(|candidate| candidate.canceled_recovery)
|
||||
&& requests.loaded.is_some()
|
||||
{
|
||||
replies.report_loss(&root, output);
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -464,7 +489,7 @@ async fn run_worker_with_prepared(
|
||||
if result.is_err()
|
||||
&& let Some(root) = canceled
|
||||
{
|
||||
replies.report_loss(&root, output);
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
let _ = command.response.send(Ok(request_result(result)));
|
||||
}
|
||||
@@ -647,7 +672,7 @@ async fn advance_prepared_resume(
|
||||
}
|
||||
}
|
||||
if let Some(root) = candidate.canceled_recovery {
|
||||
replies.report_loss(&root, output);
|
||||
replies.report_loss(&root, /*reason*/ None, output);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user