From 59ac39addbbcd3499319c805a916846ca8ac5fef Mon Sep 17 00:00:00 2001 From: Joe Savona Date: Tue, 17 Nov 2026 06:09:16 +0000 Subject: [PATCH] Surface status line discovery failures and lookup timeouts (#61993) ## Why Status line provider discovery failures were silently ignored, and startup lookup timeouts appeared as unavailable providers. ## What changed - Show incomplete provider discovery in the warnings viewer, with sanitized server names, bounded summaries, and no raw RPC error details. Preserve the warning across canceled scans and clear it after a successful scan. - Share a single five-second startup deadline across provider lookup retries and report timed-out lookups as `Status line timed out`. ## Testing Add coverage for discovery warning rendering and lifecycle, and for a hung lookup retry exhausting the remaining startup deadline. Update the startup timeout expectation. GitOrigin-RevId: 0403af2ed688c31c175d1b5c47d2cdd4cd355653 --- ...s__statusline_discovery_warning_in_f2.snap | 16 +++++ codex-rs/tui/src/app/statusline_picker.rs | 63 ++++++++++++++++++- .../tui/src/app/statusline_picker_tests.rs | 34 ++++++++++ .../tui/src/chatwidget/statusline_reader.rs | 51 ++++++++------- .../chatwidget/statusline_reader/resource.rs | 6 +- .../src/chatwidget/statusline_reader_tests.rs | 29 ++++++++- codex-rs/tui/src/chatwidget/warnings.rs | 28 ++++++++- codex-rs/tui/src/history_cell/warnings.rs | 1 + codex-rs/tui/src/statusline_provider.rs | 13 +++- .../tui/src/statusline_provider/discovery.rs | 1 - .../statusline_provider/discovery_tests.rs | 1 + codex-rs/tui/src/statusline_provider_tests.rs | 13 +++- 12 files changed, 220 insertions(+), 36 deletions(-) create mode 100644 codex-rs/tui/src/app/snapshots/codex_tui__app__statusline_picker__tests__statusline_discovery_warning_in_f2.snap diff --git a/codex-rs/tui/src/app/snapshots/codex_tui__app__statusline_picker__tests__statusline_discovery_warning_in_f2.snap b/codex-rs/tui/src/app/snapshots/codex_tui__app__statusline_picker__tests__statusline_discovery_warning_in_f2.snap new file mode 100644 index 0000000000..29ba8256ab --- /dev/null +++ b/codex-rs/tui/src/app/snapshots/codex_tui__app__statusline_picker__tests__statusline_discovery_warning_in_f2.snap @@ -0,0 +1,16 @@ +--- +source: tui/src/app/statusline_picker_tests.rs +expression: warning +--- + Warnings · 1 of 1 · Status line provider discovery + + Global provider inventory: request failed. + beta: no result by the 5-second discovery deadline. + mail: no result by the 5-second discovery deadline. + chat: no result by the 5-second discovery deadline. + docs: no result by the 5-second discovery deadline. + Additional affected servers: 1. + + + + k keep & next · esc dismiss & close · ⌃o copy · ←/→ warning diff --git a/codex-rs/tui/src/app/statusline_picker.rs b/codex-rs/tui/src/app/statusline_picker.rs index ee343fdf1a..22c16a50ab 100644 --- a/codex-rs/tui/src/app/statusline_picker.rs +++ b/codex-rs/tui/src/app/statusline_picker.rs @@ -5,7 +5,9 @@ use crate::backend_id::BackendId; use crate::bottom_pane::ColumnWidthMode; use crate::bottom_pane::SelectionKeyAction; use crate::config_update::format_config_error; +use crate::statusline_provider::McpServerName; use crate::statusline_provider::ProviderId; +use crate::statusline_provider::ProviderLookupError; use crate::statusline_provider::StatuslineProvider; use crate::statusline_provider::discovery::DiscoveryUpdate; use crate::statusline_provider::discovery::refresh_statusline_providers; @@ -27,6 +29,52 @@ pub(super) struct StatuslinePicker { worker: Option>, } +#[derive(Default)] +struct DiscoveryFailures { + servers: Vec, + omitted: usize, + inventory: Option<&'static str>, +} + +impl DiscoveryFailures { + fn record_server(&mut self, server: &McpServerName, error: &ProviderLookupError) { + if self.servers.len() < 4 { + let name: String = label(server.as_str().to_owned()).chars().take(80).collect(); + let reason = discovery_failure_reason(error); + self.servers.push(format!("{name}: {reason}.")); + } else { + self.omitted += 1; + } + } + + fn into_details(mut self) -> Option { + if self.servers.is_empty() && self.inventory.is_none() { + return None; + } + tracing::warn!( + affected_servers = self.servers.len() + self.omitted, + inventory = self.inventory, + "Status line provider discovery was incomplete" + ); + if let Some(reason) = self.inventory { + self.servers + .insert(0, format!("Global provider inventory: {reason}.")); + } + if self.omitted > 0 { + self.servers + .push(format!("Additional affected servers: {}.", self.omitted)); + } + Some(self.servers.join("\n")) + } +} + +fn discovery_failure_reason(error: &ProviderLookupError) -> &'static str { + match error { + ProviderLookupError::TimedOut => "no result by the 5-second discovery deadline", + ProviderLookupError::Request(_) => "request failed", + } +} + fn label(text: String) -> String { crate::ui_tree::sanitize_leaf(text).replace(['\n', '\t'], " ") } @@ -205,10 +253,19 @@ impl App { if picker.worker.take().is_none() { return; } + let mut failures = DiscoveryFailures::default(); for update in updates { - let DiscoveryUpdate::Server(_server, Ok(mut providers)) = update else { - continue; + let (server, result) = match update { + DiscoveryUpdate::Server(server, result) => (server, result), + DiscoveryUpdate::InventoryFailed(error) => { + failures.inventory = Some(discovery_failure_reason(&error)); + continue; + } }; + let mut providers = result.unwrap_or_else(|error| { + failures.record_server(&server, &error); + Vec::new() + }); for provider in &mut providers { provider.title = label(std::mem::take(&mut provider.title)); provider.plugin_label = provider.plugin_label.take().map(label); @@ -219,6 +276,8 @@ impl App { .providers .sort_unstable_by(|a, b| (&a.title, &a.id).cmp(&(&b.title, &b.id))); let view = picker.view(selected_index); + self.chat_widget + .set_statusline_discovery_warning(scan, failures.into_details()); self.chat_widget .replace_selection_view_if_present(PICKER, view); } diff --git a/codex-rs/tui/src/app/statusline_picker_tests.rs b/codex-rs/tui/src/app/statusline_picker_tests.rs index 4463016786..eb93d0588f 100644 --- a/codex-rs/tui/src/app/statusline_picker_tests.rs +++ b/codex-rs/tui/src/app/statusline_picker_tests.rs @@ -211,3 +211,37 @@ async fn discovery_only_starts_a_reader_on_explicit_selection() -> anyhow::Resul } Ok(()) } + +#[tokio::test] +async fn discovery_shows_a_bounded_warning_without_rpc_details() -> anyhow::Result<()> { + let (mut app, owner, _requests, _events) = picker().await?; + app.open_statusline_picker(&owner); + let scan = app.statusline_picker.as_ref().unwrap().id; + let private_error = color_eyre::eyre::eyre!("private credentials in an RPC message"); + let inventory = DiscoveryUpdate::InventoryFailed(private_error.into()); + let mut updates = vec![inventory]; + for server in ["\x1b[31mbeta\x1b[0m", "mail", "chat", "docs", "links"] { + let update = DiscoveryUpdate::Server(server.into(), Err(ProviderLookupError::TimedOut)); + updates.push(update); + } + app.finish_statusline_discovery(scan, updates, &owner); + app.chat_widget.handle_key_event(KeyCode::Esc.into()); + + app.open_statusline_picker(&owner); + let canceled_scan = app.statusline_picker.as_ref().unwrap().id; + app.chat_widget.handle_key_event(KeyCode::Esc.into()); + // A canceled rescan and its queued completion leave the last warning available. + app.finish_statusline_discovery(canceled_scan, Vec::new(), &owner); + app.chat_widget.open_warnings(&[]); + let warning = render_bottom_popup(&app.chat_widget, /*width*/ 80); + insta::assert_snapshot!("statusline_discovery_warning_in_f2", warning); + + app.chat_widget.handle_key_event(KeyCode::Esc.into()); + app.open_statusline_picker(&owner); + let successful_scan = app.statusline_picker.as_ref().unwrap().id; + app.finish_statusline_discovery(successful_scan, Vec::new(), &owner); + app.chat_widget.handle_key_event(KeyCode::Esc.into()); + app.chat_widget.open_warnings(&[]); + assert!(render_bottom_popup(&app.chat_widget, /*width*/ 80).contains("No warnings")); + Ok(()) +} diff --git a/codex-rs/tui/src/chatwidget/statusline_reader.rs b/codex-rs/tui/src/chatwidget/statusline_reader.rs index 2323e24a8f..3df38f8e3f 100644 --- a/codex-rs/tui/src/chatwidget/statusline_reader.rs +++ b/codex-rs/tui/src/chatwidget/statusline_reader.rs @@ -11,7 +11,9 @@ use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::app_server_session::AppServerSession; use crate::backend_id::BackendId; +use crate::statusline_provider::LOOKUP_TIMEOUT; use crate::statusline_provider::ProviderId; +use crate::statusline_provider::ProviderLookupError; use crate::statusline_provider::ProviderSelection; use crate::statusline_provider::ResolvedStatuslineProvider; use crate::statusline_provider::StatuslineProvider; @@ -28,7 +30,6 @@ use tokio::sync::oneshot; use tokio::sync::watch; use tokio::time::Instant; use tokio::time::sleep; -use tokio::time::timeout; use tokio::time::timeout_at; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -201,8 +202,10 @@ impl ChatWidget { footer.cancel.cancel(); tracing::warn!(?failure, "Status line provider stopped"); let message = match failure { - Failure::Read(ReadIssue::TimedOut) => "Status line timed out", - Failure::Lookup => "Status line unavailable", + Failure::LookupTimedOut | Failure::Read(ReadIssue::TimedOut) => { + "Status line timed out" + } + Failure::LookupUnavailable => "Status line unavailable", Failure::Snapshot | Failure::Read(_) => "Status line failed", }; UiTree(Some(UiNode::String(format!("{message} · /statusline")))) @@ -248,28 +251,30 @@ impl ActiveStatuslineRead { tokio::spawn(async move { let result: Option> = cancel .run_until_cancelled(async { - let cwd = cwd.ok_or(Failure::Lookup)?; - let provider = timeout(Duration::from_secs(/*secs*/ 5), async { - loop { - if let Ok(Some(provider)) = ResolvedStatuslineProvider::lookup( - &client, - thread, - &cwd, - provider_from_picker.as_ref().map_or( - ProviderSelection::Saved(&id), - ProviderSelection::Discovered, - ), - ) - .await - { - return Ok::<_, Failure>(provider); + let cwd = cwd.ok_or(Failure::LookupUnavailable)?; + let deadline = Instant::now() + LOOKUP_TIMEOUT; + let provider = loop { + let choice = provider_from_picker + .as_ref() + .map_or(ProviderSelection::Saved(&id), ProviderSelection::Discovered); + match ResolvedStatuslineProvider::lookup( + &client, thread, &cwd, choice, deadline, + ) + .await + { + Ok(Some(provider)) => break provider, + Err(ProviderLookupError::TimedOut) => { + return Err(Failure::LookupTimedOut); } - // A cold catalog may arrive before the selected MCP server finishes starting. - ready.changed().await.map_err(|_| Failure::Lookup)?; + Ok(None) | Err(ProviderLookupError::Request(_)) => {} } - }) - .await - .map_err(|_| Failure::Lookup)??; + // A cold catalog can arrive before this server finishes starting. + if !matches!(timeout_at(deadline, ready.changed()).await, Ok(Ok(()))) + || Instant::now() >= deadline + { + return Err(Failure::LookupUnavailable); + } + }; loop { let (reply, snapshot) = oneshot::channel(); StatuslineReadEvent::send( diff --git a/codex-rs/tui/src/chatwidget/statusline_reader/resource.rs b/codex-rs/tui/src/chatwidget/statusline_reader/resource.rs index ccd761e9fd..1f7e28fa57 100644 --- a/codex-rs/tui/src/chatwidget/statusline_reader/resource.rs +++ b/codex-rs/tui/src/chatwidget/statusline_reader/resource.rs @@ -36,8 +36,10 @@ impl From for ReadIssue { #[derive(Debug, thiserror::Error)] pub(super) enum Failure { - #[error("The selected provider could not be resolved for this thread.")] - Lookup, + #[error("The selected provider is unavailable.")] + LookupUnavailable, + #[error("The selected provider lookup timed out.")] + LookupTimedOut, #[error("The displayed thread could not supply its current status line state.")] Snapshot, #[error("Resource read: {0}.")] diff --git a/codex-rs/tui/src/chatwidget/statusline_reader_tests.rs b/codex-rs/tui/src/chatwidget/statusline_reader_tests.rs index fb0b44fe34..ae588c819f 100644 --- a/codex-rs/tui/src/chatwidget/statusline_reader_tests.rs +++ b/codex-rs/tui/src/chatwidget/statusline_reader_tests.rs @@ -482,7 +482,7 @@ async fn statusline_startup_timeout_stops_before_the_first_read() { assert_eq!(started.elapsed(), Duration::from_secs(5)); assert_eq!( footer(&reader.chat).as_deref(), - Some("Status line unavailable · /statusline") + Some("Status line timed out · /statusline") ); assert!(held.is_closed()); reader.chat.refresh_status_line(); @@ -490,6 +490,33 @@ async fn statusline_startup_timeout_stops_before_the_first_read() { reader.assert_idle(); } +#[tokio::test(start_paused = true)] +async fn statusline_hung_retry_reports_the_remaining_startup_deadline() { + let mut reader = Reader::new().await; + let started = Instant::now(); + let (_, first) = reader.request().await; + first + .send(Err(std::io::Error::other("private config error"))) + .unwrap(); + advance(/*millis*/ 2_000).await; + reader + .chat + .statusline_read + .as_ref() + .unwrap() + .server_ready("fixture"); + let (ClientRequest::ConfigRead { .. }, held) = reader.request().await else { + panic!("expected a retry after the selected server became ready"); + }; + advance(/*millis*/ 3_000).await; + reader.dispatch().await; + assert!(held.is_closed()); + assert_eq!(started.elapsed(), LOOKUP_TIMEOUT); + insta::assert_snapshot!( + footer(&reader.chat).unwrap(), @"Status line timed out · /statusline" + ); +} + #[tokio::test(start_paused = true)] async fn statusline_changing_provider_or_reconnecting_discards_the_previous_read() { enum Change { diff --git a/codex-rs/tui/src/chatwidget/warnings.rs b/codex-rs/tui/src/chatwidget/warnings.rs index 51ceab801f..3aea8163f6 100644 --- a/codex-rs/tui/src/chatwidget/warnings.rs +++ b/codex-rs/tui/src/chatwidget/warnings.rs @@ -17,6 +17,7 @@ const FALLBACK_MODEL_METADATA_WARNING_SUFFIX: &str = #[derive(Default)] pub(crate) struct WarningDisplayState { pub(super) count: usize, + discovery: Option, /// Completed resume history does not end startup; active work does. pub(super) startup_complete: bool, /// Initialization config warnings may also arrive as ordinary thread warnings. @@ -35,6 +36,7 @@ impl WarningDisplayState { fn visible_entries(&self, cells: &[Arc]) -> Vec { crate::history_cell::warning_entries(cells) .into_iter() + .chain(self.discovery.clone()) .filter(|entry| self.dismissed.get(&entry.id) != Some(&entry.details)) .collect() } @@ -78,7 +80,7 @@ impl super::ChatWidget { .warning_display_state .startup_config_warnings .contains(message), - WarningId::McpServer(_) => false, + WarningId::McpServer(_) | WarningId::StatuslineDiscovery(_) => false, }) }); self.bottom_pane.show_warnings( @@ -88,6 +90,21 @@ impl super::ChatWidget { ); } + pub(crate) fn set_statusline_discovery_warning( + &mut self, + scan: uuid::Uuid, + details: Option, + ) { + let state = &mut self.warning_display_state; + state.discovery = details.map(|details| WarningEntry { + id: WarningId::StatuslineDiscovery(scan), + source: "Status line provider discovery".into(), + details, + }); + state.synced_cells = None; + self.request_redraw(); + } + pub(crate) fn sync_warnings(&mut self, cells: &[Arc]) { let state = &mut self.warning_display_state; if state.synced_cells.as_ref().is_some_and(|synced| { @@ -99,8 +116,15 @@ impl super::ChatWidget { }) { return; } + state.dismissed.retain(|id, _| match id { + WarningId::StatuslineDiscovery(_) => state + .discovery + .as_ref() + .is_some_and(|entry| id == &entry.id), + WarningId::Message(_) | WarningId::McpServer(_) => true, + }); state.count = if state.dismissed.is_empty() { - crate::history_cell::warning_count(cells) + crate::history_cell::warning_count(cells) + usize::from(state.discovery.is_some()) } else { state.visible_entries(cells).len() }; diff --git a/codex-rs/tui/src/history_cell/warnings.rs b/codex-rs/tui/src/history_cell/warnings.rs index e74fde9314..1be54496fd 100644 --- a/codex-rs/tui/src/history_cell/warnings.rs +++ b/codex-rs/tui/src/history_cell/warnings.rs @@ -11,6 +11,7 @@ use std::sync::Arc; pub(crate) enum WarningId { Message(String), McpServer(String), + StatuslineDiscovery(uuid::Uuid), } /// A retained diagnostic. Identity is independent of wrapping and duplicate delivery. diff --git a/codex-rs/tui/src/statusline_provider.rs b/codex-rs/tui/src/statusline_provider.rs index db92d8b462..0e215e2b22 100644 --- a/codex-rs/tui/src/statusline_provider.rs +++ b/codex-rs/tui/src/statusline_provider.rs @@ -22,9 +22,10 @@ pub(crate) use descriptor::StatuslineProvider; use serde_json::Value; use std::collections::HashMap; use tokio::time::Duration; +use tokio::time::Instant; use uuid::Uuid; -const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); +pub(crate) const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Debug, thiserror::Error)] pub(crate) enum ProviderLookupError { @@ -107,11 +108,17 @@ impl ResolvedStatuslineProvider { thread: ThreadId, cwd: &PathUri, selection: ProviderSelection<'_>, + deadline: Instant, ) -> Result, ProviderLookupError> { tokio::select! { biased; - () = tokio::time::sleep(LOOKUP_TIMEOUT) => Err(ProviderLookupError::TimedOut), - result = Self::resolve(client, thread, cwd, selection) => Ok(result?), + () = tokio::time::sleep_until(deadline) => Err(ProviderLookupError::TimedOut), + result = Self::resolve(client, thread, cwd, selection) => { + if Instant::now() >= deadline { + return Err(ProviderLookupError::TimedOut); + } + Ok(result?) + } } } diff --git a/codex-rs/tui/src/statusline_provider/discovery.rs b/codex-rs/tui/src/statusline_provider/discovery.rs index f4f232de9e..672ee804cd 100644 --- a/codex-rs/tui/src/statusline_provider/discovery.rs +++ b/codex-rs/tui/src/statusline_provider/discovery.rs @@ -26,7 +26,6 @@ pub(crate) enum DiscoveryUpdate { McpServerName, Result, ProviderLookupError>, ), - #[cfg_attr(not(test), expect(dead_code, reason = "picker diagnostics are staged"))] InventoryFailed(ProviderLookupError), } diff --git a/codex-rs/tui/src/statusline_provider/discovery_tests.rs b/codex-rs/tui/src/statusline_provider/discovery_tests.rs index e7b258daf2..9b58da7674 100644 --- a/codex-rs/tui/src/statusline_provider/discovery_tests.rs +++ b/codex-rs/tui/src/statusline_provider/discovery_tests.rs @@ -238,6 +238,7 @@ async fn a_cold_picker_and_configured_provider_have_independent_deadlines() { ThreadId::new(), &cwd, ProviderSelection::Saved(&id), + selected_start + LOOKUP_TIMEOUT, ) .await }); diff --git a/codex-rs/tui/src/statusline_provider_tests.rs b/codex-rs/tui/src/statusline_provider_tests.rs index add3add164..cf4ec44af8 100644 --- a/codex-rs/tui/src/statusline_provider_tests.rs +++ b/codex-rs/tui/src/statusline_provider_tests.rs @@ -126,6 +126,7 @@ async fn picker_choice_reads_the_displayed_threads_uri_with_the_snapshot() -> Re thread, &cwd, ProviderSelection::Discovered(&known), + Instant::now() + LOOKUP_TIMEOUT, ) .await? .unwrap(); @@ -183,6 +184,7 @@ async fn visible_project_registration_blocks_the_selected_provider() -> Result<( ThreadId::new(), &cwd, ProviderSelection::Discovered(&known), + Instant::now() + LOOKUP_TIMEOUT, ); let (provider, ()) = tokio::join!(lookup, host); assert!(provider?.is_none()); @@ -252,8 +254,15 @@ async fn selecting_a_plugin_requires_the_globally_advertised_tool_and_the_connec server: "calendar".into(), tool: "status".into(), }; - let lookup = - || ResolvedStatuslineProvider::lookup(&client, thread, &cwd, ProviderSelection::Saved(&id)); + let lookup = || { + ResolvedStatuslineProvider::lookup( + &client, + thread, + &cwd, + ProviderSelection::Saved(&id), + Instant::now() + LOOKUP_TIMEOUT, + ) + }; for _ in 0..rejected { assert!(lookup().await?.is_none()); }