Keep thread storage available when diagnostic logs fail (#60466)

## Why

Failure to open the diagnostic logs database should not disable thread storage or database-only history listing. Thread deletion must still preserve saved conversations when diagnostic log cleanup cannot complete.

## What changed

- Allow state initialization to continue after non-corruption errors opening the logs database, while preserving corruption recovery.
- Report the original diagnostic database error after tracing is installed in the TUI and app server, and return it from log operations.
- Reject thread deletion before destructive cleanup when diagnostic storage failed to initialize.
- Continue periodic cleanup of legacy logs when the current logs database is unavailable.

## Testing

Extend coverage for unavailable log directories, database-only history listing, startup diagnostics, deletion preserving saved conversations, corrupt log headers, and periodic legacy log cleanup during a diagnostic storage outage.

GitOrigin-RevId: a5184580ae6c39ed0c4eefa8cb48ee7731a991c3
This commit is contained in:
Dmitriy Kovalenko
2026-11-09 23:22:08 +00:00
committed by copyberry
parent e1cc96712f
commit caae161dd0
11 changed files with 257 additions and 90 deletions
+55 -25
View File
@@ -102,7 +102,8 @@ pub struct StateRuntime {
sqlite: SqliteConfig,
default_provider: String,
pool: Arc<sqlx::SqlitePool>,
logs_pool: Arc<sqlx::SqlitePool>,
logs_pool: Option<Arc<sqlx::SqlitePool>>,
diagnostic_logs_error: Option<String>,
thread_goals: GoalStore,
memories: MemoryStore,
memories_v2: Arc<tokio::sync::OnceCell<MemoryStore>>,
@@ -159,16 +160,22 @@ impl StateRuntime {
}
};
let has_memories_v2 = tokio::fs::try_exists(sqlite.memories_v2_db_path()).await?;
let logs_pool = match sqlite
let (logs_pool, diagnostic_logs_error) = match sqlite
.open_logs_db(&logs_migrator, telemetry_override)
.await
{
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open logs db at {}: {err}", logs_path.display());
close_sqlite_pools(&[pool.as_ref()]).await;
Ok(db) => (Some(Arc::new(db)), None),
Err(err) if is_sqlite_corruption_error(&err) => {
pool.close().await;
return Err(err);
}
Err(err) => (
None,
Some(format!(
"failed to open logs db at {}: {err:#}",
logs_path.display()
)),
),
};
let goals_pool = match sqlite
.open_goals_db(&goals_migrator, telemetry_override)
@@ -177,7 +184,7 @@ impl StateRuntime {
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open goals db at {}: {err}", goals_path.display());
close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref()]).await;
close_sqlite_pools(&[Some(pool.as_ref()), logs_pool.as_deref()]).await;
return Err(err);
}
};
@@ -191,7 +198,12 @@ impl StateRuntime {
"failed to open memories db at {}: {err}",
memories_path.display()
);
close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref(), goals_pool.as_ref()]).await;
close_sqlite_pools(&[
Some(pool.as_ref()),
logs_pool.as_deref(),
Some(goals_pool.as_ref()),
])
.await;
return Err(err);
}
};
@@ -203,10 +215,10 @@ impl StateRuntime {
Err(err) => {
warn!("failed to open queue db at {}: {err}", queue_path.display());
close_sqlite_pools(&[
pool.as_ref(),
logs_pool.as_ref(),
goals_pool.as_ref(),
memories_pool.as_ref(),
Some(pool.as_ref()),
logs_pool.as_deref(),
Some(goals_pool.as_ref()),
Some(memories_pool.as_ref()),
])
.await;
return Err(err);
@@ -223,11 +235,11 @@ impl StateRuntime {
);
if let Err(err) = backfill_state_result {
close_sqlite_pools(&[
pool.as_ref(),
logs_pool.as_ref(),
goals_pool.as_ref(),
memories_pool.as_ref(),
queue_pool.as_ref(),
Some(pool.as_ref()),
logs_pool.as_deref(),
Some(goals_pool.as_ref()),
Some(memories_pool.as_ref()),
Some(queue_pool.as_ref()),
])
.await;
return Err(err);
@@ -252,11 +264,11 @@ impl StateRuntime {
Ok(value) => value,
Err(err) => {
close_sqlite_pools(&[
pool.as_ref(),
logs_pool.as_ref(),
goals_pool.as_ref(),
memories_pool.as_ref(),
queue_pool.as_ref(),
Some(pool.as_ref()),
logs_pool.as_deref(),
Some(goals_pool.as_ref()),
Some(memories_pool.as_ref()),
Some(queue_pool.as_ref()),
])
.await;
return Err(err);
@@ -272,6 +284,7 @@ impl StateRuntime {
thread_queue: SqliteQueueStore::new(queue_pool),
pool,
logs_pool,
diagnostic_logs_error,
sqlite,
default_provider,
thread_updated_at_millis: Arc::new(AtomicI64::new(thread_updated_at_millis)),
@@ -309,6 +322,21 @@ impl StateRuntime {
&self.thread_queue
}
/// Startup failure to report after tracing is installed and before destructive cleanup.
pub fn diagnostic_logs_error(&self) -> Option<&str> {
self.diagnostic_logs_error.as_deref()
}
fn diagnostic_logs_pool(&self) -> anyhow::Result<&SqlitePool> {
self.logs_pool.as_deref().ok_or_else(|| {
anyhow::anyhow!(
"diagnostic database unavailable: {}",
self.diagnostic_logs_error()
.unwrap_or("logs pool was not initialized")
)
})
}
/// Close all SQLite pools and wait for outstanding pool workers to exit.
pub async fn close(&self) {
self.reclamation.close().await;
@@ -318,7 +346,9 @@ impl StateRuntime {
memories.close().await;
}
self.thread_goals.close().await;
self.logs_pool.close().await;
if let Some(pool) = &self.logs_pool {
pool.close().await;
}
self.pool.close().await;
}
@@ -355,8 +385,8 @@ impl StateRuntime {
}
}
async fn close_sqlite_pools(pools: &[&SqlitePool]) {
for pool in pools {
async fn close_sqlite_pools(pools: &[Option<&SqlitePool>]) {
for pool in pools.iter().flatten() {
pool.close().await;
}
}