Files
codex/codex-rs/thread-store/src/local/helpers.rs
T
Charlie Marsh b8cbdbfeff Preserve SQLite row metadata in thread listings (#61866)
## Why

Direct SQLite listings already contain thread titles and section ordering metadata. Converting those rows to rollout summaries and fetching metadata again discards the original per-row values.

## What changed

Build thread listings directly from the collected SQLite rows, retaining their titles, section positions, and section entry times. Preserve row order and each occurrence's metadata when a thread appears more than once during pagination refill. Continue resolving legacy names from the session index with the existing title precedence.

Keep rollout listings and search name enrichment best effort, and avoid redundant metadata queries for direct SQLite listings.

## Testing

Add coverage for repeated legacy rows retaining indexed names and distinct timestamps and section positions. Extend listing tests to check SQLite titles and section metadata in recent-thread results.

GitOrigin-RevId: 838036250d38e6ca1b403741de3e6dd3eafdd04b
2026-11-16 19:30:11 +00:00

404 lines
13 KiB
Rust

use std::collections::HashMap;
use std::collections::HashSet;
use std::ffi::OsStr;
use std::fs::FileTimes;
use std::fs::OpenOptions;
use std::path::Path;
use std::path::PathBuf;
use std::time::SystemTime;
use chrono::DateTime;
use chrono::Utc;
use codex_git_utils::GitSha;
use codex_protocol::SanitizedGitUrl;
use codex_protocol::ThreadId;
use codex_protocol::models::PermissionProfile;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::NetworkAccess;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::ARCHIVED_SESSIONS_SUBDIR;
use codex_rollout::RolloutReferenceIndex;
use codex_rollout::ThreadItem;
use codex_rollout::find_thread_names_by_ids;
use codex_state::ThreadMetadata;
use super::LocalThreadStore;
use crate::StoredThread;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
pub(super) fn scoped_rollout_path(
root: PathBuf,
rollout_path: &Path,
root_name: &str,
) -> ThreadStoreResult<PathBuf> {
let canonical_root =
std::fs::canonicalize(&root).map_err(|err| ThreadStoreError::Internal {
message: format!(
"failed to resolve {root_name} directory `{}`: {err}",
root.display()
),
})?;
let canonical_rollout_path =
std::fs::canonicalize(rollout_path).map_err(|_| ThreadStoreError::InvalidRequest {
message: format!(
"rollout path `{}` must be in {root_name} directory",
rollout_path.display()
),
})?;
if canonical_rollout_path.starts_with(&canonical_root) {
Ok(canonical_rollout_path)
} else {
Err(ThreadStoreError::InvalidRequest {
message: format!(
"rollout path `{}` must be in {root_name} directory",
rollout_path.display()
),
})
}
}
pub(super) fn rollout_path_is_archived(codex_home: &Path, path: &Path) -> bool {
path.starts_with(codex_home.join(ARCHIVED_SESSIONS_SUBDIR))
|| path
.components()
.any(|component| component.as_os_str() == OsStr::new(ARCHIVED_SESSIONS_SUBDIR))
}
pub(super) fn owned_rollout_paths_from_index(
index: &RolloutReferenceIndex,
thread_id: ThreadId,
) -> Vec<PathBuf> {
index
.rollouts_for_thread(thread_id)
.map(|(_, path)| path.to_path_buf())
.collect()
}
pub(super) fn validated_rollout_file_name(
rollout_path: &Path,
display_path: &Path,
) -> ThreadStoreResult<std::ffi::OsString> {
let Some(file_name) = rollout_path.file_name().map(OsStr::to_owned) else {
return Err(ThreadStoreError::InvalidRequest {
message: format!(
"rollout path `{}` missing file name",
display_path.display()
),
});
};
if codex_rollout::rollout_id_from_path(rollout_path).is_some() {
Ok(file_name)
} else {
Err(ThreadStoreError::InvalidRequest {
message: format!(
"rollout path `{}` has an invalid filename",
display_path.display()
),
})
}
}
pub(super) fn touch_modified_time(path: &Path) -> std::io::Result<()> {
let times = FileTimes::new().set_modified(SystemTime::now());
OpenOptions::new().append(true).open(path)?.set_times(times)
}
pub(super) fn restore_rollout_moves(moves: &[(PathBuf, PathBuf)]) -> std::io::Result<()> {
for (source, destination) in moves.iter().rev() {
std::fs::rename(destination, source)?;
}
Ok(())
}
pub(super) fn stored_thread_from_rollout_item(
item: ThreadItem,
archived: bool,
default_provider: &str,
) -> Option<StoredThread> {
let thread_id = item
.thread_id
.or_else(|| thread_id_from_rollout_path(item.path.as_path()))?;
let created_at = parse_rfc3339(item.created_at.as_deref()).unwrap_or_else(Utc::now);
let updated_at = parse_rfc3339(item.updated_at.as_deref()).unwrap_or(created_at);
let recency_at = parse_rfc3339(item.recency_at.as_deref()).unwrap_or(updated_at);
let archived_at = archived.then_some(updated_at);
let git_info = git_info_from_parts(
item.git_sha.clone(),
item.git_branch.clone(),
item.git_origin_url.clone(),
);
let source = item.source.unwrap_or(SessionSource::Unknown);
let preview = item
.preview
.clone()
.or_else(|| item.first_user_message.clone())
.unwrap_or_default();
let rollout_path = codex_rollout::plain_rollout_path(item.path.as_path());
Some(StoredThread {
creator_user_id: None,
creator_account_id: None,
thread_id,
extra_config: None,
rollout_path: Some(rollout_path),
forked_from_id: None,
attached_to_thread_id: item.attached_to_thread_id,
parent_thread_id: item.parent_thread_id,
preview,
name: codex_state::is_guardian_review_source(&source)
.then(|| codex_state::GUARDIAN_THREAD_TITLE.to_string()),
model_provider: item
.model_provider
.filter(|provider| !provider.is_empty())
.unwrap_or_else(|| default_provider.to_string()),
model: item.model,
reasoning_effort: item.reasoning_effort,
created_at,
updated_at,
recency_at,
archived_at,
section: item.section,
section_position: None,
section_entered_at: None,
project_id: item.project_id,
daybreak_enabled: item.daybreak_enabled,
cwd: item.cwd.unwrap_or_default(),
cli_version: item.cli_version.unwrap_or_default(),
originator: item.originator,
source,
history_mode: item.history_mode,
thread_source: None,
agent_nickname: item.agent_nickname,
agent_role: item.agent_role,
agent_path: None,
git_info,
approval_mode: AskForApproval::OnRequest,
permission_profile: PermissionProfile::read_only(),
token_usage: None,
first_user_message: item.first_user_message,
history: None,
})
}
pub(super) fn permission_profile_from_metadata_value(value: &str, cwd: &Path) -> PermissionProfile {
serde_json::from_str::<PermissionProfile>(value)
.or_else(|_| {
parse_legacy_sandbox_policy(value)
.map(|policy| PermissionProfile::from_legacy_sandbox_policy_for_cwd(&policy, cwd))
})
.unwrap_or_else(|_| PermissionProfile::read_only())
}
pub(super) fn permission_profile_to_metadata_value(
permission_profile: &PermissionProfile,
) -> String {
match serde_json::to_string(permission_profile) {
Ok(value) => value,
Err(err) => {
tracing::warn!("failed to serialize permission profile metadata: {err}");
String::new()
}
}
}
pub(super) fn sqlite_thread_name(metadata: &ThreadMetadata) -> Option<String> {
metadata
.name
.as_deref()
.map(str::trim)
.filter(|name| !name.is_empty())
.map(str::to_string)
}
pub(super) async fn resolve_thread_section_metadata(
state_db: &codex_state::StateRuntime,
thread_ids: &[ThreadId],
) -> HashMap<ThreadId, (Option<i64>, Option<DateTime<Utc>>)> {
if thread_ids.is_empty() {
return HashMap::new();
}
state_db
.get_thread_section_ordering(thread_ids)
.await
.unwrap_or_default()
}
/// Enriches rollout listings and search results with names on a best-effort basis.
pub(super) async fn resolve_thread_names(
store: &LocalThreadStore,
thread_history_modes: &HashMap<ThreadId, ThreadHistoryMode>,
) -> HashMap<ThreadId, String> {
let legacy_thread_ids = thread_history_modes
.iter()
.filter_map(|(&thread_id, &history_mode)| {
(history_mode == ThreadHistoryMode::Legacy).then_some(thread_id)
})
.collect::<HashSet<_>>();
let mut names = find_thread_names_by_ids(store.config.codex_home.as_path(), &legacy_thread_ids)
.await
.unwrap_or_default();
if let Some(state_db_ctx) = store.state_db().await {
let thread_ids = thread_history_modes.keys().copied().collect::<Vec<_>>();
let Ok(metadata_by_id) = state_db_ctx.get_threads(&thread_ids).await else {
return names;
};
for (&thread_id, &history_mode) in thread_history_modes {
let Some(metadata) = metadata_by_id.get(&thread_id) else {
continue;
};
if let Some(name) = listed_thread_name(history_mode, metadata, names.remove(&thread_id))
{
names.insert(thread_id, name);
}
}
}
names
}
pub(super) async fn indexed_thread_names(
store: &LocalThreadStore,
legacy_thread_ids: &HashSet<ThreadId>,
use_state_db_only: bool,
) -> ThreadStoreResult<HashMap<ThreadId, String>> {
match find_thread_names_by_ids(store.config.codex_home.as_path(), legacy_thread_ids).await {
Ok(names) => Ok(names),
Err(err) if use_state_db_only => Err(ThreadStoreError::Internal {
message: format!("failed to read indexed thread names: {err}"),
}),
Err(_) => Ok(HashMap::new()),
}
}
pub(super) fn listed_thread_name(
history_mode: ThreadHistoryMode,
metadata: &ThreadMetadata,
indexed_name: Option<String>,
) -> Option<String> {
match history_mode {
ThreadHistoryMode::Legacy => {
let title = distinct_thread_metadata_title(metadata);
if has_guardian_default_title(metadata) {
indexed_name.or(title)
} else {
title.or(indexed_name)
}
}
ThreadHistoryMode::Paginated => sqlite_thread_name(metadata),
}
}
/// Identifies the derived Guardian label, which must yield to legacy indexed names.
pub(super) fn has_guardian_default_title(metadata: &ThreadMetadata) -> bool {
metadata.title.trim() == codex_state::GUARDIAN_THREAD_TITLE
&& serde_json::from_str::<SessionSource>(&metadata.source)
.as_ref()
.is_ok_and(codex_state::is_guardian_review_source)
}
pub(super) fn distinct_thread_metadata_title(metadata: &ThreadMetadata) -> Option<String> {
let title = metadata.title.trim();
if title.is_empty() || metadata.first_user_message.as_deref().map(str::trim) == Some(title) {
None
} else {
Some(title.to_string())
}
}
pub(super) fn set_thread_name(thread: &mut StoredThread, name: String) {
if thread.history_mode == ThreadHistoryMode::Paginated || thread.preview.trim() != name.trim() {
thread.name = Some(name);
}
}
fn parse_rfc3339(value: Option<&str>) -> Option<DateTime<Utc>> {
DateTime::parse_from_rfc3339(value?)
.ok()
.map(|dt| dt.with_timezone(&Utc))
}
fn parse_legacy_sandbox_policy(value: &str) -> serde_json::Result<SandboxPolicy> {
serde_json::from_str(value)
.or_else(|_| serde_json::from_value(serde_json::Value::String(value.to_string())))
.or_else(|_| match value {
"danger-full-access" => Ok(SandboxPolicy::DangerFullAccess),
"read-only" => Ok(SandboxPolicy::new_read_only_policy()),
"workspace-write" => Ok(SandboxPolicy::new_workspace_write_policy()),
"external-sandbox" => Ok(SandboxPolicy::ExternalSandbox {
network_access: NetworkAccess::Restricted,
}),
_ => serde_json::from_value(serde_json::Value::String(value.to_string())),
})
}
pub(super) fn git_info_from_parts(
sha: Option<String>,
branch: Option<String>,
origin_url: Option<SanitizedGitUrl>,
) -> Option<GitInfo> {
if sha.is_none() && branch.is_none() && origin_url.is_none() {
return None;
}
Some(GitInfo {
commit_hash: sha.as_deref().map(GitSha::new),
branch,
repository_url: origin_url,
})
}
fn thread_id_from_rollout_path(path: &Path) -> Option<ThreadId> {
let file_name = path.file_name()?.to_str()?;
let file_name = file_name.strip_suffix(".zst").unwrap_or(file_name);
let stem = file_name.strip_suffix(".jsonl")?;
if stem.len() < 37 {
return None;
}
let uuid_start = stem.len().saturating_sub(36);
if !stem[..uuid_start].ends_with('-') {
return None;
}
ThreadId::from_string(&stem[uuid_start..]).ok()
}
#[cfg(test)]
mod tests {
use codex_protocol::protocol::SubAgentSource;
use codex_rollout::ThreadItem;
use pretty_assertions::assert_eq;
use uuid::Uuid;
use super::*;
#[test]
fn guardian_rollout_conversion_preserves_name_and_logical_path() {
let uuid = Uuid::from_u128(1);
let compressed_path = PathBuf::from(format!(
"/tmp/sessions/2025/01/03/rollout-2025-01-03T12-00-00-{uuid}.jsonl.zst"
));
let thread = stored_thread_from_rollout_item(
ThreadItem {
path: compressed_path.clone(),
source: Some(SessionSource::SubAgent(SubAgentSource::Other(
"guardian".to_string(),
))),
..Default::default()
},
/*archived*/ false,
"test-provider",
)
.expect("stored thread");
assert_eq!(thread.name.as_deref(), Some("Guardian review"));
assert_eq!(
thread.rollout_path,
Some(
compressed_path.with_file_name(format!("rollout-2025-01-03T12-00-00-{uuid}.jsonl"))
)
);
}
}