codex: move local path reads into thread store

This commit is contained in:
Tom Wiltzius
2026-04-16 13:02:36 -07:00
parent 9198963975
commit b501d9ea98
8 changed files with 573 additions and 739 deletions
@@ -1,7 +1,5 @@
use crate::codex_message_processor::ApiVersion;
use crate::codex_message_processor::read_rollout_items_from_rollout;
use crate::codex_message_processor::read_summary_from_rollout;
use crate::codex_message_processor::summary_to_thread;
use crate::codex_message_processor::thread_from_stored_thread;
use crate::error_code::INTERNAL_ERROR_CODE;
use crate::error_code::INVALID_REQUEST_ERROR_CODE;
use crate::outgoing_message::ClientRequestResult;
@@ -111,7 +109,6 @@ use codex_app_server_protocol::convert_patch_changes;
use codex_app_server_protocol::guardian_auto_approval_review_notification;
use codex_core::CodexThread;
use codex_core::ThreadManager;
use codex_core::find_thread_name_by_id;
use codex_core::review_format::format_review_findings_block;
use codex_core::review_prompts;
use codex_protocol::ThreadId;
@@ -140,6 +137,10 @@ use codex_protocol::request_user_input::RequestUserInputAnswer as CoreRequestUse
use codex_protocol::request_user_input::RequestUserInputResponse as CoreRequestUserInputResponse;
use codex_sandboxing::policy_transforms::intersect_permission_profiles;
use codex_shell_command::parse_command::shlex_join;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::ReadThreadParams as StoreReadThreadParams;
use codex_thread_store::ThreadReadSource as StoreThreadReadSource;
use codex_thread_store::ThreadStore;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::collections::HashMap;
use std::path::Path;
@@ -147,7 +148,6 @@ use std::sync::Arc;
use tokio::sync::Mutex;
use tokio::sync::oneshot;
use tracing::error;
use tracing::warn;
type JsonValue = serde_json::Value;
@@ -175,7 +175,8 @@ pub(crate) async fn apply_bespoke_event_handling(
thread_watch_manager: ThreadWatchManager,
api_version: ApiVersion,
fallback_model_provider: String,
codex_home: &Path,
thread_store: &LocalThreadStore,
_codex_home: &Path,
) {
let Event {
id: event_turn_id,
@@ -1827,46 +1828,39 @@ pub(crate) async fn apply_bespoke_event_handling(
outgoing.send_error(request_id, error).await;
return;
};
let response = match read_summary_from_rollout(
rollout_path.as_path(),
fallback_model_provider.as_str(),
)
.await
let response = match thread_store
.read_thread(StoreReadThreadParams {
source: StoreThreadReadSource::LocalPath {
rollout_path: rollout_path.clone(),
},
include_history: true,
})
.await
{
Ok(summary) => {
Ok(stored_thread) => {
let fallback_cwd = conversation.config_snapshot().await.cwd;
let mut thread = summary_to_thread(summary, &fallback_cwd);
match read_rollout_items_from_rollout(rollout_path.as_path()).await {
Ok(items) => {
thread.turns = build_turns_from_rollout_items(&items);
thread.status = thread_watch_manager
.loaded_status_for_thread(&thread.id)
.await;
match find_thread_name_by_id(codex_home, &conversation_id).await {
Ok(name) => {
thread.name = name;
}
Err(err) => {
warn!(
"Failed to read thread name for {conversation_id}: {err}"
);
}
}
ThreadRollbackResponse { thread }
}
Err(err) => {
let error = JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!(
"failed to load rollout `{}`: {err}",
rollout_path.display()
),
data: None,
};
outgoing.send_error(request_id.clone(), error).await;
return;
}
}
let (mut thread, history) = thread_from_stored_thread(
stored_thread,
fallback_model_provider.as_str(),
&fallback_cwd,
);
let Some(history) = history else {
let error = JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!(
"failed to load rollout `{}`",
rollout_path.display()
),
data: None,
};
outgoing.send_error(request_id.clone(), error).await;
return;
};
thread.turns = build_turns_from_rollout_items(&history.items);
thread.status = thread_watch_manager
.loaded_status_for_thread(&thread.id)
.await;
ThreadRollbackResponse { thread }
}
Err(err) => {
let error = JSONRPCErrorError {
@@ -3119,6 +3113,7 @@ mod tests {
thread_state: Arc<Mutex<ThreadState>>,
thread_watch_manager: ThreadWatchManager,
analytics_events_client: AnalyticsEventsClient,
thread_store: LocalThreadStore,
codex_home: PathBuf,
}
@@ -3139,6 +3134,7 @@ mod tests {
self.thread_watch_manager.clone(),
ApiVersion::V2,
"test-provider".to_string(),
&self.thread_store,
&self.codex_home,
)
.await;
@@ -3429,6 +3425,7 @@ mod tests {
async fn guardian_command_execution_notifications_wrap_review_lifecycle() -> Result<()> {
let codex_home = TempDir::new()?;
let config = load_default_config_for_test(&codex_home).await;
let thread_store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&config));
let thread_manager = Arc::new(
codex_core::test_support::thread_manager_with_models_provider_and_home(
CodexAuth::create_dummy_chatgpt_auth_for_testing(),
@@ -3467,6 +3464,7 @@ mod tests {
"http://localhost".to_string(),
Some(false),
),
thread_store,
codex_home: codex_home.path().to_path_buf(),
};
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -8,10 +8,10 @@ use codex_core::CodexThread;
use codex_core::ThreadConfigSnapshot;
use codex_protocol::ThreadId;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Weak;
use tokio::sync::Mutex;
@@ -27,7 +27,7 @@ type PendingInterruptQueue = Vec<(
pub(crate) struct PendingThreadResumeRequest {
pub(crate) request_id: ConnectionRequestId,
pub(crate) rollout_path: PathBuf,
pub(crate) history_items: Vec<RolloutItem>,
pub(crate) config_snapshot: ThreadConfigSnapshot,
pub(crate) instruction_sources: Vec<AbsolutePathBuf>,
pub(crate) thread_summary: codex_app_server_protocol::Thread,
+1
View File
@@ -32,5 +32,6 @@ pub use types::StoredThreadHistory;
pub use types::ThreadEventPersistenceMode;
pub use types::ThreadMetadataPatch;
pub use types::ThreadPage;
pub use types::ThreadReadSource;
pub use types::ThreadSortKey;
pub use types::UpdateThreadMetadataParams;
+2 -2
View File
@@ -62,9 +62,9 @@ impl ThreadStore for LocalThreadStore {
async fn load_history(
&self,
_params: LoadThreadHistoryParams,
params: LoadThreadHistoryParams,
) -> ThreadStoreResult<StoredThreadHistory> {
unsupported("load_history")
read_thread::load_history(self, params.source).await
}
async fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreResult<StoredThread> {
+155 -55
View File
@@ -14,6 +14,7 @@ use super::helpers::stored_thread_from_rollout_item;
use crate::ReadThreadParams;
use crate::StoredThread;
use crate::StoredThreadHistory;
use crate::ThreadReadSource;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
@@ -25,19 +26,34 @@ pub(super) async fn read_thread(
// with list/read behavior for normal persisted sessions. SQLite is a fallback
// for metadata rows that refer to rollouts not discoverable under codex_home,
// such as explicit-path resumes or legacy state.
let Some(path) = resolve_rollout_path(store, &params).await? else {
return read_thread_from_sqlite_fallback(store, params).await;
let (path, include_archived) = match params.source {
ThreadReadSource::ThreadId {
thread_id,
include_archived,
} => {
let Some(path) = resolve_rollout_path(store, thread_id, include_archived).await? else {
return read_thread_from_sqlite_fallback(
store,
thread_id,
include_archived,
params.include_history,
)
.await;
};
(path, include_archived)
}
ThreadReadSource::LocalPath { rollout_path } => (rollout_path, true),
};
read_thread_from_rollout_path(store, params, path).await
read_thread_from_rollout_path(store, path, include_archived, params.include_history).await
}
async fn resolve_rollout_path(
store: &LocalThreadStore,
params: &ReadThreadParams,
thread_id: codex_protocol::ThreadId,
include_archived: bool,
) -> ThreadStoreResult<Option<std::path::PathBuf>> {
let thread_id = params.thread_id;
let path = if params.include_archived {
let path = if include_archived {
match find_thread_path_by_id_str(store.config.codex_home.as_path(), &thread_id.to_string())
.await
.map_err(|err| ThreadStoreError::InvalidRequest {
@@ -65,35 +81,47 @@ async fn resolve_rollout_path(
async fn read_thread_from_rollout_path(
store: &LocalThreadStore,
params: ReadThreadParams,
path: std::path::PathBuf,
include_archived: bool,
include_history: bool,
) -> ThreadStoreResult<StoredThread> {
let thread_id = params.thread_id;
let archived = path.starts_with(
store
.config
.codex_home
.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
);
if archived && !include_archived {
return Err(ThreadStoreError::InvalidRequest {
message: format!("archived rollout is not eligible: {}", path.display()),
});
}
let Some(item) = read_thread_item_from_rollout(path.clone()).await else {
// Some materialized sessions, such as standalone shell-command turns, have
// valid persisted history and SQLite metadata before they have a user-message
// preview that the rollout summary reader can use.
let session_meta_thread =
stored_thread_from_session_meta(store, path.clone(), archived).await?;
let thread_id = session_meta_thread.thread_id;
let mut thread =
if let Ok(Some(mut metadata)) = read_sqlite_metadata(store, thread_id).await {
metadata.rollout_path = path.clone();
stored_thread_from_sqlite_metadata(store, metadata).await
} else {
stored_thread_from_session_meta(store, path.clone(), archived).await?
session_meta_thread
};
if params.include_history {
if include_history {
let Some(path) = thread.rollout_path.clone() else {
return Err(ThreadStoreError::Internal {
message: format!("failed to load thread history for thread {thread_id}"),
});
};
let items = load_history_items(&path).await?;
thread.history = Some(StoredThreadHistory { thread_id, items });
thread.history = Some(StoredThreadHistory {
thread_id,
rollout_path: Some(path),
items,
});
}
return Ok(thread);
};
@@ -102,6 +130,13 @@ async fn read_thread_from_rollout_path(
.ok_or_else(|| ThreadStoreError::Internal {
message: format!("failed to read thread id from {}", path.display()),
})?;
if let Some(updated_at) = read_rollout_updated_at(path.as_path()) {
thread.updated_at = updated_at;
if archived {
thread.archived_at = Some(updated_at);
}
}
let thread_id = thread.thread_id;
thread.forked_from_id = read_session_meta_line(path.as_path())
.await
.ok()
@@ -122,21 +157,26 @@ async fn read_thread_from_rollout_path(
{
set_thread_name_from_title(&mut thread, title);
}
if params.include_history {
if include_history {
let items = load_history_items(&path).await?;
thread.history = Some(StoredThreadHistory { thread_id, items });
thread.history = Some(StoredThreadHistory {
thread_id,
rollout_path: Some(path),
items,
});
}
Ok(thread)
}
async fn read_thread_from_sqlite_fallback(
store: &LocalThreadStore,
params: ReadThreadParams,
thread_id: codex_protocol::ThreadId,
include_archived: bool,
include_history: bool,
) -> ThreadStoreResult<StoredThread> {
let thread_id = params.thread_id;
let metadata = read_sqlite_metadata(store, thread_id).await?;
let Some(metadata) = metadata
.filter(|metadata| params.include_archived || !metadata_is_archived(store, metadata))
let Some(metadata) =
metadata.filter(|metadata| include_archived || !metadata_is_archived(store, metadata))
else {
return Err(ThreadStoreError::InvalidRequest {
message: format!("no rollout found for thread id {thread_id}"),
@@ -144,18 +184,42 @@ async fn read_thread_from_sqlite_fallback(
};
let mut thread = stored_thread_from_sqlite_metadata(store, metadata).await;
if params.include_history {
if include_history {
let Some(path) = thread.rollout_path.clone() else {
return Err(ThreadStoreError::Internal {
message: format!("failed to load thread history for thread {thread_id}"),
});
};
let items = load_history_items(&path).await?;
thread.history = Some(StoredThreadHistory { thread_id, items });
thread.history = Some(StoredThreadHistory {
thread_id,
rollout_path: Some(path),
items,
});
}
Ok(thread)
}
pub(super) async fn load_history(
store: &LocalThreadStore,
source: ThreadReadSource,
) -> ThreadStoreResult<StoredThreadHistory> {
let thread = read_thread(
store,
ReadThreadParams {
source,
include_history: true,
},
)
.await?;
thread.history.ok_or_else(|| ThreadStoreError::Internal {
message: format!(
"failed to load thread history for thread {}",
thread.thread_id
),
})
}
async fn read_sqlite_metadata(
store: &LocalThreadStore,
thread_id: codex_protocol::ThreadId,
@@ -395,8 +459,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: true,
})
.await
@@ -422,8 +488,10 @@ mod tests {
let active_only_err = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -438,8 +506,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: true,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: true,
},
include_history: false,
})
.await
@@ -465,8 +535,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: true,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: true,
},
include_history: false,
})
.await
@@ -499,8 +571,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -540,8 +614,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -585,8 +661,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -634,8 +712,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -657,8 +737,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -717,8 +799,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: true,
})
.await
@@ -768,8 +852,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: true,
})
.await
@@ -844,8 +930,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -912,8 +1000,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: true,
})
.await
@@ -953,8 +1043,10 @@ mod tests {
let active_only_err = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -969,8 +1061,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: true,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: true,
},
include_history: false,
})
.await
@@ -1011,8 +1105,10 @@ mod tests {
let active_only_err = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
@@ -1027,8 +1123,10 @@ mod tests {
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: true,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: true,
},
include_history: false,
})
.await
@@ -1047,8 +1145,10 @@ mod tests {
let err = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
source: ThreadReadSource::ThreadId {
thread_id,
include_archived: false,
},
include_history: false,
})
.await
+54 -4
View File
@@ -14,6 +14,7 @@ use crate::SetThreadNameParams;
use crate::StoredThread;
use crate::StoredThreadHistory;
use crate::ThreadPage;
use crate::ThreadReadSource;
use crate::ThreadRecorder;
use crate::ThreadStore;
use crate::ThreadStoreError;
@@ -69,13 +70,19 @@ impl ThreadStore for RemoteThreadStore {
async fn load_history(
&self,
_params: LoadThreadHistoryParams,
params: LoadThreadHistoryParams,
) -> ThreadStoreResult<StoredThreadHistory> {
Err(not_implemented("load_history"))
Err(unsupported_or_not_implemented(
"load_history",
&params.source,
))
}
async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult<StoredThread> {
Err(not_implemented("read_thread"))
async fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreResult<StoredThread> {
Err(unsupported_or_not_implemented(
"read_thread",
&params.source,
))
}
async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult<ThreadPage> {
@@ -110,3 +117,46 @@ fn not_implemented(method: &str) -> ThreadStoreError {
message: format!("remote thread store does not implement {method} yet"),
}
}
fn unsupported_or_not_implemented(method: &str, source: &ThreadReadSource) -> ThreadStoreError {
match source {
ThreadReadSource::LocalPath { .. } => ThreadStoreError::InvalidRequest {
message: format!(
"remote thread store does not support local rollout paths for {method}"
),
},
ThreadReadSource::ThreadId { .. } => not_implemented(method),
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::ReadThreadParams;
use crate::RemoteThreadStore;
use crate::ThreadReadSource;
use crate::ThreadStore;
use crate::ThreadStoreError;
#[tokio::test]
async fn read_thread_rejects_local_paths() {
let store = RemoteThreadStore::new("http://localhost:1");
let err = store
.read_thread(ReadThreadParams {
source: ThreadReadSource::LocalPath {
rollout_path: PathBuf::from("/tmp/rollout.jsonl"),
},
include_history: false,
})
.await
.expect_err("local paths should not be supported by remote store");
assert!(matches!(
err,
ThreadStoreError::InvalidRequest { message }
if message == "remote thread store does not support local rollout paths for read_thread"
));
}
}
+23 -8
View File
@@ -62,13 +62,28 @@ pub struct AppendThreadItemsParams {
pub items: Vec<RolloutItem>,
}
/// Identifies a persisted thread either by durable thread id or by a local rollout path.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ThreadReadSource {
/// Preferred storage-neutral lookup by thread id.
ThreadId {
/// Thread id to read.
thread_id: ThreadId,
/// Whether archived threads are eligible.
include_archived: bool,
},
/// Local-only compatibility lookup for callers that still hold rollout paths.
LocalPath {
/// Local rollout path to read.
rollout_path: PathBuf,
},
}
/// Parameters for loading persisted history for resume, fork, rollback, and memory jobs.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LoadThreadHistoryParams {
/// Thread id to load.
pub thread_id: ThreadId,
/// Whether archived threads are eligible.
pub include_archived: bool,
/// Thread source to load.
pub source: ThreadReadSource,
}
/// Persisted rollout history for a thread, without any filesystem path requirement.
@@ -76,6 +91,8 @@ pub struct LoadThreadHistoryParams {
pub struct StoredThreadHistory {
/// Thread id represented by the history.
pub thread_id: ThreadId,
/// Local rollout path when the backing store is filesystem-based.
pub rollout_path: Option<PathBuf>,
/// Persisted rollout items in replay order.
pub items: Vec<RolloutItem>,
}
@@ -83,10 +100,8 @@ pub struct StoredThreadHistory {
/// Parameters for reading a thread summary and optionally its replay history.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReadThreadParams {
/// Thread id to read.
pub thread_id: ThreadId,
/// Whether archived threads are eligible.
pub include_archived: bool,
/// Thread source to read.
pub source: ThreadReadSource,
/// Whether persisted rollout items should be included in the response.
pub include_history: bool,
}