mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Move live thread metadata handling above recorder
This commit is contained in:
@@ -726,6 +726,7 @@ impl ThreadRequestProcessor {
|
||||
async fn prepare_thread_for_archive(&self, thread_id: ThreadId) {
|
||||
let removed_conversation = self.thread_manager.remove_thread(&thread_id).await;
|
||||
if let Some(conversation) = removed_conversation {
|
||||
conversation.ensure_rollout_materialized().await;
|
||||
info!("thread {thread_id} was active; shutting down");
|
||||
match wait_for_thread_shutdown(&conversation).await {
|
||||
ThreadShutdownResult::Complete => {}
|
||||
@@ -1279,6 +1280,7 @@ impl ThreadRequestProcessor {
|
||||
}
|
||||
|
||||
let mut archive_thread_ids = Vec::new();
|
||||
let mut prepared_parent_for_archive = false;
|
||||
match self
|
||||
.thread_store
|
||||
.read_thread(StoreReadThreadParams {
|
||||
@@ -1293,7 +1295,29 @@ impl ThreadRequestProcessor {
|
||||
archive_thread_ids.push(thread_id);
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(thread_store_archive_error("archive", err)),
|
||||
Err(err) => {
|
||||
if self.thread_manager.get_thread(thread_id).await.is_err() {
|
||||
return Err(thread_store_archive_error("archive", err));
|
||||
}
|
||||
self.prepare_thread_for_archive(thread_id).await;
|
||||
prepared_parent_for_archive = true;
|
||||
match self
|
||||
.thread_store
|
||||
.read_thread(StoreReadThreadParams {
|
||||
thread_id,
|
||||
include_archived: false,
|
||||
include_history: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(thread) => {
|
||||
if thread.archived_at.is_none() {
|
||||
archive_thread_ids.push(thread_id);
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(thread_store_archive_error("archive", err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
for descendant_thread_id in thread_ids.into_iter().skip(1) {
|
||||
match self
|
||||
@@ -1311,9 +1335,38 @@ impl ThreadRequestProcessor {
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read spawned descendant thread {descendant_thread_id} while archiving {thread_id}: {err}"
|
||||
);
|
||||
if self
|
||||
.thread_manager
|
||||
.get_thread(descendant_thread_id)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.prepare_thread_for_archive(descendant_thread_id).await;
|
||||
match self
|
||||
.thread_store
|
||||
.read_thread(StoreReadThreadParams {
|
||||
thread_id: descendant_thread_id,
|
||||
include_archived: true,
|
||||
include_history: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(thread) => {
|
||||
if thread.archived_at.is_none() {
|
||||
archive_thread_ids.push(descendant_thread_id);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read live spawned descendant thread {descendant_thread_id} while archiving {thread_id}: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"failed to read spawned descendant thread {descendant_thread_id} while archiving {thread_id}: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1324,7 +1377,9 @@ impl ThreadRequestProcessor {
|
||||
return Ok((ThreadArchiveResponse {}, archived_thread_ids));
|
||||
};
|
||||
|
||||
self.prepare_thread_for_archive(*parent_thread_id).await;
|
||||
if !prepared_parent_for_archive {
|
||||
self.prepare_thread_for_archive(*parent_thread_id).await;
|
||||
}
|
||||
match self
|
||||
.thread_store
|
||||
.archive_thread(StoreArchiveThreadParams {
|
||||
@@ -1413,17 +1468,25 @@ impl ThreadRequestProcessor {
|
||||
};
|
||||
|
||||
let _thread_list_state_permit = self.acquire_thread_list_state_permit().await?;
|
||||
self.thread_store
|
||||
.update_thread_metadata(StoreUpdateThreadMetadataParams {
|
||||
thread_id,
|
||||
patch: StoreThreadMetadataPatch {
|
||||
name: Some(name.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
include_archived: false,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| thread_store_write_error("set thread name", err))?;
|
||||
let patch = StoreThreadMetadataPatch {
|
||||
name: Some(name.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Ok(thread) = self.thread_manager.get_thread(thread_id).await {
|
||||
thread
|
||||
.update_thread_metadata(patch, /*include_archived*/ false)
|
||||
.await
|
||||
.map_err(|err| thread_store_write_error("set thread name", err))?;
|
||||
} else {
|
||||
self.thread_store
|
||||
.update_thread_metadata(StoreUpdateThreadMetadataParams {
|
||||
thread_id,
|
||||
patch,
|
||||
include_archived: false,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| thread_store_write_error("set thread name", err))?;
|
||||
}
|
||||
|
||||
Ok((
|
||||
ThreadSetNameResponse {},
|
||||
|
||||
@@ -3,7 +3,6 @@ use app_test_support::McpProcess;
|
||||
use app_test_support::create_fake_rollout;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::to_response;
|
||||
use codex_app_server_protocol::JSONRPCError;
|
||||
use codex_app_server_protocol::JSONRPCResponse;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ThreadArchiveParams;
|
||||
@@ -33,7 +32,8 @@ use tokio::time::timeout;
|
||||
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_archive_requires_materialized_rollout() -> Result<()> {
|
||||
async fn thread_archive_materializes_initial_metadata_for_unmaterialized_live_thread() -> Result<()>
|
||||
{
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
@@ -69,61 +69,6 @@ async fn thread_archive_requires_materialized_rollout() -> Result<()> {
|
||||
"thread id should not be discoverable before rollout materialization"
|
||||
);
|
||||
|
||||
// Archive should fail before the rollout is materialized.
|
||||
let archive_id = mcp
|
||||
.send_thread_archive_request(ThreadArchiveParams {
|
||||
thread_id: thread.id.clone(),
|
||||
})
|
||||
.await?;
|
||||
let archive_err: JSONRPCError = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_error_message(RequestId::Integer(archive_id)),
|
||||
)
|
||||
.await??;
|
||||
assert!(
|
||||
archive_err
|
||||
.error
|
||||
.message
|
||||
.contains("no rollout found for thread id"),
|
||||
"unexpected archive error: {}",
|
||||
archive_err.error.message
|
||||
);
|
||||
|
||||
// Materialize rollout via a real user turn and confirm archive succeeds.
|
||||
let turn_start_id = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
input: vec![UserInput::Text {
|
||||
text: "materialize".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_start_response: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_start_id)),
|
||||
)
|
||||
.await??;
|
||||
let _: TurnStartResponse = to_response::<TurnStartResponse>(turn_start_response)?;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
assert!(
|
||||
rollout_path.exists(),
|
||||
"expected rollout path {} to exist after first user message",
|
||||
rollout_path.display()
|
||||
);
|
||||
|
||||
let discovered_path =
|
||||
find_thread_path_by_id_str(codex_home.path(), &thread.id, /*state_db_ctx*/ None)
|
||||
.await?
|
||||
.expect("expected rollout path for thread id to exist after materialization");
|
||||
assert_paths_match_on_disk(&discovered_path, &rollout_path)?;
|
||||
|
||||
let archive_id = mcp
|
||||
.send_thread_archive_request(ThreadArchiveParams {
|
||||
thread_id: thread.id.clone(),
|
||||
@@ -147,14 +92,14 @@ async fn thread_archive_requires_materialized_rollout() -> Result<()> {
|
||||
)?;
|
||||
assert_eq!(archived_notification.thread_id, thread.id);
|
||||
|
||||
// Verify file moved.
|
||||
// Archiving provides a persistence barrier, so the initial metadata rollout is materialized
|
||||
// directly in the archive location.
|
||||
let archived_directory = codex_home.path().join(ARCHIVED_SESSIONS_SUBDIR);
|
||||
// The archived file keeps the original filename (rollout-...-<id>.jsonl).
|
||||
let archived_rollout_path =
|
||||
archived_directory.join(rollout_path.file_name().expect("rollout file name"));
|
||||
assert!(
|
||||
!rollout_path.exists(),
|
||||
"expected rollout path {} to be moved",
|
||||
"expected rollout path {} to remain absent",
|
||||
rollout_path.display()
|
||||
);
|
||||
assert!(
|
||||
@@ -648,10 +593,3 @@ stream_max_retries = 0
|
||||
"#
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_paths_match_on_disk(actual: &Path, expected: &Path) -> std::io::Result<()> {
|
||||
let actual = actual.canonicalize()?;
|
||||
let expected = expected.canonicalize()?;
|
||||
assert_eq!(actual, expected);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -955,6 +955,74 @@ async fn thread_name_set_is_reflected_in_read_list_and_resume() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_name_set_materializes_active_unmaterialized_thread() -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let mut mcp = McpProcess::new(codex_home.path()).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let start_id = mcp
|
||||
.send_thread_start_request(ThreadStartParams {
|
||||
model: Some("mock-model".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let start_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(start_id)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
|
||||
let rollout_path = thread.path.clone().expect("thread path");
|
||||
assert!(!rollout_path.exists());
|
||||
|
||||
let new_name = "Active renamed thread";
|
||||
let set_id = mcp
|
||||
.send_thread_set_name_request(ThreadSetNameParams {
|
||||
thread_id: thread.id.clone(),
|
||||
name: new_name.to_string(),
|
||||
})
|
||||
.await?;
|
||||
let set_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(set_id)),
|
||||
)
|
||||
.await??;
|
||||
let _: ThreadSetNameResponse = to_response::<ThreadSetNameResponse>(set_resp)?;
|
||||
let notification = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("thread/name/updated"),
|
||||
)
|
||||
.await??;
|
||||
let notification: ThreadNameUpdatedNotification =
|
||||
serde_json::from_value(notification.params.expect("thread/name/updated params"))?;
|
||||
assert_eq!(notification.thread_id, thread.id);
|
||||
assert_eq!(notification.thread_name.as_deref(), Some(new_name));
|
||||
|
||||
let read_id = mcp
|
||||
.send_thread_read_request(ThreadReadParams {
|
||||
thread_id: thread.id.clone(),
|
||||
include_turns: false,
|
||||
})
|
||||
.await?;
|
||||
let read_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(read_id)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadReadResponse { thread } = to_response::<ThreadReadResponse>(read_resp)?;
|
||||
assert_eq!(thread.name.as_deref(), Some(new_name));
|
||||
assert!(
|
||||
rollout_path.exists(),
|
||||
"active rename should materialize initial session metadata"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_read_include_turns_rejects_unmaterialized_loaded_thread() -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
|
||||
@@ -7779,10 +7779,23 @@ async fn pending_request_user_input_does_not_spawn_extra_goal_continuation() ->
|
||||
})
|
||||
.await;
|
||||
assert_eq!(3, responses.requests().len());
|
||||
let next_event = timeout(Duration::from_millis(200), async {
|
||||
loop {
|
||||
let event = test.codex.next_event().await;
|
||||
if !matches!(
|
||||
event,
|
||||
Ok(Event {
|
||||
msg: EventMsg::TokenCount(_),
|
||||
..
|
||||
})
|
||||
) {
|
||||
return event;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
timeout(Duration::from_millis(200), test.codex.next_event())
|
||||
.await
|
||||
.is_err(),
|
||||
next_event.is_err(),
|
||||
"waiting for request_user_input should keep the turn open without emitting more events"
|
||||
);
|
||||
assert_eq!(
|
||||
|
||||
@@ -891,7 +891,7 @@ async fn handle_response_item_records_tool_result_for_custom_tool_call() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TokenCount(_))).await;
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
logs_assert(|lines: &[&str]| {
|
||||
let line = lines
|
||||
@@ -966,7 +966,7 @@ async fn handle_response_item_records_tool_result_for_function_call() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TokenCount(_))).await;
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
logs_assert(|lines: &[&str]| {
|
||||
let line = lines
|
||||
|
||||
@@ -4,7 +4,6 @@ use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use chrono::Utc;
|
||||
use codex_core::EventPersistenceMode;
|
||||
use codex_core::RolloutRecorder;
|
||||
use codex_core::RolloutRecorderParams;
|
||||
use codex_core::config::ConfigBuilder;
|
||||
@@ -12,7 +11,9 @@ use codex_core::find_archived_thread_path_by_id_str;
|
||||
use codex_core::find_thread_meta_by_name_str;
|
||||
use codex_core::find_thread_path_by_id_str;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_rollout::StateDbHandle;
|
||||
use codex_state::StateRuntime;
|
||||
@@ -180,22 +181,29 @@ async fn find_locates_rollout_file_written_by_recorder() -> std::io::Result<()>
|
||||
.await?;
|
||||
let thread_id = ThreadId::new();
|
||||
let thread_name = "named thread";
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id: thread_id,
|
||||
forked_from_id: None,
|
||||
source: SessionSource::Exec,
|
||||
originator: "test_originator".to_string(),
|
||||
thread_source: None,
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
event_persistence_mode: EventPersistenceMode::Limited,
|
||||
},
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::new(thread_id)).await?;
|
||||
recorder
|
||||
.record_items(&[RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
forked_from_id: None,
|
||||
timestamp: "2024-01-01T00:00:00Z".to_string(),
|
||||
cwd: home.path().to_path_buf(),
|
||||
originator: "test".to_string(),
|
||||
cli_version: "test".to_string(),
|
||||
source: SessionSource::Exec,
|
||||
thread_source: None,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
model_provider: Some("test-provider".to_string()),
|
||||
base_instructions: None,
|
||||
dynamic_tools: None,
|
||||
memory_mode: None,
|
||||
},
|
||||
git: None,
|
||||
})])
|
||||
.await?;
|
||||
recorder.persist().await?;
|
||||
recorder.flush().await?;
|
||||
|
||||
|
||||
@@ -8,16 +8,9 @@ use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::DateTime;
|
||||
use chrono::SecondsFormat;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_utils_string::truncate_middle_chars;
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::FormatItem;
|
||||
@@ -47,25 +40,16 @@ use super::list::get_threads_in_root;
|
||||
use super::list::parse_cursor;
|
||||
use super::list::parse_timestamp_uuid_from_filename;
|
||||
use super::metadata;
|
||||
use super::policy::EventPersistenceMode;
|
||||
use super::policy::is_persisted_rollout_item;
|
||||
use super::session_index::find_thread_names_by_ids;
|
||||
use crate::config::RolloutConfigView;
|
||||
use crate::state_db;
|
||||
use crate::state_db::StateDbHandle;
|
||||
use codex_git_utils::collect_git_info;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::GitInfo as ProtocolGitInfo;
|
||||
use codex_protocol::protocol::InitialHistory;
|
||||
use codex_protocol::protocol::ResumedHistory;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadSource;
|
||||
use codex_state::StateRuntime;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
use codex_utils_path as path_utils;
|
||||
|
||||
/// Records all [`ResponseItem`]s for a session and flushes them to disk after
|
||||
@@ -82,25 +66,12 @@ pub struct RolloutRecorder {
|
||||
tx: Sender<RolloutCmd>,
|
||||
writer_task: Arc<RolloutWriterTask>,
|
||||
pub(crate) rollout_path: PathBuf,
|
||||
event_persistence_mode: EventPersistenceMode,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum RolloutRecorderParams {
|
||||
Create {
|
||||
conversation_id: ThreadId,
|
||||
forked_from_id: Option<ThreadId>,
|
||||
source: SessionSource,
|
||||
originator: String,
|
||||
thread_source: Option<ThreadSource>,
|
||||
base_instructions: BaseInstructions,
|
||||
dynamic_tools: Vec<DynamicToolSpec>,
|
||||
event_persistence_mode: EventPersistenceMode,
|
||||
},
|
||||
Resume {
|
||||
path: PathBuf,
|
||||
event_persistence_mode: EventPersistenceMode,
|
||||
},
|
||||
Create { conversation_id: ThreadId },
|
||||
Resume { path: PathBuf },
|
||||
}
|
||||
|
||||
enum RolloutCmd {
|
||||
@@ -165,38 +136,12 @@ fn clone_io_error(err: &IoError) -> IoError {
|
||||
}
|
||||
|
||||
impl RolloutRecorderParams {
|
||||
pub fn resume(path: PathBuf, event_persistence_mode: EventPersistenceMode) -> Self {
|
||||
Self::Resume {
|
||||
path,
|
||||
event_persistence_mode,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES: usize = 10_000;
|
||||
|
||||
fn sanitize_rollout_item_for_persistence(
|
||||
item: RolloutItem,
|
||||
mode: EventPersistenceMode,
|
||||
) -> RolloutItem {
|
||||
if mode != EventPersistenceMode::Extended {
|
||||
return item;
|
||||
pub fn new(conversation_id: ThreadId) -> Self {
|
||||
Self::Create { conversation_id }
|
||||
}
|
||||
|
||||
match item {
|
||||
RolloutItem::EventMsg(EventMsg::ExecCommandEnd(mut event)) => {
|
||||
// Persist only a bounded aggregated summary of command output.
|
||||
event.aggregated_output = truncate_middle_chars(
|
||||
&event.aggregated_output,
|
||||
PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES,
|
||||
);
|
||||
// Drop unnecessary fields from rollout storage since aggregated_output is all we need.
|
||||
event.stdout.clear();
|
||||
event.stderr.clear();
|
||||
event.formatted_output.clear();
|
||||
RolloutItem::EventMsg(EventMsg::ExecCommandEnd(event))
|
||||
}
|
||||
_ => item,
|
||||
pub fn resume(path: PathBuf) -> Self {
|
||||
Self::Resume { path }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -645,84 +590,25 @@ impl RolloutRecorder {
|
||||
pub async fn new(
|
||||
config: &impl RolloutConfigView,
|
||||
params: RolloutRecorderParams,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
state_builder: Option<ThreadMetadataBuilder>,
|
||||
) -> std::io::Result<Self> {
|
||||
let (file, deferred_log_file_info, rollout_path, meta, event_persistence_mode) =
|
||||
match params {
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id,
|
||||
forked_from_id,
|
||||
source,
|
||||
originator,
|
||||
thread_source,
|
||||
base_instructions,
|
||||
dynamic_tools,
|
||||
event_persistence_mode,
|
||||
} => {
|
||||
let log_file_info = precompute_log_file_info(config, conversation_id)?;
|
||||
let path = log_file_info.path.clone();
|
||||
let session_id = log_file_info.conversation_id;
|
||||
let started_at = log_file_info.timestamp;
|
||||
let (file, deferred_log_file_info, rollout_path) = match params {
|
||||
RolloutRecorderParams::Create { conversation_id } => {
|
||||
let log_file_info = precompute_log_file_info(config, conversation_id)?;
|
||||
let path = log_file_info.path.clone();
|
||||
|
||||
let timestamp_format: &[FormatItem] = format_description!(
|
||||
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
|
||||
);
|
||||
let timestamp = started_at
|
||||
.to_offset(time::UtcOffset::UTC)
|
||||
.format(timestamp_format)
|
||||
.map_err(|e| IoError::other(format!("failed to format timestamp: {e}")))?;
|
||||
|
||||
let session_meta = SessionMeta {
|
||||
id: session_id,
|
||||
forked_from_id,
|
||||
timestamp,
|
||||
cwd: config.cwd().to_path_buf(),
|
||||
originator,
|
||||
cli_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
agent_nickname: source.get_nickname(),
|
||||
agent_role: source.get_agent_role(),
|
||||
agent_path: source.get_agent_path().map(Into::into),
|
||||
source,
|
||||
thread_source,
|
||||
model_provider: Some(config.model_provider_id().to_string()),
|
||||
base_instructions: Some(base_instructions),
|
||||
dynamic_tools: if dynamic_tools.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(dynamic_tools)
|
||||
},
|
||||
memory_mode: (!config.generate_memories())
|
||||
.then_some("disabled".to_string()),
|
||||
};
|
||||
|
||||
(
|
||||
None,
|
||||
Some(log_file_info),
|
||||
path,
|
||||
Some(session_meta),
|
||||
event_persistence_mode,
|
||||
)
|
||||
}
|
||||
RolloutRecorderParams::Resume {
|
||||
path,
|
||||
event_persistence_mode,
|
||||
} => (
|
||||
Some(
|
||||
tokio::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.await?,
|
||||
),
|
||||
None,
|
||||
path,
|
||||
None,
|
||||
event_persistence_mode,
|
||||
(None, Some(log_file_info), path)
|
||||
}
|
||||
RolloutRecorderParams::Resume { path } => (
|
||||
Some(
|
||||
tokio::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.await?,
|
||||
),
|
||||
};
|
||||
|
||||
// Clone the cwd for the spawned task to collect git info asynchronously
|
||||
let cwd = config.cwd().to_path_buf();
|
||||
None,
|
||||
path,
|
||||
),
|
||||
};
|
||||
|
||||
// A reasonably-sized bounded channel. If the buffer fills up the send
|
||||
// future will yield, which is fine – we only need to ensure we do not
|
||||
@@ -734,21 +620,12 @@ impl RolloutRecorder {
|
||||
let writer_task = Arc::new(RolloutWriterTask::new());
|
||||
let writer_task_for_spawn = Arc::clone(&writer_task);
|
||||
let rollout_path_for_spawn = rollout_path.clone();
|
||||
let default_provider = config.model_provider_id().to_string();
|
||||
let generate_memories = config.generate_memories();
|
||||
let state_db_ctx_for_spawn = state_db_ctx.clone();
|
||||
let handle = tokio::task::spawn(async move {
|
||||
let result = rollout_writer(
|
||||
file,
|
||||
deferred_log_file_info,
|
||||
rx,
|
||||
meta,
|
||||
cwd,
|
||||
rollout_path_for_spawn.clone(),
|
||||
state_db_ctx_for_spawn,
|
||||
state_builder,
|
||||
default_provider,
|
||||
generate_memories,
|
||||
)
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
@@ -767,7 +644,6 @@ impl RolloutRecorder {
|
||||
tx,
|
||||
writer_task,
|
||||
rollout_path,
|
||||
event_persistence_mode,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -776,23 +652,11 @@ impl RolloutRecorder {
|
||||
}
|
||||
|
||||
pub async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
|
||||
let mut filtered = Vec::new();
|
||||
for item in items {
|
||||
// Note that function calls may look a bit strange if they are
|
||||
// "fully qualified MCP tool calls," so we could consider
|
||||
// reformatting them in that case.
|
||||
if is_persisted_rollout_item(item, self.event_persistence_mode) {
|
||||
filtered.push(sanitize_rollout_item_for_persistence(
|
||||
item.clone(),
|
||||
self.event_persistence_mode,
|
||||
));
|
||||
}
|
||||
}
|
||||
if filtered.is_empty() {
|
||||
if items.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.tx
|
||||
.send(RolloutCmd::AddItems(filtered))
|
||||
.send(RolloutCmd::AddItems(items.to_vec()))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
self.writer_task.terminal_failure().unwrap_or_else(|| {
|
||||
@@ -1344,12 +1208,6 @@ fn cursor_from_thread_item(item: &ThreadItem, sort_key: ThreadSortKey) -> Option
|
||||
struct LogFileInfo {
|
||||
/// Full path to the rollout file.
|
||||
path: PathBuf,
|
||||
|
||||
/// Session ID (also embedded in filename).
|
||||
conversation_id: ThreadId,
|
||||
|
||||
/// Timestamp for the start of the session.
|
||||
timestamp: OffsetDateTime,
|
||||
}
|
||||
|
||||
fn precompute_log_file_info(
|
||||
@@ -1377,11 +1235,7 @@ fn precompute_log_file_info(
|
||||
|
||||
let path = dir.join(filename);
|
||||
|
||||
Ok(LogFileInfo {
|
||||
path,
|
||||
conversation_id,
|
||||
timestamp,
|
||||
})
|
||||
Ok(LogFileInfo { path })
|
||||
}
|
||||
|
||||
fn open_log_file(path: &Path) -> std::io::Result<File> {
|
||||
@@ -1407,63 +1261,21 @@ struct RolloutWriterState {
|
||||
writer: Option<JsonlWriter>,
|
||||
deferred_log_file_info: Option<LogFileInfo>,
|
||||
pending_items: Vec<RolloutItem>,
|
||||
meta: Option<SessionMeta>,
|
||||
cwd: PathBuf,
|
||||
rollout_path: PathBuf,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
state_builder: Option<ThreadMetadataBuilder>,
|
||||
default_provider: String,
|
||||
generate_memories: bool,
|
||||
thread_updated_at_touch: ThreadUpdatedAtTouch,
|
||||
last_logged_error: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
const THREAD_UPDATED_AT_TOUCH_INTERVAL: Duration = Duration::from_secs(5);
|
||||
#[cfg(test)]
|
||||
const THREAD_UPDATED_AT_TOUCH_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
#[derive(Default)]
|
||||
struct ThreadUpdatedAtTouch {
|
||||
last_persisted_at: Option<Instant>,
|
||||
pending_touch: Option<(ThreadId, DateTime<Utc>)>,
|
||||
}
|
||||
|
||||
impl ThreadUpdatedAtTouch {
|
||||
fn mark_persisted(&mut self, now: Instant) {
|
||||
self.last_persisted_at = Some(now);
|
||||
self.pending_touch = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl RolloutWriterState {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn new(
|
||||
file: Option<tokio::fs::File>,
|
||||
deferred_log_file_info: Option<LogFileInfo>,
|
||||
meta: Option<SessionMeta>,
|
||||
cwd: PathBuf,
|
||||
rollout_path: PathBuf,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
mut state_builder: Option<ThreadMetadataBuilder>,
|
||||
default_provider: String,
|
||||
generate_memories: bool,
|
||||
) -> Self {
|
||||
if let Some(builder) = state_builder.as_mut() {
|
||||
builder.rollout_path = rollout_path.clone();
|
||||
}
|
||||
Self {
|
||||
writer: file.map(|file| JsonlWriter { file }),
|
||||
deferred_log_file_info,
|
||||
pending_items: Vec::new(),
|
||||
meta,
|
||||
cwd,
|
||||
rollout_path,
|
||||
state_db_ctx,
|
||||
state_builder,
|
||||
default_provider,
|
||||
generate_memories,
|
||||
thread_updated_at_touch: ThreadUpdatedAtTouch::default(),
|
||||
last_logged_error: None,
|
||||
}
|
||||
}
|
||||
@@ -1496,19 +1308,7 @@ impl RolloutWriterState {
|
||||
if self.is_deferred() && self.pending_items.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.write_pending_with_recovery("shutdown").await?;
|
||||
if let Some((thread_id, updated_at)) = self.thread_updated_at_touch.pending_touch.take()
|
||||
&& state_db::touch_thread_updated_at(
|
||||
self.state_db_ctx.as_deref(),
|
||||
Some(thread_id),
|
||||
updated_at,
|
||||
"rollout_writer_shutdown",
|
||||
)
|
||||
.await
|
||||
{
|
||||
self.thread_updated_at_touch.mark_persisted(Instant::now());
|
||||
}
|
||||
Ok(())
|
||||
self.write_pending_with_recovery("shutdown").await
|
||||
}
|
||||
|
||||
async fn write_pending_with_recovery(&mut self, operation: &str) -> std::io::Result<()> {
|
||||
@@ -1572,29 +1372,8 @@ impl RolloutWriterState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_session_meta_if_needed(&mut self) -> std::io::Result<()> {
|
||||
let Some(session_meta) = self.meta.as_ref().cloned() else {
|
||||
return Ok(());
|
||||
};
|
||||
write_session_meta(
|
||||
self.writer.as_mut(),
|
||||
session_meta,
|
||||
&self.cwd,
|
||||
&self.rollout_path,
|
||||
self.state_db_ctx.as_deref(),
|
||||
&mut self.state_builder,
|
||||
self.default_provider.as_str(),
|
||||
self.generate_memories,
|
||||
&mut self.thread_updated_at_touch,
|
||||
)
|
||||
.await?;
|
||||
self.meta = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_pending_once(&mut self) -> std::io::Result<()> {
|
||||
self.ensure_writer_open().await?;
|
||||
self.write_session_meta_if_needed().await?;
|
||||
|
||||
self.write_pending_items_once().await?;
|
||||
|
||||
@@ -1619,49 +1398,19 @@ impl RolloutWriterState {
|
||||
written_count += 1;
|
||||
}
|
||||
|
||||
if written_count > 0 {
|
||||
let written_items: Vec<RolloutItem> =
|
||||
self.pending_items.drain(..written_count).collect();
|
||||
sync_thread_state_after_write(
|
||||
self.state_db_ctx.as_deref(),
|
||||
&self.rollout_path,
|
||||
self.state_builder.as_ref(),
|
||||
written_items.as_slice(),
|
||||
self.default_provider.as_str(),
|
||||
/*new_thread_memory_mode*/ None,
|
||||
&mut self.thread_updated_at_touch,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.pending_items.drain(..written_count);
|
||||
|
||||
write_result
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn rollout_writer(
|
||||
file: Option<tokio::fs::File>,
|
||||
deferred_log_file_info: Option<LogFileInfo>,
|
||||
mut rx: mpsc::Receiver<RolloutCmd>,
|
||||
meta: Option<SessionMeta>,
|
||||
cwd: PathBuf,
|
||||
rollout_path: PathBuf,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
state_builder: Option<ThreadMetadataBuilder>,
|
||||
default_provider: String,
|
||||
generate_memories: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let mut state = RolloutWriterState::new(
|
||||
file,
|
||||
deferred_log_file_info,
|
||||
meta,
|
||||
cwd,
|
||||
rollout_path,
|
||||
state_db_ctx,
|
||||
state_builder,
|
||||
default_provider,
|
||||
generate_memories,
|
||||
);
|
||||
let mut state = RolloutWriterState::new(file, deferred_log_file_info, rollout_path);
|
||||
|
||||
// Process rollout commands
|
||||
while let Some(cmd) = rx.recv().await {
|
||||
@@ -1691,117 +1440,11 @@ async fn rollout_writer(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn write_session_meta(
|
||||
mut writer: Option<&mut JsonlWriter>,
|
||||
session_meta: SessionMeta,
|
||||
cwd: &Path,
|
||||
rollout_path: &Path,
|
||||
state_db_ctx: Option<&StateRuntime>,
|
||||
state_builder: &mut Option<ThreadMetadataBuilder>,
|
||||
default_provider: &str,
|
||||
generate_memories: bool,
|
||||
thread_updated_at_touch: &mut ThreadUpdatedAtTouch,
|
||||
) -> std::io::Result<()> {
|
||||
let git_info = collect_git_info(cwd).await.map(|info| ProtocolGitInfo {
|
||||
commit_hash: info.commit_hash,
|
||||
branch: info.branch,
|
||||
repository_url: info.repository_url,
|
||||
});
|
||||
let session_meta_line = SessionMetaLine {
|
||||
meta: session_meta,
|
||||
git: git_info,
|
||||
};
|
||||
if state_db_ctx.is_some() {
|
||||
*state_builder = metadata::builder_from_session_meta(&session_meta_line, rollout_path);
|
||||
}
|
||||
|
||||
let rollout_item = RolloutItem::SessionMeta(session_meta_line);
|
||||
if let Some(writer) = writer.as_mut() {
|
||||
writer.write_rollout_item(&rollout_item).await?;
|
||||
}
|
||||
sync_thread_state_after_write(
|
||||
state_db_ctx,
|
||||
rollout_path,
|
||||
state_builder.as_ref(),
|
||||
std::slice::from_ref(&rollout_item),
|
||||
default_provider,
|
||||
(!generate_memories).then_some("disabled"),
|
||||
thread_updated_at_touch,
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_thread_state_after_write(
|
||||
state_db_ctx: Option<&StateRuntime>,
|
||||
rollout_path: &Path,
|
||||
state_builder: Option<&ThreadMetadataBuilder>,
|
||||
items: &[RolloutItem],
|
||||
default_provider: &str,
|
||||
new_thread_memory_mode: Option<&str>,
|
||||
thread_updated_at_touch: &mut ThreadUpdatedAtTouch,
|
||||
) {
|
||||
let updated_at = Utc::now();
|
||||
let now = Instant::now();
|
||||
if new_thread_memory_mode.is_some()
|
||||
|| items
|
||||
.iter()
|
||||
.any(codex_state::rollout_item_affects_thread_metadata)
|
||||
{
|
||||
state_db::apply_rollout_items(
|
||||
state_db_ctx,
|
||||
rollout_path,
|
||||
default_provider,
|
||||
state_builder,
|
||||
items,
|
||||
"rollout_writer",
|
||||
new_thread_memory_mode,
|
||||
Some(updated_at),
|
||||
)
|
||||
.await;
|
||||
thread_updated_at_touch.mark_persisted(now);
|
||||
return;
|
||||
}
|
||||
|
||||
let thread_id = state_builder
|
||||
.map(|builder| builder.id)
|
||||
.or_else(|| metadata::builder_from_items(items, rollout_path).map(|builder| builder.id));
|
||||
if thread_updated_at_touch
|
||||
.last_persisted_at
|
||||
.is_some_and(|last_persisted_at| {
|
||||
now.duration_since(last_persisted_at) < THREAD_UPDATED_AT_TOUCH_INTERVAL
|
||||
})
|
||||
{
|
||||
thread_updated_at_touch.pending_touch = thread_id.map(|thread_id| (thread_id, updated_at));
|
||||
return;
|
||||
}
|
||||
|
||||
if state_db::touch_thread_updated_at(state_db_ctx, thread_id, updated_at, "rollout_writer")
|
||||
.await
|
||||
{
|
||||
thread_updated_at_touch.mark_persisted(now);
|
||||
return;
|
||||
}
|
||||
state_db::apply_rollout_items(
|
||||
state_db_ctx,
|
||||
rollout_path,
|
||||
default_provider,
|
||||
state_builder,
|
||||
items,
|
||||
"rollout_writer",
|
||||
new_thread_memory_mode,
|
||||
Some(updated_at),
|
||||
)
|
||||
.await;
|
||||
thread_updated_at_touch.mark_persisted(now);
|
||||
}
|
||||
|
||||
/// Append one already-filtered rollout item to an existing rollout JSONL file.
|
||||
///
|
||||
/// This is for metadata updates to unloaded threads. Live sessions should use
|
||||
/// `RolloutRecorder::record_items` so rollout and SQLite updates remain ordered
|
||||
/// with the rest of the session stream.
|
||||
/// This is for unloaded-thread helpers that have already prepared canonical
|
||||
/// rollout JSONL. Live sessions should use `LiveThread` so rollout and metadata
|
||||
/// updates remain ordered with the rest of the session stream.
|
||||
pub async fn append_rollout_item_to_path(
|
||||
rollout_path: &Path,
|
||||
item: &RolloutItem,
|
||||
|
||||
@@ -37,6 +37,29 @@ fn test_config(codex_home: &Path) -> RolloutConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn session_meta_item(thread_id: ThreadId, cwd: &Path) -> RolloutItem {
|
||||
RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
forked_from_id: None,
|
||||
timestamp: "2026-01-27T12:34:56Z".to_string(),
|
||||
cwd: cwd.to_path_buf(),
|
||||
originator: "test".to_string(),
|
||||
cli_version: "test".to_string(),
|
||||
source: SessionSource::Cli,
|
||||
thread_source: None,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
model_provider: Some("test-provider".to_string()),
|
||||
base_instructions: None,
|
||||
dynamic_tools: None,
|
||||
memory_mode: None,
|
||||
},
|
||||
git: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result<PathBuf> {
|
||||
let day_dir = root.join("sessions/2025/01/03");
|
||||
fs::create_dir_all(&day_dir)?;
|
||||
@@ -363,22 +386,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let thread_id = ThreadId::new();
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id: thread_id,
|
||||
forked_from_id: None,
|
||||
source: SessionSource::Exec,
|
||||
originator: "test_originator".to_string(),
|
||||
thread_source: None,
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
event_persistence_mode: EventPersistenceMode::Limited,
|
||||
},
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::new(thread_id)).await?;
|
||||
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
assert!(
|
||||
@@ -419,10 +427,6 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
assert!(rollout_path.exists(), "rollout file should be materialized");
|
||||
|
||||
let text = std::fs::read_to_string(&rollout_path)?;
|
||||
assert!(
|
||||
text.contains("\"type\":\"session_meta\""),
|
||||
"expected session metadata in rollout"
|
||||
);
|
||||
let buffered_idx = text
|
||||
.find("buffered-event")
|
||||
.expect("buffered event in rollout");
|
||||
@@ -445,22 +449,7 @@ async fn persist_reports_filesystem_error_and_retries_buffered_items() -> std::i
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let thread_id = ThreadId::new();
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id: thread_id,
|
||||
forked_from_id: None,
|
||||
source: SessionSource::Exec,
|
||||
originator: "test_originator".to_string(),
|
||||
thread_source: None,
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
event_persistence_mode: EventPersistenceMode::Limited,
|
||||
},
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::new(thread_id)).await?;
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
|
||||
recorder
|
||||
@@ -500,20 +489,13 @@ async fn persist_reports_filesystem_error_and_retries_buffered_items() -> std::i
|
||||
#[tokio::test]
|
||||
async fn writer_state_retries_write_error_before_reporting_flush_success() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
File::create(&rollout_path)?;
|
||||
let read_only_file = std::fs::OpenOptions::new().read(true).open(&rollout_path)?;
|
||||
let mut state = RolloutWriterState::new(
|
||||
Some(tokio::fs::File::from_std(read_only_file)),
|
||||
/*deferred_log_file_info*/ None,
|
||||
/*meta*/ None,
|
||||
home.path().to_path_buf(),
|
||||
rollout_path.clone(),
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
config.model_provider_id.clone(),
|
||||
config.generate_memories,
|
||||
);
|
||||
state.add_items(vec![RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
@@ -532,217 +514,6 @@ async fn writer_state_retries_write_error_before_reporting_flush_success() -> st
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_irrelevant_events_coalesce_state_db_updated_at() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
state_db
|
||||
.mark_backfill_complete(/*last_watermark*/ None)
|
||||
.await
|
||||
.expect("backfill should be complete");
|
||||
|
||||
let thread_id = ThreadId::new();
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id: thread_id,
|
||||
forked_from_id: None,
|
||||
source: SessionSource::Cli,
|
||||
originator: "test_originator".to_string(),
|
||||
thread_source: None,
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
event_persistence_mode: EventPersistenceMode::Limited,
|
||||
},
|
||||
Some(state_db.clone()),
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
message: "first-user-message".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.persist().await?;
|
||||
recorder.flush().await?;
|
||||
let initial_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load")
|
||||
.expect("thread should exist");
|
||||
let initial_updated_at = initial_thread.updated_at;
|
||||
let initial_title = initial_thread.title.clone();
|
||||
let initial_first_user_message = initial_thread.first_user_message.clone();
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.flush().await?;
|
||||
|
||||
let updated_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after agent message")
|
||||
.expect("thread should still exist");
|
||||
|
||||
assert_eq!(updated_thread.updated_at, initial_updated_at);
|
||||
assert_eq!(updated_thread.title, initial_title);
|
||||
assert_eq!(
|
||||
updated_thread.first_user_message,
|
||||
initial_first_user_message
|
||||
);
|
||||
|
||||
tokio::time::sleep(THREAD_UPDATED_AT_TOUCH_INTERVAL + Duration::from_millis(10)).await;
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "more assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.flush().await?;
|
||||
|
||||
let refreshed_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after refresh")
|
||||
.expect("thread should still exist");
|
||||
assert!(refreshed_thread.updated_at > initial_updated_at);
|
||||
assert_eq!(refreshed_thread.title, initial_title);
|
||||
assert_eq!(
|
||||
refreshed_thread.first_user_message,
|
||||
initial_first_user_message
|
||||
);
|
||||
|
||||
recorder.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_flushes_pending_metadata_irrelevant_updated_at() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
state_db
|
||||
.mark_backfill_complete(/*last_watermark*/ None)
|
||||
.await
|
||||
.expect("backfill should be complete");
|
||||
|
||||
let thread_id = ThreadId::new();
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
let initial_updated_at = Utc.with_ymd_and_hms(2026, 5, 7, 7, 37, 8).unwrap();
|
||||
let builder = ThreadMetadataBuilder::new(
|
||||
thread_id,
|
||||
rollout_path.clone(),
|
||||
initial_updated_at,
|
||||
SessionSource::Cli,
|
||||
);
|
||||
state_db
|
||||
.upsert_thread(&builder.build(config.model_provider_id.as_str()))
|
||||
.await
|
||||
.expect("thread should be inserted");
|
||||
|
||||
File::create(&rollout_path)?;
|
||||
let rollout_file = std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&rollout_path)?;
|
||||
let mut state = RolloutWriterState::new(
|
||||
Some(tokio::fs::File::from_std(rollout_file)),
|
||||
/*deferred_log_file_info*/ None,
|
||||
/*meta*/ None,
|
||||
home.path().to_path_buf(),
|
||||
rollout_path,
|
||||
Some(state_db.clone()),
|
||||
Some(builder),
|
||||
config.model_provider_id.clone(),
|
||||
config.generate_memories,
|
||||
);
|
||||
let pending_updated_at = initial_updated_at + chrono::Duration::seconds(1);
|
||||
state.thread_updated_at_touch.pending_touch = Some((thread_id, pending_updated_at));
|
||||
|
||||
state.shutdown().await?;
|
||||
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after shutdown")
|
||||
.expect("thread should still exist")
|
||||
.updated_at,
|
||||
pending_updated_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() -> std::io::Result<()>
|
||||
{
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
let thread_id = ThreadId::new();
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
let builder = ThreadMetadataBuilder::new(
|
||||
thread_id,
|
||||
rollout_path.clone(),
|
||||
Utc::now(),
|
||||
SessionSource::Cli,
|
||||
);
|
||||
let items = vec![RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))];
|
||||
|
||||
let mut thread_updated_at_touch = ThreadUpdatedAtTouch::default();
|
||||
sync_thread_state_after_write(
|
||||
Some(state_db.as_ref()),
|
||||
rollout_path.as_path(),
|
||||
Some(&builder),
|
||||
items.as_slice(),
|
||||
config.model_provider_id.as_str(),
|
||||
/*new_thread_memory_mode*/ None,
|
||||
&mut thread_updated_at_touch,
|
||||
)
|
||||
.await;
|
||||
|
||||
let thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after fallback")
|
||||
.expect("thread should be inserted after fallback");
|
||||
assert_eq!(thread.id, thread_id);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
@@ -229,8 +229,8 @@ pub struct InMemoryThreadStoreCalls {
|
||||
pub read_thread: usize,
|
||||
pub read_thread_by_rollout_path: usize,
|
||||
pub list_threads: usize,
|
||||
pub update_thread_metadata: usize,
|
||||
pub apply_thread_metadata: usize,
|
||||
pub update_thread_metadata: usize,
|
||||
pub archive_thread: usize,
|
||||
pub unarchive_thread: usize,
|
||||
}
|
||||
@@ -479,9 +479,7 @@ fn stored_thread_from_state(
|
||||
let git_info = update
|
||||
.and_then(|update| update.git_info.clone())
|
||||
.unwrap_or(git_info);
|
||||
let rollout_path = state
|
||||
.metadata_updates
|
||||
.get(&thread_id)
|
||||
let rollout_path = update
|
||||
.and_then(|update| update.rollout_path.clone())
|
||||
.or_else(|| {
|
||||
state
|
||||
|
||||
@@ -9,6 +9,7 @@ mod in_memory;
|
||||
mod live_thread;
|
||||
mod local;
|
||||
mod store;
|
||||
mod thread_metadata_handler;
|
||||
mod types;
|
||||
|
||||
pub use error::ThreadStoreError;
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::AppendThreadItemsParams;
|
||||
use crate::ApplyThreadMetadataParams;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::LoadThreadHistoryParams;
|
||||
use crate::LocalThreadStore;
|
||||
@@ -15,9 +20,13 @@ use crate::ResumeThreadParams;
|
||||
use crate::StoredThread;
|
||||
use crate::StoredThreadHistory;
|
||||
use crate::ThreadMetadataPatch;
|
||||
use crate::ThreadMetadataUpdate;
|
||||
use crate::ThreadStore;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
use crate::UpdateThreadMetadataParams;
|
||||
use crate::thread_metadata_handler::PreparedThreadMetadata;
|
||||
use crate::thread_metadata_handler::ThreadMetadataHandler;
|
||||
use crate::thread_metadata_handler::protocol_git_info;
|
||||
|
||||
/// Handle for an active thread's persistence lifecycle.
|
||||
///
|
||||
@@ -28,6 +37,15 @@ use crate::UpdateThreadMetadataParams;
|
||||
pub struct LiveThread {
|
||||
thread_id: ThreadId,
|
||||
thread_store: Arc<dyn ThreadStore>,
|
||||
metadata_handler: Arc<Mutex<ThreadMetadataHandler>>,
|
||||
operation_semaphore: Arc<Semaphore>,
|
||||
pending_metadata_updates: Arc<Mutex<VecDeque<ThreadMetadataUpdate>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MetadataFailurePolicy {
|
||||
Propagate,
|
||||
WarnAndContinueForLocal,
|
||||
}
|
||||
|
||||
/// Owns a live thread while session initialization is still fallible.
|
||||
@@ -85,11 +103,16 @@ impl LiveThread {
|
||||
params: CreateThreadParams,
|
||||
) -> ThreadStoreResult<Self> {
|
||||
let thread_id = params.thread_id;
|
||||
let metadata_handler = Arc::new(Mutex::new(ThreadMetadataHandler::for_create(¶ms)));
|
||||
thread_store.create_thread(params).await?;
|
||||
Ok(Self {
|
||||
let live_thread = Self {
|
||||
thread_id,
|
||||
thread_store,
|
||||
})
|
||||
metadata_handler,
|
||||
operation_semaphore: Arc::new(Semaphore::new(1)),
|
||||
pending_metadata_updates: Arc::new(Mutex::new(VecDeque::new())),
|
||||
};
|
||||
Ok(live_thread)
|
||||
}
|
||||
|
||||
pub async fn resume(
|
||||
@@ -97,31 +120,217 @@ impl LiveThread {
|
||||
params: ResumeThreadParams,
|
||||
) -> ThreadStoreResult<Self> {
|
||||
let thread_id = params.thread_id;
|
||||
thread_store.resume_thread(params).await?;
|
||||
thread_store.resume_thread(params.clone()).await?;
|
||||
let mut handler_params = params;
|
||||
if handler_params.history.is_none() {
|
||||
match thread_store
|
||||
.load_history(LoadThreadHistoryParams {
|
||||
thread_id,
|
||||
include_archived: handler_params.include_archived,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(history) => {
|
||||
handler_params.history = Some(history.items);
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = thread_store.discard_thread(thread_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
let metadata_handler = Arc::new(Mutex::new(ThreadMetadataHandler::for_resume(
|
||||
&handler_params,
|
||||
)));
|
||||
Ok(Self {
|
||||
thread_id,
|
||||
thread_store,
|
||||
metadata_handler,
|
||||
operation_semaphore: Arc::new(Semaphore::new(1)),
|
||||
pending_metadata_updates: Arc::new(Mutex::new(VecDeque::new())),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn append_items(&self, items: &[RolloutItem]) -> ThreadStoreResult<()> {
|
||||
let _operation_permit = self.acquire_operation_permit().await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
let prepared = {
|
||||
let handler = self.metadata_handler.lock().await;
|
||||
let Some(prepared) = handler.prepare_items(items) else {
|
||||
return Ok(());
|
||||
};
|
||||
prepared
|
||||
};
|
||||
let initial = self.initial_metadata().await;
|
||||
|
||||
self.append_initial_metadata(initial).await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
|
||||
let update = self.append_prepared_items(&prepared).await?;
|
||||
self.metadata_handler
|
||||
.lock()
|
||||
.await
|
||||
.commit_prepared(&prepared);
|
||||
self.push_pending_metadata_update(update).await;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn emit_initial_metadata(&self) -> ThreadStoreResult<()> {
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
let Some(prepared) = self.initial_metadata().await else {
|
||||
return Ok(());
|
||||
};
|
||||
self.append_initial_metadata(Some(prepared)).await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn initial_metadata(&self) -> Option<PreparedThreadMetadata> {
|
||||
let git_cwd = self.metadata_handler.lock().await.take_initial_git_cwd();
|
||||
let git_info = match git_cwd.as_ref() {
|
||||
Some(cwd) => protocol_git_info(cwd.as_path()).await,
|
||||
None => None,
|
||||
};
|
||||
let mut handler = self.metadata_handler.lock().await;
|
||||
if git_cwd.is_some() {
|
||||
handler.set_initial_git_info(git_info);
|
||||
}
|
||||
handler.initial_metadata()
|
||||
}
|
||||
|
||||
async fn acquire_operation_permit(&self) -> ThreadStoreResult<OwnedSemaphorePermit> {
|
||||
self.operation_semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| ThreadStoreError::Internal {
|
||||
message: format!("live thread {} operation semaphore closed", self.thread_id),
|
||||
})
|
||||
}
|
||||
|
||||
async fn append_prepared_items(
|
||||
&self,
|
||||
prepared: &PreparedThreadMetadata,
|
||||
) -> ThreadStoreResult<ThreadMetadataUpdate> {
|
||||
let update = prepared.update.clone();
|
||||
self.thread_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: self.thread_id,
|
||||
items: items.to_vec(),
|
||||
items: prepared.items.clone(),
|
||||
})
|
||||
.await?;
|
||||
Ok(update)
|
||||
}
|
||||
|
||||
async fn append_initial_metadata(
|
||||
&self,
|
||||
prepared: Option<PreparedThreadMetadata>,
|
||||
) -> ThreadStoreResult<Option<ThreadMetadataUpdate>> {
|
||||
let Some(prepared) = prepared else {
|
||||
return Ok(None);
|
||||
};
|
||||
let update = self.append_prepared_items(&prepared).await?;
|
||||
self.mark_initial_metadata_emitted().await;
|
||||
self.push_pending_metadata_update(update.clone()).await;
|
||||
Ok(Some(update))
|
||||
}
|
||||
|
||||
async fn mark_initial_metadata_emitted(&self) {
|
||||
self.metadata_handler
|
||||
.lock()
|
||||
.await
|
||||
.mark_initial_metadata_emitted();
|
||||
}
|
||||
|
||||
async fn push_pending_metadata_update(&self, update: ThreadMetadataUpdate) {
|
||||
if !update.is_empty() {
|
||||
self.pending_metadata_updates.lock().await.push_back(update);
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_pending_metadata_updates(
|
||||
&self,
|
||||
failure_policy: MetadataFailurePolicy,
|
||||
) -> ThreadStoreResult<()> {
|
||||
loop {
|
||||
let Some(update) = self.pending_metadata_updates.lock().await.pop_front() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(err) = self
|
||||
.apply_metadata_update(update.clone(), failure_policy)
|
||||
.await
|
||||
{
|
||||
self.pending_metadata_updates
|
||||
.lock()
|
||||
.await
|
||||
.push_front(update);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_metadata_update(
|
||||
&self,
|
||||
update: ThreadMetadataUpdate,
|
||||
failure_policy: MetadataFailurePolicy,
|
||||
) -> ThreadStoreResult<()> {
|
||||
if update.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let result = self
|
||||
.thread_store
|
||||
.apply_thread_metadata(ApplyThreadMetadataParams {
|
||||
thread_id: self.thread_id,
|
||||
update,
|
||||
})
|
||||
.await;
|
||||
if matches!(
|
||||
failure_policy,
|
||||
MetadataFailurePolicy::WarnAndContinueForLocal
|
||||
) && self.thread_store.as_any().is::<LocalThreadStore>()
|
||||
&& let Err(err) = result
|
||||
{
|
||||
warn!("failed to apply local thread metadata update: {err}");
|
||||
return Ok(());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn append_prepared_metadata(
|
||||
&self,
|
||||
prepared: PreparedThreadMetadata,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let update = self.append_prepared_items(&prepared).await?;
|
||||
self.metadata_handler
|
||||
.lock()
|
||||
.await
|
||||
.commit_prepared(&prepared);
|
||||
self.push_pending_metadata_update(update).await;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::Propagate)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn persist(&self) -> ThreadStoreResult<()> {
|
||||
let _operation_permit = self.acquire_operation_permit().await?;
|
||||
self.emit_initial_metadata().await?;
|
||||
self.thread_store.persist_thread(self.thread_id).await
|
||||
}
|
||||
|
||||
pub async fn flush(&self) -> ThreadStoreResult<()> {
|
||||
let _operation_permit = self.acquire_operation_permit().await?;
|
||||
self.emit_initial_metadata().await?;
|
||||
self.thread_store.flush_thread(self.thread_id).await
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self) -> ThreadStoreResult<()> {
|
||||
let _operation_permit = self.acquire_operation_permit().await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
self.thread_store.shutdown_thread(self.thread_id).await
|
||||
}
|
||||
|
||||
@@ -160,16 +369,14 @@ impl LiveThread {
|
||||
mode: ThreadMemoryMode,
|
||||
include_archived: bool,
|
||||
) -> ThreadStoreResult<()> {
|
||||
self.thread_store
|
||||
.update_thread_metadata(UpdateThreadMetadataParams {
|
||||
thread_id: self.thread_id,
|
||||
patch: ThreadMetadataPatch {
|
||||
memory_mode: Some(mode),
|
||||
..Default::default()
|
||||
},
|
||||
include_archived,
|
||||
})
|
||||
.await?;
|
||||
self.update_metadata(
|
||||
ThreadMetadataPatch {
|
||||
memory_mode: Some(mode),
|
||||
..Default::default()
|
||||
},
|
||||
include_archived,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -178,15 +385,63 @@ impl LiveThread {
|
||||
patch: ThreadMetadataPatch,
|
||||
include_archived: bool,
|
||||
) -> ThreadStoreResult<StoredThread> {
|
||||
if patch.name.is_some() || patch.memory_mode.is_some() || patch.git_info.is_some() {
|
||||
let _operation_permit = self.acquire_operation_permit().await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
self.reject_archived_metadata_update(include_archived)
|
||||
.await?;
|
||||
let initial = self.initial_metadata().await;
|
||||
let prepared = {
|
||||
let handler = self.metadata_handler.lock().await;
|
||||
handler.prepare_metadata_patch(&patch)
|
||||
};
|
||||
|
||||
self.append_initial_metadata(initial).await?;
|
||||
self.apply_pending_metadata_updates(MetadataFailurePolicy::WarnAndContinueForLocal)
|
||||
.await?;
|
||||
if !prepared.items.is_empty() {
|
||||
self.append_prepared_metadata(prepared).await?;
|
||||
} else {
|
||||
self.apply_metadata_update(prepared.update, MetadataFailurePolicy::Propagate)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
self.thread_store
|
||||
.update_thread_metadata(UpdateThreadMetadataParams {
|
||||
.read_thread(ReadThreadParams {
|
||||
thread_id: self.thread_id,
|
||||
patch,
|
||||
include_archived,
|
||||
include_history: false,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reject_archived_metadata_update(
|
||||
&self,
|
||||
include_archived: bool,
|
||||
) -> ThreadStoreResult<()> {
|
||||
if include_archived {
|
||||
return Ok(());
|
||||
}
|
||||
match self
|
||||
.thread_store
|
||||
.read_thread(ReadThreadParams {
|
||||
thread_id: self.thread_id,
|
||||
include_archived,
|
||||
include_history: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Err(ThreadStoreError::InvalidRequest { message })
|
||||
if message.contains(" is archived") =>
|
||||
{
|
||||
Err(ThreadStoreError::InvalidRequest { message })
|
||||
}
|
||||
Ok(_) | Err(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the live local rollout path for legacy local-only callers.
|
||||
///
|
||||
/// Remote stores do not expose rollout files, so they return `Ok(None)`.
|
||||
|
||||
@@ -523,6 +523,17 @@ mod tests {
|
||||
.await
|
||||
.expect("read thread name");
|
||||
assert_eq!(name.as_deref(), Some(""));
|
||||
|
||||
store
|
||||
.apply_thread_metadata(ApplyThreadMetadataParams {
|
||||
thread_id,
|
||||
update: ThreadMetadataUpdate {
|
||||
git_info: Some(None),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await
|
||||
.expect("git metadata is represented by appended JSONL without sqlite");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
use super::LocalThreadStore;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::ThreadEventPersistenceMode;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use codex_rollout::EventPersistenceMode;
|
||||
use codex_rollout::RolloutConfig;
|
||||
use codex_rollout::RolloutRecorder;
|
||||
use codex_rollout::RolloutRecorderParams;
|
||||
|
||||
pub(super) async fn create_thread(
|
||||
store: &LocalThreadStore,
|
||||
params: CreateThreadParams,
|
||||
) -> ThreadStoreResult<RolloutRecorder> {
|
||||
let cwd = params
|
||||
.metadata
|
||||
.cwd
|
||||
.clone()
|
||||
.ok_or_else(|| ThreadStoreError::InvalidRequest {
|
||||
message: "local thread store requires a cwd".to_string(),
|
||||
})?;
|
||||
let config = RolloutConfig {
|
||||
codex_home: store.config.codex_home.clone(),
|
||||
sqlite_home: store.config.sqlite_home.clone(),
|
||||
cwd,
|
||||
model_provider_id: params.metadata.model_provider.clone(),
|
||||
generate_memories: matches!(params.metadata.memory_mode, ThreadMemoryMode::Enabled),
|
||||
};
|
||||
let state_db_ctx = store.state_db().await;
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id: params.thread_id,
|
||||
forked_from_id: params.forked_from_id,
|
||||
source: params.source,
|
||||
originator: params.originator,
|
||||
thread_source: params.thread_source,
|
||||
base_instructions: params.base_instructions,
|
||||
dynamic_tools: params.dynamic_tools,
|
||||
event_persistence_mode: event_persistence_mode(params.event_persistence_mode),
|
||||
},
|
||||
state_db_ctx,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to initialize local thread recorder: {err}"),
|
||||
})?;
|
||||
|
||||
Ok(recorder)
|
||||
}
|
||||
|
||||
pub(super) fn event_persistence_mode(mode: ThreadEventPersistenceMode) -> EventPersistenceMode {
|
||||
match mode {
|
||||
ThreadEventPersistenceMode::Limited => EventPersistenceMode::Limited,
|
||||
ThreadEventPersistenceMode::Extended => EventPersistenceMode::Extended,
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,8 @@ use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use codex_rollout::RolloutConfig;
|
||||
use codex_rollout::RolloutRecorder;
|
||||
use codex_rollout::RolloutRecorderParams;
|
||||
use codex_rollout::builder_from_items;
|
||||
|
||||
use super::LocalThreadStore;
|
||||
use super::create_thread;
|
||||
use crate::AppendThreadItemsParams;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::ReadThreadParams;
|
||||
@@ -22,41 +20,6 @@ pub(super) async fn create_thread(
|
||||
) -> ThreadStoreResult<()> {
|
||||
let thread_id = params.thread_id;
|
||||
store.ensure_live_recorder_absent(thread_id).await?;
|
||||
let recorder = create_thread::create_thread(store, params).await?;
|
||||
store.insert_live_recorder(thread_id, recorder).await
|
||||
}
|
||||
|
||||
pub(super) async fn resume_thread(
|
||||
store: &LocalThreadStore,
|
||||
params: ResumeThreadParams,
|
||||
) -> ThreadStoreResult<()> {
|
||||
store.ensure_live_recorder_absent(params.thread_id).await?;
|
||||
let (rollout_path, history) = match (params.rollout_path, params.history) {
|
||||
(Some(rollout_path), history) => (rollout_path, history),
|
||||
(None, history) => {
|
||||
let thread = super::read_thread::read_thread(
|
||||
store,
|
||||
ReadThreadParams {
|
||||
thread_id: params.thread_id,
|
||||
include_archived: params.include_archived,
|
||||
include_history: history.is_none(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let rollout_path = thread
|
||||
.rollout_path
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: format!("thread {} does not have a rollout path", params.thread_id),
|
||||
})?;
|
||||
(
|
||||
rollout_path,
|
||||
history.or_else(|| thread.history.map(|history| history.items)),
|
||||
)
|
||||
}
|
||||
};
|
||||
let state_builder = history
|
||||
.as_deref()
|
||||
.and_then(|items| builder_from_items(items, rollout_path.as_path()));
|
||||
let cwd = params
|
||||
.metadata
|
||||
.cwd
|
||||
@@ -71,20 +34,57 @@ pub(super) async fn resume_thread(
|
||||
model_provider_id: params.metadata.model_provider.clone(),
|
||||
generate_memories: matches!(params.metadata.memory_mode, ThreadMemoryMode::Enabled),
|
||||
};
|
||||
let state_db_ctx = store.state_db().await;
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::resume(
|
||||
rollout_path,
|
||||
create_thread::event_persistence_mode(params.event_persistence_mode),
|
||||
),
|
||||
state_db_ctx,
|
||||
state_builder,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to resume local thread recorder: {err}"),
|
||||
})?;
|
||||
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::new(thread_id))
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to initialize local thread recorder: {err}"),
|
||||
})?;
|
||||
store.insert_live_recorder(thread_id, recorder).await
|
||||
}
|
||||
|
||||
pub(super) async fn resume_thread(
|
||||
store: &LocalThreadStore,
|
||||
params: ResumeThreadParams,
|
||||
) -> ThreadStoreResult<()> {
|
||||
store.ensure_live_recorder_absent(params.thread_id).await?;
|
||||
let rollout_path = match params.rollout_path {
|
||||
Some(rollout_path) => rollout_path,
|
||||
None => {
|
||||
let thread = super::read_thread::read_thread(
|
||||
store,
|
||||
ReadThreadParams {
|
||||
thread_id: params.thread_id,
|
||||
include_archived: params.include_archived,
|
||||
include_history: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
thread
|
||||
.rollout_path
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: format!("thread {} does not have a rollout path", params.thread_id),
|
||||
})?
|
||||
}
|
||||
};
|
||||
let cwd = params
|
||||
.metadata
|
||||
.cwd
|
||||
.clone()
|
||||
.ok_or_else(|| ThreadStoreError::InvalidRequest {
|
||||
message: "local thread store requires a cwd".to_string(),
|
||||
})?;
|
||||
let config = RolloutConfig {
|
||||
codex_home: store.config.codex_home.clone(),
|
||||
sqlite_home: store.config.sqlite_home.clone(),
|
||||
cwd,
|
||||
model_provider_id: params.metadata.model_provider.clone(),
|
||||
generate_memories: matches!(params.metadata.memory_mode, ThreadMemoryMode::Enabled),
|
||||
};
|
||||
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::resume(rollout_path))
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to resume local thread recorder: {err}"),
|
||||
})?;
|
||||
store.insert_live_recorder(params.thread_id, recorder).await
|
||||
}
|
||||
|
||||
@@ -92,12 +92,12 @@ pub(super) async fn append_items(
|
||||
store: &LocalThreadStore,
|
||||
params: AppendThreadItemsParams,
|
||||
) -> ThreadStoreResult<()> {
|
||||
store
|
||||
.live_recorder(params.thread_id)
|
||||
.await?
|
||||
let recorder = store.live_recorder(params.thread_id).await?;
|
||||
recorder
|
||||
.record_items(params.items.as_slice())
|
||||
.await
|
||||
.map_err(thread_store_io_error)
|
||||
.map_err(thread_store_io_error)?;
|
||||
recorder.flush().await.map_err(thread_store_io_error)
|
||||
}
|
||||
|
||||
pub(super) async fn persist_thread(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
mod apply_thread_metadata;
|
||||
mod archive_thread;
|
||||
mod create_thread;
|
||||
mod helpers;
|
||||
mod list_threads;
|
||||
mod live_writer;
|
||||
@@ -297,17 +296,27 @@ impl ThreadStore for LocalThreadStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::GitInfoPatch;
|
||||
use crate::LiveThread;
|
||||
use crate::SortDirection;
|
||||
use crate::ThreadEventPersistenceMode;
|
||||
use crate::ThreadMetadataPatch;
|
||||
use crate::ThreadPersistenceMetadata;
|
||||
use crate::ThreadSortKey;
|
||||
use crate::local::test_support::test_config;
|
||||
use crate::local::test_support::write_archived_session_file;
|
||||
use crate::local::test_support::write_session_file;
|
||||
@@ -431,7 +440,10 @@ mod tests {
|
||||
first_store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
items: vec![user_message_item("before resume")],
|
||||
items: vec![
|
||||
session_meta_item(thread_id),
|
||||
user_message_item("before resume"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append initial item");
|
||||
@@ -727,7 +739,7 @@ mod tests {
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
items: vec![user_message_item("path read")],
|
||||
items: vec![session_meta_item(thread_id), user_message_item("path read")],
|
||||
})
|
||||
.await
|
||||
.expect("append item");
|
||||
@@ -759,6 +771,241 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_thread_writes_single_session_meta_and_updates_sqlite() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let runtime = codex_state::StateRuntime::init(
|
||||
home.path().to_path_buf(),
|
||||
config.default_model_provider_id.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
runtime
|
||||
.mark_backfill_complete(/*last_watermark*/ None)
|
||||
.await
|
||||
.expect("backfill should be complete");
|
||||
let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
|
||||
let thread_id = ThreadId::default();
|
||||
let mut params = create_thread_params(thread_id);
|
||||
params.metadata.memory_mode = ThreadMemoryMode::Disabled;
|
||||
params.dynamic_tools = vec![DynamicToolSpec {
|
||||
namespace: Some("test".to_string()),
|
||||
name: "tool".to_string(),
|
||||
description: "tool description".to_string(),
|
||||
input_schema: serde_json::json!({"type": "object"}),
|
||||
defer_loading: true,
|
||||
}];
|
||||
|
||||
let live_thread = LiveThread::create(store.clone(), params)
|
||||
.await
|
||||
.expect("create live thread");
|
||||
live_thread
|
||||
.append_items(&[user_message_item("Hello from LiveThread")])
|
||||
.await
|
||||
.expect("append item");
|
||||
live_thread.flush().await.expect("flush thread");
|
||||
|
||||
let rollout_path = live_thread
|
||||
.local_rollout_path()
|
||||
.await
|
||||
.expect("rollout path")
|
||||
.expect("local rollout path");
|
||||
assert_eq!(count_rollout_items(&rollout_path, "session_meta"), 1);
|
||||
let thread = live_thread
|
||||
.read_thread(
|
||||
/*include_archived*/ false, /*include_history*/ false,
|
||||
)
|
||||
.await
|
||||
.expect("read thread");
|
||||
assert_eq!(thread.thread_id, thread_id);
|
||||
assert_eq!(thread.cli_version, env!("CARGO_PKG_VERSION"));
|
||||
assert_eq!(
|
||||
thread.first_user_message.as_deref(),
|
||||
Some("Hello from LiveThread")
|
||||
);
|
||||
let memory_mode = runtime
|
||||
.get_thread_memory_mode(thread_id)
|
||||
.await
|
||||
.expect("memory mode");
|
||||
assert_eq!(memory_mode.as_deref(), Some("disabled"));
|
||||
let dynamic_tools = runtime
|
||||
.get_dynamic_tools(thread_id)
|
||||
.await
|
||||
.expect("dynamic tools")
|
||||
.expect("dynamic tools stored");
|
||||
assert_eq!(dynamic_tools.len(), 1);
|
||||
assert_eq!(dynamic_tools[0].name, "tool");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_thread_writes_jsonl_format_readable_without_sqlite() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = Arc::new(LocalThreadStore::new(
|
||||
test_config(home.path()),
|
||||
/*state_db*/ None,
|
||||
));
|
||||
let thread_id = ThreadId::default();
|
||||
let message = "jsonl fallback read";
|
||||
|
||||
let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create live thread");
|
||||
live_thread
|
||||
.append_items(&[user_message_item(message)])
|
||||
.await
|
||||
.expect("append item");
|
||||
live_thread.flush().await.expect("flush thread");
|
||||
|
||||
let rollout_path = live_thread
|
||||
.local_rollout_path()
|
||||
.await
|
||||
.expect("rollout path")
|
||||
.expect("local rollout path");
|
||||
let lines = std::fs::read_to_string(rollout_path)
|
||||
.expect("read rollout")
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("jsonl item"))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert_eq!(lines[0]["type"], "session_meta");
|
||||
assert_eq!(lines[0]["payload"]["id"], thread_id.to_string());
|
||||
assert_eq!(lines[0]["payload"]["originator"], "test_originator");
|
||||
assert_eq!(
|
||||
lines[0]["payload"]["cli_version"],
|
||||
env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
assert_eq!(lines[0]["payload"]["source"], "exec");
|
||||
assert_eq!(lines[0]["payload"]["model_provider"], "test-provider");
|
||||
assert_eq!(
|
||||
lines
|
||||
.iter()
|
||||
.filter(|line| line["type"] == "session_meta")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(lines[1]["type"], "event_msg");
|
||||
assert_eq!(lines[1]["payload"]["type"], "user_message");
|
||||
assert_eq!(lines[1]["payload"]["message"], message);
|
||||
|
||||
let read_thread = live_thread
|
||||
.read_thread(
|
||||
/*include_archived*/ false, /*include_history*/ true,
|
||||
)
|
||||
.await
|
||||
.expect("read thread");
|
||||
assert_eq!(read_thread.thread_id, thread_id);
|
||||
assert_eq!(read_thread.cli_version, env!("CARGO_PKG_VERSION"));
|
||||
assert_eq!(read_thread.first_user_message.as_deref(), Some(message));
|
||||
assert!(
|
||||
read_thread
|
||||
.history
|
||||
.expect("history")
|
||||
.items
|
||||
.iter()
|
||||
.any(|item| matches!(
|
||||
item,
|
||||
RolloutItem::EventMsg(EventMsg::UserMessage(event))
|
||||
if event.message == message
|
||||
))
|
||||
);
|
||||
|
||||
let page = store
|
||||
.list_threads(ListThreadsParams {
|
||||
page_size: 10,
|
||||
cursor: None,
|
||||
sort_key: ThreadSortKey::UpdatedAt,
|
||||
sort_direction: SortDirection::Desc,
|
||||
allowed_sources: vec![SessionSource::Exec],
|
||||
model_providers: None,
|
||||
cwd_filters: None,
|
||||
archived: false,
|
||||
search_term: None,
|
||||
use_state_db_only: false,
|
||||
})
|
||||
.await
|
||||
.expect("list threads");
|
||||
let listed_thread = page
|
||||
.items
|
||||
.iter()
|
||||
.find(|thread| thread.thread_id == thread_id)
|
||||
.expect("listed thread");
|
||||
assert_eq!(listed_thread.cli_version, env!("CARGO_PKG_VERSION"));
|
||||
assert_eq!(listed_thread.first_user_message.as_deref(), Some(message));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_thread_git_update_is_readable_without_sqlite() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = Arc::new(LocalThreadStore::new(
|
||||
test_config(home.path()),
|
||||
/*state_db*/ None,
|
||||
));
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
let live_thread = LiveThread::create(store, create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create live thread");
|
||||
live_thread
|
||||
.flush()
|
||||
.await
|
||||
.expect("materialize initial metadata");
|
||||
live_thread
|
||||
.update_metadata(
|
||||
ThreadMetadataPatch {
|
||||
git_info: Some(GitInfoPatch {
|
||||
sha: Some(Some("abcdef".to_string())),
|
||||
branch: Some(Some("main".to_string())),
|
||||
origin_url: Some(Some("https://example.com/repo.git".to_string())),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
/*include_archived*/ false,
|
||||
)
|
||||
.await
|
||||
.expect("update git metadata");
|
||||
|
||||
let thread = live_thread
|
||||
.read_thread(
|
||||
/*include_archived*/ false, /*include_history*/ false,
|
||||
)
|
||||
.await
|
||||
.expect("read thread");
|
||||
let git_info = thread.git_info.expect("git info");
|
||||
assert_eq!(
|
||||
git_info.commit_hash.as_ref().map(|sha| sha.0.as_str()),
|
||||
Some("abcdef")
|
||||
);
|
||||
assert_eq!(git_info.branch.as_deref(), Some("main"));
|
||||
assert_eq!(
|
||||
git_info.repository_url.as_deref(),
|
||||
Some("https://example.com/repo.git")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_thread_shutdown_without_items_keeps_rollout_deferred() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = Arc::new(LocalThreadStore::new(
|
||||
test_config(home.path()),
|
||||
/*state_db*/ None,
|
||||
));
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
let live_thread = LiveThread::create(store, create_thread_params(thread_id))
|
||||
.await
|
||||
.expect("create live thread");
|
||||
let rollout_path = live_thread
|
||||
.local_rollout_path()
|
||||
.await
|
||||
.expect("rollout path")
|
||||
.expect("local rollout path");
|
||||
live_thread.shutdown().await.expect("shutdown thread");
|
||||
|
||||
assert!(!rollout_path.exists());
|
||||
}
|
||||
|
||||
fn create_thread_params(thread_id: ThreadId) -> CreateThreadParams {
|
||||
CreateThreadParams {
|
||||
thread_id,
|
||||
@@ -790,6 +1037,38 @@ mod tests {
|
||||
}))
|
||||
}
|
||||
|
||||
fn session_meta_item(thread_id: ThreadId) -> RolloutItem {
|
||||
RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
forked_from_id: None,
|
||||
timestamp: "2026-01-27T12:34:56Z".to_string(),
|
||||
cwd: std::env::current_dir().expect("cwd"),
|
||||
originator: "test_originator".to_string(),
|
||||
cli_version: "test_version".to_string(),
|
||||
source: SessionSource::Exec,
|
||||
thread_source: None,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
model_provider: Some("test-provider".to_string()),
|
||||
base_instructions: None,
|
||||
dynamic_tools: None,
|
||||
memory_mode: None,
|
||||
},
|
||||
git: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn count_rollout_items(path: &std::path::Path, item_type: &str) -> usize {
|
||||
std::fs::read_to_string(path)
|
||||
.expect("read rollout")
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("jsonl item"))
|
||||
.filter(|item| item["type"] == item_type)
|
||||
.count()
|
||||
}
|
||||
|
||||
async fn assert_rollout_contains_message(path: &std::path::Path, expected: &str) {
|
||||
let (items, _, _) = RolloutRecorder::load_rollout_items(path)
|
||||
.await
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::GitInfo;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
@@ -225,6 +228,11 @@ async fn read_thread_from_rollout_path(
|
||||
thread.model_provider = model_provider;
|
||||
}
|
||||
}
|
||||
if store.state_db().await.is_none()
|
||||
&& let Ok(Some(git_info)) = latest_session_meta_git(path.as_path(), thread.thread_id).await
|
||||
{
|
||||
thread.git_info = Some(git_info);
|
||||
}
|
||||
if let Ok(Some(title)) =
|
||||
find_thread_name_by_id(store.config.codex_home.as_path(), &thread.thread_id).await
|
||||
{
|
||||
@@ -344,15 +352,68 @@ async fn stored_thread_from_session_meta(
|
||||
store: &LocalThreadStore,
|
||||
path: std::path::PathBuf,
|
||||
) -> ThreadStoreResult<StoredThread> {
|
||||
let meta_line = read_session_meta_line(path.as_path())
|
||||
let first_meta_line = read_session_meta_line(path.as_path())
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to read thread {}: {err}", path.display()),
|
||||
})?;
|
||||
let latest_git = if store.state_db().await.is_none() {
|
||||
latest_session_meta_git(path.as_path(), first_meta_line.meta.id).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let archived = rollout_path_is_archived(store.config.codex_home.as_path(), path.as_path());
|
||||
Ok(stored_thread_from_meta_line(
|
||||
store, meta_line, path, archived,
|
||||
))
|
||||
let mut thread = stored_thread_from_meta_line(store, first_meta_line, path, archived);
|
||||
if let Some(git_info) = latest_git {
|
||||
thread.git_info = Some(git_info);
|
||||
}
|
||||
Ok(thread)
|
||||
}
|
||||
|
||||
async fn latest_session_meta_git(
|
||||
path: &std::path::Path,
|
||||
thread_id: codex_protocol::ThreadId,
|
||||
) -> ThreadStoreResult<Option<GitInfo>> {
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
|
||||
let file = tokio::fs::File::open(path)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to read session metadata {}: {err}", path.display()),
|
||||
})?;
|
||||
let reader = tokio::io::BufReader::new(file);
|
||||
let mut lines = reader.lines();
|
||||
let mut latest_git = None;
|
||||
while let Some(line) = lines
|
||||
.next_line()
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to read session metadata {}: {err}", path.display()),
|
||||
})?
|
||||
{
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Ok(value) = serde_json::from_str::<serde_json::Value>(trimmed) else {
|
||||
continue;
|
||||
};
|
||||
if value.get("type").and_then(serde_json::Value::as_str) != Some("session_meta") {
|
||||
continue;
|
||||
}
|
||||
let Ok(rollout_line) = serde_json::from_value::<RolloutLine>(value) else {
|
||||
continue;
|
||||
};
|
||||
let RolloutItem::SessionMeta(meta_line) = rollout_line.item else {
|
||||
continue;
|
||||
};
|
||||
if meta_line.meta.id == thread_id
|
||||
&& let Some(git_info) = meta_line.git
|
||||
{
|
||||
latest_git = Some(git_info);
|
||||
}
|
||||
}
|
||||
Ok(latest_git)
|
||||
}
|
||||
|
||||
fn stored_thread_from_meta_line(
|
||||
|
||||
@@ -0,0 +1,695 @@
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use chrono::Duration;
|
||||
use chrono::SecondsFormat;
|
||||
use chrono::Utc;
|
||||
use codex_git_utils::collect_git_info;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::GitInfo;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_rollout::EventPersistenceMode;
|
||||
use codex_rollout::persisted_rollout_items;
|
||||
use codex_state::ThreadMetadata;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
use codex_state::rollout_item_affects_thread_metadata;
|
||||
|
||||
use crate::CreateThreadParams;
|
||||
use crate::ResumeThreadParams;
|
||||
use crate::ThreadEventPersistenceMode;
|
||||
use crate::ThreadMetadataUpdate;
|
||||
|
||||
const NON_METADATA_TOUCH_INTERVAL: Duration = Duration::seconds(1);
|
||||
|
||||
/// Prepares canonical rollout items and explicit metadata updates for a live thread.
|
||||
///
|
||||
/// `LiveThread` owns one handler per active thread so metadata inference stays above the raw
|
||||
/// `ThreadStore` append API.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ThreadMetadataHandler {
|
||||
thread_id: ThreadId,
|
||||
metadata: ThreadMetadata,
|
||||
latest_session_meta: Option<SessionMetaLine>,
|
||||
initial_item: Option<RolloutItem>,
|
||||
initial_git_cwd: Option<PathBuf>,
|
||||
event_persistence_mode: EventPersistenceMode,
|
||||
memory_mode: ThreadMemoryMode,
|
||||
dynamic_tools: Option<Vec<codex_protocol::dynamic_tools::DynamicToolSpec>>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
}
|
||||
|
||||
/// Result of applying metadata policy to an incoming live-thread operation.
|
||||
///
|
||||
/// `items` are the canonical rollout items to append, and `update` is the explicit metadata delta
|
||||
/// to pass to `ThreadStore::apply_thread_metadata` after those items are accepted.
|
||||
pub(crate) struct PreparedThreadMetadata {
|
||||
pub(crate) items: Vec<RolloutItem>,
|
||||
pub(crate) update: ThreadMetadataUpdate,
|
||||
}
|
||||
|
||||
impl ThreadMetadataHandler {
|
||||
pub(crate) fn for_create(params: &CreateThreadParams) -> Self {
|
||||
let created_at = Utc::now();
|
||||
let cwd = params.metadata.cwd.clone().unwrap_or_default();
|
||||
let session_meta = SessionMeta {
|
||||
id: params.thread_id,
|
||||
forked_from_id: params.forked_from_id,
|
||||
timestamp: created_at.to_rfc3339_opts(SecondsFormat::Millis, true),
|
||||
cwd,
|
||||
originator: params.originator.clone(),
|
||||
cli_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
source: params.source.clone(),
|
||||
thread_source: params.thread_source,
|
||||
agent_nickname: params.source.get_nickname(),
|
||||
agent_role: params.source.get_agent_role(),
|
||||
agent_path: params.source.get_agent_path().map(Into::into),
|
||||
model_provider: Some(params.metadata.model_provider.clone()),
|
||||
base_instructions: Some(params.base_instructions.clone()),
|
||||
dynamic_tools: if params.dynamic_tools.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(params.dynamic_tools.clone())
|
||||
},
|
||||
memory_mode: memory_mode_as_str(params.metadata.memory_mode).map(str::to_string),
|
||||
};
|
||||
let session_meta_line = SessionMetaLine {
|
||||
meta: session_meta,
|
||||
git: None,
|
||||
};
|
||||
let metadata = metadata_from_session_meta(
|
||||
&session_meta_line,
|
||||
created_at,
|
||||
params.metadata.model_provider.as_str(),
|
||||
);
|
||||
Self {
|
||||
thread_id: params.thread_id,
|
||||
metadata,
|
||||
latest_session_meta: Some(session_meta_line.clone()),
|
||||
initial_item: Some(RolloutItem::SessionMeta(session_meta_line)),
|
||||
initial_git_cwd: params.metadata.cwd.clone(),
|
||||
event_persistence_mode: event_persistence_mode(params.event_persistence_mode),
|
||||
memory_mode: params.metadata.memory_mode,
|
||||
dynamic_tools: (!params.dynamic_tools.is_empty()).then(|| params.dynamic_tools.clone()),
|
||||
token_usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_resume(params: &ResumeThreadParams) -> Self {
|
||||
let created_at = Utc::now();
|
||||
let mut builder = ThreadMetadataBuilder::new(
|
||||
params.thread_id,
|
||||
params.rollout_path.clone().unwrap_or_default(),
|
||||
created_at,
|
||||
SessionSource::Unknown,
|
||||
);
|
||||
builder.model_provider = Some(params.metadata.model_provider.clone());
|
||||
builder.cwd = params.metadata.cwd.clone().unwrap_or_default();
|
||||
let metadata = builder.build(params.metadata.model_provider.as_str());
|
||||
let mut handler = Self {
|
||||
thread_id: params.thread_id,
|
||||
metadata,
|
||||
latest_session_meta: None,
|
||||
initial_item: None,
|
||||
initial_git_cwd: None,
|
||||
event_persistence_mode: event_persistence_mode(params.event_persistence_mode),
|
||||
memory_mode: params.metadata.memory_mode,
|
||||
dynamic_tools: None,
|
||||
token_usage: None,
|
||||
};
|
||||
if let Some(history) = params.history.as_deref() {
|
||||
handler.observe_items(history, /*updated_at_override*/ None);
|
||||
if let Some(created_at) = handler
|
||||
.latest_session_meta
|
||||
.as_ref()
|
||||
.and_then(|meta_line| parse_timestamp(meta_line.meta.timestamp.as_str()))
|
||||
{
|
||||
handler.metadata.created_at = created_at;
|
||||
}
|
||||
}
|
||||
handler
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn take_initial_metadata(&mut self) -> Option<PreparedThreadMetadata> {
|
||||
let prepared = self.initial_metadata()?;
|
||||
self.mark_initial_metadata_emitted();
|
||||
Some(prepared)
|
||||
}
|
||||
|
||||
pub(crate) fn take_initial_git_cwd(&mut self) -> Option<PathBuf> {
|
||||
self.initial_item
|
||||
.as_ref()
|
||||
.and_then(|_| self.initial_git_cwd.take())
|
||||
}
|
||||
|
||||
pub(crate) fn set_initial_git_info(&mut self, git: Option<GitInfo>) {
|
||||
let Some(RolloutItem::SessionMeta(session_meta)) = self.initial_item.as_mut() else {
|
||||
return;
|
||||
};
|
||||
session_meta.git = git;
|
||||
self.latest_session_meta = Some(session_meta.clone());
|
||||
let default_provider = self.metadata.model_provider.clone();
|
||||
codex_state::apply_rollout_item(
|
||||
&mut self.metadata,
|
||||
&RolloutItem::SessionMeta(session_meta.clone()),
|
||||
default_provider.as_str(),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn initial_metadata(&self) -> Option<PreparedThreadMetadata> {
|
||||
let item = self.initial_item.clone()?;
|
||||
Some(PreparedThreadMetadata {
|
||||
items: vec![item],
|
||||
update: self.current_update(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn mark_initial_metadata_emitted(&mut self) {
|
||||
self.initial_item = None;
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_items(&self, items: &[RolloutItem]) -> Option<PreparedThreadMetadata> {
|
||||
let persisted = persisted_rollout_items(items, self.event_persistence_mode);
|
||||
if persisted.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let affects_metadata = persisted.iter().any(rollout_item_affects_thread_metadata);
|
||||
let mut update = if affects_metadata {
|
||||
let mut next = self.clone();
|
||||
next.observe_items(persisted.as_slice(), /*updated_at_override*/ None);
|
||||
next.current_update()
|
||||
} else {
|
||||
let updated_at = Utc::now();
|
||||
if updated_at.signed_duration_since(self.metadata.updated_at)
|
||||
>= NON_METADATA_TOUCH_INTERVAL
|
||||
{
|
||||
ThreadMetadataUpdate {
|
||||
updated_at: Some(updated_at),
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
ThreadMetadataUpdate::default()
|
||||
}
|
||||
};
|
||||
let updates_session_metadata = persisted.iter().any(|item| {
|
||||
matches!(item, RolloutItem::SessionMeta(meta_line) if meta_line.meta.id == self.thread_id)
|
||||
});
|
||||
if !updates_session_metadata {
|
||||
update.git_info = None;
|
||||
update.memory_mode = None;
|
||||
update.dynamic_tools = None;
|
||||
}
|
||||
Some(PreparedThreadMetadata {
|
||||
items: persisted,
|
||||
update,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn commit_prepared(&mut self, prepared: &PreparedThreadMetadata) {
|
||||
if prepared
|
||||
.items
|
||||
.iter()
|
||||
.any(rollout_item_affects_thread_metadata)
|
||||
{
|
||||
self.observe_items(prepared.items.as_slice(), /*updated_at_override*/ None);
|
||||
} else if let Some(updated_at) = prepared.update.updated_at {
|
||||
self.metadata.updated_at = updated_at;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_metadata_patch(
|
||||
&self,
|
||||
patch: &crate::ThreadMetadataPatch,
|
||||
) -> PreparedThreadMetadata {
|
||||
let mut items = Vec::new();
|
||||
if (patch.memory_mode.is_some() || patch.git_info.is_some())
|
||||
&& let Some(mut session_meta) = self.latest_session_meta.clone()
|
||||
{
|
||||
if let Some(memory_mode) = patch.memory_mode {
|
||||
session_meta.meta.memory_mode = Some(memory_mode_to_str(memory_mode).to_string());
|
||||
}
|
||||
if let Some(git_info) = patch.git_info.clone() {
|
||||
session_meta.git = Some(resolve_git_info_patch(
|
||||
protocol_git_info_from_metadata(&self.metadata),
|
||||
git_info,
|
||||
));
|
||||
}
|
||||
items.push(RolloutItem::SessionMeta(session_meta));
|
||||
}
|
||||
let mut next = self.clone();
|
||||
next.commit_prepared(&PreparedThreadMetadata {
|
||||
items: items.clone(),
|
||||
update: ThreadMetadataUpdate::default(),
|
||||
});
|
||||
let mut update = ThreadMetadataUpdate {
|
||||
updated_at: Some(Utc::now()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(name) = patch.name.clone() {
|
||||
update.name = Some(Some(name));
|
||||
}
|
||||
if let Some(memory_mode) = patch.memory_mode {
|
||||
update.memory_mode = Some(memory_mode);
|
||||
}
|
||||
if patch.git_info.is_some() {
|
||||
update.git_info = Some(protocol_git_info_from_metadata(&next.metadata));
|
||||
}
|
||||
PreparedThreadMetadata { items, update }
|
||||
}
|
||||
|
||||
fn observe_items(
|
||||
&mut self,
|
||||
items: &[RolloutItem],
|
||||
updated_at_override: Option<chrono::DateTime<Utc>>,
|
||||
) {
|
||||
let updated_at = updated_at_override.unwrap_or_else(Utc::now);
|
||||
for item in items {
|
||||
let default_provider = self.metadata.model_provider.clone();
|
||||
codex_state::apply_rollout_item(&mut self.metadata, item, default_provider.as_str());
|
||||
match item {
|
||||
RolloutItem::SessionMeta(meta_line) if meta_line.meta.id == self.thread_id => {
|
||||
self.latest_session_meta = Some(meta_line.clone());
|
||||
if let Some(memory_mode) = meta_line.meta.memory_mode.as_deref()
|
||||
&& let Some(mode) = parse_memory_mode(memory_mode)
|
||||
{
|
||||
self.memory_mode = mode;
|
||||
}
|
||||
self.dynamic_tools = meta_line.meta.dynamic_tools.clone();
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::TokenCount(token_count)) => {
|
||||
if let Some(info) = token_count.info.as_ref() {
|
||||
self.token_usage = Some(info.total_token_usage.clone());
|
||||
}
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::UserMessage(_)) | RolloutItem::TurnContext(_) => {}
|
||||
RolloutItem::SessionMeta(_)
|
||||
| RolloutItem::EventMsg(_)
|
||||
| RolloutItem::ResponseItem(_)
|
||||
| RolloutItem::Compacted(_) => {}
|
||||
}
|
||||
}
|
||||
self.metadata.updated_at = updated_at;
|
||||
}
|
||||
|
||||
fn current_update(&self) -> ThreadMetadataUpdate {
|
||||
ThreadMetadataUpdate {
|
||||
rollout_path: (!self.metadata.rollout_path.as_os_str().is_empty())
|
||||
.then(|| self.metadata.rollout_path.clone()),
|
||||
preview: (!self.metadata.title.is_empty()).then(|| self.metadata.title.clone()),
|
||||
name: None,
|
||||
model_provider: Some(self.metadata.model_provider.clone()),
|
||||
model: Some(self.metadata.model.clone()),
|
||||
reasoning_effort: Some(self.metadata.reasoning_effort),
|
||||
created_at: Some(self.metadata.created_at),
|
||||
updated_at: Some(self.metadata.updated_at),
|
||||
source: Some(parse_session_source(self.metadata.source.as_str())),
|
||||
thread_source: Some(self.metadata.thread_source),
|
||||
agent_nickname: Some(self.metadata.agent_nickname.clone()),
|
||||
agent_role: Some(self.metadata.agent_role.clone()),
|
||||
agent_path: Some(self.metadata.agent_path.clone()),
|
||||
cwd: Some(self.metadata.cwd.clone()),
|
||||
cli_version: Some(self.metadata.cli_version.clone()),
|
||||
approval_mode: parse_json_field(self.metadata.approval_mode.as_str()),
|
||||
sandbox_policy: parse_json_field(self.metadata.sandbox_policy.as_str()),
|
||||
token_usage: Some(Some(self.token_usage.clone().unwrap_or(TokenUsage {
|
||||
total_tokens: self.metadata.tokens_used,
|
||||
..Default::default()
|
||||
}))),
|
||||
first_user_message: Some(self.metadata.first_user_message.clone()),
|
||||
git_info: Some(protocol_git_info_from_metadata(&self.metadata)),
|
||||
memory_mode: Some(self.memory_mode),
|
||||
dynamic_tools: Some(self.dynamic_tools.clone().unwrap_or_default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata_from_session_meta(
|
||||
session_meta_line: &SessionMetaLine,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
default_provider: &str,
|
||||
) -> ThreadMetadata {
|
||||
let mut builder = ThreadMetadataBuilder::new(
|
||||
session_meta_line.meta.id,
|
||||
PathBuf::new(),
|
||||
created_at,
|
||||
session_meta_line.meta.source.clone(),
|
||||
);
|
||||
builder.thread_source = session_meta_line.meta.thread_source;
|
||||
builder.agent_nickname = session_meta_line.meta.agent_nickname.clone();
|
||||
builder.agent_role = session_meta_line.meta.agent_role.clone();
|
||||
builder.agent_path = session_meta_line.meta.agent_path.clone();
|
||||
builder.model_provider = session_meta_line.meta.model_provider.clone();
|
||||
builder.cwd = session_meta_line.meta.cwd.clone();
|
||||
builder.cli_version = Some(session_meta_line.meta.cli_version.clone());
|
||||
if let Some(git) = session_meta_line.git.as_ref() {
|
||||
builder.git_sha = git.commit_hash.as_ref().map(|sha| sha.0.clone());
|
||||
builder.git_branch = git.branch.clone();
|
||||
builder.git_origin_url = git.repository_url.clone();
|
||||
}
|
||||
builder.build(default_provider)
|
||||
}
|
||||
|
||||
fn event_persistence_mode(mode: ThreadEventPersistenceMode) -> EventPersistenceMode {
|
||||
match mode {
|
||||
ThreadEventPersistenceMode::Limited => EventPersistenceMode::Limited,
|
||||
ThreadEventPersistenceMode::Extended => EventPersistenceMode::Extended,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn protocol_git_info(cwd: &Path) -> Option<GitInfo> {
|
||||
collect_git_info(cwd).await.map(|info| GitInfo {
|
||||
commit_hash: info.commit_hash,
|
||||
branch: info.branch,
|
||||
repository_url: info.repository_url,
|
||||
})
|
||||
}
|
||||
|
||||
fn protocol_git_info_from_metadata(metadata: &ThreadMetadata) -> Option<GitInfo> {
|
||||
if metadata.git_sha.is_none()
|
||||
&& metadata.git_branch.is_none()
|
||||
&& metadata.git_origin_url.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(GitInfo {
|
||||
commit_hash: metadata
|
||||
.git_sha
|
||||
.as_deref()
|
||||
.map(codex_git_utils::GitSha::new),
|
||||
branch: metadata.git_branch.clone(),
|
||||
repository_url: metadata.git_origin_url.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_session_source(value: &str) -> SessionSource {
|
||||
parse_json_field(value).unwrap_or(SessionSource::Unknown)
|
||||
}
|
||||
|
||||
fn parse_timestamp(value: &str) -> Option<chrono::DateTime<Utc>> {
|
||||
chrono::DateTime::parse_from_rfc3339(value)
|
||||
.map(|timestamp| timestamp.with_timezone(&Utc))
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn parse_json_field<T>(value: &str) -> Option<T>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
serde_json::from_str::<T>(value)
|
||||
.or_else(|_| serde_json::from_value(serde_json::Value::String(value.to_string())))
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn memory_mode_as_str(mode: ThreadMemoryMode) -> Option<&'static str> {
|
||||
match mode {
|
||||
ThreadMemoryMode::Enabled => None,
|
||||
ThreadMemoryMode::Disabled => Some("disabled"),
|
||||
}
|
||||
}
|
||||
|
||||
fn memory_mode_to_str(mode: ThreadMemoryMode) -> &'static str {
|
||||
match mode {
|
||||
ThreadMemoryMode::Enabled => "enabled",
|
||||
ThreadMemoryMode::Disabled => "disabled",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_memory_mode(value: &str) -> Option<ThreadMemoryMode> {
|
||||
match value {
|
||||
"enabled" => Some(ThreadMemoryMode::Enabled),
|
||||
"disabled" => Some(ThreadMemoryMode::Disabled),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_git_info_patch(existing: Option<GitInfo>, patch: crate::GitInfoPatch) -> GitInfo {
|
||||
let existing_sha = existing
|
||||
.as_ref()
|
||||
.and_then(|git| git.commit_hash.as_ref().map(|sha| sha.0.clone()));
|
||||
let existing_branch = existing.as_ref().and_then(|git| git.branch.clone());
|
||||
let existing_origin_url = existing.and_then(|git| git.repository_url);
|
||||
GitInfo {
|
||||
commit_hash: patch
|
||||
.sha
|
||||
.unwrap_or(existing_sha)
|
||||
.as_deref()
|
||||
.map(codex_git_utils::GitSha::new),
|
||||
branch: patch.branch.unwrap_or(existing_branch),
|
||||
repository_url: patch.origin_url.unwrap_or(existing_origin_url),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadMemoryMode;
|
||||
use codex_protocol::protocol::TokenUsageInfo;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::GitInfoPatch;
|
||||
use crate::ResumeThreadParams;
|
||||
use crate::ThreadEventPersistenceMode;
|
||||
use crate::ThreadMetadataPatch;
|
||||
use crate::ThreadPersistenceMetadata;
|
||||
|
||||
#[tokio::test]
|
||||
async fn initial_metadata_synthesizes_session_meta_and_explicit_update() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut params = create_params(temp.path().to_path_buf());
|
||||
params.metadata.memory_mode = ThreadMemoryMode::Disabled;
|
||||
params.dynamic_tools = vec![DynamicToolSpec {
|
||||
namespace: Some("ns".to_string()),
|
||||
name: "lookup".to_string(),
|
||||
description: "lookup".to_string(),
|
||||
input_schema: serde_json::json!({"type": "object"}),
|
||||
defer_loading: false,
|
||||
}];
|
||||
let mut handler = ThreadMetadataHandler::for_create(¶ms);
|
||||
|
||||
let prepared = handler.take_initial_metadata().expect("initial metadata");
|
||||
|
||||
assert_eq!(prepared.items.len(), 1);
|
||||
let RolloutItem::SessionMeta(session_meta) = &prepared.items[0] else {
|
||||
panic!("expected session metadata");
|
||||
};
|
||||
assert_eq!(session_meta.meta.id, params.thread_id);
|
||||
assert_eq!(session_meta.meta.memory_mode.as_deref(), Some("disabled"));
|
||||
assert_eq!(
|
||||
session_meta
|
||||
.meta
|
||||
.dynamic_tools
|
||||
.as_ref()
|
||||
.expect("dynamic tools")[0]
|
||||
.name,
|
||||
"lookup"
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.update.memory_mode,
|
||||
Some(ThreadMemoryMode::Disabled)
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.update.dynamic_tools.expect("dynamic tools update")[0].name,
|
||||
"lookup"
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.update.cli_version.as_deref(),
|
||||
Some(env!("CARGO_PKG_VERSION"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn user_message_updates_first_message_metadata() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut handler =
|
||||
ThreadMetadataHandler::for_create(&create_params(temp.path().to_path_buf()));
|
||||
let _ = handler.take_initial_metadata();
|
||||
|
||||
let prepared = handler
|
||||
.prepare_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
message: "hello metadata".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
))])
|
||||
.expect("prepared items");
|
||||
|
||||
assert_eq!(prepared.items.len(), 1);
|
||||
assert_eq!(
|
||||
prepared.update.first_user_message.flatten().as_deref(),
|
||||
Some("hello metadata")
|
||||
);
|
||||
assert_eq!(prepared.update.preview.as_deref(), Some("hello metadata"));
|
||||
assert_eq!(prepared.update.memory_mode, None);
|
||||
assert_eq!(prepared.update.dynamic_tools, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_metadata_items_throttle_updated_at_touches() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut handler =
|
||||
ThreadMetadataHandler::for_create(&create_params(temp.path().to_path_buf()));
|
||||
let _ = handler.take_initial_metadata();
|
||||
|
||||
let prepared = handler
|
||||
.prepare_items(&[RolloutItem::ResponseItem(ResponseItem::Message {
|
||||
id: None,
|
||||
role: "assistant".to_string(),
|
||||
content: Vec::new(),
|
||||
phase: None,
|
||||
})])
|
||||
.expect("prepared items");
|
||||
|
||||
assert_eq!(prepared.items.len(), 1);
|
||||
assert!(prepared.update.updated_at.is_none());
|
||||
assert!(prepared.update.preview.is_none());
|
||||
assert_eq!(prepared.update.memory_mode, None);
|
||||
assert_eq!(prepared.update.dynamic_tools, None);
|
||||
|
||||
handler.metadata.updated_at -= NON_METADATA_TOUCH_INTERVAL;
|
||||
let prepared = handler
|
||||
.prepare_items(&[RolloutItem::ResponseItem(ResponseItem::Message {
|
||||
id: None,
|
||||
role: "assistant".to_string(),
|
||||
content: Vec::new(),
|
||||
phase: None,
|
||||
})])
|
||||
.expect("prepared items");
|
||||
|
||||
assert!(prepared.update.updated_at.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn token_count_preserves_usage_breakdown() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut handler =
|
||||
ThreadMetadataHandler::for_create(&create_params(temp.path().to_path_buf()));
|
||||
let _ = handler.take_initial_metadata();
|
||||
let total_token_usage = TokenUsage {
|
||||
input_tokens: 10,
|
||||
cached_input_tokens: 4,
|
||||
output_tokens: 6,
|
||||
reasoning_output_tokens: 2,
|
||||
total_tokens: 16,
|
||||
};
|
||||
|
||||
let prepared = handler
|
||||
.prepare_items(&[RolloutItem::EventMsg(EventMsg::TokenCount(
|
||||
codex_protocol::protocol::TokenCountEvent {
|
||||
info: Some(TokenUsageInfo {
|
||||
total_token_usage: total_token_usage.clone(),
|
||||
last_token_usage: TokenUsage::default(),
|
||||
model_context_window: Some(128_000),
|
||||
}),
|
||||
rate_limits: None,
|
||||
},
|
||||
))])
|
||||
.expect("prepared items");
|
||||
|
||||
assert_eq!(prepared.update.token_usage, Some(Some(total_token_usage)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resume_observes_existing_session_metadata() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut create_params = create_params(temp.path().to_path_buf());
|
||||
create_params.metadata.memory_mode = ThreadMemoryMode::Disabled;
|
||||
let mut created = ThreadMetadataHandler::for_create(&create_params);
|
||||
let initial = created.take_initial_metadata().expect("initial metadata");
|
||||
let thread_id = create_params.thread_id;
|
||||
|
||||
let resumed = ThreadMetadataHandler::for_resume(&ResumeThreadParams {
|
||||
thread_id,
|
||||
rollout_path: Some(temp.path().join("rollout.jsonl")),
|
||||
history: Some(initial.items),
|
||||
include_archived: true,
|
||||
metadata: create_params.metadata,
|
||||
event_persistence_mode: ThreadEventPersistenceMode::Limited,
|
||||
});
|
||||
|
||||
let update = resumed.current_update();
|
||||
assert_eq!(update.memory_mode, Some(ThreadMemoryMode::Disabled));
|
||||
assert_eq!(
|
||||
update.cli_version.as_deref(),
|
||||
Some(env!("CARGO_PKG_VERSION"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_patch_synthesizes_session_meta_and_update() {
|
||||
let temp = TempDir::new().expect("temp dir");
|
||||
let mut handler =
|
||||
ThreadMetadataHandler::for_create(&create_params(temp.path().to_path_buf()));
|
||||
let _ = handler.take_initial_metadata();
|
||||
|
||||
let prepared = handler.prepare_metadata_patch(&ThreadMetadataPatch {
|
||||
memory_mode: Some(ThreadMemoryMode::Disabled),
|
||||
git_info: Some(GitInfoPatch {
|
||||
sha: Some(Some("abcdef".to_string())),
|
||||
branch: Some(Some("main".to_string())),
|
||||
origin_url: Some(Some("https://example.com/repo.git".to_string())),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(prepared.items.len(), 1);
|
||||
let RolloutItem::SessionMeta(session_meta) = &prepared.items[0] else {
|
||||
panic!("expected session metadata");
|
||||
};
|
||||
assert_eq!(session_meta.meta.memory_mode.as_deref(), Some("disabled"));
|
||||
assert_eq!(
|
||||
session_meta
|
||||
.git
|
||||
.as_ref()
|
||||
.and_then(|git| git.branch.as_deref()),
|
||||
Some("main")
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.update.memory_mode,
|
||||
Some(ThreadMemoryMode::Disabled)
|
||||
);
|
||||
assert_eq!(
|
||||
prepared
|
||||
.update
|
||||
.git_info
|
||||
.flatten()
|
||||
.and_then(|git| git.repository_url),
|
||||
Some("https://example.com/repo.git".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
fn create_params(cwd: std::path::PathBuf) -> CreateThreadParams {
|
||||
CreateThreadParams {
|
||||
thread_id: ThreadId::new(),
|
||||
forked_from_id: None,
|
||||
source: SessionSource::Exec,
|
||||
originator: "test_originator".to_string(),
|
||||
thread_source: None,
|
||||
base_instructions: BaseInstructions::default(),
|
||||
dynamic_tools: Vec::new(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(cwd),
|
||||
model_provider: "test-provider".to_string(),
|
||||
memory_mode: ThreadMemoryMode::Enabled,
|
||||
},
|
||||
event_persistence_mode: ThreadEventPersistenceMode::Limited,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,33 @@ pub struct ThreadMetadataUpdate {
|
||||
pub dynamic_tools: Option<Vec<DynamicToolSpec>>,
|
||||
}
|
||||
|
||||
impl ThreadMetadataUpdate {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rollout_path.is_none()
|
||||
&& self.preview.is_none()
|
||||
&& self.name.is_none()
|
||||
&& self.model_provider.is_none()
|
||||
&& self.model.is_none()
|
||||
&& self.reasoning_effort.is_none()
|
||||
&& self.created_at.is_none()
|
||||
&& self.updated_at.is_none()
|
||||
&& self.source.is_none()
|
||||
&& self.thread_source.is_none()
|
||||
&& self.agent_nickname.is_none()
|
||||
&& self.agent_role.is_none()
|
||||
&& self.agent_path.is_none()
|
||||
&& self.cwd.is_none()
|
||||
&& self.cli_version.is_none()
|
||||
&& self.approval_mode.is_none()
|
||||
&& self.sandbox_policy.is_none()
|
||||
&& self.token_usage.is_none()
|
||||
&& self.first_user_message.is_none()
|
||||
&& self.git_info.is_none()
|
||||
&& self.memory_mode.is_none()
|
||||
&& self.dynamic_tools.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for applying metadata inferred by the live thread layer.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ApplyThreadMetadataParams {
|
||||
|
||||
Reference in New Issue
Block a user