Files
codex/codex-rs/rollout/src/state_db_tests.rs
T
Owen Lin 5267e805fb feat(app-server): add history_mode to thread (#29927)
## Description

This PR adds a new `historyMode = "legacy" | "paginated"` to `Thread`.
This will be stored in `SessionMeta` in the JSONL rollout file and as a
new column in the SQLite thread_metadata table, and exposed on
`thread/start` and on the `Thread` object in app-server.

## What changed

- Added canonical `ThreadHistoryMode` with `legacy` and `paginated`,
defaulting old and new SessionMeta to `legacy`.
- Carried `history_mode` through core session config, ThreadStore stored
metadata, local/in-memory stores, rollout metadata extraction, and the
existing SQLite `threads` table.
- Added experimental `historyMode` to app-server v2 `Thread` and
`thread/start`.
- Made paginated stored threads metadata-discoverable but unsupported
for legacy full-history reads, `load_history`, live resume, and create
paths.
- Regenerated app-server schema fixtures and added
protocol/state/thread-store/app-server coverage for persistence and
fail-closed behavior.

## Compatibility floor
Because users may be running various versions of Codex binaries on the
same machine (TUI, Codex App, etc.), we will need to establish a
compatibility floor for upcoming paginated threads, which will change
how thread storage reads and writes work.

The overall plan here:
```
Release N:
- Add historyMode to SessionMeta / Thread / SQLite metadata.
- Teach binaries to understand paginated threads.
- If a binary sees `historyMode="paginated"` but does not support the paginated contract, it refuses to resume/mutate the thread.
- Default remains `"legacy"`.

Release N+1:
- First-party clients start opting into paginated threads where appropriate.
- Internal dogfood / staged rollout.
- Measure old-client usage and paginated-thread unsupported errors.

Release N+2:
- Only after Release N+ is overwhelmingly deployed, make paginated the default.
- Accept that a small tail of N-1-or-older binaries may not understand paginated threads.
```

The important behavior change is fail-closed handling for a binary that
encounters a persisted `paginated` thread before it knows how to fully
support paginated history. In app-server, if a thread is `paginated`, we
will:

- allow metadata-only discovery paths like `thread/list` and
`thread/read(includeTurns=false)`, so clients can still see the thread
and inspect its `historyMode`
- reject legacy full-history/live-thread paths like
`thread/read(includeTurns=true)` and `thread/resume` with an unsupported
JSON-RPC error
- avoid silently treating an unknown or future `historyMode` as `legacy`

Under the hood, the ThreadStore layer also rejects legacy operations
that would need to load or replay the full thread history for a
paginated thread. That gives us the behavior we want for Release N:
future paginated threads are visible, but this binary fails closed
instead of trying to operate on them as if they were legacy threads.
2026-06-26 09:12:42 -07:00

202 lines
6.7 KiB
Rust

#![allow(warnings, clippy::all)]
use super::*;
use crate::list::parse_cursor;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
use chrono::Utc;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::UserMessageEvent;
use pretty_assertions::assert_eq;
use std::path::Path;
use tempfile::TempDir;
#[test]
fn cursor_to_anchor_normalizes_timestamp_format() {
let ts_str = "2026-01-27T12-34-56";
let cursor = parse_cursor(ts_str).expect("cursor should parse");
let anchor = cursor_to_anchor(Some(&cursor)).expect("anchor should parse");
let naive =
NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H-%M-%S").expect("ts should parse");
let expected_ts = DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc)
.with_nanosecond(0)
.expect("nanosecond");
assert_eq!(anchor.ts, expected_ts);
assert_eq!(anchor.id, None);
}
#[test]
fn cursor_to_anchor_preserves_recency_tie_breaker() {
let id = ThreadId::from_string("00000000-0000-0000-0000-000000000123")
.expect("thread id should parse");
let token = format!("2026-01-27T12:34:56Z|{id}");
let cursor = parse_cursor(&token).expect("cursor should parse");
let anchor = cursor_to_anchor(Some(&cursor)).expect("anchor should parse");
assert_eq!(anchor.id, Some(id));
assert_eq!(
serde_json::to_string(&cursor).expect("cursor should serialize"),
format!("\"{token}\"")
);
}
#[tokio::test]
async fn try_init_waits_for_concurrent_startup_backfill() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let claimed = runtime.try_claim_backfill(/*lease_seconds*/ 60).await?;
assert!(claimed);
let runtime_for_completion = runtime.clone();
let complete_backfill = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
runtime_for_completion
.mark_backfill_complete(/*last_watermark*/ None)
.await
});
let initialized = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
.await?;
complete_backfill.await??;
assert_eq!(
initialized.get_backfill_state().await?.status,
codex_state::BackfillStatus::Complete
);
Ok(())
}
#[tokio::test]
async fn try_init_times_out_waiting_for_stuck_startup_backfill() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let claimed = runtime.try_claim_backfill(/*lease_seconds*/ 60).await?;
assert!(claimed);
let result = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
.await;
let err = match result {
Ok(_) => panic!("state db init should not wait forever for incomplete backfill"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("timed out waiting for state db backfill"),
"unexpected error: {err}"
);
Ok(())
}
#[tokio::test]
async fn reconcile_rollout_preserves_existing_explicit_title() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let thread_id = ThreadId::new();
let rollout_path = write_rollout_with_user_message(home.path(), thread_id, "Hey")?;
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let mut metadata =
metadata::extract_metadata_from_rollout(rollout_path.as_path(), "test-provider")
.await?
.metadata;
assert_eq!(metadata.title, "Hey");
assert_eq!(metadata.first_user_message.as_deref(), Some("Hey"));
metadata.title = "math".to_string();
runtime.upsert_thread(&metadata).await?;
reconcile_rollout(
Some(runtime.as_ref()),
rollout_path.as_path(),
"test-provider",
/*builder*/ None,
&[],
/*archived_only*/ Some(false),
/*new_thread_memory_mode*/ None,
)
.await;
let persisted = runtime
.get_thread(thread_id)
.await?
.expect("thread should exist");
assert_eq!(persisted.title, "math");
assert_eq!(persisted.first_user_message.as_deref(), Some("Hey"));
Ok(())
}
fn write_rollout_with_user_message(
home: &Path,
thread_id: ThreadId,
message: &str,
) -> anyhow::Result<std::path::PathBuf> {
let dir = home.join("sessions/2026/06/01");
std::fs::create_dir_all(dir.as_path())?;
let path = dir.join(format!("rollout-2026-06-01T14-26-25-{thread_id}.jsonl"));
let lines = [
RolloutLine {
timestamp: "2026-06-01T14:26:25Z".to_string(),
item: RolloutItem::SessionMeta(SessionMetaLine {
meta: SessionMeta {
session_id: thread_id.into(),
id: thread_id,
forked_from_id: None,
parent_thread_id: None,
timestamp: "2026-06-01T14:26:25Z".to_string(),
cwd: home.to_path_buf(),
originator: "test".to_string(),
cli_version: "test".to_string(),
source: SessionSource::Cli,
thread_source: None,
agent_nickname: None,
agent_role: None,
agent_path: None,
model_provider: Some("test-provider".to_string()),
base_instructions: None,
dynamic_tools: None,
selected_capability_roots: Vec::new(),
memory_mode: None,
history_mode: Default::default(),
multi_agent_version: None,
context_window: None,
},
git: None,
}),
},
RolloutLine {
timestamp: "2026-06-01T14:26:26Z".to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: message.to_string(),
..Default::default()
})),
},
];
let jsonl = lines
.iter()
.map(serde_json::to_string)
.collect::<Result<Vec<_>, _>>()?
.join("\n");
std::fs::write(path.as_path(), format!("{jsonl}\n"))?;
Ok(path)
}