Files
codex/codex-rs/rollout/src/state_db_tests.rs
T
Michael Bolin 01f89c8c59 core: persist initial context window metadata (#29519)
## Why

PR #29494 made context-window IDs visible to the model by wrapping the
token-budget window payload in `<context_window>`, but rollout JSONL
consumers still could not see the initial window identity by tailing the
session file. Compacted rollout items carry window IDs only after
compaction has happened, so a session with no compaction had no durable
JSONL record for window 0.

This change gives tailing consumers a stable initial-window record at
session creation time.

## What Changed

- Added `session_meta.context_window.window_id` for the initial
context-window identity.
- `CreateThreadParams` now requires `initial_window_id: String`, so
thread-store callers cannot accidentally create new threads without
window-0 metadata.
- Live thread creation derives the persisted initial window ID from the
same `AutoCompactWindowIds` used to initialize `SessionState`, keeping
runtime state and JSONL metadata aligned.
- Rollout reconstruction uses `session_meta.context_window.window_id` as
the initial-window fallback and derives `window_number = 0`,
`first_window_id = window_id`, and `previous_window_id = None`
internally.
- Fork reconstruction intentionally uses the same rollout reconstruction
path; consumers that need to distinguish copied initial-window metadata
can use the rollout `thread_id`.
- Legacy compactions without `window_number` still use compaction-count
fallback accounting instead of being reset to window 0 by the
initial-window fallback.
- Compacted rollout metadata still takes precedence once compaction
records exist, preserving the richer chain fields there.

## JSONL Shape

Real rollout JSONL is one object per line. This example is expanded for
readability, but shows the new initial `session_meta.context_window`
record followed by the existing compacted rollout item shape that also
carries window IDs:

```jsonl
{
  "timestamp": "2026-06-22T12:00:00.000Z",
  "type": "session_meta",
  "payload": {
    "session_id": "<THREAD_ID>",
    "id": "<THREAD_ID>",
    "timestamp": "2026-06-22T12:00:00.000Z",
    "cwd": "/repo",
    "originator": "codex",
    "cli_version": "0.0.0",
    "source": "cli",
    "model_provider": "<MODEL_PROVIDER>",
    "context_window": {
      "window_id": "<INITIAL_WINDOW_ID>"
    }
  }
}
...
{
  "timestamp": "2026-06-22T12:34:56.000Z",
  "type": "compacted",
  "payload": {
    "message": "<COMPACTION_SUMMARY>",
    "replacement_history": [
      "..."
    ],
    "window_number": 1,
    "first_window_id": "<INITIAL_WINDOW_ID>",
    "previous_window_id": "<INITIAL_WINDOW_ID>",
    "window_id": "<NEXT_WINDOW_ID>"
  }
}
```

The nested `context_window` object is intentional: it gives rollout
consumers a stable namespace for context-window metadata while only
writing the non-derivable initial `window_id`. For the initial window,
`window_number`, `first_window_id`, and `previous_window_id` are derived
internally instead of being written to the rollout.

## Verification

- `just test -p codex-protocol`
- `just test -p codex-rollout
recorder_materializes_on_flush_with_pending_items`
- `just test -p codex-core reconstruct_history`
- `just test -p codex-core
record_initial_history_reconstructs_forked_transcript`
- `just test -p codex-thread-store`
- `just test -p codex-state`
- `just test -p codex-app-server
thread_read_returns_summary_without_turns`
- `just test -p codex-rollout persistence_metrics`
2026-06-23 21:50:50 +00:00

200 lines
6.6 KiB
Rust

#![allow(warnings, clippy::all)]
use super::*;
use crate::list::parse_cursor;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
use chrono::Utc;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::UserMessageEvent;
use pretty_assertions::assert_eq;
use std::path::Path;
use tempfile::TempDir;
#[test]
fn cursor_to_anchor_normalizes_timestamp_format() {
let ts_str = "2026-01-27T12-34-56";
let cursor = parse_cursor(ts_str).expect("cursor should parse");
let anchor = cursor_to_anchor(Some(&cursor)).expect("anchor should parse");
let naive =
NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H-%M-%S").expect("ts should parse");
let expected_ts = DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc)
.with_nanosecond(0)
.expect("nanosecond");
assert_eq!(anchor.ts, expected_ts);
assert_eq!(anchor.id, None);
}
#[test]
fn cursor_to_anchor_preserves_recency_tie_breaker() {
let id = ThreadId::from_string("00000000-0000-0000-0000-000000000123")
.expect("thread id should parse");
let token = format!("2026-01-27T12:34:56Z|{id}");
let cursor = parse_cursor(&token).expect("cursor should parse");
let anchor = cursor_to_anchor(Some(&cursor)).expect("anchor should parse");
assert_eq!(anchor.id, Some(id));
assert_eq!(
serde_json::to_string(&cursor).expect("cursor should serialize"),
format!("\"{token}\"")
);
}
#[tokio::test]
async fn try_init_waits_for_concurrent_startup_backfill() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let claimed = runtime.try_claim_backfill(/*lease_seconds*/ 60).await?;
assert!(claimed);
let runtime_for_completion = runtime.clone();
let complete_backfill = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
runtime_for_completion
.mark_backfill_complete(/*last_watermark*/ None)
.await
});
let initialized = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
.await?;
complete_backfill.await??;
assert_eq!(
initialized.get_backfill_state().await?.status,
codex_state::BackfillStatus::Complete
);
Ok(())
}
#[tokio::test]
async fn try_init_times_out_waiting_for_stuck_startup_backfill() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let claimed = runtime.try_claim_backfill(/*lease_seconds*/ 60).await?;
assert!(claimed);
let result = try_init_with_roots_and_backfill_lease(
home.path().to_path_buf(),
home.path().to_path_buf(),
"test-provider".to_string(),
/*backfill_lease_seconds*/ 60,
)
.await;
let err = match result {
Ok(_) => panic!("state db init should not wait forever for incomplete backfill"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("timed out waiting for state db backfill"),
"unexpected error: {err}"
);
Ok(())
}
#[tokio::test]
async fn reconcile_rollout_preserves_existing_explicit_title() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let thread_id = ThreadId::new();
let rollout_path = write_rollout_with_user_message(home.path(), thread_id, "Hey")?;
let runtime =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".to_string())
.await?;
let mut metadata =
metadata::extract_metadata_from_rollout(rollout_path.as_path(), "test-provider")
.await?
.metadata;
assert_eq!(metadata.title, "Hey");
assert_eq!(metadata.first_user_message.as_deref(), Some("Hey"));
metadata.title = "math".to_string();
runtime.upsert_thread(&metadata).await?;
reconcile_rollout(
Some(runtime.as_ref()),
rollout_path.as_path(),
"test-provider",
/*builder*/ None,
&[],
/*archived_only*/ Some(false),
/*new_thread_memory_mode*/ None,
)
.await;
let persisted = runtime
.get_thread(thread_id)
.await?
.expect("thread should exist");
assert_eq!(persisted.title, "math");
assert_eq!(persisted.first_user_message.as_deref(), Some("Hey"));
Ok(())
}
fn write_rollout_with_user_message(
home: &Path,
thread_id: ThreadId,
message: &str,
) -> anyhow::Result<std::path::PathBuf> {
let dir = home.join("sessions/2026/06/01");
std::fs::create_dir_all(dir.as_path())?;
let path = dir.join(format!("rollout-2026-06-01T14-26-25-{thread_id}.jsonl"));
let lines = [
RolloutLine {
timestamp: "2026-06-01T14:26:25Z".to_string(),
item: RolloutItem::SessionMeta(SessionMetaLine {
meta: SessionMeta {
session_id: thread_id.into(),
id: thread_id,
forked_from_id: None,
parent_thread_id: None,
timestamp: "2026-06-01T14:26:25Z".to_string(),
cwd: home.to_path_buf(),
originator: "test".to_string(),
cli_version: "test".to_string(),
source: SessionSource::Cli,
thread_source: None,
agent_nickname: None,
agent_role: None,
agent_path: None,
model_provider: Some("test-provider".to_string()),
base_instructions: None,
dynamic_tools: None,
memory_mode: None,
multi_agent_version: None,
context_window: None,
},
git: None,
}),
},
RolloutLine {
timestamp: "2026-06-01T14:26:26Z".to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: message.to_string(),
..Default::default()
})),
},
];
let jsonl = lines
.iter()
.map(serde_json::to_string)
.collect::<Result<Vec<_>, _>>()?
.join("\n");
std::fs::write(path.as_path(), format!("{jsonl}\n"))?;
Ok(path)
}