Gate local realtime frame history on paginated storage (#62271)

## Why

Legacy rollout migration rewrites rolled-back history and can disrupt
the realtime stream. Enroll local threads once they use paginated history
so the stream stays intact.

## What changed

- Add the off-by-default `features.realtime_frames_local` flag for
  paginated local threads with a state database. Accept runtime updates
  through the app server and keep threads enrolled once they have a
  persisted canonical session start.
- Add thread-store APIs to recover a realtime session and look up the
  latest session ID from durable history. Route frame completion through
  a store seal so completion acknowledges durable writes.
- Add a local seal for accepted model input that writes source frames
  before the input and lets callers retry an uncertain flush without
  appending the input twice.

## Testing

Cover enrollment for legacy and paginated threads, continued enrollment
after disabling the flag, and retrying a local input seal after a failed
directory sync.

GitOrigin-RevId: 2012feeca3857da90f9b026f077b31a26d15191b
This commit is contained in:
Zahan Malkani
2026-11-18 12:55:14 +00:00
committed by copyberry
parent 4389a526d5
commit 29cb67623c
29 changed files with 693 additions and 41 deletions
@@ -6161,7 +6161,7 @@
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
@@ -25675,7 +25675,7 @@
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/v2/ThreadRealtimeVoiceTranscript"
@@ -23371,7 +23371,7 @@
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
@@ -61,7 +61,7 @@
]
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
@@ -61,7 +61,7 @@
]
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
@@ -5,6 +5,6 @@ import type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscr
/**
* A frame holds one or more payloads, at most one per channel. They share one ID,
* observation time, and seal. V1 requires its only channel, `voice_transcript`.
* observation time, and seal. Currently, `voice_transcript` is required.
*/
export type ThreadRealtimeFrameChannels = { voice_transcript: ThreadRealtimeVoiceTranscript, };
@@ -122,7 +122,7 @@ pub enum ThreadRealtimeSessionOutcome {
}
/// A frame holds one or more payloads, at most one per channel. They share one ID,
/// observation time, and seal. V1 requires its only channel, `voice_transcript`.
/// observation time, and seal. Currently, `voice_transcript` is required.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeFrameChannels {
@@ -209,7 +209,7 @@ impl ThreadRealtimeItem {
}
},
},
CoreRealtimeItemContent::RealtimeSessionClosed { outcome } => {
CoreRealtimeItemContent::RealtimeSessionClosed { outcome, .. } => {
ThreadRealtimeItemContent::RealtimeSessionClosed {
outcome: match outcome {
CoreRealtimeSessionOutcome::Ended => ThreadRealtimeSessionOutcome::Ended,
@@ -79,6 +79,7 @@ const SUPPORTED_EXPERIMENTAL_FEATURE_ENABLEMENT: &[&str] = &[
"mcp_shared_repl_cwd",
"memories",
"mentions_v2",
"realtime_frames_local",
"remote_control",
"remote_plugin",
"tool_suggest",
@@ -391,6 +392,9 @@ impl ConfigRequestProcessor {
self.thread_manager.set_api_key_model_discovery_enabled(
config.features.enabled(Feature::ApiKeyModelDiscovery),
);
self.thread_manager.set_realtime_frames_local_enabled(
config.features.enabled(Feature::RealtimeFramesLocal),
);
if should_start_background_rollout_migration && config.features.enabled(feature) {
self.thread_manager.start_background_rollout_migration();
}
@@ -6,6 +6,8 @@ use app_test_support::create_command_execution_sse_response;
use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use codex_app_server_protocol::CommandExecutionStatus;
use codex_app_server_protocol::ExperimentalFeatureEnablementSetParams;
use codex_app_server_protocol::ExperimentalFeatureEnablementSetResponse;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::JSONRPCError;
@@ -706,6 +708,7 @@ async fn realtime_conversation_streams_timeline_items() -> Result<()> {
.await?;
mcp.initialize().await?;
login_with_api_key(&mut mcp, "sk-test-key").await?;
enable_local_realtime_frames(&mut mcp).await?;
let thread_request = mcp
.send_thread_start_request_with_auto_env(ThreadStartParams {
@@ -1250,6 +1253,7 @@ async fn realtime_timeline_splits_accepted_steering_and_persists_promoted_artifa
])]),
)
.await?;
enable_local_realtime_frames(&mut harness.mcp).await?;
let thread_request = harness
.mcp
.send_thread_start_request_with_auto_env(ThreadStartParams {
@@ -4399,6 +4403,17 @@ async fn login_with_api_key(mcp: &mut TestAppServer, api_key: &str) -> Result<()
Ok(())
}
async fn enable_local_realtime_frames(mcp: &mut TestAppServer) -> Result<()> {
let request = mcp
.send_experimental_feature_enablement_set_request(ExperimentalFeatureEnablementSetParams {
enablement: BTreeMap::from([("realtime_frames_local".to_string(), true)]),
})
.await?;
let _: ExperimentalFeatureEnablementSetResponse =
timeout(DEFAULT_TIMEOUT, mcp.read_response(request)).await??;
Ok(())
}
async fn wait_for_started_command_execution(
mcp: &mut TestAppServer,
) -> Result<ItemStartedNotification> {
+6
View File
@@ -1286,6 +1286,9 @@
"realtime_conversation": {
"type": "boolean"
},
"realtime_frames_local": {
"type": "boolean"
},
"reasoning_effort_override": {
"type": "boolean"
},
@@ -8598,6 +8601,9 @@
"realtime_conversation": {
"type": "boolean"
},
"realtime_frames_local": {
"type": "boolean"
},
"reasoning_effort_override": {
"type": "boolean"
},
+1
View File
@@ -308,6 +308,7 @@ impl RealtimeHistoryState {
.frames_enabled()
.then(|| self.observation_clock.now_ms()),
content: RealtimeItemContent::RealtimeSessionClosed {
final_frame_sequence: None,
outcome: if self.failed {
RealtimeSessionOutcome::Failed
} else {
+26 -10
View File
@@ -23,7 +23,24 @@ impl Session {
.realtime_frames_enabled(self.thread_id())
.await
.map_err(std::io::Error::other)?;
history.lock().await.frame_enrollment = Some(enabled);
let floor = if enabled
&& matches!(
self.history_initialization,
codex_history::HistoryInitialization::ColdResume
) {
self.services
.thread_store
.realtime_frame_clock_floor(self.thread_id())
.await
.map_err(std::io::Error::other)?
} else {
None
};
let mut history = history.lock().await;
if let Some(floor) = floor {
history.observation_clock.observe(floor);
}
history.frame_enrollment = Some(enabled);
Ok(enabled)
}
@@ -54,16 +71,15 @@ impl Session {
}
let live_thread = self.live_thread_for_persistence("append realtime history")?;
let items = effects.items;
let records = items
.iter()
.cloned()
.map(RolloutItem::RealtimeItem)
.collect::<Vec<_>>();
live_thread.append_items(&records).await?;
// A completed notification acknowledges durable history, including on hosts
// whose append operation only admits work to an asynchronous writer.
if effects.seal_frames {
live_thread.flush().await?;
live_thread.seal_realtime_items(items.clone()).await?;
} else {
let records = items
.iter()
.cloned()
.map(RolloutItem::RealtimeItem)
.collect::<Vec<_>>();
live_thread.append_items(&records).await?;
}
for item in items {
if !matches!(
+12
View File
@@ -1034,6 +1034,18 @@ impl ThreadManager {
Arc::clone(&self.state.thread_store)
}
/// Updates local realtime enrollment after a runtime feature decision changes.
pub fn set_realtime_frames_local_enabled(&self, enabled: bool) {
if let Some(store) = self
.state
.thread_store
.as_any()
.downcast_ref::<LocalThreadStore>()
{
store.set_realtime_frames_local_enabled(enabled);
}
}
/// Starts the local rollout migration path after a runtime feature enablement.
///
/// Startup config handles the initial launch in [`crate::ThreadStoreBuilder::build`]. This covers
@@ -78,6 +78,9 @@ impl<'config> ThreadStoreBuilder<'config> {
})
}),
);
store.set_realtime_frames_local_enabled(
config.features.enabled(Feature::RealtimeFramesLocal),
);
if has_state_db && background_migration_enabled {
let startup_store = Arc::clone(&store);
let codex_home = config.codex_home.to_path_buf();
@@ -11,6 +11,7 @@ use codex_core::StartThreadOptions;
use codex_core::TurnInputRequest;
use codex_core::TurnInputSubmission;
use codex_core::test_support::auth_manager_from_auth;
use codex_features::Feature;
use codex_history::InitialHistory;
use codex_history::ResponseItemEnvelope;
use codex_history::RolloutItem;
@@ -497,6 +498,10 @@ async fn conversation_records_history_without_an_event_observer(
.with_config({
let url = realtime_server.uri().to_string();
move |config| {
config
.features
.enable(Feature::RealtimeFramesLocal)
.expect("enable local realtime frames");
config.experimental_realtime_ws_base_url = Some(url);
config.realtime.version = RealtimeWsVersion::V2;
}
@@ -3947,6 +3952,10 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime(ephemeral: bool) ->
.with_config({
let realtime_base_url = realtime_server.uri().to_string();
move |config| {
config
.features
.enable(Feature::RealtimeFramesLocal)
.expect("enable local realtime frames");
config.ephemeral = ephemeral;
config.realtime.version = RealtimeWsVersion::V2;
config.experimental_realtime_ws_base_url = Some(realtime_base_url);
+8
View File
@@ -234,6 +234,8 @@ pub enum Feature {
LocalThreadStoreSharedCompression,
/// Migrate legacy local rollout files to paginated history in the background.
BackgroundPaginatedRolloutMigration,
/// Enroll paginated local threads in canonical realtime frame history.
RealtimeFramesLocal,
/// Enable the Chronicle sidecar for passive screen-context memories.
Chronicle,
/// Compress request bodies (zstd) when sending streaming requests to codex-backend.
@@ -1505,6 +1507,12 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::RealtimeFramesLocal,
key: "realtime_frames_local",
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::Chronicle,
key: "chronicle",
+5 -1
View File
@@ -18,7 +18,7 @@ pub struct RealtimeItem {
}
/// A frame holds one or more payloads, at most one per channel. They share one ID,
/// observation time, and seal. V1 requires its only channel, `voice_transcript`.
/// observation time, and seal. Currently, `voice_transcript` is required.
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)]
pub struct RealtimeFrameChannels {
pub voice_transcript: RealtimeVoiceTranscript,
@@ -51,6 +51,10 @@ pub enum RealtimeItemContent {
},
RealtimeSessionClosed {
outcome: RealtimeSessionOutcome,
/// Hosted frame-table sequence of the session's last frame (zero for a silent call).
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(type = "number | null", optional)]
final_frame_sequence: Option<u64>,
},
/// Durable acceptance of the exact bounded input used by an agent turn.
/// The enclosing item ID is the acceptance identity; source IDs never use
+51 -19
View File
@@ -1,3 +1,4 @@
use codex_rollout::ResponseItemEnvelope;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
@@ -5,6 +6,7 @@ use std::sync::Arc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_protocol::realtime::RealtimeItem;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutPersistenceTelemetry;
use codex_rollout::measure_and_filter_rollout_items;
@@ -246,25 +248,7 @@ impl LiveThread {
if items.is_empty() {
return Ok(());
}
let update = self
.metadata_sync
.lock()
.await
.observe_appended_items(items.as_slice());
if let Some(update) = update {
self.thread_store
.record_thread_metadata(UpdateThreadMetadataParams {
thread_id: self.thread_id,
patch: update.patch.clone(),
include_archived: true,
})
.await?;
self.metadata_sync
.lock()
.await
.mark_pending_update_applied(&update);
}
Ok(())
self.observe_appended_metadata(&items).await
}
/// Returns the filtered items if sampling already needed to collect them.
@@ -309,6 +293,54 @@ impl LiveThread {
self.apply_pending_metadata_update(update, context).await
}
pub async fn seal_realtime_items(&self, items: Vec<RealtimeItem>) -> ThreadStoreResult<()> {
if self.thread_store.uses_separate_realtime_frame_store() {
return self
.thread_store
.seal_realtime_items(self.thread_id, items)
.await;
}
let records: Vec<_> = items.into_iter().map(RolloutItem::RealtimeItem).collect();
self.append_items(&records).await?;
self.flush().await
}
pub async fn seal_realtime_input(
&self,
frames: Vec<RealtimeItem>,
input: ResponseItemEnvelope,
) -> ThreadStoreResult<()> {
let record = RolloutItem::ResponseItem(input.clone());
self.thread_store
.seal_realtime_input(self.thread_id, frames, input)
.await?;
// The store owns the idempotent acceptance barrier. Apply append-derived
// metadata afterward so retrying a failed metadata update cannot append again.
self.observe_appended_metadata(&[record]).await
}
async fn observe_appended_metadata(&self, items: &[RolloutItem]) -> ThreadStoreResult<()> {
let update = self
.metadata_sync
.lock()
.await
.observe_appended_items(items);
if let Some(update) = update {
self.thread_store
.record_thread_metadata(UpdateThreadMetadataParams {
thread_id: self.thread_id,
patch: update.patch.clone(),
include_archived: true,
})
.await?;
self.metadata_sync
.lock()
.await
.mark_pending_update_applied(&update);
}
Ok(())
}
pub async fn flush(&self) -> ThreadStoreResult<()> {
self.thread_store.flush_thread(self.thread_id).await?;
self.flush_pending_metadata_update_for_existing_history()
@@ -199,6 +199,79 @@ pub(super) async fn append_items(
.await
}
pub(super) async fn seal_realtime_input(
store: &LocalThreadStore,
thread_id: ThreadId,
frames: Vec<codex_protocol::realtime::RealtimeItem>,
input: codex_rollout::ResponseItemEnvelope,
) -> ThreadStoreResult<()> {
let id = input
.realtime_handoff_item()
.ok_or_else(|| ThreadStoreError::InvalidRequest {
message: "realtime input is missing handoff metadata".to_owned(),
})?
.id;
let records = frames
.into_iter()
.map(RolloutItem::RealtimeItem)
.chain(std::iter::once(RolloutItem::ResponseItem(input)))
.collect();
seal_realtime_records(store, thread_id, id, records).await
}
/// An uncertain flush keeps ownership of the already queued model-facing seal.
/// Retrying confirms that batch instead of appending it again.
async fn seal_realtime_records(
store: &LocalThreadStore,
thread_id: ThreadId,
id: String,
items: Vec<RolloutItem>,
) -> ThreadStoreResult<()> {
let mut cursor = store.live_writer_locks.lock(thread_id).await;
let (recorder, rollout_id, history_mode, seals) = {
let entries = store.live_recorders.lock().await;
let entry = entries
.get(&thread_id)
.ok_or(ThreadStoreError::ThreadNotFound { thread_id })?;
(
entry.recorder.clone(),
entry.rollout_id,
entry.history_mode,
entry.realtime_seals.clone(),
)
};
let mut seals = seals.lock_owned().await;
if let Some(pending) = seals.pending.clone() {
recorder.flush().await.map_err(thread_store_io_error)?;
seals.pending = None;
seals.committed.insert(pending);
}
if !seals.committed.contains(&id) {
let mut items = into_persisted_rollout_items(items, history_mode);
let next = codex_rollout::stamp_item_ordinals(*cursor, &mut items);
recorder
.record_canonical_items(&items)
.await
.map_err(thread_store_io_error)?;
*cursor = next;
seals.pending = Some(id.clone());
recorder.flush().await.map_err(thread_store_io_error)?;
seals.pending = None;
seals.committed.insert(id);
}
if history_mode == ThreadHistoryMode::Paginated
&& let Err(err) = super::thread_history_materialization::materialize_to_sqlite(
store,
rollout_id,
recorder.rollout_path(),
)
.await
{
warn!("failed to project durable rollout for {thread_id}: {err}");
}
sync_materialized_rollout_path(store, thread_id, recorder.rollout_path()).await
}
pub(super) async fn persist_thread(
store: &LocalThreadStore,
thread_id: ThreadId,
+119 -2
View File
@@ -38,6 +38,8 @@ mod pending_thread_metadata_tests;
#[cfg(test)]
#[path = "read_thread_tests.rs"]
mod read_thread_tests;
#[cfg(all(test, unix))]
mod realtime_input_tests;
#[cfg(test)]
mod test_support;
#[cfg(test)]
@@ -46,15 +48,20 @@ mod timestamp_metadata_tests;
use codex_protocol::ThreadId;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::realtime::RealtimeItem;
use codex_rollout::ResponseItemEnvelope;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutRecorder;
use codex_rollout::StateDbHandle;
use codex_rollout::WriterLockCoordinator;
use codex_state::SqliteConfig;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::hash_map::Entry;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use tokio::sync::Mutex;
use tokio::sync::OnceCell;
use tokio::sync::OwnedMutexGuard;
@@ -158,6 +165,7 @@ pub struct LocalThreadStore {
/// can use the other connections while SQLite's single writer is busy.
thread_history_writer: Arc<Semaphore>,
thread_data_cleanup: Option<Arc<ThreadDataCleanup>>,
realtime_frames_local_enabled: Arc<AtomicBool>,
#[cfg(test)]
startup_migration_retry_gate: Option<Arc<StartupMigrationRetryGate>>,
}
@@ -176,6 +184,13 @@ struct LiveRecorderEntry {
// so missing SQLite rows can still be seeded.
history_mode: ThreadHistoryMode,
writer_lock: WriterLockGuard,
realtime_seals: Arc<Mutex<RealtimeSeals>>,
}
#[derive(Default)]
struct RealtimeSeals {
pending: Option<String>,
committed: HashSet<String>,
}
#[derive(Default)]
@@ -287,6 +302,7 @@ impl LocalThreadStore {
thread_history_db: Arc::new(OnceCell::new()),
thread_history_writer: Arc::new(Semaphore::new(/* permits */ 1)),
thread_data_cleanup: None,
realtime_frames_local_enabled: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
startup_migration_retry_gate: None,
}
@@ -303,6 +319,12 @@ impl LocalThreadStore {
self
}
/// Controls new realtime-frame enrollment without changing existing enrolled threads.
pub fn set_realtime_frames_local_enabled(&self, enabled: bool) {
self.realtime_frames_local_enabled
.store(enabled, Ordering::Relaxed);
}
/// Return the state DB handle used by local rollout writers.
pub async fn state_db(&self) -> Option<StateDbHandle> {
self.state_db.clone()
@@ -426,6 +448,7 @@ impl LocalThreadStore {
rollout_id,
history_mode,
writer_lock,
realtime_seals: Arc::new(Mutex::new(RealtimeSeals::default())),
});
Ok(())
}
@@ -524,6 +547,17 @@ impl LocalThreadStore {
}
impl ThreadStore for LocalThreadStore {
fn seal_realtime_input(
&self,
thread_id: ThreadId,
frames: Vec<RealtimeItem>,
input: ResponseItemEnvelope,
) -> ThreadStoreFuture<'_, ()> {
Box::pin(
async move { live_writer::seal_realtime_input(self, thread_id, frames, input).await },
)
}
fn model_ids(&self) -> Option<Arc<dyn codex_state::SessionModelIdStore>> {
self.state_db
.clone()
@@ -806,8 +840,22 @@ impl ThreadStore for LocalThreadStore {
Box::pin(async move { projects::delete_project(self, project_id).await })
}
fn realtime_frames_enabled(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> {
Box::pin(async { Ok(self.state_db.is_some()) })
fn realtime_frames_enabled(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> {
Box::pin(async move {
if self.state_db.is_none() {
return Ok(false);
}
// Legacy rollout migration rewrites rolled-back history. Enroll only once
// the live thread is already paginated, so the realtime stream stays intact.
let (_, _, history_mode) = live_writer::live_writer_parts(self, thread_id).await?;
if history_mode != ThreadHistoryMode::Paginated {
return Ok(false);
}
if self.realtime_frames_local_enabled.load(Ordering::Relaxed) {
return Ok(true);
}
thread_history::has_canonical_realtime_session(self, thread_id).await
})
}
fn supports_paginated_history_lists(&self) -> bool {
@@ -833,6 +881,25 @@ impl ThreadStore for LocalThreadStore {
Box::pin(LocalThreadStore::list_realtime(self, params))
}
fn load_realtime_session(
&self,
thread_id: ThreadId,
realtime_session_id: String,
) -> ThreadStoreFuture<'_, Option<Vec<codex_protocol::realtime::RealtimeItem>>> {
Box::pin(thread_history::load_realtime_session(
self,
thread_id,
realtime_session_id,
))
}
fn load_realtime_session_head(
&self,
thread_id: ThreadId,
) -> ThreadStoreFuture<'_, Option<String>> {
Box::pin(thread_history::load_realtime_session_head(self, thread_id))
}
fn search_threads(
&self,
params: SearchThreadsParams,
@@ -963,6 +1030,56 @@ mod tests {
use crate::local::test_support::write_session_file;
use crate::local::test_support::write_session_file_with_history_mode;
#[tokio::test]
async fn realtime_frames_require_enabled_paginated_local_history() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let runtime = codex_state::StateRuntime::init(
config.sqlite.clone(),
config.default_model_provider_id.clone(),
)
.await
.expect("state db should initialize");
let store = Arc::new(LocalThreadStore::new(config, Some(runtime)));
for (history_mode, expected) in [
(ThreadHistoryMode::Legacy, false),
(ThreadHistoryMode::Paginated, true),
] {
let thread_id = ThreadId::default();
let mut params = create_thread_params(thread_id);
params.history_mode = history_mode;
let live_thread = LiveThread::create(store.clone(), params)
.await
.expect("create live thread");
assert!(!store.realtime_frames_enabled(thread_id).await.unwrap());
store.set_realtime_frames_local_enabled(/*enabled*/ true);
assert_eq!(
store.realtime_frames_enabled(thread_id).await.unwrap(),
expected
);
if expected {
live_thread
.append_items(&[RolloutItem::RealtimeItem(RealtimeItem {
id: "realtime-session-start:call-1".to_string(),
realtime_session_id: "call-1".to_string(),
observed_at_ms: Some(1),
content:
codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted,
})])
.await
.unwrap();
live_thread.persist(PersistContext::Standard).await.unwrap();
live_thread.flush().await.unwrap();
}
store.set_realtime_frames_local_enabled(/*enabled*/ false);
assert_eq!(
store.realtime_frames_enabled(thread_id).await.unwrap(),
expected
);
}
}
#[tokio::test]
async fn live_writer_lifecycle_writes_and_closes() {
let home = TempDir::new().expect("temp dir");
@@ -0,0 +1,96 @@
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadMemoryMode;
use codex_rollout::ResponseItemEnvelope;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutRecorder;
use tempfile::TempDir;
use uuid::Uuid;
use super::LocalThreadStore;
use super::test_support::test_config;
use super::test_support::write_session_file_with_history_mode;
use crate::ResumeThreadParams;
use crate::ThreadPersistenceMetadata;
use crate::ThreadStore;
/// A failed directory sync happens after the file contains the accepted input.
/// Confirming that same seal must not append another model-facing input.
#[tokio::test]
async fn retry_after_uncertain_local_input_seal_does_not_append_again() {
let home = TempDir::new().unwrap();
let uuid = Uuid::now_v7();
let thread_id = codex_protocol::ThreadId::from_string(&uuid.to_string()).unwrap();
let path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
uuid,
ThreadHistoryMode::Paginated,
)
.unwrap();
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
store
.resume_thread(ResumeThreadParams {
history_revision: None,
thread_id,
rollout_path: Some(path.clone()),
history: None,
include_archived: true,
metadata: ThreadPersistenceMetadata {
cwd: Some(home.path().to_path_buf()),
model_provider: "test-provider".into(),
memory_mode: ThreadMemoryMode::Enabled,
},
})
.await
.unwrap();
let mut response = ResponseItem::Message {
encrypted_content: None,
status: None,
id: Some(ResponseItemId::from_server("msg_voice".into())),
role: "user".into(),
content: vec![ContentItem::InputText {
text: "voice input".into(),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
response.set_turn_id_if_missing("turn-1");
let input = ResponseItemEnvelope {
item: response,
metadata: Some(
serde_json::from_value(serde_json::json!({
"realtime_handoff": {
"event_id": "event-1",
"realtime_session_id": "call-1",
"observed_at_ms": 1,
"handoff_id": "handoff-1",
"mode": "start",
"source_event_ids": []
}
}))
.unwrap(),
),
};
let parent = path.parent().unwrap();
let moved = home.path().join("moved");
std::fs::rename(parent, &moved).unwrap();
assert!(
store
.seal_realtime_input(thread_id, vec![], input.clone())
.await
.is_err()
);
std::fs::rename(&moved, parent).unwrap();
store
.seal_realtime_input(thread_id, vec![], input)
.await
.unwrap();
let (items, _, _) = RolloutRecorder::load_rollout_items(&path).await.unwrap();
let accepted = items.iter().filter(|item| {
matches!(item, RolloutItem::ResponseItem(envelope) if envelope.realtime_handoff_item().is_some())
});
assert_eq!(accepted.count(), 1);
}
@@ -21,7 +21,10 @@ mod turn_lookup;
pub(super) use read::list_items;
pub(super) use read::list_turns;
pub(super) use realtime::list_timeline;
pub(super) use realtime_stream::has_canonical_realtime_session;
pub(super) use realtime_stream::list_realtime;
pub(super) use realtime_stream::load_realtime_session;
pub(super) use realtime_stream::load_realtime_session_head;
pub(super) use search::search_thread_occurrences;
pub(super) use turn_lookup::find_source_turn;
pub(super) use turn_lookup::find_visible_turn;
@@ -644,6 +644,7 @@ async fn timeline_interleaves_items_and_restores_page_boundary_session_state() {
realtime_session_id: "voice-1".to_string(),
observed_at_ms: None,
content: RealtimeItemContent::RealtimeSessionClosed {
final_frame_sequence: None,
outcome: RealtimeSessionOutcome::Ended,
},
},
@@ -920,6 +921,7 @@ async fn timeline_page_state_excludes_realtime_events_after_fork_cutoff() {
realtime_session_id: "voice".to_string(),
observed_at_ms: None,
content: RealtimeItemContent::RealtimeSessionClosed {
final_frame_sequence: None,
outcome: RealtimeSessionOutcome::Ended,
},
},
@@ -130,7 +130,7 @@ ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7
SELECT rollout_ordinal, 2 AS kind, item_id AS id, NULL AS turn_id, item_json
FROM thread_realtime_items
WHERE thread_id = ?1 AND rollout_ordinal >= ?2
AND item_type NOT IN ('turn_input_accepted', 'session_end_input_saved')
AND item_type NOT IN ('turn_input_accepted', 'session_input_saved')
AND rollout_ordinal < ?3 AND rollout_ordinal <= ?4
AND (rollout_ordinal, 2, item_id) < (?4, ?5, ?6)
ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7
@@ -116,6 +116,149 @@ pub(in crate::local) async fn list_realtime(
.await
}
/// Resolve and project the complete durable prefix while the caller holds the writer lock.
async fn recovery_snapshot(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<Option<(ThreadId, u64, u64)>> {
let Some(resolved) =
super::super::thread_rollout_resolver::resolve_current_including_archived(store, thread_id)
.await?
else {
// A fresh paginated thread has a live writer before its rollout or SQLite row exists.
return Ok(None);
};
validate_thread_for_paginated_reads(
store,
thread_id,
/*include_archived*/ true,
"load_realtime_session",
)
.await?;
let meta = codex_rollout::read_session_meta_line(&resolved.path)
.await
.map_err(thread_history_error)?
.meta;
let first = meta
.subagent_history_start_ordinal
.or_else(|| meta.history_base.map(|base| base.end_ordinal_exclusive));
if meta.forked_from_id.is_some() && first.is_none() {
return Err(ThreadStoreError::Unsupported {
operation: "load_realtime_session",
});
}
super::super::thread_history_materialization::materialize_to_sqlite(
store,
resolved.rollout_id,
&resolved.path,
)
.await?;
let end = super::projection_state(store, resolved.rollout_id)
.await?
.map_or(0, |state| state.next_ordinal);
Ok(Some((resolved.rollout_id, first.unwrap_or(0), end)))
}
/// Recover a logical call from the same authoritative durable-prefix projection as list_realtime.
pub(in crate::local) async fn load_realtime_session(
store: &LocalThreadStore,
thread_id: ThreadId,
session_id: String,
) -> ThreadStoreResult<Option<Vec<RealtimeItem>>> {
let _writer_guard = store.live_writer_locks.lock(thread_id).await;
let Some((rollout_id, first, end)) = recovery_snapshot(store, thread_id).await? else {
return Ok(None);
};
let rows = sqlx::query_scalar::<_, String>(
r#"
SELECT item_json
FROM thread_realtime_items
WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ?
AND json_extract(item_json, '$.realtime_session_id') = ?
ORDER BY rollout_ordinal
"#,
)
.bind(rollout_id.to_string())
.bind(sqlite_integer(first, "first realtime position")?)
.bind(sqlite_integer(end, "realtime snapshot end")?)
.bind(session_id)
.fetch_all(store.thread_history_db().await?)
.await
.map_err(thread_history_error)?;
let items: Vec<RealtimeItem> = rows
.into_iter()
.map(|row| serde_json::from_str(&row).map_err(thread_history_error))
.collect::<ThreadStoreResult<_>>()?;
Ok(items
.iter()
.any(|item| {
matches!(
item.content,
codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted
)
})
.then_some(items))
}
pub(in crate::local) async fn load_realtime_session_head(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<Option<String>> {
let _writer_guard = store.live_writer_locks.lock(thread_id).await;
let Some((rollout_id, first, end)) = recovery_snapshot(store, thread_id).await? else {
return Ok(None);
};
sqlx::query_scalar::<_, String>(
r#"
SELECT json_extract(item_json, '$.realtime_session_id')
FROM thread_realtime_items
WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ?
AND item_type = 'realtime_session_started'
ORDER BY rollout_ordinal DESC
LIMIT 1
"#,
)
.bind(rollout_id.to_string())
.bind(sqlite_integer(first, "first realtime position")?)
.bind(sqlite_integer(end, "realtime snapshot end")?)
.fetch_optional(store.thread_history_db().await?)
.await
.map_err(thread_history_error)
}
/// A canonical start marker makes enrollment sticky when the rollout flag is disabled.
pub(in crate::local) async fn has_canonical_realtime_session(
store: &LocalThreadStore,
thread_id: ThreadId,
) -> ThreadStoreResult<bool> {
let _writer_guard = store.live_writer_locks.lock(thread_id).await;
let snapshot = match recovery_snapshot(store, thread_id).await {
Err(ThreadStoreError::Unsupported {
operation: "load_realtime_session",
}) => return Ok(false),
result => result?,
};
let Some((rollout_id, first, end)) = snapshot else {
return Ok(false);
};
sqlx::query_scalar::<_, bool>(
r#"
SELECT EXISTS (
SELECT 1 FROM thread_realtime_items
WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ?
AND item_type = 'realtime_session_started'
AND json_extract(item_json, '$.observed_at_ms') IS NOT NULL
)
"#,
)
.bind(rollout_id.to_string())
.bind(sqlite_integer(first, "first realtime position")?)
.bind(sqlite_integer(end, "realtime snapshot end")?)
.fetch_one(store.thread_history_db().await?)
.await
.map_err(thread_history_error)
}
async fn read_page(
pool: &sqlx::SqlitePool,
params: ListRealtimeParams,
@@ -701,6 +701,7 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
realtime_session_id: "voice".to_string(),
observed_at_ms: None,
content: RealtimeItemContent::RealtimeSessionClosed {
final_frame_sequence: None,
outcome: RealtimeSessionOutcome::Ended,
},
}),
+107
View File
@@ -1,5 +1,7 @@
use codex_protocol::ThreadId;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::realtime::RealtimeItem;
use codex_rollout::ResponseItemEnvelope;
use codex_rollout::RolloutItem;
use futures::future::BoxFuture;
use std::any::Any;
@@ -286,6 +288,67 @@ pub trait ThreadStore: Any + Send + Sync {
params: LoadThreadHistoryParams,
) -> ThreadStoreFuture<'_, StoredThreadHistory>;
/// Loads one logical call from authoritative persisted history. Separated-frame stores
/// must override this method to include their frame table; a lagging projection must fail
/// rather than report that the session is absent.
fn load_realtime_session(
&self,
thread_id: ThreadId,
realtime_session_id: String,
) -> ThreadStoreFuture<'_, Option<Vec<codex_protocol::realtime::RealtimeItem>>> {
Box::pin(async move {
let history = self
.load_history(LoadThreadHistoryParams {
thread_id,
include_archived: true,
})
.await?;
let items: Vec<_> = history
.items
.iter()
.filter_map(|item| item.realtime_item().map(std::borrow::Cow::into_owned))
.filter(|item| item.realtime_session_id == realtime_session_id)
.collect();
Ok(items
.iter()
.any(|item| {
matches!(
item.content,
codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted
)
})
.then_some(items))
})
}
/// Returns the latest logical session ID from authoritative start records, including a
/// closed session. Used only when a newly requested ID has no persisted session yet.
fn load_realtime_session_head(
&self,
thread_id: ThreadId,
) -> ThreadStoreFuture<'_, Option<String>> {
Box::pin(async move {
let history = self
.load_history(LoadThreadHistoryParams {
thread_id,
include_archived: true,
})
.await?;
Ok(history
.items
.iter()
.rev()
.filter_map(RolloutItem::realtime_item)
.find_map(|item| {
matches!(
item.content,
codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted
)
.then(|| item.realtime_session_id.clone())
}))
})
}
/// Captures a complete tail, optionally reading originals after a saved position.
/// Local storage uses one reader for the cutoff and records; compressed rollouts
/// are decoded once per call. It supports append-only paginated roots;
@@ -558,6 +621,50 @@ pub trait ThreadStore: Any + Send + Sync {
})
}
/// Hosted frame stores can acknowledge a seal independently of ordinary rollout writes.
fn uses_separate_realtime_frame_store(&self) -> bool {
false
}
/// Seal in input order. A durable receipt implies that every preceding source
/// frame is durable. Hosted stores may commit frames outside the rollout.
fn seal_realtime_items(
&self,
thread_id: ThreadId,
items: Vec<RealtimeItem>,
) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move {
let items = items.into_iter().map(RolloutItem::RealtimeItem).collect();
self.append_items(AppendThreadItemsParams { thread_id, items })
.await?;
self.flush_thread(thread_id).await
})
}
/// Commit an accepted model input after its realtime source frames are durable.
/// Implementations must safely accept retries of the same input after an uncertain
/// append or flush outcome without appending the model input more than once.
fn seal_realtime_input(
&self,
_thread_id: ThreadId,
_frames: Vec<RealtimeItem>,
_input: ResponseItemEnvelope,
) -> ThreadStoreFuture<'_, ()> {
Box::pin(async {
Err(ThreadStoreError::Unsupported {
operation: "thread/realtime/sealInput",
})
})
}
/// Highest durable observation time outside the rollout, if the store has one.
fn realtime_frame_clock_floor(
&self,
_thread_id: ThreadId,
) -> ThreadStoreFuture<'_, Option<i64>> {
Box::pin(async { Ok(None) })
}
/// Whether this live thread writes canonical realtime frames and handoff receipts.
fn realtime_frames_enabled(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> {
Box::pin(async { Ok(false) })