diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 70e34a5c18..01022d6ecc 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -6161,7 +6161,7 @@ "type": "object" }, "ThreadRealtimeFrameChannels": { - "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.", + "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.", "properties": { "voice_transcript": { "$ref": "#/definitions/ThreadRealtimeVoiceTranscript" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 3d138c25ae..157b1240b0 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -25675,7 +25675,7 @@ "type": "object" }, "ThreadRealtimeFrameChannels": { - "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.", + "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.", "properties": { "voice_transcript": { "$ref": "#/definitions/v2/ThreadRealtimeVoiceTranscript" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index eed0e8ef7f..dcf74c89f2 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -23371,7 +23371,7 @@ "type": "object" }, "ThreadRealtimeFrameChannels": { - "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.", + "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.", "properties": { "voice_transcript": { "$ref": "#/definitions/ThreadRealtimeVoiceTranscript" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json index 8edd31598e..b9e2964407 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json @@ -61,7 +61,7 @@ ] }, "ThreadRealtimeFrameChannels": { - "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.", + "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.", "properties": { "voice_transcript": { "$ref": "#/definitions/ThreadRealtimeVoiceTranscript" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json index 59839f61b3..fa2865b1d3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json @@ -61,7 +61,7 @@ ] }, "ThreadRealtimeFrameChannels": { - "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.", + "description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. Currently, `voice_transcript` is required.", "properties": { "voice_transcript": { "$ref": "#/definitions/ThreadRealtimeVoiceTranscript" diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 12935c308c..770bc6ca3e 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 95ea9c74ed..1a6e375272 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeFrameChannels.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeFrameChannels.ts index f22b1c393e..f7caeff11f 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeFrameChannels.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeFrameChannels.ts @@ -5,6 +5,6 @@ import type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscr /** * A frame holds one or more payloads, at most one per channel. They share one ID, - * observation time, and seal. V1 requires its only channel, `voice_transcript`. + * observation time, and seal. Currently, `voice_transcript` is required. */ export type ThreadRealtimeFrameChannels = { voice_transcript: ThreadRealtimeVoiceTranscript, }; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 5234e039c7..2fdc0ca149 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -122,7 +122,7 @@ pub enum ThreadRealtimeSessionOutcome { } /// A frame holds one or more payloads, at most one per channel. They share one ID, -/// observation time, and seal. V1 requires its only channel, `voice_transcript`. +/// observation time, and seal. Currently, `voice_transcript` is required. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[ts(export_to = "v2/")] pub struct ThreadRealtimeFrameChannels { @@ -209,7 +209,7 @@ impl ThreadRealtimeItem { } }, }, - CoreRealtimeItemContent::RealtimeSessionClosed { outcome } => { + CoreRealtimeItemContent::RealtimeSessionClosed { outcome, .. } => { ThreadRealtimeItemContent::RealtimeSessionClosed { outcome: match outcome { CoreRealtimeSessionOutcome::Ended => ThreadRealtimeSessionOutcome::Ended, diff --git a/codex-rs/app-server/src/request_processors/config_processor.rs b/codex-rs/app-server/src/request_processors/config_processor.rs index 4a6349ba0f..1d80bfedb4 100644 --- a/codex-rs/app-server/src/request_processors/config_processor.rs +++ b/codex-rs/app-server/src/request_processors/config_processor.rs @@ -79,6 +79,7 @@ const SUPPORTED_EXPERIMENTAL_FEATURE_ENABLEMENT: &[&str] = &[ "mcp_shared_repl_cwd", "memories", "mentions_v2", + "realtime_frames_local", "remote_control", "remote_plugin", "tool_suggest", @@ -391,6 +392,9 @@ impl ConfigRequestProcessor { self.thread_manager.set_api_key_model_discovery_enabled( config.features.enabled(Feature::ApiKeyModelDiscovery), ); + self.thread_manager.set_realtime_frames_local_enabled( + config.features.enabled(Feature::RealtimeFramesLocal), + ); if should_start_background_rollout_migration && config.features.enabled(feature) { self.thread_manager.start_background_rollout_migration(); } diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index b84d6ee7ef..d0fa2e9245 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -6,6 +6,8 @@ use app_test_support::create_command_execution_sse_response; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_sequence_unchecked; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::ExperimentalFeatureEnablementSetParams; +use codex_app_server_protocol::ExperimentalFeatureEnablementSetResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCError; @@ -706,6 +708,7 @@ async fn realtime_conversation_streams_timeline_items() -> Result<()> { .await?; mcp.initialize().await?; login_with_api_key(&mut mcp, "sk-test-key").await?; + enable_local_realtime_frames(&mut mcp).await?; let thread_request = mcp .send_thread_start_request_with_auto_env(ThreadStartParams { @@ -1250,6 +1253,7 @@ async fn realtime_timeline_splits_accepted_steering_and_persists_promoted_artifa ])]), ) .await?; + enable_local_realtime_frames(&mut harness.mcp).await?; let thread_request = harness .mcp .send_thread_start_request_with_auto_env(ThreadStartParams { @@ -4399,6 +4403,17 @@ async fn login_with_api_key(mcp: &mut TestAppServer, api_key: &str) -> Result<() Ok(()) } +async fn enable_local_realtime_frames(mcp: &mut TestAppServer) -> Result<()> { + let request = mcp + .send_experimental_feature_enablement_set_request(ExperimentalFeatureEnablementSetParams { + enablement: BTreeMap::from([("realtime_frames_local".to_string(), true)]), + }) + .await?; + let _: ExperimentalFeatureEnablementSetResponse = + timeout(DEFAULT_TIMEOUT, mcp.read_response(request)).await??; + Ok(()) +} + async fn wait_for_started_command_execution( mcp: &mut TestAppServer, ) -> Result { diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 6745a09472..3d1f72dc14 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1286,6 +1286,9 @@ "realtime_conversation": { "type": "boolean" }, + "realtime_frames_local": { + "type": "boolean" + }, "reasoning_effort_override": { "type": "boolean" }, @@ -8598,6 +8601,9 @@ "realtime_conversation": { "type": "boolean" }, + "realtime_frames_local": { + "type": "boolean" + }, "reasoning_effort_override": { "type": "boolean" }, diff --git a/codex-rs/core/src/realtime_history.rs b/codex-rs/core/src/realtime_history.rs index d85ae8702c..4178647aae 100644 --- a/codex-rs/core/src/realtime_history.rs +++ b/codex-rs/core/src/realtime_history.rs @@ -308,6 +308,7 @@ impl RealtimeHistoryState { .frames_enabled() .then(|| self.observation_clock.now_ms()), content: RealtimeItemContent::RealtimeSessionClosed { + final_frame_sequence: None, outcome: if self.failed { RealtimeSessionOutcome::Failed } else { diff --git a/codex-rs/core/src/session/realtime_history.rs b/codex-rs/core/src/session/realtime_history.rs index 33107be221..c39dfef8dd 100644 --- a/codex-rs/core/src/session/realtime_history.rs +++ b/codex-rs/core/src/session/realtime_history.rs @@ -23,7 +23,24 @@ impl Session { .realtime_frames_enabled(self.thread_id()) .await .map_err(std::io::Error::other)?; - history.lock().await.frame_enrollment = Some(enabled); + let floor = if enabled + && matches!( + self.history_initialization, + codex_history::HistoryInitialization::ColdResume + ) { + self.services + .thread_store + .realtime_frame_clock_floor(self.thread_id()) + .await + .map_err(std::io::Error::other)? + } else { + None + }; + let mut history = history.lock().await; + if let Some(floor) = floor { + history.observation_clock.observe(floor); + } + history.frame_enrollment = Some(enabled); Ok(enabled) } @@ -54,16 +71,15 @@ impl Session { } let live_thread = self.live_thread_for_persistence("append realtime history")?; let items = effects.items; - let records = items - .iter() - .cloned() - .map(RolloutItem::RealtimeItem) - .collect::>(); - live_thread.append_items(&records).await?; - // A completed notification acknowledges durable history, including on hosts - // whose append operation only admits work to an asynchronous writer. if effects.seal_frames { - live_thread.flush().await?; + live_thread.seal_realtime_items(items.clone()).await?; + } else { + let records = items + .iter() + .cloned() + .map(RolloutItem::RealtimeItem) + .collect::>(); + live_thread.append_items(&records).await?; } for item in items { if !matches!( diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 3d40069a1e..cc00dab5b0 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1034,6 +1034,18 @@ impl ThreadManager { Arc::clone(&self.state.thread_store) } + /// Updates local realtime enrollment after a runtime feature decision changes. + pub fn set_realtime_frames_local_enabled(&self, enabled: bool) { + if let Some(store) = self + .state + .thread_store + .as_any() + .downcast_ref::() + { + store.set_realtime_frames_local_enabled(enabled); + } + } + /// Starts the local rollout migration path after a runtime feature enablement. /// /// Startup config handles the initial launch in [`crate::ThreadStoreBuilder::build`]. This covers diff --git a/codex-rs/core/src/thread_store_builder.rs b/codex-rs/core/src/thread_store_builder.rs index fb10b8bb1b..47c2f1cce0 100644 --- a/codex-rs/core/src/thread_store_builder.rs +++ b/codex-rs/core/src/thread_store_builder.rs @@ -78,6 +78,9 @@ impl<'config> ThreadStoreBuilder<'config> { }) }), ); + store.set_realtime_frames_local_enabled( + config.features.enabled(Feature::RealtimeFramesLocal), + ); if has_state_db && background_migration_enabled { let startup_store = Arc::clone(&store); let codex_home = config.codex_home.to_path_buf(); diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index ed5b22cfc9..00996cce28 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -11,6 +11,7 @@ use codex_core::StartThreadOptions; use codex_core::TurnInputRequest; use codex_core::TurnInputSubmission; use codex_core::test_support::auth_manager_from_auth; +use codex_features::Feature; use codex_history::InitialHistory; use codex_history::ResponseItemEnvelope; use codex_history::RolloutItem; @@ -497,6 +498,10 @@ async fn conversation_records_history_without_an_event_observer( .with_config({ let url = realtime_server.uri().to_string(); move |config| { + config + .features + .enable(Feature::RealtimeFramesLocal) + .expect("enable local realtime frames"); config.experimental_realtime_ws_base_url = Some(url); config.realtime.version = RealtimeWsVersion::V2; } @@ -3947,6 +3952,10 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime(ephemeral: bool) -> .with_config({ let realtime_base_url = realtime_server.uri().to_string(); move |config| { + config + .features + .enable(Feature::RealtimeFramesLocal) + .expect("enable local realtime frames"); config.ephemeral = ephemeral; config.realtime.version = RealtimeWsVersion::V2; config.experimental_realtime_ws_base_url = Some(realtime_base_url); diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 724a796d94..154e4f6539 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -234,6 +234,8 @@ pub enum Feature { LocalThreadStoreSharedCompression, /// Migrate legacy local rollout files to paginated history in the background. BackgroundPaginatedRolloutMigration, + /// Enroll paginated local threads in canonical realtime frame history. + RealtimeFramesLocal, /// Enable the Chronicle sidecar for passive screen-context memories. Chronicle, /// Compress request bodies (zstd) when sending streaming requests to codex-backend. @@ -1505,6 +1507,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::RealtimeFramesLocal, + key: "realtime_frames_local", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::Chronicle, key: "chronicle", diff --git a/codex-rs/protocol/src/realtime.rs b/codex-rs/protocol/src/realtime.rs index fde949c529..81066b92fb 100644 --- a/codex-rs/protocol/src/realtime.rs +++ b/codex-rs/protocol/src/realtime.rs @@ -18,7 +18,7 @@ pub struct RealtimeItem { } /// A frame holds one or more payloads, at most one per channel. They share one ID, -/// observation time, and seal. V1 requires its only channel, `voice_transcript`. +/// observation time, and seal. Currently, `voice_transcript` is required. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] pub struct RealtimeFrameChannels { pub voice_transcript: RealtimeVoiceTranscript, @@ -51,6 +51,10 @@ pub enum RealtimeItemContent { }, RealtimeSessionClosed { outcome: RealtimeSessionOutcome, + /// Hosted frame-table sequence of the session's last frame (zero for a silent call). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "number | null", optional)] + final_frame_sequence: Option, }, /// Durable acceptance of the exact bounded input used by an agent turn. /// The enclosing item ID is the acceptance identity; source IDs never use diff --git a/codex-rs/thread-store/src/live_thread.rs b/codex-rs/thread-store/src/live_thread.rs index bb11518b5c..e5879042de 100644 --- a/codex-rs/thread-store/src/live_thread.rs +++ b/codex-rs/thread-store/src/live_thread.rs @@ -1,3 +1,4 @@ +use codex_rollout::ResponseItemEnvelope; use std::future::Future; use std::path::PathBuf; use std::sync::Arc; @@ -5,6 +6,7 @@ use std::sync::Arc; use codex_protocol::ThreadId; use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; +use codex_protocol::realtime::RealtimeItem; use codex_rollout::RolloutItem; use codex_rollout::RolloutPersistenceTelemetry; use codex_rollout::measure_and_filter_rollout_items; @@ -246,25 +248,7 @@ impl LiveThread { if items.is_empty() { return Ok(()); } - let update = self - .metadata_sync - .lock() - .await - .observe_appended_items(items.as_slice()); - if let Some(update) = update { - self.thread_store - .record_thread_metadata(UpdateThreadMetadataParams { - thread_id: self.thread_id, - patch: update.patch.clone(), - include_archived: true, - }) - .await?; - self.metadata_sync - .lock() - .await - .mark_pending_update_applied(&update); - } - Ok(()) + self.observe_appended_metadata(&items).await } /// Returns the filtered items if sampling already needed to collect them. @@ -309,6 +293,54 @@ impl LiveThread { self.apply_pending_metadata_update(update, context).await } + pub async fn seal_realtime_items(&self, items: Vec) -> ThreadStoreResult<()> { + if self.thread_store.uses_separate_realtime_frame_store() { + return self + .thread_store + .seal_realtime_items(self.thread_id, items) + .await; + } + let records: Vec<_> = items.into_iter().map(RolloutItem::RealtimeItem).collect(); + self.append_items(&records).await?; + self.flush().await + } + + pub async fn seal_realtime_input( + &self, + frames: Vec, + input: ResponseItemEnvelope, + ) -> ThreadStoreResult<()> { + let record = RolloutItem::ResponseItem(input.clone()); + self.thread_store + .seal_realtime_input(self.thread_id, frames, input) + .await?; + // The store owns the idempotent acceptance barrier. Apply append-derived + // metadata afterward so retrying a failed metadata update cannot append again. + self.observe_appended_metadata(&[record]).await + } + + async fn observe_appended_metadata(&self, items: &[RolloutItem]) -> ThreadStoreResult<()> { + let update = self + .metadata_sync + .lock() + .await + .observe_appended_items(items); + if let Some(update) = update { + self.thread_store + .record_thread_metadata(UpdateThreadMetadataParams { + thread_id: self.thread_id, + patch: update.patch.clone(), + include_archived: true, + }) + .await?; + self.metadata_sync + .lock() + .await + .mark_pending_update_applied(&update); + } + Ok(()) + } + pub async fn flush(&self) -> ThreadStoreResult<()> { self.thread_store.flush_thread(self.thread_id).await?; self.flush_pending_metadata_update_for_existing_history() diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index ed0d66a353..2b2bbed1c1 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -199,6 +199,79 @@ pub(super) async fn append_items( .await } +pub(super) async fn seal_realtime_input( + store: &LocalThreadStore, + thread_id: ThreadId, + frames: Vec, + input: codex_rollout::ResponseItemEnvelope, +) -> ThreadStoreResult<()> { + let id = input + .realtime_handoff_item() + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: "realtime input is missing handoff metadata".to_owned(), + })? + .id; + let records = frames + .into_iter() + .map(RolloutItem::RealtimeItem) + .chain(std::iter::once(RolloutItem::ResponseItem(input))) + .collect(); + seal_realtime_records(store, thread_id, id, records).await +} + +/// An uncertain flush keeps ownership of the already queued model-facing seal. +/// Retrying confirms that batch instead of appending it again. +async fn seal_realtime_records( + store: &LocalThreadStore, + thread_id: ThreadId, + id: String, + items: Vec, +) -> ThreadStoreResult<()> { + let mut cursor = store.live_writer_locks.lock(thread_id).await; + let (recorder, rollout_id, history_mode, seals) = { + let entries = store.live_recorders.lock().await; + let entry = entries + .get(&thread_id) + .ok_or(ThreadStoreError::ThreadNotFound { thread_id })?; + ( + entry.recorder.clone(), + entry.rollout_id, + entry.history_mode, + entry.realtime_seals.clone(), + ) + }; + let mut seals = seals.lock_owned().await; + if let Some(pending) = seals.pending.clone() { + recorder.flush().await.map_err(thread_store_io_error)?; + seals.pending = None; + seals.committed.insert(pending); + } + if !seals.committed.contains(&id) { + let mut items = into_persisted_rollout_items(items, history_mode); + let next = codex_rollout::stamp_item_ordinals(*cursor, &mut items); + recorder + .record_canonical_items(&items) + .await + .map_err(thread_store_io_error)?; + *cursor = next; + seals.pending = Some(id.clone()); + recorder.flush().await.map_err(thread_store_io_error)?; + seals.pending = None; + seals.committed.insert(id); + } + if history_mode == ThreadHistoryMode::Paginated + && let Err(err) = super::thread_history_materialization::materialize_to_sqlite( + store, + rollout_id, + recorder.rollout_path(), + ) + .await + { + warn!("failed to project durable rollout for {thread_id}: {err}"); + } + sync_materialized_rollout_path(store, thread_id, recorder.rollout_path()).await +} + pub(super) async fn persist_thread( store: &LocalThreadStore, thread_id: ThreadId, diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index f4647d18a0..fea3c87613 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -38,6 +38,8 @@ mod pending_thread_metadata_tests; #[cfg(test)] #[path = "read_thread_tests.rs"] mod read_thread_tests; +#[cfg(all(test, unix))] +mod realtime_input_tests; #[cfg(test)] mod test_support; #[cfg(test)] @@ -46,15 +48,20 @@ mod timestamp_metadata_tests; use codex_protocol::ThreadId; use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::realtime::RealtimeItem; +use codex_rollout::ResponseItemEnvelope; use codex_rollout::RolloutItem; use codex_rollout::RolloutRecorder; use codex_rollout::StateDbHandle; use codex_rollout::WriterLockCoordinator; use codex_state::SqliteConfig; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::Entry; use std::path::PathBuf; use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use tokio::sync::Mutex; use tokio::sync::OnceCell; use tokio::sync::OwnedMutexGuard; @@ -158,6 +165,7 @@ pub struct LocalThreadStore { /// can use the other connections while SQLite's single writer is busy. thread_history_writer: Arc, thread_data_cleanup: Option>, + realtime_frames_local_enabled: Arc, #[cfg(test)] startup_migration_retry_gate: Option>, } @@ -176,6 +184,13 @@ struct LiveRecorderEntry { // so missing SQLite rows can still be seeded. history_mode: ThreadHistoryMode, writer_lock: WriterLockGuard, + realtime_seals: Arc>, +} + +#[derive(Default)] +struct RealtimeSeals { + pending: Option, + committed: HashSet, } #[derive(Default)] @@ -287,6 +302,7 @@ impl LocalThreadStore { thread_history_db: Arc::new(OnceCell::new()), thread_history_writer: Arc::new(Semaphore::new(/* permits */ 1)), thread_data_cleanup: None, + realtime_frames_local_enabled: Arc::new(AtomicBool::new(false)), #[cfg(test)] startup_migration_retry_gate: None, } @@ -303,6 +319,12 @@ impl LocalThreadStore { self } + /// Controls new realtime-frame enrollment without changing existing enrolled threads. + pub fn set_realtime_frames_local_enabled(&self, enabled: bool) { + self.realtime_frames_local_enabled + .store(enabled, Ordering::Relaxed); + } + /// Return the state DB handle used by local rollout writers. pub async fn state_db(&self) -> Option { self.state_db.clone() @@ -426,6 +448,7 @@ impl LocalThreadStore { rollout_id, history_mode, writer_lock, + realtime_seals: Arc::new(Mutex::new(RealtimeSeals::default())), }); Ok(()) } @@ -524,6 +547,17 @@ impl LocalThreadStore { } impl ThreadStore for LocalThreadStore { + fn seal_realtime_input( + &self, + thread_id: ThreadId, + frames: Vec, + input: ResponseItemEnvelope, + ) -> ThreadStoreFuture<'_, ()> { + Box::pin( + async move { live_writer::seal_realtime_input(self, thread_id, frames, input).await }, + ) + } + fn model_ids(&self) -> Option> { self.state_db .clone() @@ -806,8 +840,22 @@ impl ThreadStore for LocalThreadStore { Box::pin(async move { projects::delete_project(self, project_id).await }) } - fn realtime_frames_enabled(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> { - Box::pin(async { Ok(self.state_db.is_some()) }) + fn realtime_frames_enabled(&self, thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> { + Box::pin(async move { + if self.state_db.is_none() { + return Ok(false); + } + // Legacy rollout migration rewrites rolled-back history. Enroll only once + // the live thread is already paginated, so the realtime stream stays intact. + let (_, _, history_mode) = live_writer::live_writer_parts(self, thread_id).await?; + if history_mode != ThreadHistoryMode::Paginated { + return Ok(false); + } + if self.realtime_frames_local_enabled.load(Ordering::Relaxed) { + return Ok(true); + } + thread_history::has_canonical_realtime_session(self, thread_id).await + }) } fn supports_paginated_history_lists(&self) -> bool { @@ -833,6 +881,25 @@ impl ThreadStore for LocalThreadStore { Box::pin(LocalThreadStore::list_realtime(self, params)) } + fn load_realtime_session( + &self, + thread_id: ThreadId, + realtime_session_id: String, + ) -> ThreadStoreFuture<'_, Option>> { + Box::pin(thread_history::load_realtime_session( + self, + thread_id, + realtime_session_id, + )) + } + + fn load_realtime_session_head( + &self, + thread_id: ThreadId, + ) -> ThreadStoreFuture<'_, Option> { + Box::pin(thread_history::load_realtime_session_head(self, thread_id)) + } + fn search_threads( &self, params: SearchThreadsParams, @@ -963,6 +1030,56 @@ mod tests { use crate::local::test_support::write_session_file; use crate::local::test_support::write_session_file_with_history_mode; + #[tokio::test] + async fn realtime_frames_require_enabled_paginated_local_history() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let runtime = codex_state::StateRuntime::init( + config.sqlite.clone(), + config.default_model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let store = Arc::new(LocalThreadStore::new(config, Some(runtime))); + + for (history_mode, expected) in [ + (ThreadHistoryMode::Legacy, false), + (ThreadHistoryMode::Paginated, true), + ] { + let thread_id = ThreadId::default(); + let mut params = create_thread_params(thread_id); + params.history_mode = history_mode; + let live_thread = LiveThread::create(store.clone(), params) + .await + .expect("create live thread"); + assert!(!store.realtime_frames_enabled(thread_id).await.unwrap()); + store.set_realtime_frames_local_enabled(/*enabled*/ true); + assert_eq!( + store.realtime_frames_enabled(thread_id).await.unwrap(), + expected + ); + if expected { + live_thread + .append_items(&[RolloutItem::RealtimeItem(RealtimeItem { + id: "realtime-session-start:call-1".to_string(), + realtime_session_id: "call-1".to_string(), + observed_at_ms: Some(1), + content: + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted, + })]) + .await + .unwrap(); + live_thread.persist(PersistContext::Standard).await.unwrap(); + live_thread.flush().await.unwrap(); + } + store.set_realtime_frames_local_enabled(/*enabled*/ false); + assert_eq!( + store.realtime_frames_enabled(thread_id).await.unwrap(), + expected + ); + } + } + #[tokio::test] async fn live_writer_lifecycle_writes_and_closes() { let home = TempDir::new().expect("temp dir"); diff --git a/codex-rs/thread-store/src/local/realtime_input_tests.rs b/codex-rs/thread-store/src/local/realtime_input_tests.rs new file mode 100644 index 0000000000..88794d89c7 --- /dev/null +++ b/codex-rs/thread-store/src/local/realtime_input_tests.rs @@ -0,0 +1,96 @@ +use codex_protocol::ResponseItemId; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_rollout::ResponseItemEnvelope; +use codex_rollout::RolloutItem; +use codex_rollout::RolloutRecorder; +use tempfile::TempDir; +use uuid::Uuid; + +use super::LocalThreadStore; +use super::test_support::test_config; +use super::test_support::write_session_file_with_history_mode; +use crate::ResumeThreadParams; +use crate::ThreadPersistenceMetadata; +use crate::ThreadStore; + +/// A failed directory sync happens after the file contains the accepted input. +/// Confirming that same seal must not append another model-facing input. +#[tokio::test] +async fn retry_after_uncertain_local_input_seal_does_not_append_again() { + let home = TempDir::new().unwrap(); + let uuid = Uuid::now_v7(); + let thread_id = codex_protocol::ThreadId::from_string(&uuid.to_string()).unwrap(); + let path = write_session_file_with_history_mode( + home.path(), + "2025-01-03T12-00-00", + uuid, + ThreadHistoryMode::Paginated, + ) + .unwrap(); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + store + .resume_thread(ResumeThreadParams { + history_revision: None, + thread_id, + rollout_path: Some(path.clone()), + history: None, + include_archived: true, + metadata: ThreadPersistenceMetadata { + cwd: Some(home.path().to_path_buf()), + model_provider: "test-provider".into(), + memory_mode: ThreadMemoryMode::Enabled, + }, + }) + .await + .unwrap(); + let mut response = ResponseItem::Message { + encrypted_content: None, + status: None, + id: Some(ResponseItemId::from_server("msg_voice".into())), + role: "user".into(), + content: vec![ContentItem::InputText { + text: "voice input".into(), + }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + response.set_turn_id_if_missing("turn-1"); + let input = ResponseItemEnvelope { + item: response, + metadata: Some( + serde_json::from_value(serde_json::json!({ + "realtime_handoff": { + "event_id": "event-1", + "realtime_session_id": "call-1", + "observed_at_ms": 1, + "handoff_id": "handoff-1", + "mode": "start", + "source_event_ids": [] + } + })) + .unwrap(), + ), + }; + let parent = path.parent().unwrap(); + let moved = home.path().join("moved"); + std::fs::rename(parent, &moved).unwrap(); + assert!( + store + .seal_realtime_input(thread_id, vec![], input.clone()) + .await + .is_err() + ); + std::fs::rename(&moved, parent).unwrap(); + store + .seal_realtime_input(thread_id, vec![], input) + .await + .unwrap(); + let (items, _, _) = RolloutRecorder::load_rollout_items(&path).await.unwrap(); + let accepted = items.iter().filter(|item| { + matches!(item, RolloutItem::ResponseItem(envelope) if envelope.realtime_handoff_item().is_some()) + }); + assert_eq!(accepted.count(), 1); +} diff --git a/codex-rs/thread-store/src/local/thread_history.rs b/codex-rs/thread-store/src/local/thread_history.rs index cff15327b0..5d4884be58 100644 --- a/codex-rs/thread-store/src/local/thread_history.rs +++ b/codex-rs/thread-store/src/local/thread_history.rs @@ -21,7 +21,10 @@ mod turn_lookup; pub(super) use read::list_items; pub(super) use read::list_turns; pub(super) use realtime::list_timeline; +pub(super) use realtime_stream::has_canonical_realtime_session; pub(super) use realtime_stream::list_realtime; +pub(super) use realtime_stream::load_realtime_session; +pub(super) use realtime_stream::load_realtime_session_head; pub(super) use search::search_thread_occurrences; pub(super) use turn_lookup::find_source_turn; pub(super) use turn_lookup::find_visible_turn; diff --git a/codex-rs/thread-store/src/local/thread_history/read_tests.rs b/codex-rs/thread-store/src/local/thread_history/read_tests.rs index b01646791f..6d33a8c8d3 100644 --- a/codex-rs/thread-store/src/local/thread_history/read_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history/read_tests.rs @@ -644,6 +644,7 @@ async fn timeline_interleaves_items_and_restores_page_boundary_session_state() { realtime_session_id: "voice-1".to_string(), observed_at_ms: None, content: RealtimeItemContent::RealtimeSessionClosed { + final_frame_sequence: None, outcome: RealtimeSessionOutcome::Ended, }, }, @@ -920,6 +921,7 @@ async fn timeline_page_state_excludes_realtime_events_after_fork_cutoff() { realtime_session_id: "voice".to_string(), observed_at_ms: None, content: RealtimeItemContent::RealtimeSessionClosed { + final_frame_sequence: None, outcome: RealtimeSessionOutcome::Ended, }, }, diff --git a/codex-rs/thread-store/src/local/thread_history/realtime.rs b/codex-rs/thread-store/src/local/thread_history/realtime.rs index f7e0907689..13f78f715a 100644 --- a/codex-rs/thread-store/src/local/thread_history/realtime.rs +++ b/codex-rs/thread-store/src/local/thread_history/realtime.rs @@ -130,7 +130,7 @@ ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7 SELECT rollout_ordinal, 2 AS kind, item_id AS id, NULL AS turn_id, item_json FROM thread_realtime_items WHERE thread_id = ?1 AND rollout_ordinal >= ?2 - AND item_type NOT IN ('turn_input_accepted', 'session_end_input_saved') + AND item_type NOT IN ('turn_input_accepted', 'session_input_saved') AND rollout_ordinal < ?3 AND rollout_ordinal <= ?4 AND (rollout_ordinal, 2, item_id) < (?4, ?5, ?6) ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7 diff --git a/codex-rs/thread-store/src/local/thread_history/realtime_stream.rs b/codex-rs/thread-store/src/local/thread_history/realtime_stream.rs index 76eaf76268..f71b8b8d0b 100644 --- a/codex-rs/thread-store/src/local/thread_history/realtime_stream.rs +++ b/codex-rs/thread-store/src/local/thread_history/realtime_stream.rs @@ -116,6 +116,149 @@ pub(in crate::local) async fn list_realtime( .await } +/// Resolve and project the complete durable prefix while the caller holds the writer lock. +async fn recovery_snapshot( + store: &LocalThreadStore, + thread_id: ThreadId, +) -> ThreadStoreResult> { + let Some(resolved) = + super::super::thread_rollout_resolver::resolve_current_including_archived(store, thread_id) + .await? + else { + // A fresh paginated thread has a live writer before its rollout or SQLite row exists. + return Ok(None); + }; + validate_thread_for_paginated_reads( + store, + thread_id, + /*include_archived*/ true, + "load_realtime_session", + ) + .await?; + let meta = codex_rollout::read_session_meta_line(&resolved.path) + .await + .map_err(thread_history_error)? + .meta; + let first = meta + .subagent_history_start_ordinal + .or_else(|| meta.history_base.map(|base| base.end_ordinal_exclusive)); + if meta.forked_from_id.is_some() && first.is_none() { + return Err(ThreadStoreError::Unsupported { + operation: "load_realtime_session", + }); + } + super::super::thread_history_materialization::materialize_to_sqlite( + store, + resolved.rollout_id, + &resolved.path, + ) + .await?; + let end = super::projection_state(store, resolved.rollout_id) + .await? + .map_or(0, |state| state.next_ordinal); + Ok(Some((resolved.rollout_id, first.unwrap_or(0), end))) +} + +/// Recover a logical call from the same authoritative durable-prefix projection as list_realtime. +pub(in crate::local) async fn load_realtime_session( + store: &LocalThreadStore, + thread_id: ThreadId, + session_id: String, +) -> ThreadStoreResult>> { + let _writer_guard = store.live_writer_locks.lock(thread_id).await; + let Some((rollout_id, first, end)) = recovery_snapshot(store, thread_id).await? else { + return Ok(None); + }; + let rows = sqlx::query_scalar::<_, String>( + r#" +SELECT item_json +FROM thread_realtime_items +WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ? + AND json_extract(item_json, '$.realtime_session_id') = ? +ORDER BY rollout_ordinal +"#, + ) + .bind(rollout_id.to_string()) + .bind(sqlite_integer(first, "first realtime position")?) + .bind(sqlite_integer(end, "realtime snapshot end")?) + .bind(session_id) + .fetch_all(store.thread_history_db().await?) + .await + .map_err(thread_history_error)?; + let items: Vec = rows + .into_iter() + .map(|row| serde_json::from_str(&row).map_err(thread_history_error)) + .collect::>()?; + Ok(items + .iter() + .any(|item| { + matches!( + item.content, + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted + ) + }) + .then_some(items)) +} + +pub(in crate::local) async fn load_realtime_session_head( + store: &LocalThreadStore, + thread_id: ThreadId, +) -> ThreadStoreResult> { + let _writer_guard = store.live_writer_locks.lock(thread_id).await; + let Some((rollout_id, first, end)) = recovery_snapshot(store, thread_id).await? else { + return Ok(None); + }; + sqlx::query_scalar::<_, String>( + r#" +SELECT json_extract(item_json, '$.realtime_session_id') +FROM thread_realtime_items +WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ? + AND item_type = 'realtime_session_started' +ORDER BY rollout_ordinal DESC +LIMIT 1 +"#, + ) + .bind(rollout_id.to_string()) + .bind(sqlite_integer(first, "first realtime position")?) + .bind(sqlite_integer(end, "realtime snapshot end")?) + .fetch_optional(store.thread_history_db().await?) + .await + .map_err(thread_history_error) +} + +/// A canonical start marker makes enrollment sticky when the rollout flag is disabled. +pub(in crate::local) async fn has_canonical_realtime_session( + store: &LocalThreadStore, + thread_id: ThreadId, +) -> ThreadStoreResult { + let _writer_guard = store.live_writer_locks.lock(thread_id).await; + let snapshot = match recovery_snapshot(store, thread_id).await { + Err(ThreadStoreError::Unsupported { + operation: "load_realtime_session", + }) => return Ok(false), + result => result?, + }; + let Some((rollout_id, first, end)) = snapshot else { + return Ok(false); + }; + sqlx::query_scalar::<_, bool>( + r#" +SELECT EXISTS ( + SELECT 1 FROM thread_realtime_items + WHERE thread_id = ? AND rollout_ordinal >= ? AND rollout_ordinal < ? + AND item_type = 'realtime_session_started' + AND json_extract(item_json, '$.observed_at_ms') IS NOT NULL +) +"#, + ) + .bind(rollout_id.to_string()) + .bind(sqlite_integer(first, "first realtime position")?) + .bind(sqlite_integer(end, "realtime snapshot end")?) + .fetch_one(store.thread_history_db().await?) + .await + .map_err(thread_history_error) +} + async fn read_page( pool: &sqlx::SqlitePool, params: ListRealtimeParams, diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs index c3dedf7411..7612fc930e 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs @@ -701,6 +701,7 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() { realtime_session_id: "voice".to_string(), observed_at_ms: None, content: RealtimeItemContent::RealtimeSessionClosed { + final_frame_sequence: None, outcome: RealtimeSessionOutcome::Ended, }, }), diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 9a6990e824..8b511d7b92 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -1,5 +1,7 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::realtime::RealtimeItem; +use codex_rollout::ResponseItemEnvelope; use codex_rollout::RolloutItem; use futures::future::BoxFuture; use std::any::Any; @@ -286,6 +288,67 @@ pub trait ThreadStore: Any + Send + Sync { params: LoadThreadHistoryParams, ) -> ThreadStoreFuture<'_, StoredThreadHistory>; + /// Loads one logical call from authoritative persisted history. Separated-frame stores + /// must override this method to include their frame table; a lagging projection must fail + /// rather than report that the session is absent. + fn load_realtime_session( + &self, + thread_id: ThreadId, + realtime_session_id: String, + ) -> ThreadStoreFuture<'_, Option>> { + Box::pin(async move { + let history = self + .load_history(LoadThreadHistoryParams { + thread_id, + include_archived: true, + }) + .await?; + let items: Vec<_> = history + .items + .iter() + .filter_map(|item| item.realtime_item().map(std::borrow::Cow::into_owned)) + .filter(|item| item.realtime_session_id == realtime_session_id) + .collect(); + Ok(items + .iter() + .any(|item| { + matches!( + item.content, + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted + ) + }) + .then_some(items)) + }) + } + + /// Returns the latest logical session ID from authoritative start records, including a + /// closed session. Used only when a newly requested ID has no persisted session yet. + fn load_realtime_session_head( + &self, + thread_id: ThreadId, + ) -> ThreadStoreFuture<'_, Option> { + Box::pin(async move { + let history = self + .load_history(LoadThreadHistoryParams { + thread_id, + include_archived: true, + }) + .await?; + Ok(history + .items + .iter() + .rev() + .filter_map(RolloutItem::realtime_item) + .find_map(|item| { + matches!( + item.content, + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted + ) + .then(|| item.realtime_session_id.clone()) + })) + }) + } + /// Captures a complete tail, optionally reading originals after a saved position. /// Local storage uses one reader for the cutoff and records; compressed rollouts /// are decoded once per call. It supports append-only paginated roots; @@ -558,6 +621,50 @@ pub trait ThreadStore: Any + Send + Sync { }) } + /// Hosted frame stores can acknowledge a seal independently of ordinary rollout writes. + fn uses_separate_realtime_frame_store(&self) -> bool { + false + } + + /// Seal in input order. A durable receipt implies that every preceding source + /// frame is durable. Hosted stores may commit frames outside the rollout. + fn seal_realtime_items( + &self, + thread_id: ThreadId, + items: Vec, + ) -> ThreadStoreFuture<'_, ()> { + Box::pin(async move { + let items = items.into_iter().map(RolloutItem::RealtimeItem).collect(); + self.append_items(AppendThreadItemsParams { thread_id, items }) + .await?; + self.flush_thread(thread_id).await + }) + } + + /// Commit an accepted model input after its realtime source frames are durable. + /// Implementations must safely accept retries of the same input after an uncertain + /// append or flush outcome without appending the model input more than once. + fn seal_realtime_input( + &self, + _thread_id: ThreadId, + _frames: Vec, + _input: ResponseItemEnvelope, + ) -> ThreadStoreFuture<'_, ()> { + Box::pin(async { + Err(ThreadStoreError::Unsupported { + operation: "thread/realtime/sealInput", + }) + }) + } + + /// Highest durable observation time outside the rollout, if the store has one. + fn realtime_frame_clock_floor( + &self, + _thread_id: ThreadId, + ) -> ThreadStoreFuture<'_, Option> { + Box::pin(async { Ok(None) }) + } + /// Whether this live thread writes canonical realtime frames and handoff receipts. fn realtime_frames_enabled(&self, _thread_id: ThreadId) -> ThreadStoreFuture<'_, bool> { Box::pin(async { Ok(false) })