Persist timestamped realtime frames before acknowledging completion (#61473)

## What changed

- Record voice transcripts as `frame` items with a `voice_transcript` channel for eligible threads, while retaining legacy transcript support and the existing app-server live and mixed timeline representation.
- Assign first-observed timestamps that never move backward, retain them through completion, and restore the clock floor from history and compaction checkpoints.
- Flush realtime records before completion notifications, including file and initial directory durability barriers for local rollouts.
- Generate a distinct default session ID for each new call, use the call ID when attaching to an existing call, and derive opening record IDs from session IDs.
- Add protocol types for turn-input acceptance and session-end input receipts, excluding those receipts from live notifications and mixed timelines. Update generated schemas and SDK types.

## Testing

Add coverage for frame serialization and required voice payloads, clock rollback and checkpoint restoration, durability barrier failures and retries, per-call session IDs, and persisted transcript availability immediately after completion.

GitOrigin-RevId: 3f6632b517d563861aee085b723475b8945c391c
This commit is contained in:
Zahan Malkani
2026-11-14 06:14:36 +00:00
committed by copyberry
parent 336b19ab26
commit 5943d7acdf
45 changed files with 1568 additions and 95 deletions
+18 -3
View File
@@ -607,6 +607,7 @@ impl Session {
thread_id: self.thread_id(),
tools: state.persisted_dynamic_tools.to_vec(),
})),
observation_clock_floor_ms: None,
multi_agent_version: self.multi_agent_version(),
last_started_turn_id: state.last_started_turn_id.clone(),
turn_attribution: state.turn_attribution.clone(),
@@ -1970,6 +1971,13 @@ impl Session {
turn_context: &Arc<TurnContext>,
rollout_items: &[RolloutItem],
) -> Option<PreviousTurnSettings> {
if let Some(history) = &self.realtime_history {
history
.lock()
.await
.observation_clock
.restore(rollout_items);
}
let rollout_reconstruction::RolloutReconstruction {
mut history,
rejected_audio_file_ids,
@@ -2953,8 +2961,8 @@ impl Session {
}
pub(crate) async fn send_event_raw(&self, event: Event) {
self.send_event_raw_with_persistence(event, /*persist*/ true)
.await;
// Keep event persistence out of callers' stack frames.
Box::pin(self.send_event_raw_with_persistence(event, /*persist*/ true)).await;
}
/// Delivers an event without creating a local rollout for a thread that has not materialized.
@@ -4879,6 +4887,10 @@ impl Session {
// Wait for accepted updates to finish persisting, then keep later updates from
// overtaking the current settings snapshot while its checkpoint is written.
let _settings_guard = thread_settings::acquire_persistence_lock(self).await;
let observation_clock_floor_ms = match &self.realtime_history {
Some(history) => history.lock().await.observation_clock.floor_ms(),
None => None,
};
let (compacted_item, world_state_baseline, started_item) = {
let mut state = self.state.lock().await;
let complete_replacement =
@@ -4958,7 +4970,10 @@ impl Session {
state
.history
.set_world_state_baseline(world_state_baseline.clone());
let resume_metadata = self.compaction_resume_metadata(&state);
let resume_metadata = CompactionResumeMetadata {
observation_clock_floor_ms,
..self.compaction_resume_metadata(&state)
};
let compacted_item = CompactedItem {
message: metadata.message,
replacement_history: Some(replacement_history),
+37 -12
View File
@@ -8,6 +8,25 @@ use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::realtime::RealtimeItemContent;
impl Session {
/// Start and attach operations run sequentially in the submission loop.
/// Cache enrollment after a successful lookup so a failed start can retry.
pub(crate) async fn initialize_realtime_history(&self) -> std::io::Result<bool> {
let Some(history) = &self.realtime_history else {
return Ok(false);
};
if let Some(enabled) = history.lock().await.frame_enrollment {
return Ok(enabled);
}
let enabled = self
.services
.thread_store
.realtime_frames_enabled(self.thread_id())
.await
.map_err(std::io::Error::other)?;
history.lock().await.frame_enrollment = Some(enabled);
Ok(enabled)
}
pub(super) async fn send_realtime_history_effects(
&self,
submission_id: &str,
@@ -33,18 +52,24 @@ impl Session {
if effects.items.is_empty() {
return Ok(());
}
self.live_thread_for_persistence("append realtime history")?
.append_items(
&effects
.items
.iter()
.cloned()
.map(RolloutItem::RealtimeItem)
.collect::<Vec<_>>(),
)
.await?;
for item in effects.items {
if !matches!(&item.content, RealtimeItemContent::TranscriptSegment { .. }) {
let live_thread = self.live_thread_for_persistence("append realtime history")?;
let items = effects.items;
let records = items
.iter()
.cloned()
.map(RolloutItem::RealtimeItem)
.collect::<Vec<_>>();
live_thread.append_items(&records).await?;
// A completed notification acknowledges durable history, including on hosts
// whose append operation only admits work to an asynchronous writer.
if effects.seal_frames {
live_thread.flush().await?;
}
for item in items {
if !matches!(
&item.content,
RealtimeItemContent::Frame { .. } | RealtimeItemContent::TranscriptSegment { .. }
) {
self.deliver_event_raw(event(RealtimeEvent::HistoryItemStarted(item.clone())))
.await;
}
@@ -1645,6 +1645,7 @@ async fn bounded_replay_matches_full_replay_after_empty_turn_compactions(
next_item_ordinal: None,
rejected_audio_file_ids: Default::default(),
dynamic_tools: None,
observation_clock_floor_ms: None,
multi_agent_version: None,
last_started_turn_id: Some(format!("wake-{window_number}")),
turn_attribution: None,
+4 -3
View File
@@ -1283,7 +1283,8 @@ impl Session {
error!("failed to initialize thread persistence: {e:#}");
e
})?;
let session_result: anyhow::Result<Arc<Self>> = async {
// Keep the large initialization future out of callers' stack frames.
let session_result: anyhow::Result<Arc<Self>> = Box::pin(async {
if let InitialHistory::Resumed(resumed) = &mut initial_history
&& let Some(history) = resume_context
{
@@ -1994,7 +1995,7 @@ impl Session {
realtime_history: (session_configuration.history_mode
== ThreadHistoryMode::Paginated
&& services.live_thread.is_some())
.then(|| Mutex::new(Default::default())),
.then(|| Mutex::new(crate::realtime_history::RealtimeHistoryState::default())),
active_turn: Mutex::new(None),
async_hook_results,
input_queue: InputQueue::with_controller(
@@ -2143,7 +2144,7 @@ impl Session {
state.queue_pending_session_start_source(session_start_source);
}
Ok(sess)
}
})
.await;
match session_result {
Ok(sess) => {
+1
View File
@@ -6177,6 +6177,7 @@ async fn compaction_persists_resume_metadata_and_companion_records() {
thread_id: session.thread_id(),
tools: Vec::new(),
})),
observation_clock_floor_ms: None,
multi_agent_version: Some(MultiAgentVersion::V2),
last_started_turn_id: Some(turn_context.sub_id.clone()),
turn_attribution: Some(codex_history::TurnAttribution {