Persist timestamped realtime frames before acknowledging completion (#61473)

## What changed

- Record voice transcripts as `frame` items with a `voice_transcript` channel for eligible threads, while retaining legacy transcript support and the existing app-server live and mixed timeline representation.
- Assign first-observed timestamps that never move backward, retain them through completion, and restore the clock floor from history and compaction checkpoints.
- Flush realtime records before completion notifications, including file and initial directory durability barriers for local rollouts.
- Generate a distinct default session ID for each new call, use the call ID when attaching to an existing call, and derive opening record IDs from session IDs.
- Add protocol types for turn-input acceptance and session-end input receipts, excluding those receipts from live notifications and mixed timelines. Update generated schemas and SDK types.

## Testing

Add coverage for frame serialization and required voice payloads, clock rollback and checkpoint restoration, durability barrier failures and retries, per-call session IDs, and persisted transcript availability immediately after completion.

GitOrigin-RevId: 3f6632b517d563861aee085b723475b8945c391c
This commit is contained in:
Zahan Malkani
2026-11-14 06:14:36 +00:00
committed by copyberry
parent 336b19ab26
commit 5943d7acdf
45 changed files with 1568 additions and 95 deletions
@@ -6141,6 +6141,25 @@
],
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
}
},
"required": [
"voice_transcript"
],
"type": "object"
},
"ThreadRealtimeHandoffMode": {
"enum": [
"start",
"steer"
],
"type": "string"
},
"ThreadRealtimeItem": {
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
"oneOf": [
@@ -6160,6 +6179,26 @@
"title": "RealtimeSessionStartedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"channels": {
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
},
"type": {
"enum": [
"frame"
],
"title": "FrameThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"channels",
"type"
],
"title": "FrameThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"role": {
@@ -6231,12 +6270,94 @@
],
"title": "RealtimeSessionClosedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"handoff_id": {
"type": "string"
},
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"mode": {
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"turn_id": {
"type": "string"
},
"type": {
"enum": [
"turnInputAccepted"
],
"title": "TurnInputAcceptedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"handoff_id",
"input",
"input_item_id",
"mode",
"source_event_ids",
"turn_id",
"type"
],
"title": "TurnInputAcceptedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"enum": [
"sessionEndInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
},
"observedAtMs": {
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
"format": "int64",
"type": [
"integer",
"null"
]
},
"realtimeSessionId": {
"type": "string"
}
@@ -6423,6 +6544,21 @@
],
"type": "string"
},
"ThreadRealtimeVoiceTranscript": {
"properties": {
"role": {
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
},
"text": {
"type": "string"
}
},
"required": [
"role",
"text"
],
"type": "object"
},
"ThreadRevertedNotification": {
"properties": {
"threadId": {
@@ -25579,6 +25579,25 @@
"title": "ThreadRealtimeErrorNotification",
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/v2/ThreadRealtimeVoiceTranscript"
}
},
"required": [
"voice_transcript"
],
"type": "object"
},
"ThreadRealtimeHandoffMode": {
"enum": [
"start",
"steer"
],
"type": "string"
},
"ThreadRealtimeInitialItem": {
"description": "EXPERIMENTAL - role-bearing text item included when a realtime V3 session starts.",
"properties": {
@@ -25614,6 +25633,26 @@
"title": "RealtimeSessionStartedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"channels": {
"$ref": "#/definitions/v2/ThreadRealtimeFrameChannels"
},
"type": {
"enum": [
"frame"
],
"title": "FrameThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"channels",
"type"
],
"title": "FrameThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"role": {
@@ -25685,12 +25724,94 @@
],
"title": "RealtimeSessionClosedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"handoff_id": {
"type": "string"
},
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"mode": {
"$ref": "#/definitions/v2/ThreadRealtimeHandoffMode"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"turn_id": {
"type": "string"
},
"type": {
"enum": [
"turnInputAccepted"
],
"title": "TurnInputAcceptedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"handoff_id",
"input",
"input_item_id",
"mode",
"source_event_ids",
"turn_id",
"type"
],
"title": "TurnInputAcceptedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"enum": [
"sessionEndInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
},
"observedAtMs": {
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
"format": "int64",
"type": [
"integer",
"null"
]
},
"realtimeSessionId": {
"type": "string"
}
@@ -25958,6 +26079,21 @@
],
"type": "string"
},
"ThreadRealtimeVoiceTranscript": {
"properties": {
"role": {
"$ref": "#/definitions/v2/ThreadRealtimeTranscriptRole"
},
"text": {
"type": "string"
}
},
"required": [
"role",
"text"
],
"type": "object"
},
"ThreadResumeInitialTurnsPageParams": {
"properties": {
"itemsView": {
@@ -23275,6 +23275,25 @@
"title": "ThreadRealtimeErrorNotification",
"type": "object"
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
}
},
"required": [
"voice_transcript"
],
"type": "object"
},
"ThreadRealtimeHandoffMode": {
"enum": [
"start",
"steer"
],
"type": "string"
},
"ThreadRealtimeInitialItem": {
"description": "EXPERIMENTAL - role-bearing text item included when a realtime V3 session starts.",
"properties": {
@@ -23310,6 +23329,26 @@
"title": "RealtimeSessionStartedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"channels": {
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
},
"type": {
"enum": [
"frame"
],
"title": "FrameThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"channels",
"type"
],
"title": "FrameThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"role": {
@@ -23381,12 +23420,94 @@
],
"title": "RealtimeSessionClosedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"handoff_id": {
"type": "string"
},
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"mode": {
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"turn_id": {
"type": "string"
},
"type": {
"enum": [
"turnInputAccepted"
],
"title": "TurnInputAcceptedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"handoff_id",
"input",
"input_item_id",
"mode",
"source_event_ids",
"turn_id",
"type"
],
"title": "TurnInputAcceptedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"enum": [
"sessionEndInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
},
"observedAtMs": {
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
"format": "int64",
"type": [
"integer",
"null"
]
},
"realtimeSessionId": {
"type": "string"
}
@@ -23654,6 +23775,21 @@
],
"type": "string"
},
"ThreadRealtimeVoiceTranscript": {
"properties": {
"role": {
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
},
"text": {
"type": "string"
}
},
"required": [
"role",
"text"
],
"type": "object"
},
"ThreadResumeInitialTurnsPageParams": {
"properties": {
"itemsView": {
@@ -60,6 +60,25 @@
}
]
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
}
},
"required": [
"voice_transcript"
],
"type": "object"
},
"ThreadRealtimeHandoffMode": {
"enum": [
"start",
"steer"
],
"type": "string"
},
"ThreadRealtimeItem": {
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
"oneOf": [
@@ -79,6 +98,26 @@
"title": "RealtimeSessionStartedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"channels": {
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
},
"type": {
"enum": [
"frame"
],
"title": "FrameThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"channels",
"type"
],
"title": "FrameThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"role": {
@@ -150,12 +189,94 @@
],
"title": "RealtimeSessionClosedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"handoff_id": {
"type": "string"
},
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"mode": {
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"turn_id": {
"type": "string"
},
"type": {
"enum": [
"turnInputAccepted"
],
"title": "TurnInputAcceptedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"handoff_id",
"input",
"input_item_id",
"mode",
"source_event_ids",
"turn_id",
"type"
],
"title": "TurnInputAcceptedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"enum": [
"sessionEndInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
},
"observedAtMs": {
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
"format": "int64",
"type": [
"integer",
"null"
]
},
"realtimeSessionId": {
"type": "string"
}
@@ -179,6 +300,21 @@
"assistant"
],
"type": "string"
},
"ThreadRealtimeVoiceTranscript": {
"properties": {
"role": {
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
},
"text": {
"type": "string"
}
},
"required": [
"role",
"text"
],
"type": "object"
}
},
"description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.",
@@ -60,6 +60,25 @@
}
]
},
"ThreadRealtimeFrameChannels": {
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
"properties": {
"voice_transcript": {
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
}
},
"required": [
"voice_transcript"
],
"type": "object"
},
"ThreadRealtimeHandoffMode": {
"enum": [
"start",
"steer"
],
"type": "string"
},
"ThreadRealtimeItem": {
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
"oneOf": [
@@ -79,6 +98,26 @@
"title": "RealtimeSessionStartedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"channels": {
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
},
"type": {
"enum": [
"frame"
],
"title": "FrameThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"channels",
"type"
],
"title": "FrameThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"role": {
@@ -150,12 +189,94 @@
],
"title": "RealtimeSessionClosedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"handoff_id": {
"type": "string"
},
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"mode": {
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"turn_id": {
"type": "string"
},
"type": {
"enum": [
"turnInputAccepted"
],
"title": "TurnInputAcceptedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"handoff_id",
"input",
"input_item_id",
"mode",
"source_event_ids",
"turn_id",
"type"
],
"title": "TurnInputAcceptedThreadRealtimeItem",
"type": "object"
},
{
"properties": {
"input": {
"type": "string"
},
"input_item_id": {
"type": "string"
},
"source_event_ids": {
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"enum": [
"sessionEndInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
},
"observedAtMs": {
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
"format": "int64",
"type": [
"integer",
"null"
]
},
"realtimeSessionId": {
"type": "string"
}
@@ -179,6 +300,21 @@
"assistant"
],
"type": "string"
},
"ThreadRealtimeVoiceTranscript": {
"properties": {
"role": {
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
},
"text": {
"type": "string"
}
},
"required": [
"role",
"text"
],
"type": "object"
}
},
"description": "EXPERIMENTAL - a realtime timeline item started before its content streams.",
@@ -0,0 +1,10 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscript";
/**
* A frame holds one or more payloads, at most one per channel. They share one ID,
* observation time, and seal. V1 requires its only channel, `voice_transcript`.
*/
export type ThreadRealtimeFrameChannels = { voice_transcript: ThreadRealtimeVoiceTranscript, };
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ThreadRealtimeHandoffMode = "start" | "steer";
@@ -2,10 +2,16 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
import type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
import type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
/**
* EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.
*/
export type ThreadRealtimeItem = { id: string, realtimeSessionId: string, } & ({ "type": "realtimeSessionStarted" } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, });
export type ThreadRealtimeItem = { id: string, realtimeSessionId: string,
/**
* Canonical observation time in Unix milliseconds for eligible voice calls.
*/
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionEndInputSaved", sourceEventIds: Array<string>, inputItemId: string, input: string, });
@@ -0,0 +1,6 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
export type ThreadRealtimeVoiceTranscript = { role: ThreadRealtimeTranscriptRole, text: string, };
@@ -585,6 +585,8 @@ export type { ThreadRealtimeAudioChunk } from "./ThreadRealtimeAudioChunk";
export type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNotification";
export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification";
export type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
export type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
export type { ThreadRealtimeInitialItem } from "./ThreadRealtimeInitialItem";
export type { ThreadRealtimeItem } from "./ThreadRealtimeItem";
export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification";
@@ -599,6 +601,7 @@ export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedN
export type { ThreadRealtimeTranscriptDeltaNotification } from "./ThreadRealtimeTranscriptDeltaNotification";
export type { ThreadRealtimeTranscriptDoneNotification } from "./ThreadRealtimeTranscriptDoneNotification";
export type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
export type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscript";
export type { ThreadResumeInitialTurnsPageParams } from "./ThreadResumeInitialTurnsPageParams";
export type { ThreadResumeParams } from "./ThreadResumeParams";
export type { ThreadResumeResponse } from "./ThreadResumeResponse";
@@ -8,6 +8,7 @@ use codex_protocol::protocol::RealtimeOutputModality;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_protocol::realtime::BemItemPresentation as CoreBemItemPresentation;
use codex_protocol::realtime::RealtimeHandoffMode as CoreRealtimeHandoffMode;
use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
@@ -24,6 +25,10 @@ use std::collections::BTreeMap;
pub struct ThreadRealtimeItem {
pub id: String,
pub realtime_session_id: String,
/// Canonical observation time in Unix milliseconds for eligible voice calls.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(type = "number", optional)]
pub observed_at_ms: Option<i64>,
#[serde(flatten)]
pub content: ThreadRealtimeItemContent,
}
@@ -38,6 +43,9 @@ pub struct ThreadRealtimeItem {
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeItemContent {
RealtimeSessionStarted,
Frame {
channels: ThreadRealtimeFrameChannels,
},
TranscriptSegment {
role: ThreadRealtimeTranscriptRole,
text: String,
@@ -50,6 +58,27 @@ pub enum ThreadRealtimeItemContent {
RealtimeSessionClosed {
outcome: ThreadRealtimeSessionOutcome,
},
TurnInputAccepted {
handoff_id: String,
turn_id: String,
mode: ThreadRealtimeHandoffMode,
source_event_ids: Vec<String>,
input_item_id: String,
input: String,
},
SessionEndInputSaved {
source_event_ids: Vec<String>,
input_item_id: String,
input: String,
},
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeHandoffMode {
Start,
Steer,
}
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
@@ -82,11 +111,52 @@ pub enum ThreadRealtimeSessionOutcome {
Failed,
}
/// A frame holds one or more payloads, at most one per channel. They share one ID,
/// observation time, and seal. V1 requires its only channel, `voice_transcript`.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeFrameChannels {
pub voice_transcript: ThreadRealtimeVoiceTranscript,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeVoiceTranscript {
pub role: ThreadRealtimeTranscriptRole,
pub text: String,
}
impl From<CoreRealtimeTranscriptRole> for ThreadRealtimeTranscriptRole {
fn from(role: CoreRealtimeTranscriptRole) -> Self {
match role {
CoreRealtimeTranscriptRole::User => Self::User,
CoreRealtimeTranscriptRole::Assistant => Self::Assistant,
}
}
}
// The existing live and mixed timeline APIs present speech as transcriptSegment.
impl From<CoreRealtimeItem> for ThreadRealtimeItem {
fn from(item: CoreRealtimeItem) -> Self {
let mut item = Self::from_persisted(item);
if let ThreadRealtimeItemContent::Frame { channels } = item.content {
let transcript = channels.voice_transcript;
item.content = ThreadRealtimeItemContent::TranscriptSegment {
role: transcript.role,
text: transcript.text,
};
}
item
}
}
impl ThreadRealtimeItem {
/// Preserve the canonical shape of a sealed record on the independent read API.
pub fn from_persisted(item: CoreRealtimeItem) -> Self {
let CoreRealtimeItem {
id,
realtime_session_id,
observed_at_ms,
content,
} = item;
let content = match content {
@@ -95,15 +165,21 @@ impl From<CoreRealtimeItem> for ThreadRealtimeItem {
}
CoreRealtimeItemContent::TranscriptSegment { role, text } => {
ThreadRealtimeItemContent::TranscriptSegment {
role: match role {
CoreRealtimeTranscriptRole::User => ThreadRealtimeTranscriptRole::User,
CoreRealtimeTranscriptRole::Assistant => {
ThreadRealtimeTranscriptRole::Assistant
}
},
role: role.into(),
text,
}
}
CoreRealtimeItemContent::Frame { channels } => {
let transcript = channels.voice_transcript;
ThreadRealtimeItemContent::Frame {
channels: ThreadRealtimeFrameChannels {
voice_transcript: ThreadRealtimeVoiceTranscript {
role: transcript.role.into(),
text: transcript.text,
},
},
}
}
CoreRealtimeItemContent::BemItemPromoted {
turn_id,
item_id,
@@ -131,10 +207,38 @@ impl From<CoreRealtimeItem> for ThreadRealtimeItem {
},
}
}
CoreRealtimeItemContent::TurnInputAccepted {
handoff_id,
turn_id,
mode,
source_event_ids,
input_item_id,
input,
} => ThreadRealtimeItemContent::TurnInputAccepted {
handoff_id,
turn_id,
mode: match mode {
CoreRealtimeHandoffMode::Start => ThreadRealtimeHandoffMode::Start,
CoreRealtimeHandoffMode::Steer => ThreadRealtimeHandoffMode::Steer,
},
source_event_ids,
input_item_id,
input,
},
CoreRealtimeItemContent::SessionEndInputSaved {
source_event_ids,
input_item_id,
input,
} => ThreadRealtimeItemContent::SessionEndInputSaved {
source_event_ids,
input_item_id,
input,
},
};
Self {
id,
realtime_session_id,
observed_at_ms,
content,
}
}
@@ -5660,3 +5660,13 @@ fn feedback_paths_preserve_native_wire_spelling_before_host_validation() {
);
}
}
#[test]
fn an_undated_live_voice_item_preserves_its_wire_shape() {
let wire = serde_json::json!({
"id": "speech", "realtimeSessionId": "voice", "type": "transcriptSegment",
"role": "assistant", "text": "hello",
});
let item: ThreadRealtimeItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}