Persist timestamped realtime frames before acknowledging completion (#61473)

## What changed

- Record voice transcripts as `frame` items with a `voice_transcript` channel for eligible threads, while retaining legacy transcript support and the existing app-server live and mixed timeline representation.
- Assign first-observed timestamps that never move backward, retain them through completion, and restore the clock floor from history and compaction checkpoints.
- Flush realtime records before completion notifications, including file and initial directory durability barriers for local rollouts.
- Generate a distinct default session ID for each new call, use the call ID when attaching to an existing call, and derive opening record IDs from session IDs.
- Add protocol types for turn-input acceptance and session-end input receipts, excluding those receipts from live notifications and mixed timelines. Update generated schemas and SDK types.

## Testing

Add coverage for frame serialization and required voice payloads, clock rollback and checkpoint restoration, durability barrier failures and retries, per-call session IDs, and persisted transcript availability immediately after completion.

GitOrigin-RevId: 3f6632b517d563861aee085b723475b8945c391c
This commit is contained in:
Zahan Malkani
2026-11-14 06:14:36 +00:00
committed by copyberry
parent 336b19ab26
commit 5943d7acdf
45 changed files with 1568 additions and 95 deletions
+117 -1
View File
@@ -6423,6 +6423,11 @@ class ThreadRealtimeErrorNotification(BaseModel):
thread_id: Annotated[str, Field(alias="threadId")]
class ThreadRealtimeHandoffMode(Enum):
start = "start"
steer = "steer"
class ThreadRealtimeInitialItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6436,6 +6441,13 @@ class RealtimeSessionStartedThreadRealtimeItem(BaseModel):
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
type: Annotated[
Literal["realtimeSessionStarted"],
@@ -6448,6 +6460,13 @@ class BemItemPromotedThreadRealtimeItem(BaseModel):
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
item_id: str
presentation: ThreadRealtimeBemItemPresentation
@@ -6457,6 +6476,51 @@ class BemItemPromotedThreadRealtimeItem(BaseModel):
]
class TurnInputAcceptedThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
handoff_id: str
input: str
input_item_id: str
mode: ThreadRealtimeHandoffMode
source_event_ids: list[str]
turn_id: str
type: Annotated[
Literal["turnInputAccepted"], Field(title="TurnInputAcceptedThreadRealtimeItemType")
]
class SessionEndInputSavedThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
input: str
input_item_id: str
source_event_ids: list[str]
type: Annotated[
Literal["sessionEndInputSaved"], Field(title="SessionEndInputSavedThreadRealtimeItemType")
]
class ThreadRealtimeItemAddedNotification(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6578,6 +6642,14 @@ class ThreadRealtimeTranscriptRole(Enum):
assistant = "assistant"
class ThreadRealtimeVoiceTranscript(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
role: ThreadRealtimeTranscriptRole
text: str
class ThreadResumeParams(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -11238,11 +11310,42 @@ class ThreadReadStateChangedNotification(BaseModel):
thread_id: Annotated[str, Field(alias="threadId")]
class ThreadRealtimeFrameChannels(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
voice_transcript: ThreadRealtimeVoiceTranscript
class FrameThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
channels: ThreadRealtimeFrameChannels
type: Annotated[Literal["frame"], Field(title="FrameThreadRealtimeItemType")]
class TranscriptSegmentThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
role: ThreadRealtimeTranscriptRole
text: str
@@ -11256,6 +11359,13 @@ class RealtimeSessionClosedThreadRealtimeItem(BaseModel):
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
outcome: ThreadRealtimeSessionOutcome
type: Annotated[
@@ -11266,16 +11376,22 @@ class RealtimeSessionClosedThreadRealtimeItem(BaseModel):
class ThreadRealtimeItem(
RootModel[
RealtimeSessionStartedThreadRealtimeItem
| FrameThreadRealtimeItem
| TranscriptSegmentThreadRealtimeItem
| BemItemPromotedThreadRealtimeItem
| RealtimeSessionClosedThreadRealtimeItem
| TurnInputAcceptedThreadRealtimeItem
| SessionEndInputSavedThreadRealtimeItem
]
):
root: Annotated[
RealtimeSessionStartedThreadRealtimeItem
| FrameThreadRealtimeItem
| TranscriptSegmentThreadRealtimeItem
| BemItemPromotedThreadRealtimeItem
| RealtimeSessionClosedThreadRealtimeItem,
| RealtimeSessionClosedThreadRealtimeItem
| TurnInputAcceptedThreadRealtimeItem
| SessionEndInputSavedThreadRealtimeItem,
Field(
description="EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline."
),