mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Persist timestamped realtime frames before acknowledging completion (#61473)
## What changed - Record voice transcripts as `frame` items with a `voice_transcript` channel for eligible threads, while retaining legacy transcript support and the existing app-server live and mixed timeline representation. - Assign first-observed timestamps that never move backward, retain them through completion, and restore the clock floor from history and compaction checkpoints. - Flush realtime records before completion notifications, including file and initial directory durability barriers for local rollouts. - Generate a distinct default session ID for each new call, use the call ID when attaching to an existing call, and derive opening record IDs from session IDs. - Add protocol types for turn-input acceptance and session-end input receipts, excluding those receipts from live notifications and mixed timelines. Update generated schemas and SDK types. ## Testing Add coverage for frame serialization and required voice payloads, clock rollback and checkpoint restoration, durability barrier failures and retries, per-call session IDs, and persisted transcript availability immediately after completion. GitOrigin-RevId: 3f6632b517d563861aee085b723475b8945c391c
This commit is contained in:
@@ -6141,6 +6141,25 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeFrameChannels": {
|
||||
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
|
||||
"properties": {
|
||||
"voice_transcript": {
|
||||
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"voice_transcript"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeHandoffMode": {
|
||||
"enum": [
|
||||
"start",
|
||||
"steer"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeItem": {
|
||||
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
|
||||
"oneOf": [
|
||||
@@ -6160,6 +6179,26 @@
|
||||
"title": "RealtimeSessionStartedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"channels": {
|
||||
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"frame"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"channels",
|
||||
"type"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"role": {
|
||||
@@ -6231,12 +6270,94 @@
|
||||
],
|
||||
"title": "RealtimeSessionClosedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"handoff_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"turn_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"turnInputAccepted"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"handoff_id",
|
||||
"input",
|
||||
"input_item_id",
|
||||
"mode",
|
||||
"source_event_ids",
|
||||
"turn_id",
|
||||
"type"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"observedAtMs": {
|
||||
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
"format": "int64",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"realtimeSessionId": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -6423,6 +6544,21 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeVoiceTranscript": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"text"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRevertedNotification": {
|
||||
"properties": {
|
||||
"threadId": {
|
||||
|
||||
+136
@@ -25579,6 +25579,25 @@
|
||||
"title": "ThreadRealtimeErrorNotification",
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeFrameChannels": {
|
||||
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
|
||||
"properties": {
|
||||
"voice_transcript": {
|
||||
"$ref": "#/definitions/v2/ThreadRealtimeVoiceTranscript"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"voice_transcript"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeHandoffMode": {
|
||||
"enum": [
|
||||
"start",
|
||||
"steer"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeInitialItem": {
|
||||
"description": "EXPERIMENTAL - role-bearing text item included when a realtime V3 session starts.",
|
||||
"properties": {
|
||||
@@ -25614,6 +25633,26 @@
|
||||
"title": "RealtimeSessionStartedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"channels": {
|
||||
"$ref": "#/definitions/v2/ThreadRealtimeFrameChannels"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"frame"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"channels",
|
||||
"type"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"role": {
|
||||
@@ -25685,12 +25724,94 @@
|
||||
],
|
||||
"title": "RealtimeSessionClosedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"handoff_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"$ref": "#/definitions/v2/ThreadRealtimeHandoffMode"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"turn_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"turnInputAccepted"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"handoff_id",
|
||||
"input",
|
||||
"input_item_id",
|
||||
"mode",
|
||||
"source_event_ids",
|
||||
"turn_id",
|
||||
"type"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"observedAtMs": {
|
||||
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
"format": "int64",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"realtimeSessionId": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -25958,6 +26079,21 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeVoiceTranscript": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"$ref": "#/definitions/v2/ThreadRealtimeTranscriptRole"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"text"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadResumeInitialTurnsPageParams": {
|
||||
"properties": {
|
||||
"itemsView": {
|
||||
|
||||
+136
@@ -23275,6 +23275,25 @@
|
||||
"title": "ThreadRealtimeErrorNotification",
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeFrameChannels": {
|
||||
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
|
||||
"properties": {
|
||||
"voice_transcript": {
|
||||
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"voice_transcript"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeHandoffMode": {
|
||||
"enum": [
|
||||
"start",
|
||||
"steer"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeInitialItem": {
|
||||
"description": "EXPERIMENTAL - role-bearing text item included when a realtime V3 session starts.",
|
||||
"properties": {
|
||||
@@ -23310,6 +23329,26 @@
|
||||
"title": "RealtimeSessionStartedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"channels": {
|
||||
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"frame"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"channels",
|
||||
"type"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"role": {
|
||||
@@ -23381,12 +23420,94 @@
|
||||
],
|
||||
"title": "RealtimeSessionClosedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"handoff_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"turn_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"turnInputAccepted"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"handoff_id",
|
||||
"input",
|
||||
"input_item_id",
|
||||
"mode",
|
||||
"source_event_ids",
|
||||
"turn_id",
|
||||
"type"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"observedAtMs": {
|
||||
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
"format": "int64",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"realtimeSessionId": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -23654,6 +23775,21 @@
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeVoiceTranscript": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"text"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadResumeInitialTurnsPageParams": {
|
||||
"properties": {
|
||||
"itemsView": {
|
||||
|
||||
+136
@@ -60,6 +60,25 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"ThreadRealtimeFrameChannels": {
|
||||
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
|
||||
"properties": {
|
||||
"voice_transcript": {
|
||||
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"voice_transcript"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeHandoffMode": {
|
||||
"enum": [
|
||||
"start",
|
||||
"steer"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeItem": {
|
||||
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
|
||||
"oneOf": [
|
||||
@@ -79,6 +98,26 @@
|
||||
"title": "RealtimeSessionStartedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"channels": {
|
||||
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"frame"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"channels",
|
||||
"type"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"role": {
|
||||
@@ -150,12 +189,94 @@
|
||||
],
|
||||
"title": "RealtimeSessionClosedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"handoff_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"turn_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"turnInputAccepted"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"handoff_id",
|
||||
"input",
|
||||
"input_item_id",
|
||||
"mode",
|
||||
"source_event_ids",
|
||||
"turn_id",
|
||||
"type"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"observedAtMs": {
|
||||
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
"format": "int64",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"realtimeSessionId": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -179,6 +300,21 @@
|
||||
"assistant"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeVoiceTranscript": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"text"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.",
|
||||
|
||||
+136
@@ -60,6 +60,25 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"ThreadRealtimeFrameChannels": {
|
||||
"description": "A frame holds one or more payloads, at most one per channel. They share one ID, observation time, and seal. V1 requires its only channel, `voice_transcript`.",
|
||||
"properties": {
|
||||
"voice_transcript": {
|
||||
"$ref": "#/definitions/ThreadRealtimeVoiceTranscript"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"voice_transcript"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeHandoffMode": {
|
||||
"enum": [
|
||||
"start",
|
||||
"steer"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeItem": {
|
||||
"description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.",
|
||||
"oneOf": [
|
||||
@@ -79,6 +98,26 @@
|
||||
"title": "RealtimeSessionStartedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"channels": {
|
||||
"$ref": "#/definitions/ThreadRealtimeFrameChannels"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"frame"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"channels",
|
||||
"type"
|
||||
],
|
||||
"title": "FrameThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"role": {
|
||||
@@ -150,12 +189,94 @@
|
||||
],
|
||||
"title": "RealtimeSessionClosedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"handoff_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"$ref": "#/definitions/ThreadRealtimeHandoffMode"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"turn_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"turnInputAccepted"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"handoff_id",
|
||||
"input",
|
||||
"input_item_id",
|
||||
"mode",
|
||||
"source_event_ids",
|
||||
"turn_id",
|
||||
"type"
|
||||
],
|
||||
"title": "TurnInputAcceptedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"input": {
|
||||
"type": "string"
|
||||
},
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"observedAtMs": {
|
||||
"description": "Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
"format": "int64",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"realtimeSessionId": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -179,6 +300,21 @@
|
||||
"assistant"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeVoiceTranscript": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"$ref": "#/definitions/ThreadRealtimeTranscriptRole"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"text"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"description": "EXPERIMENTAL - a realtime timeline item started before its content streams.",
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+10
@@ -0,0 +1,10 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscript";
|
||||
|
||||
/**
|
||||
* A frame holds one or more payloads, at most one per channel. They share one ID,
|
||||
* observation time, and seal. V1 requires its only channel, `voice_transcript`.
|
||||
*/
|
||||
export type ThreadRealtimeFrameChannels = { voice_transcript: ThreadRealtimeVoiceTranscript, };
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type ThreadRealtimeHandoffMode = "start" | "steer";
|
||||
@@ -2,10 +2,16 @@
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
|
||||
import type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
|
||||
import type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
|
||||
import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
|
||||
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
|
||||
|
||||
/**
|
||||
* EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.
|
||||
*/
|
||||
export type ThreadRealtimeItem = { id: string, realtimeSessionId: string, } & ({ "type": "realtimeSessionStarted" } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, });
|
||||
export type ThreadRealtimeItem = { id: string, realtimeSessionId: string,
|
||||
/**
|
||||
* Canonical observation time in Unix milliseconds for eligible voice calls.
|
||||
*/
|
||||
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionEndInputSaved", sourceEventIds: Array<string>, inputItemId: string, input: string, });
|
||||
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
|
||||
|
||||
export type ThreadRealtimeVoiceTranscript = { role: ThreadRealtimeTranscriptRole, text: string, };
|
||||
@@ -585,6 +585,8 @@ export type { ThreadRealtimeAudioChunk } from "./ThreadRealtimeAudioChunk";
|
||||
export type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
|
||||
export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNotification";
|
||||
export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification";
|
||||
export type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
|
||||
export type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
|
||||
export type { ThreadRealtimeInitialItem } from "./ThreadRealtimeInitialItem";
|
||||
export type { ThreadRealtimeItem } from "./ThreadRealtimeItem";
|
||||
export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification";
|
||||
@@ -599,6 +601,7 @@ export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedN
|
||||
export type { ThreadRealtimeTranscriptDeltaNotification } from "./ThreadRealtimeTranscriptDeltaNotification";
|
||||
export type { ThreadRealtimeTranscriptDoneNotification } from "./ThreadRealtimeTranscriptDoneNotification";
|
||||
export type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
|
||||
export type { ThreadRealtimeVoiceTranscript } from "./ThreadRealtimeVoiceTranscript";
|
||||
export type { ThreadResumeInitialTurnsPageParams } from "./ThreadResumeInitialTurnsPageParams";
|
||||
export type { ThreadResumeParams } from "./ThreadResumeParams";
|
||||
export type { ThreadResumeResponse } from "./ThreadResumeResponse";
|
||||
|
||||
@@ -8,6 +8,7 @@ use codex_protocol::protocol::RealtimeOutputModality;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::RealtimeVoicesList;
|
||||
use codex_protocol::realtime::BemItemPresentation as CoreBemItemPresentation;
|
||||
use codex_protocol::realtime::RealtimeHandoffMode as CoreRealtimeHandoffMode;
|
||||
use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
|
||||
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
|
||||
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
|
||||
@@ -24,6 +25,10 @@ use std::collections::BTreeMap;
|
||||
pub struct ThreadRealtimeItem {
|
||||
pub id: String,
|
||||
pub realtime_session_id: String,
|
||||
/// Canonical observation time in Unix milliseconds for eligible voice calls.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(type = "number", optional)]
|
||||
pub observed_at_ms: Option<i64>,
|
||||
#[serde(flatten)]
|
||||
pub content: ThreadRealtimeItemContent,
|
||||
}
|
||||
@@ -38,6 +43,9 @@ pub struct ThreadRealtimeItem {
|
||||
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeItemContent {
|
||||
RealtimeSessionStarted,
|
||||
Frame {
|
||||
channels: ThreadRealtimeFrameChannels,
|
||||
},
|
||||
TranscriptSegment {
|
||||
role: ThreadRealtimeTranscriptRole,
|
||||
text: String,
|
||||
@@ -50,6 +58,27 @@ pub enum ThreadRealtimeItemContent {
|
||||
RealtimeSessionClosed {
|
||||
outcome: ThreadRealtimeSessionOutcome,
|
||||
},
|
||||
TurnInputAccepted {
|
||||
handoff_id: String,
|
||||
turn_id: String,
|
||||
mode: ThreadRealtimeHandoffMode,
|
||||
source_event_ids: Vec<String>,
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
},
|
||||
SessionEndInputSaved {
|
||||
source_event_ids: Vec<String>,
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeHandoffMode {
|
||||
Start,
|
||||
Steer,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
|
||||
@@ -82,11 +111,52 @@ pub enum ThreadRealtimeSessionOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// A frame holds one or more payloads, at most one per channel. They share one ID,
|
||||
/// observation time, and seal. V1 requires its only channel, `voice_transcript`.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeFrameChannels {
|
||||
pub voice_transcript: ThreadRealtimeVoiceTranscript,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeVoiceTranscript {
|
||||
pub role: ThreadRealtimeTranscriptRole,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
impl From<CoreRealtimeTranscriptRole> for ThreadRealtimeTranscriptRole {
|
||||
fn from(role: CoreRealtimeTranscriptRole) -> Self {
|
||||
match role {
|
||||
CoreRealtimeTranscriptRole::User => Self::User,
|
||||
CoreRealtimeTranscriptRole::Assistant => Self::Assistant,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The existing live and mixed timeline APIs present speech as transcriptSegment.
|
||||
impl From<CoreRealtimeItem> for ThreadRealtimeItem {
|
||||
fn from(item: CoreRealtimeItem) -> Self {
|
||||
let mut item = Self::from_persisted(item);
|
||||
if let ThreadRealtimeItemContent::Frame { channels } = item.content {
|
||||
let transcript = channels.voice_transcript;
|
||||
item.content = ThreadRealtimeItemContent::TranscriptSegment {
|
||||
role: transcript.role,
|
||||
text: transcript.text,
|
||||
};
|
||||
}
|
||||
item
|
||||
}
|
||||
}
|
||||
|
||||
impl ThreadRealtimeItem {
|
||||
/// Preserve the canonical shape of a sealed record on the independent read API.
|
||||
pub fn from_persisted(item: CoreRealtimeItem) -> Self {
|
||||
let CoreRealtimeItem {
|
||||
id,
|
||||
realtime_session_id,
|
||||
observed_at_ms,
|
||||
content,
|
||||
} = item;
|
||||
let content = match content {
|
||||
@@ -95,15 +165,21 @@ impl From<CoreRealtimeItem> for ThreadRealtimeItem {
|
||||
}
|
||||
CoreRealtimeItemContent::TranscriptSegment { role, text } => {
|
||||
ThreadRealtimeItemContent::TranscriptSegment {
|
||||
role: match role {
|
||||
CoreRealtimeTranscriptRole::User => ThreadRealtimeTranscriptRole::User,
|
||||
CoreRealtimeTranscriptRole::Assistant => {
|
||||
ThreadRealtimeTranscriptRole::Assistant
|
||||
}
|
||||
},
|
||||
role: role.into(),
|
||||
text,
|
||||
}
|
||||
}
|
||||
CoreRealtimeItemContent::Frame { channels } => {
|
||||
let transcript = channels.voice_transcript;
|
||||
ThreadRealtimeItemContent::Frame {
|
||||
channels: ThreadRealtimeFrameChannels {
|
||||
voice_transcript: ThreadRealtimeVoiceTranscript {
|
||||
role: transcript.role.into(),
|
||||
text: transcript.text,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
CoreRealtimeItemContent::BemItemPromoted {
|
||||
turn_id,
|
||||
item_id,
|
||||
@@ -131,10 +207,38 @@ impl From<CoreRealtimeItem> for ThreadRealtimeItem {
|
||||
},
|
||||
}
|
||||
}
|
||||
CoreRealtimeItemContent::TurnInputAccepted {
|
||||
handoff_id,
|
||||
turn_id,
|
||||
mode,
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
} => ThreadRealtimeItemContent::TurnInputAccepted {
|
||||
handoff_id,
|
||||
turn_id,
|
||||
mode: match mode {
|
||||
CoreRealtimeHandoffMode::Start => ThreadRealtimeHandoffMode::Start,
|
||||
CoreRealtimeHandoffMode::Steer => ThreadRealtimeHandoffMode::Steer,
|
||||
},
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
},
|
||||
CoreRealtimeItemContent::SessionEndInputSaved {
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
} => ThreadRealtimeItemContent::SessionEndInputSaved {
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
},
|
||||
};
|
||||
Self {
|
||||
id,
|
||||
realtime_session_id,
|
||||
observed_at_ms,
|
||||
content,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5660,3 +5660,13 @@ fn feedback_paths_preserve_native_wire_spelling_before_host_validation() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_undated_live_voice_item_preserves_its_wire_shape() {
|
||||
let wire = serde_json::json!({
|
||||
"id": "speech", "realtimeSessionId": "voice", "type": "transcriptSegment",
|
||||
"role": "assistant", "text": "hello",
|
||||
});
|
||||
let item: ThreadRealtimeItem = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(item).unwrap(), wire);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user