From 001cdd274ec8567d9cbce4fb736128def9755a65 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Mon, 13 Apr 2026 16:04:12 -0700 Subject: [PATCH] Split realtime transcript notifications --- .../schema/json/ServerNotification.json | 68 +++++++---- .../codex_app_server_protocol.schemas.json | 72 ++++++++---- .../codex_app_server_protocol.v2.schemas.json | 72 ++++++++---- ...adRealtimeTranscriptDeltaNotification.json | 23 ++++ ...eadRealtimeTranscriptDoneNotification.json | 23 ++++ ...RealtimeTranscriptUpdatedNotification.json | 36 ------ .../RealtimeTranscriptUpdateKind.ts | 5 - .../schema/typescript/ServerNotification.ts | 5 +- .../schema/typescript/index.ts | 1 - ...readRealtimeTranscriptDeltaNotification.ts | 13 +++ ...hreadRealtimeTranscriptDoneNotification.ts | 13 +++ ...adRealtimeTranscriptUpdatedNotification.ts | 14 --- .../schema/typescript/v2/index.ts | 3 +- .../src/protocol/common.rs | 6 +- .../app-server-protocol/src/protocol/v2.rs | 22 +++- codex-rs/app-server/README.md | 3 +- .../app-server/src/bespoke_event_handling.rs | 41 +++++-- .../tests/suite/v2/realtime_conversation.rs | 107 ++++++++---------- codex-rs/codex-api/src/endpoint/mod.rs | 1 - .../endpoint/realtime_websocket/methods.rs | 47 +++----- .../src/endpoint/realtime_websocket/mod.rs | 1 - .../endpoint/realtime_websocket/protocol.rs | 2 - .../realtime_websocket/protocol_common.rs | 23 ++-- .../realtime_websocket/protocol_v2.rs | 14 +-- codex-rs/codex-api/src/lib.rs | 1 - codex-rs/core/src/realtime_conversation.rs | 2 + codex-rs/protocol/src/protocol.rs | 15 ++- codex-rs/tui/src/app/app_server_adapter.rs | 5 +- codex-rs/tui/src/chatwidget.rs | 3 +- codex-rs/tui/src/chatwidget/realtime.rs | 2 + 30 files changed, 383 insertions(+), 260 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDeltaNotification.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDoneNotification.json delete mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptUpdatedNotification.json delete mode 100644 codex-rs/app-server-protocol/schema/typescript/RealtimeTranscriptUpdateKind.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDeltaNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDoneNotification.ts delete mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptUpdatedNotification.ts diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index df92b988ea..4edff15748 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2111,13 +2111,6 @@ ], "type": "string" }, - "RealtimeTranscriptUpdateKind": { - "enum": [ - "delta", - "done" - ], - "type": "string" - }, "ReasoningEffort": { "description": "See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning", "enum": [ @@ -3391,28 +3384,45 @@ ], "type": "object" }, - "ThreadRealtimeTranscriptUpdatedNotification": { - "description": "EXPERIMENTAL - flat transcript update emitted whenever realtime transcript text changes or completes.", + "ThreadRealtimeTranscriptDeltaNotification": { + "description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.", + "properties": { + "delta": { + "description": "Live transcript delta from the realtime event.", + "type": "string" + }, + "role": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "role", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeTranscriptDoneNotification": { + "description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.", "properties": { "role": { "type": "string" }, "text": { - "description": "Delta text for delta updates; final complete text for done updates.", + "description": "Final complete text for the transcript part.", "type": "string" }, "threadId": { "type": "string" - }, - "updateKind": { - "$ref": "#/definitions/RealtimeTranscriptUpdateKind" } }, "required": [ "role", "text", - "threadId", - "updateKind" + "threadId" ], "type": "object" }, @@ -4961,20 +4971,40 @@ "properties": { "method": { "enum": [ - "thread/realtime/transcriptUpdated" + "thread/realtime/transcript/delta" ], - "title": "Thread/realtime/transcriptUpdatedNotificationMethod", + "title": "Thread/realtime/transcript/deltaNotificationMethod", "type": "string" }, "params": { - "$ref": "#/definitions/ThreadRealtimeTranscriptUpdatedNotification" + "$ref": "#/definitions/ThreadRealtimeTranscriptDeltaNotification" } }, "required": [ "method", "params" ], - "title": "Thread/realtime/transcriptUpdatedNotification", + "title": "Thread/realtime/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/transcript/done" + ], + "title": "Thread/realtime/transcript/doneNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeTranscriptDoneNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/transcript/doneNotification", "type": "object" }, { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 7446569e2c..c7123429d0 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -4330,20 +4330,40 @@ "properties": { "method": { "enum": [ - "thread/realtime/transcriptUpdated" + "thread/realtime/transcript/delta" ], - "title": "Thread/realtime/transcriptUpdatedNotificationMethod", + "title": "Thread/realtime/transcript/deltaNotificationMethod", "type": "string" }, "params": { - "$ref": "#/definitions/v2/ThreadRealtimeTranscriptUpdatedNotification" + "$ref": "#/definitions/v2/ThreadRealtimeTranscriptDeltaNotification" } }, "required": [ "method", "params" ], - "title": "Thread/realtime/transcriptUpdatedNotification", + "title": "Thread/realtime/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/transcript/done" + ], + "title": "Thread/realtime/transcript/doneNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeTranscriptDoneNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/transcript/doneNotification", "type": "object" }, { @@ -10622,13 +10642,6 @@ ], "type": "string" }, - "RealtimeTranscriptUpdateKind": { - "enum": [ - "delta", - "done" - ], - "type": "string" - }, "RealtimeVoice": { "enum": [ "alloy", @@ -13975,31 +13988,50 @@ "title": "ThreadRealtimeStartedNotification", "type": "object" }, - "ThreadRealtimeTranscriptUpdatedNotification": { + "ThreadRealtimeTranscriptDeltaNotification": { "$schema": "http://json-schema.org/draft-07/schema#", - "description": "EXPERIMENTAL - flat transcript update emitted whenever realtime transcript text changes or completes.", + "description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.", + "properties": { + "delta": { + "description": "Live transcript delta from the realtime event.", + "type": "string" + }, + "role": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "role", + "threadId" + ], + "title": "ThreadRealtimeTranscriptDeltaNotification", + "type": "object" + }, + "ThreadRealtimeTranscriptDoneNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.", "properties": { "role": { "type": "string" }, "text": { - "description": "Delta text for delta updates; final complete text for done updates.", + "description": "Final complete text for the transcript part.", "type": "string" }, "threadId": { "type": "string" - }, - "updateKind": { - "$ref": "#/definitions/v2/RealtimeTranscriptUpdateKind" } }, "required": [ "role", "text", - "threadId", - "updateKind" + "threadId" ], - "title": "ThreadRealtimeTranscriptUpdatedNotification", + "title": "ThreadRealtimeTranscriptDoneNotification", "type": "object" }, "ThreadResumeParams": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 139125006d..e2ce7d7f69 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -7418,13 +7418,6 @@ ], "type": "string" }, - "RealtimeTranscriptUpdateKind": { - "enum": [ - "delta", - "done" - ], - "type": "string" - }, "RealtimeVoice": { "enum": [ "alloy", @@ -9594,20 +9587,40 @@ "properties": { "method": { "enum": [ - "thread/realtime/transcriptUpdated" + "thread/realtime/transcript/delta" ], - "title": "Thread/realtime/transcriptUpdatedNotificationMethod", + "title": "Thread/realtime/transcript/deltaNotificationMethod", "type": "string" }, "params": { - "$ref": "#/definitions/ThreadRealtimeTranscriptUpdatedNotification" + "$ref": "#/definitions/ThreadRealtimeTranscriptDeltaNotification" } }, "required": [ "method", "params" ], - "title": "Thread/realtime/transcriptUpdatedNotification", + "title": "Thread/realtime/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/transcript/done" + ], + "title": "Thread/realtime/transcript/doneNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeTranscriptDoneNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/transcript/doneNotification", "type": "object" }, { @@ -11823,31 +11836,50 @@ "title": "ThreadRealtimeStartedNotification", "type": "object" }, - "ThreadRealtimeTranscriptUpdatedNotification": { + "ThreadRealtimeTranscriptDeltaNotification": { "$schema": "http://json-schema.org/draft-07/schema#", - "description": "EXPERIMENTAL - flat transcript update emitted whenever realtime transcript text changes or completes.", + "description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.", + "properties": { + "delta": { + "description": "Live transcript delta from the realtime event.", + "type": "string" + }, + "role": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "role", + "threadId" + ], + "title": "ThreadRealtimeTranscriptDeltaNotification", + "type": "object" + }, + "ThreadRealtimeTranscriptDoneNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.", "properties": { "role": { "type": "string" }, "text": { - "description": "Delta text for delta updates; final complete text for done updates.", + "description": "Final complete text for the transcript part.", "type": "string" }, "threadId": { "type": "string" - }, - "updateKind": { - "$ref": "#/definitions/RealtimeTranscriptUpdateKind" } }, "required": [ "role", "text", - "threadId", - "updateKind" + "threadId" ], - "title": "ThreadRealtimeTranscriptUpdatedNotification", + "title": "ThreadRealtimeTranscriptDoneNotification", "type": "object" }, "ThreadResumeParams": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDeltaNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDeltaNotification.json new file mode 100644 index 0000000000..22ad778eb2 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDeltaNotification.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.", + "properties": { + "delta": { + "description": "Live transcript delta from the realtime event.", + "type": "string" + }, + "role": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "role", + "threadId" + ], + "title": "ThreadRealtimeTranscriptDeltaNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDoneNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDoneNotification.json new file mode 100644 index 0000000000..2f4199fdb9 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDoneNotification.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.", + "properties": { + "role": { + "type": "string" + }, + "text": { + "description": "Final complete text for the transcript part.", + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "role", + "text", + "threadId" + ], + "title": "ThreadRealtimeTranscriptDoneNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptUpdatedNotification.json deleted file mode 100644 index f936e58172..0000000000 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptUpdatedNotification.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "RealtimeTranscriptUpdateKind": { - "enum": [ - "delta", - "done" - ], - "type": "string" - } - }, - "description": "EXPERIMENTAL - flat transcript update emitted whenever realtime transcript text changes or completes.", - "properties": { - "role": { - "type": "string" - }, - "text": { - "description": "Delta text for delta updates; final complete text for done updates.", - "type": "string" - }, - "threadId": { - "type": "string" - }, - "updateKind": { - "$ref": "#/definitions/RealtimeTranscriptUpdateKind" - } - }, - "required": [ - "role", - "text", - "threadId", - "updateKind" - ], - "title": "ThreadRealtimeTranscriptUpdatedNotification", - "type": "object" -} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/RealtimeTranscriptUpdateKind.ts b/codex-rs/app-server-protocol/schema/typescript/RealtimeTranscriptUpdateKind.ts deleted file mode 100644 index 1bc6d6ed8e..0000000000 --- a/codex-rs/app-server-protocol/schema/typescript/RealtimeTranscriptUpdateKind.ts +++ /dev/null @@ -1,5 +0,0 @@ -// GENERATED CODE! DO NOT MODIFY BY HAND! - -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export type RealtimeTranscriptUpdateKind = "delta" | "done"; diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index a985914134..1db7027feb 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -43,7 +43,8 @@ import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeIte import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification"; import type { ThreadRealtimeSdpNotification } from "./v2/ThreadRealtimeSdpNotification"; import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification"; -import type { ThreadRealtimeTranscriptUpdatedNotification } from "./v2/ThreadRealtimeTranscriptUpdatedNotification"; +import type { ThreadRealtimeTranscriptDeltaNotification } from "./v2/ThreadRealtimeTranscriptDeltaNotification"; +import type { ThreadRealtimeTranscriptDoneNotification } from "./v2/ThreadRealtimeTranscriptDoneNotification"; import type { ThreadStartedNotification } from "./v2/ThreadStartedNotification"; import type { ThreadStatusChangedNotification } from "./v2/ThreadStatusChangedNotification"; import type { ThreadTokenUsageUpdatedNotification } from "./v2/ThreadTokenUsageUpdatedNotification"; @@ -58,4 +59,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index fe9c81917b..7bbb417fdc 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -50,7 +50,6 @@ export type { Personality } from "./Personality"; export type { PlanType } from "./PlanType"; export type { RealtimeConversationVersion } from "./RealtimeConversationVersion"; export type { RealtimeOutputModality } from "./RealtimeOutputModality"; -export type { RealtimeTranscriptUpdateKind } from "./RealtimeTranscriptUpdateKind"; export type { RealtimeVoice } from "./RealtimeVoice"; export type { RealtimeVoicesList } from "./RealtimeVoicesList"; export type { ReasoningEffort } from "./ReasoningEffort"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDeltaNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDeltaNotification.ts new file mode 100644 index 0000000000..805eeddd76 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDeltaNotification.ts @@ -0,0 +1,13 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - flat transcript delta emitted whenever realtime + * transcript text changes. + */ +export type ThreadRealtimeTranscriptDeltaNotification = { threadId: string, role: string, +/** + * Live transcript delta from the realtime event. + */ +delta: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDoneNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDoneNotification.ts new file mode 100644 index 0000000000..d4667ad039 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptDoneNotification.ts @@ -0,0 +1,13 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - final transcript text emitted when realtime completes + * a transcript part. + */ +export type ThreadRealtimeTranscriptDoneNotification = { threadId: string, role: string, +/** + * Final complete text for the transcript part. + */ +text: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptUpdatedNotification.ts deleted file mode 100644 index bf1b1e3439..0000000000 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptUpdatedNotification.ts +++ /dev/null @@ -1,14 +0,0 @@ -// GENERATED CODE! DO NOT MODIFY BY HAND! - -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { RealtimeTranscriptUpdateKind } from "../RealtimeTranscriptUpdateKind"; - -/** - * EXPERIMENTAL - flat transcript update emitted whenever realtime - * transcript text changes or completes. - */ -export type ThreadRealtimeTranscriptUpdatedNotification = { threadId: string, role: string, -/** - * Delta text for delta updates; final complete text for done updates. - */ -text: string, updateKind: RealtimeTranscriptUpdateKind, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index f815fee3e9..16294d28aa 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -301,7 +301,8 @@ export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtim export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification"; export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport"; export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification"; -export type { ThreadRealtimeTranscriptUpdatedNotification } from "./ThreadRealtimeTranscriptUpdatedNotification"; +export type { ThreadRealtimeTranscriptDeltaNotification } from "./ThreadRealtimeTranscriptDeltaNotification"; +export type { ThreadRealtimeTranscriptDoneNotification } from "./ThreadRealtimeTranscriptDoneNotification"; export type { ThreadResumeParams } from "./ThreadResumeParams"; export type { ThreadResumeResponse } from "./ThreadResumeResponse"; export type { ThreadRollbackParams } from "./ThreadRollbackParams"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 3846d5984f..8d318d4271 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1017,8 +1017,10 @@ server_notification_definitions! { ThreadRealtimeStarted => "thread/realtime/started" (v2::ThreadRealtimeStartedNotification), #[experimental("thread/realtime/itemAdded")] ThreadRealtimeItemAdded => "thread/realtime/itemAdded" (v2::ThreadRealtimeItemAddedNotification), - #[experimental("thread/realtime/transcriptUpdated")] - ThreadRealtimeTranscriptUpdated => "thread/realtime/transcriptUpdated" (v2::ThreadRealtimeTranscriptUpdatedNotification), + #[experimental("thread/realtime/transcript/delta")] + ThreadRealtimeTranscriptDelta => "thread/realtime/transcript/delta" (v2::ThreadRealtimeTranscriptDeltaNotification), + #[experimental("thread/realtime/transcript/done")] + ThreadRealtimeTranscriptDone => "thread/realtime/transcript/done" (v2::ThreadRealtimeTranscriptDoneNotification), #[experimental("thread/realtime/outputAudio/delta")] ThreadRealtimeOutputAudioDelta => "thread/realtime/outputAudio/delta" (v2::ThreadRealtimeOutputAudioDeltaNotification), #[experimental("thread/realtime/sdp")] diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 2c91b7d728..8a12a2a373 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -75,7 +75,6 @@ use codex_protocol::protocol::ReadOnlyAccess as CoreReadOnlyAccess; use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; -use codex_protocol::protocol::RealtimeTranscriptUpdateKind; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; use codex_protocol::protocol::ReviewDecision as CoreReviewDecision; @@ -4076,17 +4075,28 @@ pub struct ThreadRealtimeItemAddedNotification { pub item: JsonValue, } -/// EXPERIMENTAL - flat transcript update emitted whenever realtime -/// transcript text changes or completes. +/// EXPERIMENTAL - flat transcript delta emitted whenever realtime +/// transcript text changes. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] -pub struct ThreadRealtimeTranscriptUpdatedNotification { +pub struct ThreadRealtimeTranscriptDeltaNotification { pub thread_id: String, pub role: String, - /// Delta text for delta updates; final complete text for done updates. + /// Live transcript delta from the realtime event. + pub delta: String, +} + +/// EXPERIMENTAL - final transcript text emitted when realtime completes +/// a transcript part. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeTranscriptDoneNotification { + pub thread_id: String, + pub role: String, + /// Final complete text for the transcript part. pub text: String, - pub update_kind: RealtimeTranscriptUpdateKind, } /// EXPERIMENTAL - streamed output audio emitted by thread realtime. diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index cf820e28aa..87c8adc72f 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -932,7 +932,8 @@ The thread realtime API emits thread-scoped notifications for session lifecycle - `thread/realtime/started` — `{ threadId, sessionId }` once realtime starts for the thread (experimental). - `thread/realtime/itemAdded` — `{ threadId, item }` for raw non-audio realtime items that do not have a dedicated typed app-server notification, including `handoff_request` (experimental). `item` is forwarded as raw JSON while the upstream websocket item schema remains unstable. -- `thread/realtime/transcriptUpdated` — `{ threadId, role, text, updateKind }` whenever realtime transcript text changes (experimental). `updateKind: "delta"` forwards the live transcript delta, and `updateKind: "done"` forwards the final full text for that output part. +- `thread/realtime/transcript/delta` — `{ threadId, role, delta }` for live realtime transcript deltas (experimental). +- `thread/realtime/transcript/done` — `{ threadId, role, text }` when realtime emits the final full text for a transcript part (experimental). - `thread/realtime/outputAudio/delta` — `{ threadId, audio }` for streamed output audio chunks (experimental). `audio` uses camelCase fields (`data`, `sampleRate`, `numChannels`, `samplesPerChannel`). - `thread/realtime/error` — `{ threadId, message }` when realtime encounters a transport or backend error (experimental). - `thread/realtime/closed` — `{ threadId, reason }` when the realtime transport closes (experimental). diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 20c8b887ad..b1202bda2c 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -82,7 +82,8 @@ use codex_app_server_protocol::ThreadRealtimeItemAddedNotification; use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification; use codex_app_server_protocol::ThreadRealtimeSdpNotification; use codex_app_server_protocol::ThreadRealtimeStartedNotification; -use codex_app_server_protocol::ThreadRealtimeTranscriptUpdatedNotification; +use codex_app_server_protocol::ThreadRealtimeTranscriptDeltaNotification; +use codex_app_server_protocol::ThreadRealtimeTranscriptDoneNotification; use codex_app_server_protocol::ThreadRollbackResponse; use codex_app_server_protocol::ThreadTokenUsage; use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; @@ -401,28 +402,50 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } RealtimeEvent::InputTranscriptDelta(event) => { - let notification = ThreadRealtimeTranscriptUpdatedNotification { + let notification = ThreadRealtimeTranscriptDeltaNotification { thread_id: conversation_id.to_string(), role: "user".to_string(), - text: event.delta, - update_kind: event.update_kind, + delta: event.delta, }; outgoing .send_server_notification( - ServerNotification::ThreadRealtimeTranscriptUpdated(notification), + ServerNotification::ThreadRealtimeTranscriptDelta(notification), + ) + .await; + } + RealtimeEvent::InputTranscriptDone(event) => { + let notification = ThreadRealtimeTranscriptDoneNotification { + thread_id: conversation_id.to_string(), + role: "user".to_string(), + text: event.text, + }; + outgoing + .send_server_notification( + ServerNotification::ThreadRealtimeTranscriptDone(notification), ) .await; } RealtimeEvent::OutputTranscriptDelta(event) => { - let notification = ThreadRealtimeTranscriptUpdatedNotification { + let notification = ThreadRealtimeTranscriptDeltaNotification { thread_id: conversation_id.to_string(), role: "assistant".to_string(), - text: event.delta, - update_kind: event.update_kind, + delta: event.delta, }; outgoing .send_server_notification( - ServerNotification::ThreadRealtimeTranscriptUpdated(notification), + ServerNotification::ThreadRealtimeTranscriptDelta(notification), + ) + .await; + } + RealtimeEvent::OutputTranscriptDone(event) => { + let notification = ThreadRealtimeTranscriptDoneNotification { + thread_id: conversation_id.to_string(), + role: "assistant".to_string(), + text: event.text, + }; + outgoing + .send_server_notification( + ServerNotification::ThreadRealtimeTranscriptDone(notification), ) .await; } diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 47d17a9aa1..66f7693f75 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -31,7 +31,8 @@ use codex_app_server_protocol::ThreadRealtimeStartTransport; use codex_app_server_protocol::ThreadRealtimeStartedNotification; use codex_app_server_protocol::ThreadRealtimeStopParams; use codex_app_server_protocol::ThreadRealtimeStopResponse; -use codex_app_server_protocol::ThreadRealtimeTranscriptUpdatedNotification; +use codex_app_server_protocol::ThreadRealtimeTranscriptDeltaNotification; +use codex_app_server_protocol::ThreadRealtimeTranscriptDoneNotification; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnCompletedNotification; @@ -40,7 +41,6 @@ use codex_features::FEATURES; use codex_features::Feature; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; -use codex_protocol::protocol::RealtimeTranscriptUpdateKind; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; use core_test_support::responses; @@ -624,45 +624,32 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { assert_eq!(item_added.thread_id, output_audio.thread_id); assert_eq!(item_added.item["type"], json!("message")); - let first_transcript_update = read_notification::( + let first_transcript_delta = read_notification::( &mut mcp, - "thread/realtime/transcriptUpdated", + "thread/realtime/transcript/delta", ) .await?; - assert_eq!(first_transcript_update.thread_id, output_audio.thread_id); - assert_eq!(first_transcript_update.role, "user"); - assert_eq!(first_transcript_update.text, "delegate now"); - assert_eq!( - first_transcript_update.update_kind, - RealtimeTranscriptUpdateKind::Delta - ); + assert_eq!(first_transcript_delta.thread_id, output_audio.thread_id); + assert_eq!(first_transcript_delta.role, "user"); + assert_eq!(first_transcript_delta.delta, "delegate now"); - let second_transcript_update = - read_notification::( - &mut mcp, - "thread/realtime/transcriptUpdated", - ) - .await?; - assert_eq!(second_transcript_update.thread_id, output_audio.thread_id); - assert_eq!(second_transcript_update.role, "assistant"); - assert_eq!(second_transcript_update.text, "working"); - assert_eq!( - second_transcript_update.update_kind, - RealtimeTranscriptUpdateKind::Delta - ); - - let final_transcript_update = read_notification::( + let second_transcript_delta = read_notification::( &mut mcp, - "thread/realtime/transcriptUpdated", + "thread/realtime/transcript/delta", ) .await?; - assert_eq!(final_transcript_update.thread_id, output_audio.thread_id); - assert_eq!(final_transcript_update.role, "assistant"); - assert_eq!(final_transcript_update.text, "working on it"); - assert_eq!( - final_transcript_update.update_kind, - RealtimeTranscriptUpdateKind::Done - ); + assert_eq!(second_transcript_delta.thread_id, output_audio.thread_id); + assert_eq!(second_transcript_delta.role, "assistant"); + assert_eq!(second_transcript_delta.delta, "working"); + + let final_transcript_done = read_notification::( + &mut mcp, + "thread/realtime/transcript/done", + ) + .await?; + assert_eq!(final_transcript_done.thread_id, output_audio.thread_id); + assert_eq!(final_transcript_done.role, "assistant"); + assert_eq!(final_transcript_done.text, "working on it"); let handoff_item_added = read_notification::( &mut mcp, @@ -799,44 +786,44 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript json!(["text"]) ); - let first_delta = read_notification::( + let first_delta = read_notification::( &mut mcp, - "thread/realtime/transcriptUpdated", + "thread/realtime/transcript/delta", ) .await?; - let second_delta = read_notification::( + let second_delta = read_notification::( &mut mcp, - "thread/realtime/transcriptUpdated", + "thread/realtime/transcript/delta", ) .await?; - let done = read_notification::( + let done = read_notification::( &mut mcp, - "thread/realtime/transcriptUpdated", + "thread/realtime/transcript/done", ) .await?; assert_eq!( - vec![first_delta, second_delta, done], + vec![first_delta, second_delta], vec![ - ThreadRealtimeTranscriptUpdatedNotification { + ThreadRealtimeTranscriptDeltaNotification { thread_id: thread_start.thread.id.clone(), role: "assistant".to_string(), - text: "hello ".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, + delta: "hello ".to_string(), }, - ThreadRealtimeTranscriptUpdatedNotification { + ThreadRealtimeTranscriptDeltaNotification { thread_id: thread_start.thread.id.clone(), role: "assistant".to_string(), - text: "world".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, - }, - ThreadRealtimeTranscriptUpdatedNotification { - thread_id: thread_start.thread.id, - role: "assistant".to_string(), - text: "hello world".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Done, + delta: "world".to_string(), }, ] ); + assert_eq!( + done, + ThreadRealtimeTranscriptDoneNotification { + thread_id: thread_start.thread.id, + role: "assistant".to_string(), + text: "hello world".to_string(), + } + ); realtime_server.shutdown().await; Ok(()) @@ -1314,12 +1301,11 @@ async fn webrtc_v2_forwards_audio_and_text_between_client_and_sideband() -> Resu harness.append_text(thread_id, "hello").await?; let transcript = harness - .read_notification::( - "thread/realtime/transcriptUpdated", + .read_notification::( + "thread/realtime/transcript/delta", ) .await?; - assert_eq!(transcript.text, "transcribed audio"); - assert_eq!(transcript.update_kind, RealtimeTranscriptUpdateKind::Delta); + assert_eq!(transcript.delta, "transcribed audio"); let output_audio = harness .read_notification::( "thread/realtime/outputAudio/delta", @@ -1404,12 +1390,11 @@ async fn webrtc_v2_text_input_is_append_only_while_response_is_active() -> Resul "first", ); let transcript = harness - .read_notification::( - "thread/realtime/transcriptUpdated", + .read_notification::( + "thread/realtime/transcript/delta", ) .await?; - assert_eq!(transcript.text, "active response started"); - assert_eq!(transcript.update_kind, RealtimeTranscriptUpdateKind::Delta); + assert_eq!(transcript.delta, "active response started"); // Phase 3: send a second text turn while `resp_active` is still open. The // user message must reach realtime without requesting another response. diff --git a/codex-rs/codex-api/src/endpoint/mod.rs b/codex-rs/codex-api/src/endpoint/mod.rs index 515487074f..c16687ff28 100644 --- a/codex-rs/codex-api/src/endpoint/mod.rs +++ b/codex-rs/codex-api/src/endpoint/mod.rs @@ -16,7 +16,6 @@ pub use realtime_websocket::RealtimeEventParser; pub use realtime_websocket::RealtimeOutputModality; pub use realtime_websocket::RealtimeSessionConfig; pub use realtime_websocket::RealtimeSessionMode; -pub use realtime_websocket::RealtimeTranscriptUpdateKind; pub use realtime_websocket::RealtimeWebsocketClient; pub use realtime_websocket::RealtimeWebsocketConnection; pub use realtime_websocket::RealtimeWebsocketEvents; diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 36bb8cecd0..dcd330322e 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -10,9 +10,7 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage; use crate::endpoint::realtime_websocket::protocol::RealtimeOutputModality; use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig; use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode; -use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptDelta; use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry; -use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptUpdateKind; use crate::endpoint::realtime_websocket::protocol::RealtimeVoice; use crate::endpoint::realtime_websocket::protocol::parse_realtime_event; use crate::error::ApiError; @@ -416,10 +414,20 @@ impl RealtimeWebsocketEvents { match event { RealtimeEvent::InputAudioSpeechStarted(_) => {} RealtimeEvent::InputTranscriptDelta(update) => { - update_active_transcript_entry(&mut active_transcript.entries, "user", update); + append_transcript_delta(&mut active_transcript.entries, "user", &update.delta); + } + RealtimeEvent::InputTranscriptDone(update) => { + complete_transcript_entry(&mut active_transcript.entries, "user", &update.text); } RealtimeEvent::OutputTranscriptDelta(update) => { - update_active_transcript_entry(&mut active_transcript.entries, "assistant", update); + append_transcript_delta(&mut active_transcript.entries, "assistant", &update.delta); + } + RealtimeEvent::OutputTranscriptDone(update) => { + complete_transcript_entry( + &mut active_transcript.entries, + "assistant", + &update.text, + ); } RealtimeEvent::HandoffRequested(handoff) => { if self.event_parser == RealtimeEventParser::V1 { @@ -456,21 +464,6 @@ fn append_transcript_delta(entries: &mut Vec, role: &st }); } -fn update_active_transcript_entry( - entries: &mut Vec, - role: &str, - update: &RealtimeTranscriptDelta, -) { - match update.update_kind { - RealtimeTranscriptUpdateKind::Delta => { - append_transcript_delta(entries, role, &update.delta) - } - RealtimeTranscriptUpdateKind::Done => { - complete_transcript_entry(entries, role, &update.delta) - } - } -} - fn complete_transcript_entry(entries: &mut Vec, role: &str, text: &str) { if text.is_empty() { return; @@ -770,13 +763,14 @@ fn normalize_realtime_path(url: &mut Url) { #[cfg(test)] mod tests { use super::*; - use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptDelta; use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry; use codex_protocol::protocol::RealtimeHandoffRequested; use codex_protocol::protocol::RealtimeInputAudioSpeechStarted; use codex_protocol::protocol::RealtimeResponseCancelled; use codex_protocol::protocol::RealtimeResponseCreated; use codex_protocol::protocol::RealtimeResponseDone; + use codex_protocol::protocol::RealtimeTranscriptDelta; + use codex_protocol::protocol::RealtimeTranscriptDone; use codex_protocol::protocol::RealtimeVoice; use http::HeaderValue; use pretty_assertions::assert_eq; @@ -891,7 +885,6 @@ mod tests { Some(RealtimeEvent::InputTranscriptDelta( RealtimeTranscriptDelta { delta: "hello ".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, } )) ); @@ -910,7 +903,6 @@ mod tests { Some(RealtimeEvent::OutputTranscriptDelta( RealtimeTranscriptDelta { delta: "hi".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, } )) ); @@ -954,7 +946,6 @@ mod tests { Some(RealtimeEvent::InputTranscriptDelta( RealtimeTranscriptDelta { delta: "hello".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, } )) ); @@ -970,10 +961,9 @@ mod tests { assert_eq!( parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2), - Some(RealtimeEvent::OutputTranscriptDelta( - RealtimeTranscriptDelta { - delta: "all done".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Done, + Some(RealtimeEvent::OutputTranscriptDone( + RealtimeTranscriptDone { + text: "all done".to_string(), } )) ); @@ -1514,7 +1504,6 @@ mod tests { input_delta_event, RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta { delta: "delegate ".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, }) ); @@ -1527,7 +1516,6 @@ mod tests { input_delta_event, RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta { delta: "now".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, }) ); @@ -1540,7 +1528,6 @@ mod tests { output_delta_event, RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { delta: "working".to_string(), - update_kind: RealtimeTranscriptUpdateKind::Delta, }) ); diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs index 254849b7fd..1fb49b2436 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs @@ -16,4 +16,3 @@ pub use protocol::RealtimeEventParser; pub use protocol::RealtimeOutputModality; pub use protocol::RealtimeSessionConfig; pub use protocol::RealtimeSessionMode; -pub use protocol::RealtimeTranscriptUpdateKind; diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs index b193de2e16..0706ea2422 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs @@ -3,9 +3,7 @@ use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2; pub use codex_protocol::protocol::RealtimeAudioFrame; pub use codex_protocol::protocol::RealtimeEvent; pub use codex_protocol::protocol::RealtimeOutputModality; -pub use codex_protocol::protocol::RealtimeTranscriptDelta; pub use codex_protocol::protocol::RealtimeTranscriptEntry; -pub use codex_protocol::protocol::RealtimeTranscriptUpdateKind; pub use codex_protocol::protocol::RealtimeVoice; use serde::Serialize; use serde_json::Value; diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_common.rs index 58639ee21f..c89c5ea4d0 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_common.rs @@ -1,6 +1,6 @@ use codex_protocol::protocol::RealtimeEvent; use codex_protocol::protocol::RealtimeTranscriptDelta; -use codex_protocol::protocol::RealtimeTranscriptUpdateKind; +use codex_protocol::protocol::RealtimeTranscriptDone; use serde_json::Value; use tracing::debug; @@ -46,20 +46,23 @@ pub(super) fn parse_session_updated_event(parsed: &Value) -> Option Option { - parse_transcript_update_event(parsed, field, RealtimeTranscriptUpdateKind::Delta) -} - -pub(super) fn parse_transcript_update_event( - parsed: &Value, - field: &str, - update_kind: RealtimeTranscriptUpdateKind, ) -> Option { parsed .get(field) .and_then(Value::as_str) .map(str::to_string) - .map(|delta| RealtimeTranscriptDelta { delta, update_kind }) + .map(|delta| RealtimeTranscriptDelta { delta }) +} + +pub(super) fn parse_transcript_done_event( + parsed: &Value, + field: &str, +) -> Option { + parsed + .get(field) + .and_then(Value::as_str) + .map(str::to_string) + .map(|text| RealtimeTranscriptDone { text }) } pub(super) fn parse_error_event(parsed: &Value) -> Option { diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_v2.rs index d91d6c1d2f..e09a60cb21 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_v2.rs @@ -2,7 +2,7 @@ use crate::endpoint::realtime_websocket::protocol_common::parse_error_event; use crate::endpoint::realtime_websocket::protocol_common::parse_realtime_payload; use crate::endpoint::realtime_websocket::protocol_common::parse_session_updated_event; use crate::endpoint::realtime_websocket::protocol_common::parse_transcript_delta_event; -use crate::endpoint::realtime_websocket::protocol_common::parse_transcript_update_event; +use crate::endpoint::realtime_websocket::protocol_common::parse_transcript_done_event; use codex_protocol::protocol::RealtimeAudioFrame; use codex_protocol::protocol::RealtimeEvent; use codex_protocol::protocol::RealtimeHandoffRequested; @@ -10,7 +10,6 @@ use codex_protocol::protocol::RealtimeInputAudioSpeechStarted; use codex_protocol::protocol::RealtimeResponseCancelled; use codex_protocol::protocol::RealtimeResponseCreated; use codex_protocol::protocol::RealtimeResponseDone; -use codex_protocol::protocol::RealtimeTranscriptUpdateKind; use serde_json::Map as JsonMap; use serde_json::Value; use tracing::debug; @@ -32,19 +31,18 @@ pub(super) fn parse_realtime_event_v2(payload: &str) -> Option { parse_transcript_delta_event(&parsed, "delta").map(RealtimeEvent::InputTranscriptDelta) } "conversation.item.input_audio_transcription.completed" => { - parse_transcript_update_event(&parsed, "transcript", RealtimeTranscriptUpdateKind::Done) - .map(RealtimeEvent::InputTranscriptDelta) + parse_transcript_done_event(&parsed, "transcript") + .map(RealtimeEvent::InputTranscriptDone) } "response.output_text.delta" | "response.output_audio_transcript.delta" => { parse_transcript_delta_event(&parsed, "delta").map(RealtimeEvent::OutputTranscriptDelta) } "response.output_text.done" => { - parse_transcript_update_event(&parsed, "text", RealtimeTranscriptUpdateKind::Done) - .map(RealtimeEvent::OutputTranscriptDelta) + parse_transcript_done_event(&parsed, "text").map(RealtimeEvent::OutputTranscriptDone) } "response.output_audio_transcript.done" => { - parse_transcript_update_event(&parsed, "transcript", RealtimeTranscriptUpdateKind::Done) - .map(RealtimeEvent::OutputTranscriptDelta) + parse_transcript_done_event(&parsed, "transcript") + .map(RealtimeEvent::OutputTranscriptDone) } "input_audio_buffer.speech_started" => Some(RealtimeEvent::InputAudioSpeechStarted( RealtimeInputAudioSpeechStarted { diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index 1a280c8bf4..f4f90b289c 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -44,7 +44,6 @@ pub use crate::endpoint::RealtimeEventParser; pub use crate::endpoint::RealtimeOutputModality; pub use crate::endpoint::RealtimeSessionConfig; pub use crate::endpoint::RealtimeSessionMode; -pub use crate::endpoint::RealtimeTranscriptUpdateKind; pub use crate::endpoint::RealtimeWebsocketClient; pub use crate::endpoint::RealtimeWebsocketConnection; pub use crate::endpoint::RealtimeWebsocketEvents; diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index be9568cb3a..e749736710 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -1232,7 +1232,9 @@ async fn handle_realtime_server_event( RealtimeEvent::Error(_) => true, RealtimeEvent::SessionUpdated { .. } | RealtimeEvent::InputTranscriptDelta(_) + | RealtimeEvent::InputTranscriptDone(_) | RealtimeEvent::OutputTranscriptDelta(_) + | RealtimeEvent::OutputTranscriptDone(_) | RealtimeEvent::ConversationItemAdded(_) | RealtimeEvent::ConversationItemDone { .. } => false, }; diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 83d1fab536..7431e5f123 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -294,17 +294,14 @@ pub struct RealtimeAudioFrame { pub item_id: Option, } -#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] -#[serde(rename_all = "snake_case")] -pub enum RealtimeTranscriptUpdateKind { - Delta, - Done, -} - #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] pub struct RealtimeTranscriptDelta { pub delta: String, - pub update_kind: RealtimeTranscriptUpdateKind, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +pub struct RealtimeTranscriptDone { + pub text: String, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] @@ -349,7 +346,9 @@ pub enum RealtimeEvent { }, InputAudioSpeechStarted(RealtimeInputAudioSpeechStarted), InputTranscriptDelta(RealtimeTranscriptDelta), + InputTranscriptDone(RealtimeTranscriptDone), OutputTranscriptDelta(RealtimeTranscriptDelta), + OutputTranscriptDone(RealtimeTranscriptDone), AudioOut(RealtimeAudioFrame), ResponseCreated(RealtimeResponseCreated), ResponseCancelled(RealtimeResponseCancelled), diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index 0bd78f353a..b5eccbfc6d 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -385,7 +385,10 @@ fn server_notification_thread_target( ServerNotification::ThreadRealtimeItemAdded(notification) => { Some(notification.thread_id.as_str()) } - ServerNotification::ThreadRealtimeTranscriptUpdated(notification) => { + ServerNotification::ThreadRealtimeTranscriptDelta(notification) => { + Some(notification.thread_id.as_str()) + } + ServerNotification::ThreadRealtimeTranscriptDone(notification) => { Some(notification.thread_id.as_str()) } ServerNotification::ThreadRealtimeOutputAudioDelta(notification) => { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index efb4c04194..4d5275a25d 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -6245,7 +6245,8 @@ impl ChatWidget { | ServerNotification::FsChanged(_) | ServerNotification::FuzzyFileSearchSessionUpdated(_) | ServerNotification::FuzzyFileSearchSessionCompleted(_) - | ServerNotification::ThreadRealtimeTranscriptUpdated(_) + | ServerNotification::ThreadRealtimeTranscriptDelta(_) + | ServerNotification::ThreadRealtimeTranscriptDone(_) | ServerNotification::WindowsWorldWritableWarning(_) | ServerNotification::WindowsSandboxSetupCompleted(_) | ServerNotification::AccountLoginCompleted(_) => {} diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs index e6c6ed49be..6357361f8e 100644 --- a/codex-rs/tui/src/chatwidget/realtime.rs +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -329,7 +329,9 @@ impl ChatWidget { } RealtimeEvent::InputAudioSpeechStarted(_) => self.interrupt_realtime_audio_playback(), RealtimeEvent::InputTranscriptDelta(_) => {} + RealtimeEvent::InputTranscriptDone(_) => {} RealtimeEvent::OutputTranscriptDelta(_) => {} + RealtimeEvent::OutputTranscriptDone(_) => {} RealtimeEvent::AudioOut(frame) => self.enqueue_realtime_audio_out(&frame), RealtimeEvent::ResponseCreated(_) => {} RealtimeEvent::ResponseCancelled(_) => self.interrupt_realtime_audio_playback(),