mirror of
https://github.com/openai/codex.git
synced 2026-11-09 20:50:33 +00:00
Add Frameless Bidi support for realtime conversations (#33261)
## What changed - Add realtime conversation version `v3`, which preserves the V1 Codex Voice behavior while using Frameless Bidi `delegation.*` events. - Translate audio, transcripts, handoffs, session context, and lifecycle events between the app server and the Frameless Bidi wire protocol. - Support `v3` over WebSocket and WebRTC, including the Frameless `/live` endpoint, session configuration, headers, and default model selection. - Update the app-server protocol schemas and documentation for the new version. ## Testing - Add unit coverage for Frameless event parsing, outbound messages, context chunking, URL construction, and call creation. - Add app-server end-to-end coverage for WebSocket delegation and WebRTC session startup. GitOrigin-RevId: 79a3307bc209e1a54582ebd2febc07c909fca016
This commit is contained in:
committed by
copyberry
parent
768330dd6c
commit
2e1607ee2f
@@ -1,3 +1,4 @@
|
||||
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
|
||||
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
|
||||
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
@@ -12,9 +13,12 @@ use serde_json::Value;
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RealtimeEventParser {
|
||||
V1,
|
||||
FramelessBidi,
|
||||
RealtimeV2,
|
||||
}
|
||||
|
||||
pub type RealtimeWireAdapter = RealtimeEventParser;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RealtimeSessionMode {
|
||||
Conversational,
|
||||
@@ -42,14 +46,42 @@ pub(super) enum RealtimeOutboundMessage {
|
||||
handoff_id: String,
|
||||
output_text: String,
|
||||
},
|
||||
#[serde(rename = "input_audio.append")]
|
||||
InputAudioAppend { audio: String },
|
||||
#[serde(rename = "delegation.context.append")]
|
||||
DelegationContextAppend {
|
||||
delegation_item_id: String,
|
||||
content: Vec<FramelessInputTextContent>,
|
||||
},
|
||||
#[serde(rename = "session.context.append")]
|
||||
SessionContextAppend {
|
||||
content: Vec<FramelessInputTextContent>,
|
||||
},
|
||||
#[serde(rename = "session.close")]
|
||||
SessionClose,
|
||||
#[serde(rename = "response.create")]
|
||||
ResponseCreate,
|
||||
#[serde(rename = "session.update")]
|
||||
SessionUpdate { session: SessionUpdateSession },
|
||||
#[serde(rename = "session.update")]
|
||||
FramelessSessionUpdate { session: Value },
|
||||
#[serde(rename = "conversation.item.create")]
|
||||
ConversationItemCreate { item: ConversationItemPayload },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct FramelessInputTextContent {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) r#type: FramelessContentType,
|
||||
pub(super) text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum FramelessContentType {
|
||||
InputText,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionUpdateSession {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -219,6 +251,7 @@ pub(super) fn parse_realtime_event(
|
||||
) -> Option<RealtimeEvent> {
|
||||
match event_parser {
|
||||
RealtimeEventParser::V1 => parse_realtime_event_v1(payload),
|
||||
RealtimeEventParser::FramelessBidi => parse_frameless_bidi_event(payload),
|
||||
RealtimeEventParser::RealtimeV2 => parse_realtime_event_v2(payload),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user