Add Frameless Bidi support for realtime conversations (#33261)

## What changed

- Add realtime conversation version `v3`, which preserves the V1 Codex Voice behavior while using Frameless Bidi `delegation.*` events.
- Translate audio, transcripts, handoffs, session context, and lifecycle events between the app server and the Frameless Bidi wire protocol.
- Support `v3` over WebSocket and WebRTC, including the Frameless `/live` endpoint, session configuration, headers, and default model selection.
- Update the app-server protocol schemas and documentation for the new version.

## Testing

- Add unit coverage for Frameless event parsing, outbound messages, context chunking, URL construction, and call creation.
- Add app-server end-to-end coverage for WebSocket delegation and WebRTC session startup.

GitOrigin-RevId: 79a3307bc209e1a54582ebd2febc07c909fca016
This commit is contained in:
jiayuhuang-openai
2026-07-15 05:59:37 +00:00
committed by copyberry
parent 768330dd6c
commit 2e1607ee2f
24 changed files with 1117 additions and 147 deletions
@@ -1,3 +1,4 @@
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
use codex_protocol::protocol::ConversationTextRole;
@@ -12,9 +13,12 @@ use serde_json::Value;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeEventParser {
V1,
FramelessBidi,
RealtimeV2,
}
pub type RealtimeWireAdapter = RealtimeEventParser;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeSessionMode {
Conversational,
@@ -42,14 +46,42 @@ pub(super) enum RealtimeOutboundMessage {
handoff_id: String,
output_text: String,
},
#[serde(rename = "input_audio.append")]
InputAudioAppend { audio: String },
#[serde(rename = "delegation.context.append")]
DelegationContextAppend {
delegation_item_id: String,
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.context.append")]
SessionContextAppend {
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.close")]
SessionClose,
#[serde(rename = "response.create")]
ResponseCreate,
#[serde(rename = "session.update")]
SessionUpdate { session: SessionUpdateSession },
#[serde(rename = "session.update")]
FramelessSessionUpdate { session: Value },
#[serde(rename = "conversation.item.create")]
ConversationItemCreate { item: ConversationItemPayload },
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct FramelessInputTextContent {
#[serde(rename = "type")]
pub(super) r#type: FramelessContentType,
pub(super) text: String,
}
#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum FramelessContentType {
InputText,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionUpdateSession {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -219,6 +251,7 @@ pub(super) fn parse_realtime_event(
) -> Option<RealtimeEvent> {
match event_parser {
RealtimeEventParser::V1 => parse_realtime_event_v1(payload),
RealtimeEventParser::FramelessBidi => parse_frameless_bidi_event(payload),
RealtimeEventParser::RealtimeV2 => parse_realtime_event_v2(payload),
}
}