openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-18 17:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Frameless Bidi support for realtime conversations (#33261)

Browse Source 
## What changed

- Add realtime conversation version `v3`, which preserves the V1 Codex Voice behavior while using Frameless Bidi `delegation.*` events.
- Translate audio, transcripts, handoffs, session context, and lifecycle events between the app server and the Frameless Bidi wire protocol.
- Support `v3` over WebSocket and WebRTC, including the Frameless `/live` endpoint, session configuration, headers, and default model selection.
- Update the app-server protocol schemas and documentation for the new version.

## Testing

- Add unit coverage for Frameless event parsing, outbound messages, context chunking, URL construction, and call creation.
- Add app-server end-to-end coverage for WebSocket delegation and WebRTC session startup.

GitOrigin-RevId: 79a3307bc209e1a54582ebd2febc07c909fca016
This commit is contained in:
jiayuhuang-openai
2026-07-15 05:59:37 +00:00
committed by copyberry
parent 768330dd6c
commit 2e1607ee2f
24 changed files with 1117 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/codex-api/src/endpoint/realtime_call.rs
+110 -17
View File
@@ -63,6 +63,17 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
"realtime/calls"
}
fn path_for_session(&self, event_parser: RealtimeEventParser) -> &'static str {
if self.uses_backend_request_shape() {
return Self::path();
}
match event_parser {
RealtimeEventParser::FramelessBidi => "live",
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => Self::path(),
}
}
fn uses_backend_request_shape(&self) -> bool {
self.session.provider().base_url.contains("/backend-api")
}
@@ -123,9 +134,11 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
) -> Result<RealtimeCallResponse, ApiError> {
trace!(target: "codex_api::realtime_websocket::wire", "realtime call request SDP: {sdp}");
// WebRTC can begin inference as soon as the peer connection comes up, so the initial
// session payload is sent with call creation. The sideband WebSocket still sends its normal
// session.update after it joins.
// session payload is sent with call creation. Legacy sidebands still send session.update
// after joining; Frameless sidebands attach to the session that is already running.
validate_avas_session_config(&session_config)?;
let event_parser = session_config.event_parser;
let path = self.path_for_session(event_parser);
let mut session = realtime_session_json(session_config)?;
if let Some(session) = session.as_object_mut() {
session.remove("id");
@@ -139,13 +152,13 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
.map_err(|err| ApiError::Stream(format!("failed to encode realtime call: {err}")))?;
let resp = self
.session
.execute_with(
Method::POST,
Self::path(),
extra_headers,
Some(body),
configure_realtime_call_request,
)
.execute_with(Method::POST, path, extra_headers, Some(body), |request| {
configure_realtime_call_request(
request,
event_parser,
/*uses_backend_request_shape*/ true,
)
})
.await?;
let sdp = decode_sdp_response(resp.body.as_ref())?;
let call_id = decode_call_id_from_location(&resp.headers)?;
@@ -172,11 +185,15 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
.session
.execute_with(
Method::POST,
Self::path(),
path,
extra_headers,
/*body*/ None,
|req| {
configure_realtime_call_request(req);
configure_realtime_call_request(
req,
event_parser,
/*uses_backend_request_shape*/ false,
);
req.headers.insert(
CONTENT_TYPE,
HeaderValue::from_static(MULTIPART_CONTENT_TYPE),
@@ -193,15 +210,23 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
}
}
fn configure_realtime_call_request(request: &mut Request) {
append_query_pair(&mut request.url, "intent", "quicksilver");
append_query_pair(&mut request.url, "architecture", "avas");
fn configure_realtime_call_request(
request: &mut Request,
event_parser: RealtimeEventParser,
uses_backend_request_shape: bool,
) {
if event_parser == RealtimeEventParser::V1
|| (uses_backend_request_shape && event_parser == RealtimeEventParser::FramelessBidi)
{
append_query_pair(&mut request.url, "intent", "quicksilver");
append_query_pair(&mut request.url, "architecture", "avas");
}
}
fn validate_avas_session_config(session_config: &RealtimeSessionConfig) -> Result<(), ApiError> {
if session_config.event_parser != RealtimeEventParser::V1 {
if session_config.event_parser == RealtimeEventParser::RealtimeV2 {
return Err(ApiError::InvalidRequest {
message: "AVAS realtime calls require realtime v1".to_string(),
message: "AVAS realtime calls require realtime v1 or v3".to_string(),
});
}
Ok(())
@@ -378,6 +403,13 @@ mod tests {
}
}
fn frameless_bidi_session_config(session_id: &str) -> RealtimeSessionConfig {
RealtimeSessionConfig {
event_parser: RealtimeEventParser::FramelessBidi,
..realtime_session_config(session_id)
}
}
#[tokio::test]
async fn sends_sdp_offer_as_raw_body() {
let transport = CapturingTransport::new();
@@ -520,6 +552,36 @@ mod tests {
);
}
#[tokio::test]
async fn sends_frameless_session_call_to_live_without_legacy_query_params() {
let transport = CapturingTransport::with_location("/v1/live/rtc_frameless");
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://api.openai.com/v1"),
Arc::new(DummyAuth),
);
let response = client
.create_with_session(
"v=offer\r\n".to_string(),
frameless_bidi_session_config("sess-api"),
)
.await
.expect("request should succeed");
assert_eq!(response.call_id, "rtc_frameless");
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(request.url, "https://api.openai.com/v1/live");
let Some(RequestBody::Raw(body)) = request.body else {
panic!("multipart body should be raw");
};
let body = std::str::from_utf8(&body).expect("multipart body should be utf-8");
assert!(body.contains("\"model\":\"gpt-realtime\""));
assert!(body.contains("\"delegation\":{\"type\":\"client\"}"));
assert!(!body.contains("\"id\":\"sess-api\""));
}
#[tokio::test]
async fn sends_session_call_with_avas_query_params() {
let transport = CapturingTransport::new();
@@ -573,7 +635,7 @@ mod tests {
assert_eq!(
err.to_string(),
"invalid request: AVAS realtime calls require realtime v1"
"invalid request: AVAS realtime calls require realtime v1 or v3"
);
assert!(transport.last_request.lock().unwrap().is_none());
}
@@ -627,6 +689,37 @@ mod tests {
);
}
#[tokio::test]
async fn sends_backend_frameless_session_call_to_realtime_calls() {
let transport = CapturingTransport::with_location("/v1/live/rtc_backend_frameless");
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://chatgpt.com/backend-api/codex"),
Arc::new(DummyAuth),
);
let response = client
.create_with_session(
"v=offer\r\n".to_string(),
frameless_bidi_session_config("sess-backend"),
)
.await
.expect("request should succeed");
assert_eq!(response.call_id, "rtc_backend_frameless");
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(
request.url,
"https://chatgpt.com/backend-api/codex/realtime/calls?intent=quicksilver&architecture=avas"
);
let Some(RequestBody::Json(body)) = request.body else {
panic!("backend request body should be JSON");
};
assert_eq!(body["session"]["delegation"]["type"], "client");
assert!(body["session"].get("id").is_none());
}
#[tokio::test]
async fn errors_when_location_is_missing() {
let transport = CapturingTransport::without_location();
codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs
+178 -28
View File
@@ -1,8 +1,13 @@
use crate::endpoint::realtime_websocket::methods_common::conversation_function_call_output_message;
use crate::endpoint::realtime_websocket::methods_common::conversation_handoff_append_message;
use crate::endpoint::realtime_websocket::methods_common::conversation_item_create_message;
use crate::endpoint::realtime_websocket::methods_common::normalized_session_mode;
use crate::endpoint::realtime_websocket::methods_common::session_update_session;
use crate::endpoint::realtime_websocket::methods_common::session_update_message;
use crate::endpoint::realtime_websocket::methods_common::standalone_handoff_message;
use crate::endpoint::realtime_websocket::methods_common::websocket_intent;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::context_append_chunks;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::delegation_context_append_message as frameless_delegation_context_append_message;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_context_append_message as frameless_session_context_append_message;
use crate::endpoint::realtime_websocket::protocol::RealtimeAudioFrame;
use crate::endpoint::realtime_websocket::protocol::RealtimeEvent;
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
@@ -25,6 +30,7 @@ use futures::StreamExt;
use http::HeaderMap;
use http::HeaderValue;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
@@ -210,6 +216,7 @@ pub struct RealtimeWebsocketWriter {
#[derive(Clone)]
pub struct RealtimeWebsocketEvents {
rx_message: async_channel::Receiver<Result<Message, WsError>>,
pending_events: Arc<Mutex<VecDeque<RealtimeEvent>>>,
active_transcript: Arc<Mutex<ActiveTranscriptState>>,
event_parser: RealtimeEventParser,
is_closed: Arc<AtomicBool>,
@@ -277,6 +284,7 @@ impl RealtimeWebsocketConnection {
},
events: RealtimeWebsocketEvents {
rx_message,
pending_events: Arc::new(Mutex::new(VecDeque::new())),
active_transcript: Arc::new(Mutex::new(ActiveTranscriptState::default())),
event_parser,
is_closed,
@@ -287,8 +295,15 @@ impl RealtimeWebsocketConnection {
impl RealtimeWebsocketWriter {
pub async fn send_audio_frame(&self, frame: RealtimeAudioFrame) -> Result<(), ApiError> {
self.send_json(&RealtimeOutboundMessage::InputAudioBufferAppend { audio: frame.data })
.await
let message = match self.event_parser {
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
RealtimeOutboundMessage::InputAudioBufferAppend { audio: frame.data }
}
RealtimeEventParser::FramelessBidi => {
RealtimeOutboundMessage::InputAudioAppend { audio: frame.data }
}
};
self.send_json(&message).await
}
pub async fn send_conversation_item_create(
@@ -309,10 +324,24 @@ impl RealtimeWebsocketWriter {
handoff_id: String,
output_text: String,
) -> Result<(), ApiError> {
self.send_json(&RealtimeOutboundMessage::ConversationHandoffAppend {
self.send_json(&conversation_handoff_append_message(
self.event_parser,
handoff_id,
output_text,
})
))
.await
}
pub async fn send_standalone_handoff(
&self,
handoff_id: String,
output_text: String,
) -> Result<(), ApiError> {
self.send_json(&standalone_handoff_message(
self.event_parser,
handoff_id,
output_text,
))
.await
}
@@ -342,21 +371,34 @@ impl RealtimeWebsocketWriter {
voice: RealtimeVoice,
) -> Result<(), ApiError> {
let session_mode = normalized_session_mode(self.event_parser, session_mode);
let session = session_update_session(
let message = session_update_message(
self.event_parser,
instructions,
session_mode,
output_modality,
voice,
);
self.send_json(&RealtimeOutboundMessage::SessionUpdate { session })
.await
self.send_json(&message).await
}
pub async fn close(&self) -> Result<(), ApiError> {
if self.is_closed.swap(true, Ordering::SeqCst) {
return Ok(());
}
if self.event_parser == RealtimeEventParser::FramelessBidi {
let payload =
serde_json::to_string(&RealtimeOutboundMessage::SessionClose).map_err(|err| {
ApiError::Stream(format!("failed to encode realtime request: {err}"))
})?;
trace!(target: REALTIME_WIRE_LOG_TARGET, "realtime websocket request: {payload}");
if let Err(err) = self.stream.send(Message::Text(payload.into())).await
&& !matches!(err, WsError::ConnectionClosed | WsError::AlreadyClosed)
{
return Err(ApiError::Stream(format!(
"failed to close frameless realtime session: {err}"
)));
}
}
if let Err(err) = self.stream.close().await
&& !matches!(err, WsError::ConnectionClosed | WsError::AlreadyClosed)
{
@@ -368,6 +410,37 @@ impl RealtimeWebsocketWriter {
}
async fn send_json(&self, message: &RealtimeOutboundMessage) -> Result<(), ApiError> {
match message {
RealtimeOutboundMessage::DelegationContextAppend {
delegation_item_id,
content,
} => {
if let Some(content) = content.first() {
for chunk in context_append_chunks(&content.text) {
self.send_json_frame(&frameless_delegation_context_append_message(
delegation_item_id.clone(),
chunk,
))
.await?;
}
return Ok(());
}
}
RealtimeOutboundMessage::SessionContextAppend { content } => {
if let Some(content) = content.first() {
for chunk in context_append_chunks(&content.text) {
self.send_json_frame(&frameless_session_context_append_message(chunk))
.await?;
}
return Ok(());
}
}
_ => {}
}
self.send_json_frame(message).await
}
async fn send_json_frame(&self, message: &RealtimeOutboundMessage) -> Result<(), ApiError> {
let payload = serde_json::to_string(message)
.map_err(|err| ApiError::Stream(format!("failed to encode realtime request: {err}")))?;
debug!(?message, "realtime websocket request");
@@ -403,6 +476,10 @@ impl RealtimeWebsocketEvents {
return Ok(None);
}
if let Some(event) = self.pending_events.lock().await.pop_front() {
return Ok(Some(event));
}
loop {
let msg = match self.rx_message.recv().await {
Ok(Ok(msg)) => msg,
@@ -449,6 +526,24 @@ impl RealtimeWebsocketEvents {
}
}
async fn wait_for_session_started(&self) -> Result<(), ApiError> {
let Some(event) = self.next_event().await? else {
return Err(ApiError::Stream(
"frameless realtime session ended before session.started".to_string(),
));
};
match &event {
RealtimeEvent::SessionUpdated { .. } => {
self.pending_events.lock().await.push_back(event);
Ok(())
}
RealtimeEvent::Error(message) => Err(ApiError::Stream(message.clone())),
_ => Err(ApiError::Stream(
"frameless realtime session received an event before session.started".to_string(),
)),
}
}
async fn update_active_transcript(&self, event: &mut RealtimeEvent) {
let mut active_transcript = self.active_transcript.lock().await;
match event {
@@ -602,8 +697,14 @@ impl RealtimeWebsocketClient {
config.event_parser,
config.session_mode,
)?;
self.connect_realtime_websocket_url(ws_url, config, extra_headers, default_headers)
.await
self.connect_realtime_websocket_url(
ws_url,
config,
extra_headers,
default_headers,
/*initialize_session*/ true,
)
.await
}
pub async fn connect_webrtc_sideband(
@@ -662,8 +763,14 @@ impl RealtimeWebsocketClient {
config.session_mode,
call_id,
)?;
self.connect_realtime_websocket_url(ws_url, config, extra_headers, default_headers)
.await
self.connect_realtime_websocket_url(
ws_url,
config,
extra_headers,
default_headers,
/*initialize_session*/ false,
)
.await
}
async fn connect_realtime_websocket_url(
@@ -672,6 +779,7 @@ impl RealtimeWebsocketClient {
config: RealtimeSessionConfig,
extra_headers: HeaderMap,
default_headers: HeaderMap,
initialize_session: bool,
) -> Result<RealtimeWebsocketConnection, ApiError> {
ensure_rustls_crypto_provider();
@@ -708,19 +816,24 @@ impl RealtimeWebsocketClient {
let (stream, rx_message) = WsStream::new(stream);
let connection = RealtimeWebsocketConnection::new(stream, rx_message, config.event_parser);
debug!(
session_id = config.session_id.as_deref().unwrap_or("<none>"),
"realtime websocket sending session.update"
);
connection
.writer
.send_session_update(
config.instructions,
config.session_mode,
config.output_modality,
config.voice,
)
.await?;
if initialize_session || config.event_parser != RealtimeEventParser::FramelessBidi {
debug!(
session_id = config.session_id.as_deref().unwrap_or("<none>"),
"realtime websocket sending session.update"
);
connection
.writer
.send_session_update(
config.instructions,
config.session_mode,
config.output_modality,
config.voice,
)
.await?;
}
if initialize_session && config.event_parser == RealtimeEventParser::FramelessBidi {
connection.events.wait_for_session_started().await?;
}
Ok(connection)
}
}
@@ -770,7 +883,7 @@ fn websocket_url_from_api_url(
let mut url = Url::parse(api_url)
.map_err(|err| ApiError::Stream(format!("failed to parse realtime api_url: {err}")))?;
normalize_realtime_path(&mut url);
normalize_realtime_path(&mut url, event_parser);
match url.scheme() {
"ws" | "wss" => {}
@@ -826,11 +939,31 @@ fn websocket_url_from_api_url_for_call(
event_parser,
session_mode,
)?;
url.query_pairs_mut().append_pair("call_id", call_id);
match event_parser {
RealtimeEventParser::FramelessBidi => {
let path = format!("{}/{}", url.path().trim_end_matches('/'), call_id);
url.set_path(&path);
}
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
url.query_pairs_mut().append_pair("call_id", call_id);
}
}
Ok(url)
}
fn normalize_realtime_path(url: &mut Url) {
fn normalize_realtime_path(url: &mut Url, event_parser: RealtimeEventParser) {
if event_parser == RealtimeEventParser::FramelessBidi {
let path = url.path().to_string();
if path.is_empty() || path == "/" || path == "/v1" || path == "/v1/" {
url.set_path("/v1/live");
} else if let Some(prefix) = path.trim_end_matches('/').strip_suffix("/realtime") {
url.set_path(&format!("{prefix}/live"));
} else if path.ends_with("/live/") {
url.set_path(path.trim_end_matches('/'));
}
return;
}
let path = url.path().to_string();
if path.is_empty() || path == "/" {
url.set_path("/v1/realtime");
@@ -974,6 +1107,7 @@ mod tests {
let (_tx_message, rx_message) = async_channel::unbounded();
let events = RealtimeWebsocketEvents {
rx_message,
pending_events: Arc::new(Mutex::new(VecDeque::new())),
active_transcript: Arc::new(Mutex::new(ActiveTranscriptState::default())),
event_parser: RealtimeEventParser::V1,
is_closed: Arc::new(AtomicBool::new(false)),
@@ -1546,6 +1680,22 @@ mod tests {
);
}
#[test]
fn frameless_websocket_url_rewrites_existing_realtime_path() {
let url = websocket_url_from_api_url(
"wss://example.com/v1/realtime?foo=bar",
/*query_params*/ None,
Some("snapshot"),
RealtimeEventParser::FramelessBidi,
RealtimeSessionMode::Conversational,
)
.expect("build Frameless websocket url");
assert_eq!(
url.as_str(),
"wss://example.com/v1/live?foo=bar&model=snapshot"
);
}
#[test]
fn websocket_url_v1_ignores_transcription_mode() {
let url = websocket_url_from_api_url(
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs
+101 -39
View File
@@ -1,3 +1,7 @@
use crate::endpoint::realtime_websocket::methods_frameless_bidi::delegation_context_append_message as frameless_delegation_context_append_message;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_context_append_message as frameless_session_context_append_message;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_json as frameless_session_json;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_update_message as frameless_session_update_message;
use crate::endpoint::realtime_websocket::methods_v1::conversation_handoff_append_message as v1_conversation_handoff_append_message;
use crate::endpoint::realtime_websocket::methods_v1::conversation_item_create_message as v1_conversation_item_create_message;
use crate::endpoint::realtime_websocket::methods_v1::session_update_session as v1_session_update_session;
@@ -6,13 +10,12 @@ use crate::endpoint::realtime_websocket::methods_v2::conversation_function_call_
use crate::endpoint::realtime_websocket::methods_v2::conversation_item_create_message as v2_conversation_item_create_message;
use crate::endpoint::realtime_websocket::methods_v2::session_update_session as v2_session_update_session;
use crate::endpoint::realtime_websocket::methods_v2::websocket_intent as v2_websocket_intent;
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutputModality;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
use crate::endpoint::realtime_websocket::protocol::RealtimeWireAdapter;
use codex_protocol::protocol::ConversationTextRole;
use serde_json::Result as JsonResult;
use serde_json::Value;
@@ -22,74 +25,133 @@ pub(super) const REALTIME_AUDIO_SAMPLE_RATE: u32 = 24_000;
const AGENT_FINAL_MESSAGE_PREFIX: &str = "\"Agent Final Message\":\n\n";
pub(super) fn normalized_session_mode(
event_parser: RealtimeEventParser,
wire_adapter: RealtimeWireAdapter,
session_mode: RealtimeSessionMode,
) -> RealtimeSessionMode {
match event_parser {
RealtimeEventParser::V1 => RealtimeSessionMode::Conversational,
RealtimeEventParser::RealtimeV2 => session_mode,
match wire_adapter {
RealtimeWireAdapter::V1 | RealtimeWireAdapter::FramelessBidi => {
RealtimeSessionMode::Conversational
}
RealtimeWireAdapter::RealtimeV2 => session_mode,
}
}
pub(super) fn conversation_item_create_message(
event_parser: RealtimeEventParser,
wire_adapter: RealtimeWireAdapter,
text: String,
role: ConversationTextRole,
) -> RealtimeOutboundMessage {
match event_parser {
RealtimeEventParser::V1 => v1_conversation_item_create_message(text, role),
RealtimeEventParser::RealtimeV2 => v2_conversation_item_create_message(text, role),
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_item_create_message(text, role),
RealtimeWireAdapter::FramelessBidi => frameless_session_context_append_message(text),
RealtimeWireAdapter::RealtimeV2 => v2_conversation_item_create_message(text, role),
}
}
pub(super) fn conversation_handoff_append_message(
wire_adapter: RealtimeWireAdapter,
handoff_id: String,
output_text: String,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
RealtimeWireAdapter::FramelessBidi => {
frameless_delegation_context_append_message(handoff_id, output_text)
}
RealtimeWireAdapter::RealtimeV2 => {
unreachable!("realtime v2 does not send conversation handoff output")
}
}
}
pub(super) fn standalone_handoff_message(
wire_adapter: RealtimeWireAdapter,
handoff_id: String,
output_text: String,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
RealtimeWireAdapter::FramelessBidi => frameless_session_context_append_message(output_text),
RealtimeWireAdapter::RealtimeV2 => {
unreachable!("realtime v2 does not send standalone handoff output")
}
}
}
pub(super) fn conversation_function_call_output_message(
event_parser: RealtimeEventParser,
wire_adapter: RealtimeWireAdapter,
call_id: String,
output_text: String,
) -> RealtimeOutboundMessage {
match event_parser {
RealtimeEventParser::V1 => v1_conversation_handoff_append_message(
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(
call_id,
format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"),
),
RealtimeEventParser::RealtimeV2 => {
RealtimeWireAdapter::FramelessBidi => frameless_delegation_context_append_message(
call_id,
format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"),
),
RealtimeWireAdapter::RealtimeV2 => {
v2_conversation_function_call_output_message(call_id, output_text)
}
}
}
pub(super) fn session_update_session(
event_parser: RealtimeEventParser,
pub(super) fn session_update_message(
wire_adapter: RealtimeWireAdapter,
instructions: String,
session_mode: RealtimeSessionMode,
output_modality: RealtimeOutputModality,
voice: RealtimeVoice,
) -> SessionUpdateSession {
let session_mode = normalized_session_mode(event_parser, session_mode);
match event_parser {
RealtimeEventParser::V1 => v1_session_update_session(instructions, voice),
RealtimeEventParser::RealtimeV2 => {
v2_session_update_session(instructions, session_mode, output_modality, voice)
}
) -> RealtimeOutboundMessage {
let session_mode = normalized_session_mode(wire_adapter, session_mode);
match wire_adapter {
RealtimeWireAdapter::V1 => RealtimeOutboundMessage::SessionUpdate {
session: v1_session_update_session(instructions, voice),
},
RealtimeWireAdapter::FramelessBidi => frameless_session_update_message(instructions, voice),
RealtimeWireAdapter::RealtimeV2 => RealtimeOutboundMessage::SessionUpdate {
session: v2_session_update_session(instructions, session_mode, output_modality, voice),
},
}
}
pub fn session_update_session_json(config: RealtimeSessionConfig) -> JsonResult<Value> {
let mut session = session_update_session(
config.event_parser,
config.instructions,
config.session_mode,
config.output_modality,
config.voice,
);
session.id = config.session_id;
session.model = config.model;
to_value(session)
}
pub(super) fn websocket_intent(event_parser: RealtimeEventParser) -> Option<&'static str> {
match event_parser {
RealtimeEventParser::V1 => v1_websocket_intent(),
RealtimeEventParser::RealtimeV2 => v2_websocket_intent(),
match config.event_parser {
RealtimeWireAdapter::V1 | RealtimeWireAdapter::RealtimeV2 => {
let mut session = match config.event_parser {
RealtimeWireAdapter::V1 => {
v1_session_update_session(config.instructions, config.voice)
}
RealtimeWireAdapter::RealtimeV2 => v2_session_update_session(
config.instructions,
config.session_mode,
config.output_modality,
config.voice,
),
RealtimeWireAdapter::FramelessBidi => unreachable!(),
};
session.id = config.session_id;
session.model = config.model;
to_value(session)
}
RealtimeWireAdapter::FramelessBidi => Ok(frameless_session_json(
config.model,
config.instructions,
config.voice,
)),
}
}
pub(super) fn websocket_intent(wire_adapter: RealtimeWireAdapter) -> Option<&'static str> {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_websocket_intent(),
RealtimeWireAdapter::FramelessBidi => None,
RealtimeWireAdapter::RealtimeV2 => v2_websocket_intent(),
}
}
#[cfg(test)]
#[path = "methods_common_tests.rs"]
mod tests;
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common_tests.rs
+88
View File
@@ -0,0 +1,88 @@
use super::conversation_function_call_output_message;
use super::conversation_handoff_append_message;
use super::standalone_handoff_message;
use crate::endpoint::realtime_websocket::protocol::RealtimeWireAdapter;
use pretty_assertions::assert_eq;
use serde_json::Value;
use serde_json::json;
use serde_json::to_value;
#[test]
fn identical_handoff_output_encodes_for_each_bidi_wire_protocol() {
let legacy = conversation_handoff_append_message(
RealtimeWireAdapter::V1,
"handoff-123".to_string(),
"The result".to_string(),
);
let frameless = conversation_handoff_append_message(
RealtimeWireAdapter::FramelessBidi,
"handoff-123".to_string(),
"The result".to_string(),
);
assert_eq!(
to_value(legacy).expect("legacy handoff should serialize"),
json!({
"type": "conversation.handoff.append",
"handoff_id": "handoff-123",
"output_text": "The result",
})
);
assert_eq!(
to_value(frameless).expect("frameless handoff should serialize"),
json!({
"type": "delegation.context.append",
"delegation_item_id": "handoff-123",
"content": [{"type": "input_text", "text": "The result"}],
})
);
}
#[test]
fn standalone_handoff_uses_session_context_for_frameless() {
let legacy = standalone_handoff_message(
RealtimeWireAdapter::V1,
"codex".to_string(),
"Speak this".to_string(),
);
let frameless = standalone_handoff_message(
RealtimeWireAdapter::FramelessBidi,
"codex".to_string(),
"Speak this".to_string(),
);
assert_eq!(
to_value(legacy).expect("legacy standalone handoff should serialize"),
json!({
"type": "conversation.handoff.append",
"handoff_id": "codex",
"output_text": "Speak this",
})
);
assert_eq!(
to_value(frameless).expect("frameless standalone handoff should serialize"),
json!({
"type": "session.context.append",
"content": [{"type": "input_text", "text": "Speak this"}],
})
);
}
#[test]
fn completed_handoff_preserves_legacy_payload_text_in_frameless() {
let expected_text = Value::String("\"Agent Final Message\":\n\nDone".to_string());
for wire_adapter in [RealtimeWireAdapter::V1, RealtimeWireAdapter::FramelessBidi] {
let encoded = to_value(conversation_function_call_output_message(
wire_adapter,
"handoff-123".to_string(),
"Done".to_string(),
))
.expect("handoff output should serialize");
let text = match wire_adapter {
RealtimeWireAdapter::V1 => &encoded["output_text"],
RealtimeWireAdapter::FramelessBidi => &encoded["content"][0]["text"],
RealtimeWireAdapter::RealtimeV2 => unreachable!(),
};
assert_eq!(text, &expected_text);
}
}
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_frameless_bidi.rs
+84
View File
@@ -0,0 +1,84 @@
use crate::endpoint::realtime_websocket::protocol::FramelessContentType;
use crate::endpoint::realtime_websocket::protocol::FramelessInputTextContent;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use serde_json::Value;
use serde_json::json;
const CONTEXT_APPEND_MAX_BYTES: usize = 500;
pub(super) fn delegation_context_append_message(
delegation_item_id: String,
text: String,
) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::DelegationContextAppend {
delegation_item_id,
content: input_text_content(text),
}
}
pub(super) fn session_context_append_message(text: String) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::SessionContextAppend {
content: input_text_content(text),
}
}
pub(super) fn session_update_message(
instructions: String,
voice: RealtimeVoice,
) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::FramelessSessionUpdate {
session: session_json(/*model*/ None, instructions, voice),
}
}
pub(super) fn session_json(
model: Option<String>,
instructions: String,
voice: RealtimeVoice,
) -> Value {
let mut session = json!({
"instructions": instructions,
"audio": {
"output": {
"voice": voice,
},
},
"delegation": {
"type": "client",
},
});
if let Some(model) = model {
session["model"] = Value::String(model);
}
session
}
fn input_text_content(text: String) -> Vec<FramelessInputTextContent> {
vec![FramelessInputTextContent {
r#type: FramelessContentType::InputText,
text,
}]
}
pub(super) fn context_append_chunks(text: &str) -> Vec<String> {
if text.len() <= CONTEXT_APPEND_MAX_BYTES {
return vec![text.to_string()];
}
let mut chunks = Vec::new();
let mut start = 0;
while start < text.len() {
let mut end = (start + CONTEXT_APPEND_MAX_BYTES).min(text.len());
while end > start && !text.is_char_boundary(end) {
end -= 1;
}
chunks.push(text[start..end].to_string());
start = end;
}
chunks
}
#[cfg(test)]
#[path = "methods_frameless_bidi_tests.rs"]
mod tests;
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_frameless_bidi_tests.rs
+15
View File
@@ -0,0 +1,15 @@
use super::CONTEXT_APPEND_MAX_BYTES;
use super::context_append_chunks;
#[test]
fn context_append_chunks_preserve_text_within_wire_limit() {
for text in ["a".repeat(1_201), "🙂".repeat(200)] {
let chunks = context_append_chunks(&text);
assert_eq!(chunks.concat(), text);
assert!(
chunks
.iter()
.all(|chunk| chunk.len() <= CONTEXT_APPEND_MAX_BYTES)
);
}
}
codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs
+2
View File
@@ -1,9 +1,11 @@
pub(crate) mod methods;
mod methods_common;
mod methods_frameless_bidi;
mod methods_v1;
mod methods_v2;
pub(crate) mod protocol;
mod protocol_common;
mod protocol_frameless_bidi;
mod protocol_v1;
mod protocol_v2;
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs
+33
View File
@@ -1,3 +1,4 @@
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
use codex_protocol::protocol::ConversationTextRole;
@@ -12,9 +13,12 @@ use serde_json::Value;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeEventParser {
V1,
FramelessBidi,
RealtimeV2,
}
pub type RealtimeWireAdapter = RealtimeEventParser;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RealtimeSessionMode {
Conversational,
@@ -42,14 +46,42 @@ pub(super) enum RealtimeOutboundMessage {
handoff_id: String,
output_text: String,
},
#[serde(rename = "input_audio.append")]
InputAudioAppend { audio: String },
#[serde(rename = "delegation.context.append")]
DelegationContextAppend {
delegation_item_id: String,
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.context.append")]
SessionContextAppend {
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.close")]
SessionClose,
#[serde(rename = "response.create")]
ResponseCreate,
#[serde(rename = "session.update")]
SessionUpdate { session: SessionUpdateSession },
#[serde(rename = "session.update")]
FramelessSessionUpdate { session: Value },
#[serde(rename = "conversation.item.create")]
ConversationItemCreate { item: ConversationItemPayload },
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct FramelessInputTextContent {
#[serde(rename = "type")]
pub(super) r#type: FramelessContentType,
pub(super) text: String,
}
#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum FramelessContentType {
InputText,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionUpdateSession {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -219,6 +251,7 @@ pub(super) fn parse_realtime_event(
) -> Option<RealtimeEvent> {
match event_parser {
RealtimeEventParser::V1 => parse_realtime_event_v1(payload),
RealtimeEventParser::FramelessBidi => parse_frameless_bidi_event(payload),
RealtimeEventParser::RealtimeV2 => parse_realtime_event_v2(payload),
}
}
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_frameless_bidi.rs
+99
View File
@@ -0,0 +1,99 @@
use crate::endpoint::realtime_websocket::protocol_common::parse_error_event;
use crate::endpoint::realtime_websocket::protocol_common::parse_realtime_payload;
use crate::endpoint::realtime_websocket::protocol_common::parse_session_updated_event;
use codex_protocol::protocol::RealtimeAudioFrame;
use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::protocol::RealtimeHandoffRequested;
use codex_protocol::protocol::RealtimeTranscriptDelta;
use codex_protocol::protocol::RealtimeTranscriptDone;
use serde_json::Value;
use tracing::debug;
const DEFAULT_AUDIO_SAMPLE_RATE: u32 = 24_000;
const DEFAULT_AUDIO_CHANNELS: u16 = 1;
pub(super) fn parse_frameless_bidi_event(payload: &str) -> Option<RealtimeEvent> {
let (parsed, message_type) = parse_realtime_payload(payload, "frameless bidi")?;
match message_type.as_str() {
"session.started" | "session.updated" => parse_session_updated_event(&parsed),
"output_audio.delta" => parse_output_audio_delta(&parsed),
"input_transcript.added" => {
parse_transcript_item(&parsed).map(RealtimeEvent::InputTranscriptDelta)
}
"output_transcript.added" => {
parse_transcript_item(&parsed).map(RealtimeEvent::OutputTranscriptDelta)
}
"turn.done" => parse_turn_done(&parsed),
"delegation.created" => parse_delegation_created(&parsed),
"error" => parse_error_event(&parsed),
_ => {
debug!(
"received unsupported frameless bidi event type: {message_type}, data: {payload}"
);
None
}
}
}
fn parse_output_audio_delta(parsed: &Value) -> Option<RealtimeEvent> {
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
data: parsed.get("audio").and_then(Value::as_str)?.to_string(),
sample_rate: DEFAULT_AUDIO_SAMPLE_RATE,
num_channels: DEFAULT_AUDIO_CHANNELS,
samples_per_channel: None,
item_id: None,
}))
}
fn parse_transcript_item(parsed: &Value) -> Option<RealtimeTranscriptDelta> {
parsed
.get("item")
.and_then(Value::as_object)
.and_then(|item| item.get("text"))
.and_then(Value::as_str)
.map(str::to_string)
.map(|delta| RealtimeTranscriptDelta { delta })
}
fn parse_turn_done(parsed: &Value) -> Option<RealtimeEvent> {
let turn = parsed.get("turn")?.as_object()?;
let role = turn.get("role").and_then(Value::as_str)?;
let text = turn
.get("transcript")
.and_then(Value::as_str)
.map(str::to_string)?;
let done = RealtimeTranscriptDone { text };
match role {
"user" => Some(RealtimeEvent::InputTranscriptDone(done)),
"assistant" => Some(RealtimeEvent::OutputTranscriptDone(done)),
_ => None,
}
}
fn parse_delegation_created(parsed: &Value) -> Option<RealtimeEvent> {
let item = parsed.get("item")?.as_object()?;
if item.get("type").and_then(Value::as_str) != Some("delegation")
|| item.get("target").and_then(Value::as_str) != Some("client")
{
return None;
}
let item_id = item.get("id").and_then(Value::as_str)?.to_string();
let input_transcript = item
.get("content")
.and_then(Value::as_array)?
.iter()
.filter(|content| content.get("type").and_then(Value::as_str) == Some("input_text"))
.filter_map(|content| content.get("text").and_then(Value::as_str))
.collect::<String>();
Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
handoff_id: item_id.clone(),
item_id,
input_transcript,
active_transcript: Vec::new(),
}))
}
#[cfg(test)]
#[path = "protocol_frameless_bidi_tests.rs"]
mod tests;
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol_frameless_bidi_tests.rs
+64
View File
@@ -0,0 +1,64 @@
use super::parse_frameless_bidi_event;
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::protocol::RealtimeHandoffRequested;
#[test]
fn legacy_and_frameless_delegations_decode_to_the_same_handoff() {
let expected = Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
handoff_id: "handoff-123".to_string(),
item_id: "handoff-123".to_string(),
input_transcript: "check the weather".to_string(),
active_transcript: Vec::new(),
}));
let legacy = r#"{
"type": "conversation.handoff.requested",
"handoff_id": "handoff-123",
"item_id": "handoff-123",
"input_transcript": "check the weather"
}"#;
let frameless = r#"{
"type": "delegation.created",
"offset_ms": 1000,
"item": {
"id": "handoff-123",
"type": "delegation",
"target": "client",
"content": [{"type": "input_text", "text": "check the weather"}]
}
}"#;
assert_eq!(parse_realtime_event_v1(legacy), expected);
assert_eq!(parse_frameless_bidi_event(frameless), expected);
}
#[test]
fn frameless_transcript_and_audio_events_reuse_existing_internal_events() {
let input = r#"{
"type": "input_transcript.added",
"item": {"id": "input-1", "type": "input_transcript", "text": "hello"}
}"#;
let done = r#"{
"type": "turn.done",
"turn": {"id": "turn-1", "role": "user", "transcript": "hello"}
}"#;
let audio = r#"{
"type": "output_audio.delta",
"audio": "AAE=",
"start_ms": 0,
"end_ms": 100
}"#;
assert!(matches!(
parse_frameless_bidi_event(input),
Some(RealtimeEvent::InputTranscriptDelta(_))
));
assert!(matches!(
parse_frameless_bidi_event(done),
Some(RealtimeEvent::InputTranscriptDone(_))
));
assert!(matches!(
parse_frameless_bidi_event(audio),
Some(RealtimeEvent::AudioOut(_))
));
}