Add realtime rtc protocol support

Adds the realtime_rtc protocol path so app-server clients can negotiate WebRTC media while core keeps realtime events and tool handling. Regenerates protocol/config schemas for the new request and response shapes.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-04-06 15:59:12 -07:00
co-authored by Codex
parent 1525bbdb9a
commit 96fee71d24
29 changed files with 872 additions and 76 deletions
+1 -1
View File
@@ -23,6 +23,7 @@ tungstenite = { workspace = true }
tracing = { workspace = true }
eventsource-stream = { workspace = true }
regex-lite = { workspace = true }
reqwest = { workspace = true, features = ["multipart"] }
tokio-util = { workspace = true, features = ["codec"] }
url = { workspace = true }
@@ -32,7 +33,6 @@ assert_matches = { workspace = true }
pretty_assertions = { workspace = true }
tokio-test = { workspace = true }
wiremock = { workspace = true }
reqwest = { workspace = true }
[lints]
workspace = true
@@ -0,0 +1,189 @@
use crate::endpoint::realtime_websocket::methods::merge_request_headers;
use crate::endpoint::realtime_websocket::methods_common::session_update_session;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
use crate::error::ApiError;
use crate::provider::Provider;
use codex_client::build_reqwest_client_with_custom_ca;
use http::HeaderMap;
use http::header::LOCATION;
use reqwest::multipart::Form;
use serde::Serialize;
use url::Url;
const REALTIME_CALLS_PATH: &str = "/v1/realtime/calls";
pub struct RealtimeCallCreateResponse {
pub call_id: String,
pub answer_sdp: String,
}
pub struct RealtimeCallClient {
provider: Provider,
}
impl RealtimeCallClient {
pub fn new(provider: Provider) -> Self {
Self { provider }
}
pub async fn create(
&self,
config: &RealtimeSessionConfig,
offer_sdp: String,
extra_headers: HeaderMap,
default_headers: HeaderMap,
) -> Result<RealtimeCallCreateResponse, ApiError> {
let client = build_reqwest_client_with_custom_ca(reqwest::Client::builder())
.map_err(|err| ApiError::Stream(format!("failed to configure realtime HTTP: {err}")))?;
let url = realtime_calls_url(&self.provider.base_url)?;
let headers = merge_request_headers(&self.provider.headers, extra_headers, default_headers);
let response = client
.post(url)
.headers(headers)
.multipart(session_form(config, offer_sdp)?)
.send()
.await
.map_err(|err| ApiError::Stream(format!("failed to create realtime call: {err}")))?;
let status = response.status();
let headers = response.headers().clone();
let answer_sdp = response.text().await.map_err(|err| {
ApiError::Stream(format!("failed to read realtime call answer: {err}"))
})?;
if !status.is_success() {
return Err(ApiError::Stream(format!(
"realtime call failed with HTTP {status}: {answer_sdp}"
)));
}
let call_id =
call_id_from_location(headers.get(LOCATION).and_then(|value| value.to_str().ok()))?;
Ok(RealtimeCallCreateResponse {
call_id,
answer_sdp,
})
}
}
fn realtime_calls_url(base_url: &str) -> Result<Url, ApiError> {
let mut url =
Url::parse(base_url).map_err(|err| ApiError::Stream(format!("invalid base URL: {err}")))?;
match url.scheme() {
"http" | "https" => {}
"ws" => {
let _ = url.set_scheme("http");
}
"wss" => {
let _ = url.set_scheme("https");
}
scheme => {
return Err(ApiError::Stream(format!(
"unsupported realtime calls URL scheme: {scheme}"
)));
}
}
url.set_path(REALTIME_CALLS_PATH);
url.set_query(None);
Ok(url)
}
fn session_form(config: &RealtimeSessionConfig, offer_sdp: String) -> Result<Form, ApiError> {
let session_json = serde_json::to_string(&session_payload(config)).map_err(|err| {
ApiError::Stream(format!("failed to serialize realtime call session: {err}"))
})?;
Ok(Form::new()
.text("sdp", offer_sdp)
.text("session", session_json))
}
fn session_payload(config: &RealtimeSessionConfig) -> RealtimeCallSession {
let session = session_update_session(
config.event_parser,
config.instructions.clone(),
config.session_mode,
);
RealtimeCallSession {
session,
model: config.model.clone(),
}
}
#[derive(Serialize)]
struct RealtimeCallSession {
#[serde(flatten)]
session: SessionUpdateSession,
#[serde(skip_serializing_if = "Option::is_none")]
model: Option<String>,
}
fn call_id_from_location(location: Option<&str>) -> Result<String, ApiError> {
let Some(location) = location else {
return Err(ApiError::Stream(
"realtime call response missing Location header".to_string(),
));
};
let call_id = location
.trim_end_matches('/')
.rsplit('/')
.next()
.filter(|call_id| !call_id.is_empty())
.ok_or_else(|| ApiError::Stream("invalid realtime call Location header".to_string()))?;
Ok(call_id.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
use pretty_assertions::assert_eq;
#[test]
fn realtime_calls_url_uses_calls_path() {
assert_eq!(
realtime_calls_url("wss://api.openai.com/v1/realtime")
.expect("url")
.as_str(),
"https://api.openai.com/v1/realtime/calls"
);
}
#[test]
fn call_id_from_location_extracts_last_path_segment() {
assert_eq!(
call_id_from_location(Some("/v1/realtime/calls/rtc_123")).expect("call id"),
"rtc_123"
);
}
#[test]
fn session_form_contains_offer_and_session() {
let form = session_form(
&RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
model: Some("gpt-realtime".to_string()),
session_id: Some("session".to_string()),
event_parser: RealtimeEventParser::RealtimeV2,
session_mode: RealtimeSessionMode::Conversational,
},
"v=0".to_string(),
)
.expect("form");
assert!(!form.boundary().is_empty());
}
#[test]
fn session_payload_includes_model() {
let payload = serde_json::to_value(session_payload(&RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
model: Some("gpt-realtime".to_string()),
session_id: Some("session".to_string()),
event_parser: RealtimeEventParser::RealtimeV2,
session_mode: RealtimeSessionMode::Conversational,
}))
.expect("session payload");
assert_eq!(payload["type"], "realtime");
assert_eq!(payload["model"], "gpt-realtime");
assert_eq!(payload["instructions"], "backend prompt");
}
}
@@ -463,7 +463,34 @@ impl RealtimeWebsocketClient {
config.event_parser,
config.session_mode,
)?;
self.connect_with_url(ws_url, config, extra_headers, default_headers)
.await
}
pub async fn connect_to_call_id(
&self,
config: RealtimeSessionConfig,
call_id: &str,
extra_headers: HeaderMap,
default_headers: HeaderMap,
) -> Result<RealtimeWebsocketConnection, ApiError> {
ensure_rustls_crypto_provider();
let ws_url = websocket_url_from_api_url_with_call_id(
self.provider.base_url.as_str(),
self.provider.query_params.as_ref(),
call_id,
)?;
self.connect_with_url(ws_url, config, extra_headers, default_headers)
.await
}
async fn connect_with_url(
&self,
ws_url: Url,
config: RealtimeSessionConfig,
extra_headers: HeaderMap,
default_headers: HeaderMap,
) -> Result<RealtimeWebsocketConnection, ApiError> {
let mut request = ws_url
.as_str()
.into_client_request()
@@ -509,7 +536,7 @@ impl RealtimeWebsocketClient {
}
}
fn merge_request_headers(
pub(super) fn merge_request_headers(
provider_headers: &HeaderMap,
extra_headers: HeaderMap,
default_headers: HeaderMap,
@@ -596,6 +623,46 @@ fn websocket_url_from_api_url(
Ok(url)
}
fn websocket_url_from_api_url_with_call_id(
api_url: &str,
query_params: Option<&HashMap<String, String>>,
call_id: &str,
) -> Result<Url, ApiError> {
let mut url = Url::parse(api_url)
.map_err(|err| ApiError::Stream(format!("failed to parse realtime api_url: {err}")))?;
normalize_realtime_path(&mut url);
match url.scheme() {
"ws" | "wss" => {}
"http" | "https" => {
let scheme = if url.scheme() == "http" { "ws" } else { "wss" };
let _ = url.set_scheme(scheme);
}
scheme => {
return Err(ApiError::Stream(format!(
"unsupported realtime api_url scheme: {scheme}"
)));
}
}
url.set_query(None);
{
let mut query = url.query_pairs_mut();
query.append_pair("call_id", call_id);
if let Some(query_params) = query_params {
for (key, value) in query_params {
if matches!(key.as_str(), "call_id" | "intent" | "model") {
continue;
}
query.append_pair(key, value);
}
}
}
Ok(url)
}
fn normalize_realtime_path(url: &mut Url) {
let path = url.path().to_string();
if path.is_empty() || path == "/" {
@@ -1094,6 +1161,25 @@ mod tests {
assert_eq!(url.as_str(), "wss://example.com/v1/realtime");
}
#[test]
fn websocket_url_with_call_id_uses_call_id_only_for_session_selectors() {
let url = websocket_url_from_api_url_with_call_id(
"https://example.com/v1/realtime?model=old&foo=bar",
Some(&HashMap::from([
("trace".to_string(), "1".to_string()),
("intent".to_string(), "ignored".to_string()),
("model".to_string(), "ignored".to_string()),
])),
"rtc_123",
)
.expect("build ws url");
assert_eq!(
url.as_str(),
"wss://example.com/v1/realtime?call_id=rtc_123&trace=1"
);
}
#[tokio::test]
async fn e2e_connect_and_exchange_events_against_mock_ws_server() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
@@ -1,3 +1,4 @@
mod calls;
pub mod methods;
mod methods_common;
mod methods_v1;
@@ -7,6 +8,8 @@ mod protocol_common;
mod protocol_v1;
mod protocol_v2;
pub use calls::RealtimeCallClient;
pub use calls::RealtimeCallCreateResponse;
pub use codex_protocol::protocol::RealtimeAudioFrame;
pub use codex_protocol::protocol::RealtimeEvent;
pub use methods::RealtimeWebsocketClient;
+2
View File
@@ -31,6 +31,8 @@ pub use crate::common::response_create_client_metadata;
pub use crate::endpoint::compact::CompactClient;
pub use crate::endpoint::memories::MemoriesClient;
pub use crate::endpoint::models::ModelsClient;
pub use crate::endpoint::realtime_websocket::RealtimeCallClient;
pub use crate::endpoint::realtime_websocket::RealtimeCallCreateResponse;
pub use crate::endpoint::realtime_websocket::RealtimeEventParser;
pub use crate::endpoint::realtime_websocket::RealtimeSessionConfig;
pub use crate::endpoint::realtime_websocket::RealtimeSessionMode;