mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Add realtime rtc protocol support
Adds the realtime_rtc protocol path so app-server clients can negotiate WebRTC media while core keeps realtime events and tool handling. Regenerates protocol/config schemas for the new request and response shapes. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -23,6 +23,7 @@ tungstenite = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
eventsource-stream = { workspace = true }
|
||||
regex-lite = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["multipart"] }
|
||||
tokio-util = { workspace = true, features = ["codec"] }
|
||||
url = { workspace = true }
|
||||
|
||||
@@ -32,7 +33,6 @@ assert_matches = { workspace = true }
|
||||
pretty_assertions = { workspace = true }
|
||||
tokio-test = { workspace = true }
|
||||
wiremock = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
use crate::endpoint::realtime_websocket::methods::merge_request_headers;
|
||||
use crate::endpoint::realtime_websocket::methods_common::session_update_session;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
|
||||
use crate::error::ApiError;
|
||||
use crate::provider::Provider;
|
||||
use codex_client::build_reqwest_client_with_custom_ca;
|
||||
use http::HeaderMap;
|
||||
use http::header::LOCATION;
|
||||
use reqwest::multipart::Form;
|
||||
use serde::Serialize;
|
||||
use url::Url;
|
||||
|
||||
const REALTIME_CALLS_PATH: &str = "/v1/realtime/calls";
|
||||
|
||||
pub struct RealtimeCallCreateResponse {
|
||||
pub call_id: String,
|
||||
pub answer_sdp: String,
|
||||
}
|
||||
|
||||
pub struct RealtimeCallClient {
|
||||
provider: Provider,
|
||||
}
|
||||
|
||||
impl RealtimeCallClient {
|
||||
pub fn new(provider: Provider) -> Self {
|
||||
Self { provider }
|
||||
}
|
||||
|
||||
pub async fn create(
|
||||
&self,
|
||||
config: &RealtimeSessionConfig,
|
||||
offer_sdp: String,
|
||||
extra_headers: HeaderMap,
|
||||
default_headers: HeaderMap,
|
||||
) -> Result<RealtimeCallCreateResponse, ApiError> {
|
||||
let client = build_reqwest_client_with_custom_ca(reqwest::Client::builder())
|
||||
.map_err(|err| ApiError::Stream(format!("failed to configure realtime HTTP: {err}")))?;
|
||||
let url = realtime_calls_url(&self.provider.base_url)?;
|
||||
let headers = merge_request_headers(&self.provider.headers, extra_headers, default_headers);
|
||||
let response = client
|
||||
.post(url)
|
||||
.headers(headers)
|
||||
.multipart(session_form(config, offer_sdp)?)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| ApiError::Stream(format!("failed to create realtime call: {err}")))?;
|
||||
let status = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let answer_sdp = response.text().await.map_err(|err| {
|
||||
ApiError::Stream(format!("failed to read realtime call answer: {err}"))
|
||||
})?;
|
||||
if !status.is_success() {
|
||||
return Err(ApiError::Stream(format!(
|
||||
"realtime call failed with HTTP {status}: {answer_sdp}"
|
||||
)));
|
||||
}
|
||||
let call_id =
|
||||
call_id_from_location(headers.get(LOCATION).and_then(|value| value.to_str().ok()))?;
|
||||
Ok(RealtimeCallCreateResponse {
|
||||
call_id,
|
||||
answer_sdp,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn realtime_calls_url(base_url: &str) -> Result<Url, ApiError> {
|
||||
let mut url =
|
||||
Url::parse(base_url).map_err(|err| ApiError::Stream(format!("invalid base URL: {err}")))?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => {}
|
||||
"ws" => {
|
||||
let _ = url.set_scheme("http");
|
||||
}
|
||||
"wss" => {
|
||||
let _ = url.set_scheme("https");
|
||||
}
|
||||
scheme => {
|
||||
return Err(ApiError::Stream(format!(
|
||||
"unsupported realtime calls URL scheme: {scheme}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
url.set_path(REALTIME_CALLS_PATH);
|
||||
url.set_query(None);
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn session_form(config: &RealtimeSessionConfig, offer_sdp: String) -> Result<Form, ApiError> {
|
||||
let session_json = serde_json::to_string(&session_payload(config)).map_err(|err| {
|
||||
ApiError::Stream(format!("failed to serialize realtime call session: {err}"))
|
||||
})?;
|
||||
Ok(Form::new()
|
||||
.text("sdp", offer_sdp)
|
||||
.text("session", session_json))
|
||||
}
|
||||
|
||||
fn session_payload(config: &RealtimeSessionConfig) -> RealtimeCallSession {
|
||||
let session = session_update_session(
|
||||
config.event_parser,
|
||||
config.instructions.clone(),
|
||||
config.session_mode,
|
||||
);
|
||||
RealtimeCallSession {
|
||||
session,
|
||||
model: config.model.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RealtimeCallSession {
|
||||
#[serde(flatten)]
|
||||
session: SessionUpdateSession,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
fn call_id_from_location(location: Option<&str>) -> Result<String, ApiError> {
|
||||
let Some(location) = location else {
|
||||
return Err(ApiError::Stream(
|
||||
"realtime call response missing Location header".to_string(),
|
||||
));
|
||||
};
|
||||
let call_id = location
|
||||
.trim_end_matches('/')
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.filter(|call_id| !call_id.is_empty())
|
||||
.ok_or_else(|| ApiError::Stream("invalid realtime call Location header".to_string()))?;
|
||||
Ok(call_id.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn realtime_calls_url_uses_calls_path() {
|
||||
assert_eq!(
|
||||
realtime_calls_url("wss://api.openai.com/v1/realtime")
|
||||
.expect("url")
|
||||
.as_str(),
|
||||
"https://api.openai.com/v1/realtime/calls"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn call_id_from_location_extracts_last_path_segment() {
|
||||
assert_eq!(
|
||||
call_id_from_location(Some("/v1/realtime/calls/rtc_123")).expect("call id"),
|
||||
"rtc_123"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_form_contains_offer_and_session() {
|
||||
let form = session_form(
|
||||
&RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
model: Some("gpt-realtime".to_string()),
|
||||
session_id: Some("session".to_string()),
|
||||
event_parser: RealtimeEventParser::RealtimeV2,
|
||||
session_mode: RealtimeSessionMode::Conversational,
|
||||
},
|
||||
"v=0".to_string(),
|
||||
)
|
||||
.expect("form");
|
||||
assert!(!form.boundary().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_payload_includes_model() {
|
||||
let payload = serde_json::to_value(session_payload(&RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
model: Some("gpt-realtime".to_string()),
|
||||
session_id: Some("session".to_string()),
|
||||
event_parser: RealtimeEventParser::RealtimeV2,
|
||||
session_mode: RealtimeSessionMode::Conversational,
|
||||
}))
|
||||
.expect("session payload");
|
||||
|
||||
assert_eq!(payload["type"], "realtime");
|
||||
assert_eq!(payload["model"], "gpt-realtime");
|
||||
assert_eq!(payload["instructions"], "backend prompt");
|
||||
}
|
||||
}
|
||||
@@ -463,7 +463,34 @@ impl RealtimeWebsocketClient {
|
||||
config.event_parser,
|
||||
config.session_mode,
|
||||
)?;
|
||||
self.connect_with_url(ws_url, config, extra_headers, default_headers)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn connect_to_call_id(
|
||||
&self,
|
||||
config: RealtimeSessionConfig,
|
||||
call_id: &str,
|
||||
extra_headers: HeaderMap,
|
||||
default_headers: HeaderMap,
|
||||
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
||||
ensure_rustls_crypto_provider();
|
||||
let ws_url = websocket_url_from_api_url_with_call_id(
|
||||
self.provider.base_url.as_str(),
|
||||
self.provider.query_params.as_ref(),
|
||||
call_id,
|
||||
)?;
|
||||
self.connect_with_url(ws_url, config, extra_headers, default_headers)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn connect_with_url(
|
||||
&self,
|
||||
ws_url: Url,
|
||||
config: RealtimeSessionConfig,
|
||||
extra_headers: HeaderMap,
|
||||
default_headers: HeaderMap,
|
||||
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
||||
let mut request = ws_url
|
||||
.as_str()
|
||||
.into_client_request()
|
||||
@@ -509,7 +536,7 @@ impl RealtimeWebsocketClient {
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_request_headers(
|
||||
pub(super) fn merge_request_headers(
|
||||
provider_headers: &HeaderMap,
|
||||
extra_headers: HeaderMap,
|
||||
default_headers: HeaderMap,
|
||||
@@ -596,6 +623,46 @@ fn websocket_url_from_api_url(
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn websocket_url_from_api_url_with_call_id(
|
||||
api_url: &str,
|
||||
query_params: Option<&HashMap<String, String>>,
|
||||
call_id: &str,
|
||||
) -> Result<Url, ApiError> {
|
||||
let mut url = Url::parse(api_url)
|
||||
.map_err(|err| ApiError::Stream(format!("failed to parse realtime api_url: {err}")))?;
|
||||
|
||||
normalize_realtime_path(&mut url);
|
||||
|
||||
match url.scheme() {
|
||||
"ws" | "wss" => {}
|
||||
"http" | "https" => {
|
||||
let scheme = if url.scheme() == "http" { "ws" } else { "wss" };
|
||||
let _ = url.set_scheme(scheme);
|
||||
}
|
||||
scheme => {
|
||||
return Err(ApiError::Stream(format!(
|
||||
"unsupported realtime api_url scheme: {scheme}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
url.set_query(None);
|
||||
{
|
||||
let mut query = url.query_pairs_mut();
|
||||
query.append_pair("call_id", call_id);
|
||||
if let Some(query_params) = query_params {
|
||||
for (key, value) in query_params {
|
||||
if matches!(key.as_str(), "call_id" | "intent" | "model") {
|
||||
continue;
|
||||
}
|
||||
query.append_pair(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn normalize_realtime_path(url: &mut Url) {
|
||||
let path = url.path().to_string();
|
||||
if path.is_empty() || path == "/" {
|
||||
@@ -1094,6 +1161,25 @@ mod tests {
|
||||
assert_eq!(url.as_str(), "wss://example.com/v1/realtime");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_url_with_call_id_uses_call_id_only_for_session_selectors() {
|
||||
let url = websocket_url_from_api_url_with_call_id(
|
||||
"https://example.com/v1/realtime?model=old&foo=bar",
|
||||
Some(&HashMap::from([
|
||||
("trace".to_string(), "1".to_string()),
|
||||
("intent".to_string(), "ignored".to_string()),
|
||||
("model".to_string(), "ignored".to_string()),
|
||||
])),
|
||||
"rtc_123",
|
||||
)
|
||||
.expect("build ws url");
|
||||
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"wss://example.com/v1/realtime?call_id=rtc_123&trace=1"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn e2e_connect_and_exchange_events_against_mock_ws_server() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod calls;
|
||||
pub mod methods;
|
||||
mod methods_common;
|
||||
mod methods_v1;
|
||||
@@ -7,6 +8,8 @@ mod protocol_common;
|
||||
mod protocol_v1;
|
||||
mod protocol_v2;
|
||||
|
||||
pub use calls::RealtimeCallClient;
|
||||
pub use calls::RealtimeCallCreateResponse;
|
||||
pub use codex_protocol::protocol::RealtimeAudioFrame;
|
||||
pub use codex_protocol::protocol::RealtimeEvent;
|
||||
pub use methods::RealtimeWebsocketClient;
|
||||
|
||||
@@ -31,6 +31,8 @@ pub use crate::common::response_create_client_metadata;
|
||||
pub use crate::endpoint::compact::CompactClient;
|
||||
pub use crate::endpoint::memories::MemoriesClient;
|
||||
pub use crate::endpoint::models::ModelsClient;
|
||||
pub use crate::endpoint::realtime_websocket::RealtimeCallClient;
|
||||
pub use crate::endpoint::realtime_websocket::RealtimeCallCreateResponse;
|
||||
pub use crate::endpoint::realtime_websocket::RealtimeEventParser;
|
||||
pub use crate::endpoint::realtime_websocket::RealtimeSessionConfig;
|
||||
pub use crate::endpoint::realtime_websocket::RealtimeSessionMode;
|
||||
|
||||
Reference in New Issue
Block a user