mirror of
https://github.com/openai/codex.git
synced 2026-11-07 20:30:33 +00:00
cooper-oai
44f1b0e867
## Why Realtime connections bypassed runtime authentication and reused captured authentication headers for sideband handshakes. They need to honor the selected credential source and its transport policy across call creation, retries, and reconnects. ## What changed - Use runtime authentication for realtime WebSocket connections, WebRTC call creation, and sideband attachment when enabled for supported OpenAI destinations. - Resolve authentication on every WebSocket handshake, preserving credential ownership and request scope while generating fresh assertions. Reject authentication failures without falling back to bearer authentication. - Keep call creation and sideband attachment bound to the selected credentials and account network policy, including when inference uses a different provider. - Avoid runtime initialization for local marketplace and MCP operations, plugin commands that do not use the remote catalog, and model debugging with a configured catalog. ## Testing Add coverage for fresh handshake credentials on retries and reconnects, credential and account changes, runtime authentication modes, destination validation, and registration failures. GitOrigin-RevId: 57e53122912fa78a524427b0f51013f76f9f7681
3289 lines
115 KiB
Rust
3289 lines
115 KiB
Rust
use crate::endpoint::realtime_websocket::methods_common::conversation_function_call_output_message;
|
|
use crate::endpoint::realtime_websocket::methods_common::conversation_handoff_append_message;
|
|
use crate::endpoint::realtime_websocket::methods_common::conversation_item_create_message;
|
|
use crate::endpoint::realtime_websocket::methods_common::normalized_session_mode;
|
|
use crate::endpoint::realtime_websocket::methods_common::session_update_message;
|
|
use crate::endpoint::realtime_websocket::methods_common::standalone_handoff_message;
|
|
use crate::endpoint::realtime_websocket::methods_common::websocket_intent;
|
|
use crate::endpoint::realtime_websocket::methods_frameless_bidi::context_append_chunks;
|
|
use crate::endpoint::realtime_websocket::methods_frameless_bidi::delegation_context_append_message as frameless_delegation_context_append_message;
|
|
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_context_append_message as frameless_session_context_append_message;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeAudioFrame;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeEvent;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeOutputModality;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
|
|
use crate::endpoint::realtime_websocket::protocol::parse_realtime_event;
|
|
use crate::error::ApiError;
|
|
use crate::provider::Provider;
|
|
use codex_client::backoff;
|
|
use codex_http_client::HttpClientFactory;
|
|
use codex_http_client::NetworkPolicyDenied;
|
|
use codex_http_client::maybe_build_rustls_client_config_with_custom_ca;
|
|
use codex_protocol::protocol::ConversationTextParams;
|
|
use codex_protocol::protocol::ConversationTextRole;
|
|
use codex_protocol::protocol::RealtimeTranscriptDelta;
|
|
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
|
|
use codex_websocket_client::WebSocketConnection;
|
|
use codex_websocket_client::WebSocketConnector;
|
|
use codex_websocket_client::WebSocketTlsMode;
|
|
use futures::SinkExt;
|
|
use futures::StreamExt;
|
|
use http::HeaderMap;
|
|
use http::HeaderValue;
|
|
use http::StatusCode;
|
|
use std::collections::HashMap;
|
|
use std::collections::HashSet;
|
|
use std::collections::VecDeque;
|
|
use std::hash::BuildHasher;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::AtomicBool;
|
|
use std::sync::atomic::Ordering;
|
|
use std::time::Duration;
|
|
use tokio::sync::Mutex;
|
|
use tokio::sync::mpsc;
|
|
use tokio::sync::oneshot;
|
|
use tokio::time::sleep;
|
|
use tokio_tungstenite::tungstenite::Error as WsError;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
|
use tracing::debug;
|
|
use tracing::error;
|
|
use tracing::info;
|
|
use tracing::trace;
|
|
use tracing::warn;
|
|
use tungstenite::protocol::CloseFrame;
|
|
use tungstenite::protocol::WebSocketConfig;
|
|
use tungstenite::protocol::frame::coding::CloseCode;
|
|
use url::Url;
|
|
|
|
const REALTIME_WIRE_LOG_TARGET: &str = "codex_api::realtime_websocket::wire";
|
|
mod sideband;
|
|
const OPENAI_REALTIME_API_BASE_URL: &str = "https://api.openai.com/v1";
|
|
|
|
struct WsStream {
|
|
tx_command: mpsc::Sender<WsCommand>,
|
|
transport_task: tokio::task::JoinHandle<()>,
|
|
sideband_input_stop: Option<tokio::sync::watch::Sender<bool>>,
|
|
terminal_policy_error: tokio::sync::watch::Receiver<Option<NetworkPolicyDenied>>,
|
|
}
|
|
|
|
enum WsCommand {
|
|
Send {
|
|
message: Message,
|
|
tx_result: oneshot::Sender<Result<(), WsError>>,
|
|
},
|
|
Close {
|
|
frame: Option<CloseFrame>,
|
|
tx_result: oneshot::Sender<Result<(), WsError>>,
|
|
},
|
|
}
|
|
|
|
impl WsStream {
|
|
fn new(
|
|
inner: WebSocketConnection,
|
|
) -> (Self, async_channel::Receiver<Result<Message, WsError>>) {
|
|
let (tx_command, mut rx_command) = mpsc::channel::<WsCommand>(32);
|
|
let (tx_message, rx_message) = async_channel::unbounded::<Result<Message, WsError>>();
|
|
|
|
let (terminal_policy_tx, terminal_policy_error) = tokio::sync::watch::channel(None);
|
|
let transport_task = tokio::spawn(async move {
|
|
let mut inner = inner;
|
|
let report_write_policy_error = |error: &WsError| {
|
|
if let Some(denied) = codex_websocket_client::network_policy_denial(error) {
|
|
terminal_policy_tx.send_replace(Some(denied));
|
|
let _ = tx_message.try_send(Err(Self::policy_error(denied)));
|
|
}
|
|
};
|
|
loop {
|
|
tokio::select! {
|
|
command = rx_command.recv() => {
|
|
let Some(command) = command else {
|
|
break;
|
|
};
|
|
match command {
|
|
WsCommand::Send { message, tx_result } => {
|
|
debug!("realtime websocket sending message");
|
|
let result = inner.send(message).await;
|
|
let should_break = result.is_err();
|
|
if let Err(err) = &result {
|
|
error!("realtime websocket send failed: {err}");
|
|
report_write_policy_error(err);
|
|
}
|
|
let _ = tx_result.send(result);
|
|
if should_break {
|
|
break;
|
|
}
|
|
}
|
|
WsCommand::Close { frame, tx_result } => {
|
|
info!("realtime websocket sending close");
|
|
let result = inner.send(Message::Close(frame)).await;
|
|
if let Err(err) = &result {
|
|
error!("realtime websocket close failed: {err}");
|
|
report_write_policy_error(err);
|
|
}
|
|
let _ = tx_result.send(result);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
message = inner.next() => {
|
|
let Some(message) = message else {
|
|
break;
|
|
};
|
|
match message {
|
|
Ok(Message::Ping(payload)) => {
|
|
trace!(payload_len = payload.len(), "realtime websocket received ping");
|
|
if let Err(err) = inner.send(Message::Pong(payload)).await {
|
|
error!("realtime websocket failed to send pong: {err}");
|
|
terminal_policy_tx.send_replace(codex_websocket_client::network_policy_denial(&err));
|
|
let _ = tx_message.send(Err(err)).await;
|
|
break;
|
|
}
|
|
}
|
|
Ok(Message::Pong(_)) => {}
|
|
Ok(message @ (Message::Text(_)
|
|
| Message::Binary(_)
|
|
| Message::Close(_)
|
|
| Message::Frame(_))) => {
|
|
let is_close = matches!(message, Message::Close(_));
|
|
match &message {
|
|
Message::Text(_) => trace!("realtime websocket received text frame"),
|
|
Message::Binary(binary) => {
|
|
error!(
|
|
payload_len = binary.len(),
|
|
"realtime websocket received unexpected binary frame"
|
|
);
|
|
}
|
|
Message::Close(frame) => info!(
|
|
"realtime websocket received close frame: code={:?} reason={:?}",
|
|
frame.as_ref().map(|frame| frame.code),
|
|
frame.as_ref().map(|frame| frame.reason.as_str())
|
|
),
|
|
Message::Frame(_) => {
|
|
trace!("realtime websocket received raw frame");
|
|
}
|
|
Message::Ping(_) | Message::Pong(_) => {}
|
|
}
|
|
if tx_message.send(Ok(message)).await.is_err() {
|
|
break;
|
|
}
|
|
if is_close {
|
|
break;
|
|
}
|
|
}
|
|
Err(err) => {
|
|
error!("realtime websocket receive failed: {err}");
|
|
terminal_policy_tx.send_replace(codex_websocket_client::network_policy_denial(&err));
|
|
let _ = tx_message.send(Err(err)).await;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
info!("realtime websocket pump exiting");
|
|
});
|
|
|
|
(
|
|
Self {
|
|
tx_command,
|
|
transport_task,
|
|
sideband_input_stop: None,
|
|
terminal_policy_error,
|
|
},
|
|
rx_message,
|
|
)
|
|
}
|
|
|
|
async fn request(
|
|
&self,
|
|
make_command: impl FnOnce(oneshot::Sender<Result<(), WsError>>) -> WsCommand,
|
|
) -> Result<(), WsError> {
|
|
let (tx_result, rx_result) = oneshot::channel();
|
|
if self.tx_command.send(make_command(tx_result)).await.is_err() {
|
|
return Err(self.closed_error());
|
|
}
|
|
rx_result.await.unwrap_or_else(|_| Err(self.closed_error()))
|
|
}
|
|
|
|
fn closed_error(&self) -> WsError {
|
|
match *self.terminal_policy_error.borrow() {
|
|
Some(denied) => Self::policy_error(denied),
|
|
None => WsError::ConnectionClosed,
|
|
}
|
|
}
|
|
|
|
fn policy_error(denied: NetworkPolicyDenied) -> WsError {
|
|
WsError::Io(std::io::Error::new(
|
|
std::io::ErrorKind::PermissionDenied,
|
|
denied,
|
|
))
|
|
}
|
|
|
|
async fn send(&self, message: Message) -> Result<(), WsError> {
|
|
self.request(|tx_result| WsCommand::Send { message, tx_result })
|
|
.await
|
|
}
|
|
|
|
/// Ask the socket pump to send a close frame to the connected peer and
|
|
/// return the write result.
|
|
async fn close(&self, frame: Option<CloseFrame>) -> Result<(), WsError> {
|
|
self.request(|tx_result| WsCommand::Close { frame, tx_result })
|
|
.await
|
|
}
|
|
}
|
|
|
|
impl Drop for WsStream {
|
|
fn drop(&mut self) {
|
|
self.transport_task.abort();
|
|
}
|
|
}
|
|
|
|
pub struct RealtimeWebsocketConnection {
|
|
writer: RealtimeWebsocketWriter,
|
|
events: RealtimeWebsocketEvents,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct RealtimeWebsocketWriter {
|
|
stream: Arc<WsStream>,
|
|
is_closed: Arc<AtomicBool>,
|
|
event_parser: RealtimeEventParser,
|
|
context_append_channel: Option<RealtimeContextAppendChannel>,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct RealtimeWebsocketEvents {
|
|
rx_message: async_channel::Receiver<Result<Message, WsError>>,
|
|
pending_events: Arc<Mutex<VecDeque<RealtimeEvent>>>,
|
|
transcript_state: RealtimeTranscriptState,
|
|
event_parser: RealtimeEventParser,
|
|
is_closed: Arc<AtomicBool>,
|
|
}
|
|
|
|
#[derive(Clone, Default)]
|
|
pub struct RealtimeTranscriptState {
|
|
active: Arc<Mutex<ActiveTranscriptState>>,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct ActiveTranscriptState {
|
|
entries: Vec<RealtimeTranscriptEntry>,
|
|
// Handoff input can echo an utterance already written to child history.
|
|
flushed_user_inputs: HashSet<u64>,
|
|
new_input_entry: bool,
|
|
new_output_entry: bool,
|
|
}
|
|
|
|
impl ActiveTranscriptState {
|
|
fn take_entries(&mut self) -> Vec<RealtimeTranscriptEntry> {
|
|
self.flushed_user_inputs.clear();
|
|
std::mem::take(&mut self.entries)
|
|
}
|
|
}
|
|
|
|
impl RealtimeWebsocketConnection {
|
|
pub async fn send_audio_frame(&self, frame: RealtimeAudioFrame) -> Result<(), ApiError> {
|
|
self.writer.send_audio_frame(frame).await
|
|
}
|
|
|
|
pub async fn send_conversation_item_create(
|
|
&self,
|
|
text: String,
|
|
role: ConversationTextRole,
|
|
) -> Result<(), ApiError> {
|
|
self.writer.send_conversation_item_create(text, role).await
|
|
}
|
|
|
|
pub async fn send_conversation_function_call_output(
|
|
&self,
|
|
call_id: String,
|
|
output_text: String,
|
|
) -> Result<(), ApiError> {
|
|
self.writer
|
|
.send_conversation_function_call_output(call_id, output_text)
|
|
.await
|
|
}
|
|
|
|
pub async fn close(&self) -> Result<(), ApiError> {
|
|
self.writer.close().await
|
|
}
|
|
|
|
pub async fn next_event(&self) -> Result<Option<RealtimeEvent>, ApiError> {
|
|
self.events.next_event().await
|
|
}
|
|
|
|
pub fn writer(&self) -> RealtimeWebsocketWriter {
|
|
self.writer.clone()
|
|
}
|
|
|
|
pub fn events(&self) -> RealtimeWebsocketEvents {
|
|
self.events.clone()
|
|
}
|
|
|
|
fn new(
|
|
stream: WsStream,
|
|
rx_message: async_channel::Receiver<Result<Message, WsError>>,
|
|
event_parser: RealtimeEventParser,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Self {
|
|
let stream = Arc::new(stream);
|
|
let is_closed = Arc::new(AtomicBool::new(false));
|
|
Self {
|
|
writer: RealtimeWebsocketWriter {
|
|
stream: Arc::clone(&stream),
|
|
is_closed: Arc::clone(&is_closed),
|
|
event_parser,
|
|
context_append_channel: None,
|
|
},
|
|
events: RealtimeWebsocketEvents {
|
|
rx_message,
|
|
pending_events: Arc::new(Mutex::new(VecDeque::new())),
|
|
transcript_state,
|
|
event_parser,
|
|
is_closed,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl RealtimeWebsocketWriter {
|
|
pub fn with_context_append_channel(mut self, channel: RealtimeContextAppendChannel) -> Self {
|
|
self.context_append_channel = Some(channel);
|
|
self
|
|
}
|
|
|
|
pub async fn send_audio_frame(&self, frame: RealtimeAudioFrame) -> Result<(), ApiError> {
|
|
let message = match self.event_parser {
|
|
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
|
|
RealtimeOutboundMessage::InputAudioBufferAppend { audio: frame.data }
|
|
}
|
|
RealtimeEventParser::FramelessBidi => {
|
|
RealtimeOutboundMessage::InputAudioAppend { audio: frame.data }
|
|
}
|
|
};
|
|
self.send_json(&message).await
|
|
}
|
|
|
|
pub async fn send_conversation_item_create(
|
|
&self,
|
|
text: String,
|
|
role: ConversationTextRole,
|
|
) -> Result<(), ApiError> {
|
|
self.send_json(&conversation_item_create_message(
|
|
self.event_parser,
|
|
text,
|
|
role,
|
|
self.context_append_channel,
|
|
))
|
|
.await
|
|
}
|
|
|
|
pub async fn send_conversation_handoff_append(
|
|
&self,
|
|
handoff_id: String,
|
|
output_text: String,
|
|
) -> Result<(), ApiError> {
|
|
self.send_json(&conversation_handoff_append_message(
|
|
self.event_parser,
|
|
handoff_id,
|
|
output_text,
|
|
self.context_append_channel,
|
|
))
|
|
.await
|
|
}
|
|
|
|
pub async fn send_standalone_handoff(
|
|
&self,
|
|
handoff_id: String,
|
|
output_text: String,
|
|
) -> Result<(), ApiError> {
|
|
self.send_json(&standalone_handoff_message(
|
|
self.event_parser,
|
|
handoff_id,
|
|
output_text,
|
|
self.context_append_channel,
|
|
))
|
|
.await
|
|
}
|
|
|
|
pub async fn send_conversation_function_call_output(
|
|
&self,
|
|
call_id: String,
|
|
output_text: String,
|
|
) -> Result<(), ApiError> {
|
|
self.send_json(&conversation_function_call_output_message(
|
|
self.event_parser,
|
|
call_id,
|
|
output_text,
|
|
self.context_append_channel,
|
|
))
|
|
.await
|
|
}
|
|
|
|
pub async fn send_response_create(&self) -> Result<(), ApiError> {
|
|
self.send_json(&RealtimeOutboundMessage::ResponseCreate)
|
|
.await
|
|
}
|
|
|
|
pub async fn send_session_update(
|
|
&self,
|
|
instructions: String,
|
|
initial_items: Vec<ConversationTextParams>,
|
|
session_mode: RealtimeSessionMode,
|
|
output_modality: RealtimeOutputModality,
|
|
voice: RealtimeVoice,
|
|
delegation_ack_filler: Option<bool>,
|
|
) -> Result<(), ApiError> {
|
|
let session_mode = normalized_session_mode(self.event_parser, session_mode);
|
|
let message = session_update_message(
|
|
self.event_parser,
|
|
instructions,
|
|
initial_items,
|
|
session_mode,
|
|
output_modality,
|
|
voice,
|
|
delegation_ack_filler,
|
|
);
|
|
self.send_json(&message).await
|
|
}
|
|
|
|
pub async fn close(&self) -> Result<(), ApiError> {
|
|
if self.is_closed.swap(true, Ordering::SeqCst) {
|
|
return Ok(());
|
|
}
|
|
if self.event_parser == RealtimeEventParser::FramelessBidi {
|
|
let payload =
|
|
serde_json::to_string(&RealtimeOutboundMessage::SessionClose).map_err(|err| {
|
|
ApiError::Stream(format!("failed to encode realtime request: {err}"))
|
|
})?;
|
|
trace!(target: REALTIME_WIRE_LOG_TARGET, "realtime websocket request: {payload}");
|
|
if let Err(err) = self.stream.send(Message::Text(payload.into())).await
|
|
&& !matches!(err, WsError::ConnectionClosed | WsError::AlreadyClosed)
|
|
{
|
|
return Err(ApiError::Stream(format!(
|
|
"failed to close frameless realtime session: {err}"
|
|
)));
|
|
}
|
|
}
|
|
if let Err(err) = self.stream.close(/*frame*/ None).await
|
|
&& !matches!(err, WsError::ConnectionClosed | WsError::AlreadyClosed)
|
|
{
|
|
return Err(ApiError::Stream(format!(
|
|
"failed to close websocket: {err}"
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Detach an observer without ending the frontend-owned realtime session.
|
|
pub async fn close_transport(&self) -> Result<(), ApiError> {
|
|
if self.is_closed.swap(true, Ordering::SeqCst) {
|
|
return Ok(());
|
|
}
|
|
self.stream
|
|
.close(Some(CloseFrame {
|
|
code: CloseCode::Normal,
|
|
reason: "".into(),
|
|
}))
|
|
.await
|
|
.map_err(|err| ApiError::Stream(format!("failed to close websocket transport: {err}")))
|
|
}
|
|
|
|
async fn send_json(&self, message: &RealtimeOutboundMessage) -> Result<(), ApiError> {
|
|
match message {
|
|
RealtimeOutboundMessage::DelegationContextAppend {
|
|
delegation_item_id,
|
|
channel,
|
|
content,
|
|
} => {
|
|
if let Some(content) = content.first() {
|
|
for chunk in context_append_chunks(&content.text) {
|
|
self.send_json_frame(&frameless_delegation_context_append_message(
|
|
delegation_item_id.clone(),
|
|
chunk,
|
|
*channel,
|
|
))
|
|
.await?;
|
|
}
|
|
return Ok(());
|
|
}
|
|
}
|
|
RealtimeOutboundMessage::SessionContextAppend { channel, content } => {
|
|
if let Some(content) = content.first() {
|
|
for chunk in context_append_chunks(&content.text) {
|
|
self.send_json_frame(&frameless_session_context_append_message(
|
|
chunk, *channel,
|
|
))
|
|
.await?;
|
|
}
|
|
return Ok(());
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
self.send_json_frame(message).await
|
|
}
|
|
|
|
async fn send_json_frame(&self, message: &RealtimeOutboundMessage) -> Result<(), ApiError> {
|
|
let payload = serde_json::to_string(message)
|
|
.map_err(|err| ApiError::Stream(format!("failed to encode realtime request: {err}")))?;
|
|
debug!(?message, "realtime websocket request");
|
|
self.send_payload(payload).await
|
|
}
|
|
|
|
pub async fn send_payload(&self, payload: String) -> Result<(), ApiError> {
|
|
if self.is_closed.load(Ordering::SeqCst) {
|
|
return Err(map_realtime_websocket_connect_error(
|
|
self.stream.closed_error(),
|
|
));
|
|
}
|
|
|
|
trace!(target: REALTIME_WIRE_LOG_TARGET, "realtime websocket request: {payload}");
|
|
self.stream
|
|
.send(Message::Text(payload.into()))
|
|
.await
|
|
.map_err(map_realtime_websocket_connect_error)?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl RealtimeWebsocketEvents {
|
|
pub fn transcript_state(&self) -> RealtimeTranscriptState {
|
|
self.transcript_state.clone()
|
|
}
|
|
|
|
pub async fn take_transcript_tail(&self) -> Vec<RealtimeTranscriptEntry> {
|
|
self.transcript_state.take_transcript_tail().await
|
|
}
|
|
|
|
pub async fn next_event(&self) -> Result<Option<RealtimeEvent>, ApiError> {
|
|
if self.is_closed.load(Ordering::SeqCst) {
|
|
return Ok(None);
|
|
}
|
|
|
|
if let Some(event) = self.pending_events.lock().await.pop_front() {
|
|
return Ok(Some(event));
|
|
}
|
|
|
|
loop {
|
|
let msg = match self.rx_message.recv().await {
|
|
Ok(Ok(msg)) => msg,
|
|
Ok(Err(err)) => {
|
|
self.is_closed.store(true, Ordering::SeqCst);
|
|
error!("realtime websocket read failed: {err}");
|
|
return Err(map_realtime_websocket_connect_error(err));
|
|
}
|
|
Err(_) => {
|
|
self.is_closed.store(true, Ordering::SeqCst);
|
|
if self.event_parser == RealtimeEventParser::FramelessBidi {
|
|
error!("realtime websocket event stream ended unexpectedly");
|
|
return Err(ApiError::Stream(
|
|
"realtime websocket event stream ended unexpectedly".to_string(),
|
|
));
|
|
}
|
|
info!("realtime websocket event stream ended");
|
|
return Ok(None);
|
|
}
|
|
};
|
|
|
|
match msg {
|
|
Message::Text(text) => {
|
|
trace!(target: REALTIME_WIRE_LOG_TARGET, "realtime websocket event: {text}");
|
|
if let Some(mut event) = parse_realtime_event(&text, self.event_parser) {
|
|
self.update_active_transcript(&mut event).await;
|
|
debug!(?event, "realtime websocket parsed event");
|
|
return Ok(Some(event));
|
|
}
|
|
debug!("realtime websocket ignored unsupported text frame");
|
|
}
|
|
Message::Close(frame) => {
|
|
self.is_closed.store(true, Ordering::SeqCst);
|
|
info!(
|
|
"realtime websocket closed: code={:?} reason={:?}",
|
|
frame.as_ref().map(|frame| frame.code),
|
|
frame.as_ref().map(|frame| frame.reason.as_str())
|
|
);
|
|
if self.event_parser != RealtimeEventParser::FramelessBidi {
|
|
return Ok(None);
|
|
}
|
|
return match frame {
|
|
Some(frame) if frame.code == CloseCode::Normal => Ok(None),
|
|
Some(frame) => Err(ApiError::Stream(format!(
|
|
"realtime websocket closed unexpectedly: code={:?} reason={:?}",
|
|
frame.code, frame.reason
|
|
))),
|
|
None => Err(ApiError::Stream(
|
|
"realtime websocket closed unexpectedly without a status code"
|
|
.to_string(),
|
|
)),
|
|
};
|
|
}
|
|
Message::Binary(_) => {
|
|
return Ok(Some(RealtimeEvent::Error(
|
|
"unexpected binary realtime websocket event".to_string(),
|
|
)));
|
|
}
|
|
Message::Frame(_) | Message::Ping(_) | Message::Pong(_) => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn wait_for_session_started(&self) -> Result<(), ApiError> {
|
|
let Some(event) = self.next_event().await? else {
|
|
return Err(ApiError::Stream(
|
|
"frameless realtime session ended before session.started".to_string(),
|
|
));
|
|
};
|
|
match &event {
|
|
RealtimeEvent::SessionUpdated { .. } => {
|
|
self.pending_events.lock().await.push_back(event);
|
|
Ok(())
|
|
}
|
|
RealtimeEvent::Error(message) => Err(ApiError::Stream(message.clone())),
|
|
_ => Err(ApiError::Stream(
|
|
"frameless realtime session received an event before session.started".to_string(),
|
|
)),
|
|
}
|
|
}
|
|
|
|
async fn update_active_transcript(&self, event: &mut RealtimeEvent) {
|
|
let mut active_transcript = self.transcript_state.active.lock().await;
|
|
match event {
|
|
RealtimeEvent::InputAudioSpeechStarted(_) => {
|
|
active_transcript.new_input_entry = true;
|
|
}
|
|
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta { delta, .. }) => {
|
|
let force_new = active_transcript.new_input_entry;
|
|
append_transcript_delta(
|
|
&mut active_transcript.entries,
|
|
"user",
|
|
delta,
|
|
force_new,
|
|
self.event_parser,
|
|
);
|
|
if !delta.is_empty() || self.event_parser != RealtimeEventParser::FramelessBidi {
|
|
active_transcript.new_input_entry = false;
|
|
}
|
|
}
|
|
RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { delta, .. }) => {
|
|
let force_new = active_transcript.new_output_entry;
|
|
append_transcript_delta(
|
|
&mut active_transcript.entries,
|
|
"assistant",
|
|
delta,
|
|
force_new,
|
|
self.event_parser,
|
|
);
|
|
if !delta.is_empty() || self.event_parser != RealtimeEventParser::FramelessBidi {
|
|
active_transcript.new_output_entry = false;
|
|
}
|
|
}
|
|
RealtimeEvent::InputTranscriptDone(done) => {
|
|
let force_new = active_transcript.new_input_entry;
|
|
apply_transcript_done(
|
|
&mut active_transcript.entries,
|
|
"user",
|
|
&done.text,
|
|
force_new,
|
|
self.event_parser,
|
|
);
|
|
active_transcript.new_input_entry =
|
|
self.event_parser == RealtimeEventParser::FramelessBidi;
|
|
}
|
|
RealtimeEvent::OutputTranscriptDone(done) => {
|
|
let force_new = active_transcript.new_output_entry;
|
|
apply_transcript_done(
|
|
&mut active_transcript.entries,
|
|
"assistant",
|
|
&done.text,
|
|
force_new,
|
|
self.event_parser,
|
|
);
|
|
active_transcript.new_output_entry =
|
|
self.event_parser == RealtimeEventParser::FramelessBidi;
|
|
}
|
|
RealtimeEvent::HandoffRequested(handoff) => {
|
|
append_handoff_input(&mut active_transcript, &handoff.input_transcript);
|
|
handoff.active_transcript = active_transcript.take_entries();
|
|
active_transcript.new_input_entry = true;
|
|
active_transcript.new_output_entry = true;
|
|
}
|
|
RealtimeEvent::ResponseCreated(_) => {
|
|
active_transcript.new_output_entry = true;
|
|
}
|
|
RealtimeEvent::SessionUpdated { .. }
|
|
| RealtimeEvent::AudioOut(_)
|
|
| RealtimeEvent::ResponseCancelled(_)
|
|
| RealtimeEvent::ResponseDone(_)
|
|
| RealtimeEvent::ConversationItemDone { .. }
|
|
| RealtimeEvent::NoopRequested(_)
|
|
| RealtimeEvent::ConversationItemAdded(_)
|
|
| RealtimeEvent::Error(_)
|
|
| RealtimeEvent::HistoryItemStarted(_)
|
|
| RealtimeEvent::HistoryTranscriptDelta { .. }
|
|
| RealtimeEvent::HistoryItemCompleted(_) => {}
|
|
}
|
|
}
|
|
|
|
pub async fn take_completed_transcript(
|
|
&self,
|
|
event: &RealtimeEvent,
|
|
threshold: usize,
|
|
) -> Vec<RealtimeTranscriptEntry> {
|
|
if self.event_parser != RealtimeEventParser::FramelessBidi
|
|
|| !matches!(
|
|
event,
|
|
RealtimeEvent::InputTranscriptDone(_) | RealtimeEvent::OutputTranscriptDone(_)
|
|
)
|
|
{
|
|
return Vec::new();
|
|
}
|
|
let mut active = self.transcript_state.active.lock().await;
|
|
// Only the other speaker can still be open after turn.done. Keep their utterance
|
|
// and everything after it in place so history stays in conversation order.
|
|
let open_role = match event {
|
|
RealtimeEvent::InputTranscriptDone(_) if !active.new_output_entry => Some("assistant"),
|
|
RealtimeEvent::OutputTranscriptDone(_) if !active.new_input_entry => Some("user"),
|
|
_ => None,
|
|
};
|
|
let entries = &active.entries;
|
|
let end = open_role
|
|
.and_then(|role| entries.iter().rposition(|entry| entry.role == role))
|
|
.unwrap_or(entries.len());
|
|
if end == 0 || transcript_entries_bytes(entries) <= threshold {
|
|
return Vec::new();
|
|
}
|
|
let completed: Vec<_> = active.entries.drain(..end).collect();
|
|
let recorded = &mut active.flushed_user_inputs;
|
|
let hash = recorded.hasher().clone();
|
|
recorded.extend(
|
|
completed
|
|
.iter()
|
|
.filter(|entry| entry.role == "user")
|
|
.map(|entry| hash.hash_one(entry.text.trim())),
|
|
);
|
|
completed
|
|
}
|
|
}
|
|
|
|
impl RealtimeTranscriptState {
|
|
pub async fn take_transcript_tail(&self) -> Vec<RealtimeTranscriptEntry> {
|
|
self.active.lock().await.take_entries()
|
|
}
|
|
}
|
|
|
|
fn transcript_entries_bytes(entries: &[RealtimeTranscriptEntry]) -> usize {
|
|
entries
|
|
.iter()
|
|
.map(|entry| entry.role.len() + entry.text.len() + 3)
|
|
.sum()
|
|
}
|
|
|
|
fn append_transcript_delta(
|
|
entries: &mut Vec<RealtimeTranscriptEntry>,
|
|
role: &str,
|
|
delta: &str,
|
|
force_new: bool,
|
|
event_parser: RealtimeEventParser,
|
|
) {
|
|
if delta.is_empty() {
|
|
return;
|
|
}
|
|
|
|
// V3 interleaves the two speakers' transcripts. Legacy protocols retain
|
|
// adjacent speaker runs and their existing response lifecycle semantics.
|
|
let entry = match event_parser {
|
|
RealtimeEventParser::FramelessBidi => {
|
|
entries.iter_mut().rev().find(|entry| entry.role == role)
|
|
}
|
|
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
|
|
entries.last_mut().filter(|entry| entry.role == role)
|
|
}
|
|
};
|
|
if !force_new && let Some(last_entry) = entry {
|
|
last_entry.text.push_str(delta);
|
|
return;
|
|
}
|
|
|
|
entries.push(RealtimeTranscriptEntry {
|
|
role: role.to_string(),
|
|
text: delta.to_string(),
|
|
});
|
|
}
|
|
|
|
fn apply_transcript_done(
|
|
entries: &mut Vec<RealtimeTranscriptEntry>,
|
|
role: &str,
|
|
text: &str,
|
|
force_new: bool,
|
|
event_parser: RealtimeEventParser,
|
|
) {
|
|
if text.is_empty() {
|
|
return;
|
|
}
|
|
|
|
// V3 interleaves the two speakers' transcripts. Legacy protocols retain
|
|
// adjacent speaker runs and their existing response lifecycle semantics.
|
|
let entry = match event_parser {
|
|
RealtimeEventParser::FramelessBidi => {
|
|
entries.iter_mut().rev().find(|entry| entry.role == role)
|
|
}
|
|
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
|
|
entries.last_mut().filter(|entry| entry.role == role)
|
|
}
|
|
};
|
|
if !force_new && let Some(last_entry) = entry {
|
|
// A delayed V3 final can arrive after deltas from the next utterance.
|
|
// Preserve accumulated speech unless the final extends it.
|
|
if event_parser != RealtimeEventParser::FramelessBidi || text.starts_with(&last_entry.text)
|
|
{
|
|
last_entry.text = text.to_string();
|
|
}
|
|
return;
|
|
}
|
|
|
|
entries.push(RealtimeTranscriptEntry {
|
|
role: role.to_string(),
|
|
text: text.to_string(),
|
|
});
|
|
}
|
|
|
|
fn append_handoff_input(active: &mut ActiveTranscriptState, input: &str) {
|
|
let input = input.trim();
|
|
let recorded = &active.flushed_user_inputs;
|
|
if input.is_empty()
|
|
|| contains_transcript_entry(&active.entries, "user", input)
|
|
|| (!recorded.is_empty() && recorded.contains(&recorded.hasher().hash_one(input)))
|
|
{
|
|
return;
|
|
}
|
|
|
|
active.entries.push(RealtimeTranscriptEntry {
|
|
role: "user".to_string(),
|
|
text: input.to_string(),
|
|
});
|
|
}
|
|
|
|
fn contains_transcript_entry(entries: &[RealtimeTranscriptEntry], role: &str, text: &str) -> bool {
|
|
entries
|
|
.iter()
|
|
.any(|entry| entry.role == role && entry.text.trim() == text.trim())
|
|
}
|
|
|
|
pub struct RealtimeWebsocketClient {
|
|
provider: Provider,
|
|
http_client_factory: HttpClientFactory,
|
|
auth: Option<crate::SharedAuthProvider>,
|
|
webrtc_sideband_base_url: String,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum RealtimeSessionInitialization {
|
|
NewSession,
|
|
LegacyWebrtcSideband,
|
|
ExistingCall,
|
|
}
|
|
|
|
impl RealtimeWebsocketClient {
|
|
pub fn new(provider: Provider, http_client_factory: HttpClientFactory) -> Self {
|
|
Self {
|
|
provider,
|
|
http_client_factory,
|
|
auth: None,
|
|
webrtc_sideband_base_url: OPENAI_REALTIME_API_BASE_URL.to_string(),
|
|
}
|
|
}
|
|
|
|
/// Use the transport captured with the selected authentication source.
|
|
pub fn with_http_client_factory(mut self, factory: HttpClientFactory) -> Self {
|
|
self.http_client_factory = factory;
|
|
self
|
|
}
|
|
|
|
/// Resolve request auth on every handshake, including sideband retries and reconnects.
|
|
pub fn with_auth(mut self, auth: crate::SharedAuthProvider) -> Self {
|
|
self.auth = Some(auth);
|
|
self
|
|
}
|
|
|
|
/// Overrides the direct WebRTC sideband URL for local development and tests.
|
|
pub fn with_webrtc_sideband_base_url(mut self, base_url: String) -> Self {
|
|
self.webrtc_sideband_base_url = base_url;
|
|
self
|
|
}
|
|
|
|
pub async fn connect(
|
|
&self,
|
|
config: RealtimeSessionConfig,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
let ws_url = websocket_url_from_api_url(
|
|
self.provider.base_url.as_str(),
|
|
self.provider.query_params.as_ref(),
|
|
config.model.as_deref(),
|
|
config.event_parser,
|
|
config.session_mode,
|
|
)?;
|
|
self.connect_realtime_websocket_url(
|
|
ws_url,
|
|
config,
|
|
extra_headers,
|
|
default_headers,
|
|
RealtimeSessionInitialization::NewSession,
|
|
RealtimeTranscriptState::default(),
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn connect_webrtc_sideband(
|
|
&self,
|
|
config: RealtimeSessionConfig,
|
|
call_id: &str,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
self.connect_sideband(
|
|
config,
|
|
call_id,
|
|
extra_headers,
|
|
default_headers,
|
|
RealtimeSessionInitialization::LegacyWebrtcSideband,
|
|
transcript_state,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Attaches to a client-created call without overwriting its session configuration.
|
|
pub async fn connect_existing_call_sideband(
|
|
&self,
|
|
config: RealtimeSessionConfig,
|
|
call_id: &str,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
// Instant Connect returns a call ID before ICE activates the call. Its sideband
|
|
// can briefly return 404 while the client connects media in parallel.
|
|
let mut activation_delays = [100, 300, 800].into_iter();
|
|
loop {
|
|
let result = self
|
|
.connect_sideband(
|
|
config.clone(),
|
|
call_id,
|
|
extra_headers.clone(),
|
|
default_headers.clone(),
|
|
RealtimeSessionInitialization::ExistingCall,
|
|
transcript_state.clone(),
|
|
)
|
|
.await;
|
|
if matches!(&result, Err(ApiError::Api { status, .. }) if *status == StatusCode::NOT_FOUND)
|
|
&& let Some(delay_ms) = activation_delays.next()
|
|
{
|
|
sleep(Duration::from_millis(delay_ms)).await;
|
|
continue;
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
async fn connect_sideband(
|
|
&self,
|
|
config: RealtimeSessionConfig,
|
|
call_id: &str,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
session_initialization: RealtimeSessionInitialization,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
// The WebRTC call already exists; this loop only retries joining its sideband control
|
|
// socket. Once joined, the returned connection is the same reader/writer state that the
|
|
// ordinary websocket start path uses.
|
|
for attempt in 0..=self.provider.retry.max_attempts {
|
|
let result = self
|
|
.connect_webrtc_sideband_once(
|
|
config.clone(),
|
|
call_id,
|
|
extra_headers.clone(),
|
|
default_headers.clone(),
|
|
session_initialization,
|
|
transcript_state.clone(),
|
|
)
|
|
.await;
|
|
match result {
|
|
Ok(connection) => return Ok(connection),
|
|
Err(err @ ApiError::Transport(codex_http_client::TransportError::Policy(_))) => {
|
|
return Err(err);
|
|
}
|
|
Err(err) if webrtc_sideband_session_ended(&err) => return Err(err),
|
|
Err(err) if attempt < self.provider.retry.max_attempts => {
|
|
let delay = backoff(self.provider.retry.base_delay, attempt + 1);
|
|
warn!(
|
|
attempt = attempt + 1,
|
|
call_id,
|
|
delay_ms = delay.as_millis(),
|
|
"realtime sideband websocket connect failed; retrying: {err}"
|
|
);
|
|
sleep(delay).await;
|
|
}
|
|
Err(err) => return Err(err),
|
|
}
|
|
}
|
|
|
|
Err(ApiError::Stream(
|
|
"realtime sideband websocket retry loop exhausted".to_string(),
|
|
))
|
|
}
|
|
|
|
async fn connect_webrtc_sideband_once(
|
|
&self,
|
|
config: RealtimeSessionConfig,
|
|
call_id: &str,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
session_initialization: RealtimeSessionInitialization,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
// Keep the parser/session query shaping from standalone realtime while replacing the model
|
|
// query with a call_id join onto an existing WebRTC session.
|
|
let ws_url = self.webrtc_sideband_url(config.event_parser, config.session_mode, call_id)?;
|
|
self.connect_realtime_websocket_url(
|
|
ws_url,
|
|
config,
|
|
extra_headers,
|
|
default_headers,
|
|
session_initialization,
|
|
transcript_state,
|
|
)
|
|
.await
|
|
}
|
|
|
|
fn webrtc_sideband_url(
|
|
&self,
|
|
event_parser: RealtimeEventParser,
|
|
session_mode: RealtimeSessionMode,
|
|
call_id: &str,
|
|
) -> Result<Url, ApiError> {
|
|
websocket_url_from_api_url_for_call(
|
|
self.webrtc_sideband_base_url.as_str(),
|
|
/*query_params*/ None,
|
|
event_parser,
|
|
session_mode,
|
|
call_id,
|
|
)
|
|
}
|
|
|
|
async fn connect_realtime_websocket_url(
|
|
&self,
|
|
ws_url: Url,
|
|
config: RealtimeSessionConfig,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
session_initialization: RealtimeSessionInitialization,
|
|
transcript_state: RealtimeTranscriptState,
|
|
) -> Result<RealtimeWebsocketConnection, ApiError> {
|
|
ensure_rustls_crypto_provider();
|
|
|
|
let mut request = ws_url
|
|
.as_str()
|
|
.into_client_request()
|
|
.map_err(|err| ApiError::Stream(format!("failed to build websocket request: {err}")))?;
|
|
let headers = merge_request_headers(
|
|
&self.provider.headers,
|
|
with_session_id_header(extra_headers, config.session_id.as_deref())?,
|
|
default_headers,
|
|
);
|
|
request.headers_mut().extend(headers);
|
|
if let Some(auth) = &self.auth {
|
|
let headers = std::mem::take(request.headers_mut());
|
|
*request.headers_mut() = auth.apply_auth_headers(headers).await?;
|
|
}
|
|
|
|
info!("connecting realtime websocket: {ws_url}");
|
|
// Realtime websocket TLS should honor the same custom-CA env vars as the rest of Codex's
|
|
// outbound HTTPS and websocket traffic.
|
|
let tls_config = maybe_build_rustls_client_config_with_custom_ca()
|
|
.map_err(|err| ApiError::Stream(format!("failed to configure websocket TLS: {err}")))?;
|
|
// A fresh Windows install may not have downloaded the server's trusted root yet.
|
|
// Use platform validation only for system trust, preserving custom CA semantics.
|
|
#[cfg(windows)]
|
|
let tls_config = match tls_config {
|
|
Some(tls_config) => Some(tls_config),
|
|
None => Some(
|
|
codex_http_client::build_windows_platform_tls_config().map_err(|err| {
|
|
ApiError::Stream(format!("failed to configure websocket TLS: {err}"))
|
|
})?,
|
|
),
|
|
};
|
|
let tls_mode = match tls_config {
|
|
Some(tls_config) => WebSocketTlsMode::Rustls(tls_config),
|
|
None => WebSocketTlsMode::TungsteniteDefault,
|
|
};
|
|
let connector = WebSocketConnector::new_with_tls_mode(&self.http_client_factory, tls_mode)
|
|
.map_err(|err| ApiError::Stream(format!("failed to configure websocket TLS: {err}")))?;
|
|
let (stream, response) = connector
|
|
.connect(request, websocket_config())
|
|
.await
|
|
.map_err(map_realtime_websocket_connect_error)?;
|
|
info!(
|
|
ws_url = %ws_url,
|
|
status = %response.status(),
|
|
"realtime websocket connected"
|
|
);
|
|
|
|
let (stream, rx_message) = WsStream::new(stream);
|
|
let connection = RealtimeWebsocketConnection::new(
|
|
stream,
|
|
rx_message,
|
|
config.event_parser,
|
|
transcript_state,
|
|
);
|
|
let initialize_session = match session_initialization {
|
|
RealtimeSessionInitialization::NewSession => true,
|
|
RealtimeSessionInitialization::LegacyWebrtcSideband => {
|
|
config.event_parser != RealtimeEventParser::FramelessBidi
|
|
}
|
|
RealtimeSessionInitialization::ExistingCall => false,
|
|
};
|
|
if initialize_session {
|
|
debug!(
|
|
session_id = config.session_id.as_deref().unwrap_or("<none>"),
|
|
"realtime websocket sending session.update"
|
|
);
|
|
connection
|
|
.writer
|
|
.send_session_update(
|
|
config.instructions,
|
|
config.initial_items,
|
|
config.session_mode,
|
|
config.output_modality,
|
|
config.voice,
|
|
config.delegation_ack_filler,
|
|
)
|
|
.await?;
|
|
}
|
|
if matches!(
|
|
session_initialization,
|
|
RealtimeSessionInitialization::NewSession
|
|
) && config.event_parser == RealtimeEventParser::FramelessBidi
|
|
{
|
|
connection.events.wait_for_session_started().await?;
|
|
}
|
|
Ok(connection)
|
|
}
|
|
}
|
|
|
|
fn webrtc_sideband_session_ended(err: &ApiError) -> bool {
|
|
matches!(
|
|
err,
|
|
ApiError::Api { status, .. }
|
|
if matches!(*status, StatusCode::NOT_FOUND | StatusCode::GONE)
|
|
)
|
|
}
|
|
|
|
fn map_realtime_websocket_connect_error(err: WsError) -> ApiError {
|
|
if let Some(denied) = codex_websocket_client::network_policy_denial(&err) {
|
|
return ApiError::Transport(codex_http_client::TransportError::Policy(denied));
|
|
}
|
|
match err {
|
|
WsError::Http(response) => ApiError::Api {
|
|
status: response.status(),
|
|
message: "realtime websocket handshake failed".to_string(),
|
|
},
|
|
err => ApiError::Stream(format!("failed to connect realtime websocket: {err}")),
|
|
}
|
|
}
|
|
|
|
fn merge_request_headers(
|
|
provider_headers: &HeaderMap,
|
|
extra_headers: HeaderMap,
|
|
default_headers: HeaderMap,
|
|
) -> HeaderMap {
|
|
let mut headers = provider_headers.clone();
|
|
headers.extend(extra_headers);
|
|
for (name, value) in &default_headers {
|
|
if let http::header::Entry::Vacant(entry) = headers.entry(name) {
|
|
entry.insert(value.clone());
|
|
}
|
|
}
|
|
headers
|
|
}
|
|
|
|
fn with_session_id_header(
|
|
mut headers: HeaderMap,
|
|
session_id: Option<&str>,
|
|
) -> Result<HeaderMap, ApiError> {
|
|
let Some(session_id) = session_id else {
|
|
return Ok(headers);
|
|
};
|
|
headers.insert(
|
|
"x-session-id",
|
|
HeaderValue::from_str(session_id).map_err(|err| {
|
|
ApiError::Stream(format!("invalid realtime session id header: {err}"))
|
|
})?,
|
|
);
|
|
Ok(headers)
|
|
}
|
|
|
|
fn websocket_config() -> WebSocketConfig {
|
|
WebSocketConfig::default()
|
|
}
|
|
|
|
fn websocket_url_from_api_url(
|
|
api_url: &str,
|
|
query_params: Option<&HashMap<String, String>>,
|
|
model: Option<&str>,
|
|
event_parser: RealtimeEventParser,
|
|
_session_mode: RealtimeSessionMode,
|
|
) -> Result<Url, ApiError> {
|
|
let mut url = Url::parse(api_url)
|
|
.map_err(|err| ApiError::Stream(format!("failed to parse realtime api_url: {err}")))?;
|
|
|
|
normalize_realtime_path(&mut url, event_parser);
|
|
|
|
match url.scheme() {
|
|
"ws" | "wss" => {}
|
|
"http" | "https" => {
|
|
let scheme = if url.scheme() == "http" { "ws" } else { "wss" };
|
|
let _ = url.set_scheme(scheme);
|
|
}
|
|
scheme => {
|
|
return Err(ApiError::Stream(format!(
|
|
"unsupported realtime api_url scheme: {scheme}"
|
|
)));
|
|
}
|
|
}
|
|
|
|
let intent = websocket_intent(event_parser);
|
|
let has_extra_query_params = query_params.is_some_and(|query_params| {
|
|
query_params
|
|
.iter()
|
|
.any(|(key, _)| key != "intent" && !(key == "model" && model.is_some()))
|
|
});
|
|
if intent.is_some() || model.is_some() || has_extra_query_params {
|
|
let mut query = url.query_pairs_mut();
|
|
if let Some(intent) = intent {
|
|
query.append_pair("intent", intent);
|
|
}
|
|
if let Some(model) = model {
|
|
query.append_pair("model", model);
|
|
}
|
|
if let Some(query_params) = query_params {
|
|
for (key, value) in query_params {
|
|
if key == "intent" || (key == "model" && model.is_some()) {
|
|
continue;
|
|
}
|
|
query.append_pair(key, value);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(url)
|
|
}
|
|
|
|
fn websocket_url_from_api_url_for_call(
|
|
api_url: &str,
|
|
query_params: Option<&HashMap<String, String>>,
|
|
event_parser: RealtimeEventParser,
|
|
session_mode: RealtimeSessionMode,
|
|
call_id: &str,
|
|
) -> Result<Url, ApiError> {
|
|
let mut url = websocket_url_from_api_url(
|
|
api_url,
|
|
query_params,
|
|
/*model*/ None,
|
|
event_parser,
|
|
session_mode,
|
|
)?;
|
|
match event_parser {
|
|
RealtimeEventParser::FramelessBidi => {
|
|
if matches!(call_id, "." | "..") {
|
|
return Err(ApiError::InvalidRequest {
|
|
message: format!("invalid realtime call id: {call_id}"),
|
|
});
|
|
}
|
|
url.path_segments_mut()
|
|
.map_err(|()| {
|
|
ApiError::Stream(
|
|
"realtime sideband URL cannot contain path segments".to_string(),
|
|
)
|
|
})?
|
|
.pop_if_empty()
|
|
.push(call_id);
|
|
}
|
|
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
|
|
url.query_pairs_mut().append_pair("call_id", call_id);
|
|
}
|
|
}
|
|
Ok(url)
|
|
}
|
|
|
|
fn normalize_realtime_path(url: &mut Url, event_parser: RealtimeEventParser) {
|
|
if event_parser == RealtimeEventParser::FramelessBidi {
|
|
let path = url.path().to_string();
|
|
if path.is_empty() || path == "/" || path == "/v1" || path == "/v1/" {
|
|
url.set_path("/v1/live");
|
|
} else if let Some(prefix) = path.trim_end_matches('/').strip_suffix("/realtime") {
|
|
url.set_path(&format!("{prefix}/live"));
|
|
} else if path.ends_with("/live/") {
|
|
url.set_path(path.trim_end_matches('/'));
|
|
}
|
|
return;
|
|
}
|
|
|
|
let path = url.path().to_string();
|
|
if path.is_empty() || path == "/" {
|
|
url.set_path("/v1/realtime");
|
|
return;
|
|
}
|
|
|
|
if path.ends_with("/realtime") {
|
|
return;
|
|
}
|
|
|
|
if path.ends_with("/realtime/") {
|
|
url.set_path(path.trim_end_matches('/'));
|
|
return;
|
|
}
|
|
|
|
if path.ends_with("/v1") {
|
|
url.set_path(&format!("{path}/realtime"));
|
|
return;
|
|
}
|
|
|
|
if path.ends_with("/v1/") {
|
|
url.set_path(&format!("{path}realtime"));
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "transcript_tests.rs"]
|
|
mod transcript_tests;
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry;
|
|
use crate::provider::RetryConfig;
|
|
use codex_http_client::OutboundProxyPolicy::ReqwestDefault;
|
|
use codex_protocol::protocol::RealtimeHandoffRequested;
|
|
use codex_protocol::protocol::RealtimeInputAudioSpeechStarted;
|
|
use codex_protocol::protocol::RealtimeNoopRequested;
|
|
use codex_protocol::protocol::RealtimeResponseCancelled;
|
|
use codex_protocol::protocol::RealtimeResponseCreated;
|
|
use codex_protocol::protocol::RealtimeResponseDone;
|
|
use codex_protocol::protocol::RealtimeTranscriptDelta;
|
|
use codex_protocol::protocol::RealtimeTranscriptDone;
|
|
use codex_protocol::protocol::RealtimeVoice;
|
|
use http::HeaderValue;
|
|
use http::StatusCode;
|
|
use pretty_assertions::assert_eq;
|
|
use serde_json::Value;
|
|
use serde_json::json;
|
|
use std::collections::HashMap;
|
|
use std::time::Duration;
|
|
use tokio::net::TcpListener;
|
|
use tokio_tungstenite::accept_async;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
use tungstenite::protocol::CloseFrame;
|
|
|
|
fn test_client(provider: Provider) -> RealtimeWebsocketClient {
|
|
RealtimeWebsocketClient::new(provider, HttpClientFactory::new(ReqwestDefault))
|
|
}
|
|
|
|
#[test]
|
|
fn parse_session_updated_event() {
|
|
let payload = json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_123", "instructions": "backend prompt"}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_123".to_string(),
|
|
instructions: Some("backend prompt".to_string()),
|
|
})
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_audio_delta_event() {
|
|
let payload = json!({
|
|
"type": "conversation.output_audio.delta",
|
|
"delta": "AAA=",
|
|
"sample_rate": 48000,
|
|
"channels": 1,
|
|
"samples_per_channel": 960
|
|
})
|
|
.to_string();
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
|
|
data: "AAA=".to_string(),
|
|
sample_rate: 48000,
|
|
num_channels: 1,
|
|
samples_per_channel: Some(960),
|
|
item_id: None,
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_conversation_item_added_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.added",
|
|
"item": {"type": "message", "seq": 7}
|
|
})
|
|
.to_string();
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::ConversationItemAdded(
|
|
json!({"type": "message", "seq": 7})
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_conversation_item_done_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.done",
|
|
"item": {"id": "item_123", "type": "message"}
|
|
})
|
|
.to_string();
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::ConversationItemDone {
|
|
item_id: "item_123".to_string(),
|
|
})
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_handoff_requested_event() {
|
|
let payload = json!({
|
|
"type": "conversation.handoff.requested",
|
|
"handoff_id": "handoff_123",
|
|
"item_id": "item_123",
|
|
"input_transcript": "delegate this"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
|
handoff_id: "handoff_123".to_string(),
|
|
item_id: "item_123".to_string(),
|
|
input_transcript: "delegate this".to_string(),
|
|
active_transcript: Vec::new(),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn takes_only_transcript_after_last_handoff_once() {
|
|
let (_tx_message, rx_message) = async_channel::unbounded();
|
|
let events = RealtimeWebsocketEvents {
|
|
rx_message,
|
|
pending_events: Arc::new(Mutex::new(VecDeque::new())),
|
|
transcript_state: RealtimeTranscriptState::default(),
|
|
event_parser: RealtimeEventParser::V1,
|
|
is_closed: Arc::new(AtomicBool::new(false)),
|
|
};
|
|
|
|
assert_eq!(events.take_transcript_tail().await, vec![]);
|
|
|
|
let mut covered = RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "already handed off".to_string(),
|
|
});
|
|
events.update_active_transcript(&mut covered).await;
|
|
let mut handoff = RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
|
handoff_id: "handoff_1".to_string(),
|
|
item_id: "item_1".to_string(),
|
|
input_transcript: "already handed off".to_string(),
|
|
active_transcript: vec![],
|
|
});
|
|
events.update_active_transcript(&mut handoff).await;
|
|
assert_eq!(
|
|
handoff,
|
|
RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
|
handoff_id: "handoff_1".to_string(),
|
|
item_id: "item_1".to_string(),
|
|
input_transcript: "already handed off".to_string(),
|
|
active_transcript: vec![RealtimeTranscriptEntry {
|
|
role: "user".to_string(),
|
|
text: "already handed off".to_string(),
|
|
}],
|
|
})
|
|
);
|
|
|
|
let mut tail = RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "tail".to_string(),
|
|
});
|
|
events.update_active_transcript(&mut tail).await;
|
|
assert_eq!(
|
|
events.take_transcript_tail().await,
|
|
vec![RealtimeTranscriptEntry {
|
|
role: "assistant".to_string(),
|
|
text: "tail".to_string(),
|
|
}]
|
|
);
|
|
assert_eq!(events.take_transcript_tail().await, vec![]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn transcript_state_survives_reconnect_and_reconciles_done() {
|
|
let transcript_state = RealtimeTranscriptState::default();
|
|
let (_first_tx, first_rx) = async_channel::unbounded();
|
|
let first_events = RealtimeWebsocketEvents {
|
|
rx_message: first_rx,
|
|
pending_events: Arc::new(Mutex::new(VecDeque::new())),
|
|
transcript_state: transcript_state.clone(),
|
|
event_parser: RealtimeEventParser::FramelessBidi,
|
|
is_closed: Arc::new(AtomicBool::new(false)),
|
|
};
|
|
let mut partial = RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "hello wor".to_string(),
|
|
});
|
|
first_events.update_active_transcript(&mut partial).await;
|
|
|
|
let (_second_tx, second_rx) = async_channel::unbounded();
|
|
let second_events = RealtimeWebsocketEvents {
|
|
rx_message: second_rx,
|
|
pending_events: Arc::new(Mutex::new(VecDeque::new())),
|
|
transcript_state,
|
|
event_parser: RealtimeEventParser::FramelessBidi,
|
|
is_closed: Arc::new(AtomicBool::new(false)),
|
|
};
|
|
let mut done = RealtimeEvent::InputTranscriptDone(RealtimeTranscriptDone {
|
|
text: "hello world".to_string(),
|
|
});
|
|
second_events.update_active_transcript(&mut done).await;
|
|
|
|
assert_eq!(
|
|
second_events.take_transcript_tail().await,
|
|
vec![RealtimeTranscriptEntry {
|
|
role: "user".to_string(),
|
|
text: "hello world".to_string(),
|
|
}]
|
|
);
|
|
assert_eq!(second_events.take_transcript_tail().await, vec![]);
|
|
}
|
|
|
|
#[test]
|
|
fn terminal_sideband_handshake_statuses_are_not_retryable() {
|
|
for status in [StatusCode::NOT_FOUND, StatusCode::GONE] {
|
|
assert!(webrtc_sideband_session_ended(&ApiError::Api {
|
|
status,
|
|
message: "session ended".to_string(),
|
|
}));
|
|
}
|
|
assert!(!webrtc_sideband_session_ended(&ApiError::Api {
|
|
status: StatusCode::INTERNAL_SERVER_ERROR,
|
|
message: "retry".to_string(),
|
|
}));
|
|
}
|
|
|
|
enum TestRealtimeTermination {
|
|
Close(Option<CloseCode>),
|
|
EventChannelEnd,
|
|
}
|
|
|
|
async fn realtime_termination_is_transport_error(
|
|
event_parser: RealtimeEventParser,
|
|
termination: TestRealtimeTermination,
|
|
) -> bool {
|
|
let (tx_message, rx_message) = async_channel::unbounded();
|
|
let events = RealtimeWebsocketEvents {
|
|
rx_message,
|
|
pending_events: Arc::new(Mutex::new(VecDeque::new())),
|
|
transcript_state: RealtimeTranscriptState::default(),
|
|
event_parser,
|
|
is_closed: Arc::new(AtomicBool::new(false)),
|
|
};
|
|
match termination {
|
|
TestRealtimeTermination::Close(close_code) => {
|
|
let frame = close_code.map(|code| CloseFrame {
|
|
code,
|
|
reason: "session ended".into(),
|
|
});
|
|
tx_message
|
|
.send(Ok(Message::Close(frame)))
|
|
.await
|
|
.expect("send close frame");
|
|
}
|
|
TestRealtimeTermination::EventChannelEnd => drop(tx_message),
|
|
}
|
|
events.next_event().await.is_err()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn writer_policy_failure_reaches_realtime_events() {
|
|
use codex_http_client::DestinationPolicy::Unrestricted;
|
|
use codex_http_client::NetworkPolicyController;
|
|
use codex_http_client::NetworkPolicyDenied::Revoked;
|
|
use codex_http_client::TransportError::Policy;
|
|
use codex_websocket_client::network_policy_denial;
|
|
use tokio::time::timeout;
|
|
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let url = format!("ws://{}/", listener.local_addr().unwrap());
|
|
let (writing, started) = oneshot::channel();
|
|
let server = tokio::spawn(async move {
|
|
let (socket, _) = listener.accept().await.unwrap();
|
|
let socket = accept_async(socket).await.unwrap();
|
|
socket.get_ref().peek(&mut [0]).await.unwrap();
|
|
writing.send(()).unwrap();
|
|
std::future::pending::<()>().await;
|
|
});
|
|
let controller = NetworkPolicyController::default();
|
|
let policy = controller.policy();
|
|
controller.publish(policy.revision(), Unrestricted);
|
|
let factory = HttpClientFactory::new(ReqwestDefault).with_network_policy(policy.clone());
|
|
let connector = WebSocketConnector::new(&factory).unwrap();
|
|
let request = url.into_client_request().unwrap();
|
|
let connected = connector.connect(request, websocket_config());
|
|
let (socket, _) = connected.await.unwrap();
|
|
let (stream, rx_message) = WsStream::new(socket);
|
|
let parser = RealtimeEventParser::V1;
|
|
let transcript = RealtimeTranscriptState::default();
|
|
let connection = RealtimeWebsocketConnection::new(stream, rx_message, parser, transcript);
|
|
let stream = Arc::clone(&connection.writer.stream);
|
|
let payload = Message::Binary(vec![0; 8 * 1024 * 1024].into());
|
|
let writer = tokio::spawn(async move {
|
|
loop {
|
|
if let Err(error) = stream.send(payload.clone()).await {
|
|
return error;
|
|
}
|
|
}
|
|
});
|
|
let deadline = Duration::from_secs(/*secs*/ 5);
|
|
timeout(deadline, started).await.unwrap().unwrap();
|
|
policy.invalidate();
|
|
let writer = timeout(deadline, writer).await.unwrap();
|
|
let error = writer.unwrap();
|
|
assert_eq!(network_policy_denial(&error), Some(Revoked));
|
|
let next = timeout(deadline, connection.next_event());
|
|
let error = next.await.unwrap().unwrap_err();
|
|
assert!(matches!(error, ApiError::Transport(Policy(Revoked))));
|
|
assert!(connection.next_event().await.unwrap().is_none());
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_termination_classification_preserves_protocol_behavior() {
|
|
use CloseCode::Away;
|
|
use CloseCode::Normal;
|
|
use RealtimeEventParser::FramelessBidi;
|
|
use RealtimeEventParser::RealtimeV2;
|
|
use RealtimeEventParser::V1;
|
|
use TestRealtimeTermination::Close;
|
|
use TestRealtimeTermination::EventChannelEnd;
|
|
|
|
for (event_parser, termination, expect_transport_error) in [
|
|
(FramelessBidi, Close(Some(Normal)), false),
|
|
(FramelessBidi, Close(Some(Away)), true),
|
|
(FramelessBidi, Close(None), true),
|
|
(FramelessBidi, EventChannelEnd, true),
|
|
(V1, Close(Some(Away)), false),
|
|
(V1, EventChannelEnd, false),
|
|
(RealtimeV2, Close(Some(Away)), false),
|
|
] {
|
|
assert_eq!(
|
|
realtime_termination_is_transport_error(event_parser, termination).await,
|
|
expect_transport_error,
|
|
"unexpected termination classification for {event_parser:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_handshake_error_preserves_http_status() {
|
|
let response = http::Response::builder()
|
|
.status(StatusCode::GONE)
|
|
.body(None)
|
|
.expect("build response");
|
|
|
|
assert!(matches!(
|
|
map_realtime_websocket_connect_error(WsError::Http(Box::new(response))),
|
|
ApiError::Api {
|
|
status: StatusCode::GONE,
|
|
..
|
|
}
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn parse_input_transcript_delta_event() {
|
|
let payload = json!({
|
|
"type": "conversation.input_transcript.delta",
|
|
"delta": "hello "
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::InputTranscriptDelta(
|
|
RealtimeTranscriptDelta {
|
|
delta: "hello ".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_input_audio_transcription_delta_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.input_audio_transcription.delta",
|
|
"item_id": "item_input_1",
|
|
"content_index": 0,
|
|
"delta": "hello"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::InputTranscriptDelta(
|
|
RealtimeTranscriptDelta {
|
|
delta: "hello".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_input_audio_transcription_completed_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.input_audio_transcription.completed",
|
|
"item_id": "item_input_1",
|
|
"content_index": 0,
|
|
"transcript": "hello world"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::InputTranscriptDone(RealtimeTranscriptDone {
|
|
text: "hello world".to_string(),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_input_transcript_turn_marked_event() {
|
|
let payload = json!({
|
|
"type": "conversation.input_transcript.turn_marked",
|
|
"transcript": "hello realtime"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::InputTranscriptDone(RealtimeTranscriptDone {
|
|
text: "hello realtime".to_string(),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_output_transcript_delta_event() {
|
|
let payload = json!({
|
|
"type": "conversation.output_transcript.delta",
|
|
"delta": "hi"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::OutputTranscriptDelta(
|
|
RealtimeTranscriptDelta {
|
|
delta: "hi".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_output_audio_transcript_delta_event() {
|
|
let payload = json!({
|
|
"type": "response.output_audio_transcript.delta",
|
|
"delta": "hi"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::OutputTranscriptDelta(
|
|
RealtimeTranscriptDelta {
|
|
delta: "hi".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_output_audio_transcript_done_event() {
|
|
let payload = json!({
|
|
"type": "response.output_audio_transcript.done",
|
|
"transcript": "hi there"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::OutputTranscriptDone(
|
|
RealtimeTranscriptDone {
|
|
text: "hi there".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_v1_item_done_output_text_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.done",
|
|
"item": {
|
|
"id": "item_output_1",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "output_text", "text": "hello"},
|
|
{"type": "output_text", "text": " world"}
|
|
]
|
|
}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::V1),
|
|
Some(RealtimeEvent::ConversationItemDone {
|
|
item_id: "item_output_1".to_string(),
|
|
})
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_handoff_tool_call_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.done",
|
|
"item": {
|
|
"id": "item_123",
|
|
"type": "function_call",
|
|
"name": "background_agent",
|
|
"call_id": "call_123",
|
|
"arguments": "{\"prompt\":\"delegate this\"}"
|
|
}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
|
handoff_id: "call_123".to_string(),
|
|
item_id: "item_123".to_string(),
|
|
input_transcript: "delegate this".to_string(),
|
|
active_transcript: Vec::new(),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_noop_tool_call_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.done",
|
|
"item": {
|
|
"id": "item_silent",
|
|
"type": "function_call",
|
|
"name": "remain_silent",
|
|
"call_id": "call_silent",
|
|
"arguments": "{}"
|
|
}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::NoopRequested(RealtimeNoopRequested {
|
|
call_id: "call_silent".to_string(),
|
|
item_id: "item_silent".to_string(),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_input_audio_transcription_delta_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.input_audio_transcription.delta",
|
|
"item_id": "item_input_1",
|
|
"content_index": 0,
|
|
"delta": "hello"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::InputTranscriptDelta(
|
|
RealtimeTranscriptDelta {
|
|
delta: "hello".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_output_audio_transcript_done_event() {
|
|
let payload = json!({
|
|
"type": "response.output_audio_transcript.done",
|
|
"transcript": "hello there"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::OutputTranscriptDone(
|
|
RealtimeTranscriptDone {
|
|
text: "hello there".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_output_text_done_event() {
|
|
let payload = json!({
|
|
"type": "response.output_text.done",
|
|
"text": "hello there"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::OutputTranscriptDone(
|
|
RealtimeTranscriptDone {
|
|
text: "hello there".to_string(),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_conversation_item_created_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.created",
|
|
"item": {"type": "message", "role": "user"}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::ConversationItemAdded(
|
|
json!({"type": "message", "role": "user"})
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_item_done_output_text_event() {
|
|
let payload = json!({
|
|
"type": "conversation.item.done",
|
|
"item": {
|
|
"id": "item_output_1",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "output_text", "text": "hello"},
|
|
{"type": "output_text", "text": " world"}
|
|
]
|
|
}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::ConversationItemDone {
|
|
item_id: "item_output_1".to_string(),
|
|
})
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_output_audio_delta_defaults_audio_shape() {
|
|
let payload = json!({
|
|
"type": "response.output_audio.delta",
|
|
"delta": "AQID"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 24_000,
|
|
num_channels: 1,
|
|
samples_per_channel: None,
|
|
item_id: None,
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_response_audio_delta_with_item_id() {
|
|
let payload = json!({
|
|
"type": "response.audio.delta",
|
|
"delta": "AQID",
|
|
"item_id": "item_audio_1"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 24_000,
|
|
num_channels: 1,
|
|
samples_per_channel: None,
|
|
item_id: Some("item_audio_1".to_string()),
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_speech_started_event() {
|
|
let payload = json!({
|
|
"type": "input_audio_buffer.speech_started",
|
|
"item_id": "item_input_1"
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::InputAudioSpeechStarted(
|
|
RealtimeInputAudioSpeechStarted {
|
|
item_id: Some("item_input_1".to_string()),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_response_cancelled_event() {
|
|
let payload = json!({
|
|
"type": "response.cancelled",
|
|
"response": {"id": "resp_cancelled_1"}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::ResponseCancelled(
|
|
RealtimeResponseCancelled {
|
|
response_id: Some("resp_cancelled_1".to_string()),
|
|
}
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_response_done_event() {
|
|
let payload = json!({
|
|
"type": "response.done",
|
|
"response": {
|
|
"output": [{
|
|
"id": "item_123",
|
|
"type": "function_call",
|
|
"name": "background_agent",
|
|
"call_id": "call_123",
|
|
"arguments": "{\"prompt\":\"delegate from done\"}"
|
|
}]
|
|
}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::ResponseDone(RealtimeResponseDone {
|
|
response_id: None
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_realtime_v2_response_created_event() {
|
|
let payload = json!({
|
|
"type": "response.created",
|
|
"response": {"id": "resp_created_1"}
|
|
})
|
|
.to_string();
|
|
|
|
assert_eq!(
|
|
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
|
Some(RealtimeEvent::ResponseCreated(RealtimeResponseCreated {
|
|
response_id: Some("resp_created_1".to_string())
|
|
}))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn merge_request_headers_matches_http_precedence() {
|
|
let mut provider_headers = HeaderMap::new();
|
|
provider_headers.insert(
|
|
"originator",
|
|
HeaderValue::from_static("provider-originator"),
|
|
);
|
|
provider_headers.insert("x-priority", HeaderValue::from_static("provider"));
|
|
|
|
let mut extra_headers = HeaderMap::new();
|
|
extra_headers.insert("x-priority", HeaderValue::from_static("extra"));
|
|
|
|
let mut default_headers = HeaderMap::new();
|
|
default_headers.insert("originator", HeaderValue::from_static("default-originator"));
|
|
default_headers.insert("x-priority", HeaderValue::from_static("default"));
|
|
default_headers.insert("x-default-only", HeaderValue::from_static("default-only"));
|
|
|
|
let merged = merge_request_headers(&provider_headers, extra_headers, default_headers);
|
|
|
|
assert_eq!(
|
|
merged.get("originator"),
|
|
Some(&HeaderValue::from_static("provider-originator"))
|
|
);
|
|
assert_eq!(
|
|
merged.get("x-priority"),
|
|
Some(&HeaderValue::from_static("extra"))
|
|
);
|
|
assert_eq!(
|
|
merged.get("x-default-only"),
|
|
Some(&HeaderValue::from_static("default-only"))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_from_http_base_defaults_to_ws_path() {
|
|
let url = websocket_url_from_api_url(
|
|
"http://127.0.0.1:8011",
|
|
/*query_params*/ None,
|
|
/*model*/ None,
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"ws://127.0.0.1:8011/v1/realtime?intent=quicksilver"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_from_ws_base_defaults_to_ws_path() {
|
|
let url = websocket_url_from_api_url(
|
|
"wss://example.com",
|
|
/*query_params*/ None,
|
|
Some("realtime-test-model"),
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/v1/realtime?intent=quicksilver&model=realtime-test-model"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_from_v1_base_appends_realtime_path() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://api.openai.com/v1",
|
|
/*query_params*/ None,
|
|
Some("snapshot"),
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://api.openai.com/v1/realtime?intent=quicksilver&model=snapshot"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_from_nested_v1_base_appends_realtime_path() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://example.com/openai/v1",
|
|
/*query_params*/ None,
|
|
Some("snapshot"),
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/openai/v1/realtime?intent=quicksilver&model=snapshot"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_preserves_existing_realtime_path_and_extra_query_params() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://example.com/v1/realtime?foo=bar",
|
|
Some(&HashMap::from([
|
|
("trace".to_string(), "1".to_string()),
|
|
("intent".to_string(), "ignored".to_string()),
|
|
])),
|
|
Some("snapshot"),
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/v1/realtime?foo=bar&intent=quicksilver&model=snapshot&trace=1"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn frameless_websocket_url_rewrites_existing_realtime_path() {
|
|
let url = websocket_url_from_api_url(
|
|
"wss://example.com/v1/realtime?foo=bar",
|
|
/*query_params*/ None,
|
|
Some("snapshot"),
|
|
RealtimeEventParser::FramelessBidi,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build Frameless websocket url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/v1/live?foo=bar&model=snapshot"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_v1_ignores_transcription_mode() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://example.com",
|
|
/*query_params*/ None,
|
|
/*model*/ None,
|
|
RealtimeEventParser::V1,
|
|
RealtimeSessionMode::Transcription,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/v1/realtime?intent=quicksilver"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_omits_intent_for_realtime_v2_conversational_mode() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://example.com/v1/realtime?foo=bar",
|
|
Some(&HashMap::from([
|
|
("trace".to_string(), "1".to_string()),
|
|
("intent".to_string(), "ignored".to_string()),
|
|
])),
|
|
Some("snapshot"),
|
|
RealtimeEventParser::RealtimeV2,
|
|
RealtimeSessionMode::Conversational,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/v1/realtime?foo=bar&model=snapshot&trace=1"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_omits_intent_for_realtime_v2_transcription_mode() {
|
|
let url = websocket_url_from_api_url(
|
|
"https://example.com",
|
|
/*query_params*/ None,
|
|
/*model*/ None,
|
|
RealtimeEventParser::RealtimeV2,
|
|
RealtimeSessionMode::Transcription,
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(url.as_str(), "wss://example.com/v1/realtime");
|
|
}
|
|
|
|
#[test]
|
|
fn websocket_url_for_call_id_joins_existing_realtime_session() {
|
|
let url = websocket_url_from_api_url_for_call(
|
|
"https://api.openai.com/v1",
|
|
/*query_params*/ None,
|
|
RealtimeEventParser::RealtimeV2,
|
|
RealtimeSessionMode::Conversational,
|
|
"rtc_test",
|
|
)
|
|
.expect("build ws url");
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://api.openai.com/v1/realtime?call_id=rtc_test"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn frameless_websocket_url_encodes_call_id_as_one_path_segment() {
|
|
let url = websocket_url_from_api_url_for_call(
|
|
"https://example.com/proxy/v1/live?trace=1",
|
|
/*query_params*/ None,
|
|
RealtimeEventParser::FramelessBidi,
|
|
RealtimeSessionMode::Conversational,
|
|
"../../admin",
|
|
)
|
|
.expect("build existing-call websocket url");
|
|
|
|
assert_eq!(
|
|
url.as_str(),
|
|
"wss://example.com/proxy/v1/live/..%2F..%2Fadmin?trace=1"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn frameless_websocket_url_rejects_dot_only_call_ids() {
|
|
for call_id in [".", ".."] {
|
|
let error = websocket_url_from_api_url_for_call(
|
|
"https://example.com/v1/live",
|
|
/*query_params*/ None,
|
|
RealtimeEventParser::FramelessBidi,
|
|
RealtimeSessionMode::Conversational,
|
|
call_id,
|
|
)
|
|
.expect_err("dot-only call IDs must not target the parent sideband URL");
|
|
|
|
assert!(matches!(
|
|
error,
|
|
ApiError::InvalidRequest { message }
|
|
if message == format!("invalid realtime call id: {call_id}")
|
|
));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn webrtc_frameless_sideband_ignores_provider_base_url() {
|
|
let client = test_client(Provider {
|
|
name: "chatgpt".to_string(),
|
|
base_url: "https://chatgpt.com/backend-api/codex".to_string(),
|
|
query_params: None,
|
|
headers: HeaderMap::new(),
|
|
retry: RetryConfig {
|
|
max_attempts: 0,
|
|
base_delay: Duration::ZERO,
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
});
|
|
|
|
let url = client
|
|
.webrtc_sideband_url(
|
|
RealtimeEventParser::FramelessBidi,
|
|
RealtimeSessionMode::Conversational,
|
|
"rtc_test",
|
|
)
|
|
.expect("build ws url");
|
|
|
|
assert_eq!(url.as_str(), "wss://api.openai.com/v1/live/rtc_test");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn e2e_connect_and_exchange_events_against_mock_ws_server() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
|
|
let server = tokio::spawn(async move {
|
|
let (stream, _) = listener.accept().await.expect("accept");
|
|
let mut ws = accept_async(stream).await.expect("accept ws");
|
|
|
|
let first = ws
|
|
.next()
|
|
.await
|
|
.expect("first msg")
|
|
.expect("first msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let first_json: Value = serde_json::from_str(&first).expect("json");
|
|
assert_eq!(first_json["type"], "session.update");
|
|
assert_eq!(
|
|
first_json["session"]["type"],
|
|
Value::String("quicksilver".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["instructions"],
|
|
Value::String("backend prompt".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["format"]["type"],
|
|
Value::String("audio/pcm".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["format"]["rate"],
|
|
Value::from(24_000)
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["output"]["voice"],
|
|
Value::String("breeze".to_string())
|
|
);
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_mock", "instructions": "backend prompt"}
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send session.updated");
|
|
|
|
let second = ws
|
|
.next()
|
|
.await
|
|
.expect("second msg")
|
|
.expect("second msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let second_json: Value = serde_json::from_str(&second).expect("json");
|
|
assert_eq!(second_json["type"], "input_audio_buffer.append");
|
|
|
|
let third = ws
|
|
.next()
|
|
.await
|
|
.expect("third msg")
|
|
.expect("third msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let third_json: Value = serde_json::from_str(&third).expect("json");
|
|
assert_eq!(third_json["type"], "conversation.item.create");
|
|
assert_eq!(third_json["item"]["role"], "developer");
|
|
assert_eq!(
|
|
third_json["item"]["content"][0]["type"],
|
|
Value::String("input_text".to_string())
|
|
);
|
|
assert_eq!(third_json["item"]["content"][0]["text"], "hello agent");
|
|
|
|
let fourth = ws
|
|
.next()
|
|
.await
|
|
.expect("fourth msg")
|
|
.expect("fourth msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
|
|
assert_eq!(fourth_json["type"], "conversation.item.create");
|
|
assert_eq!(fourth_json["item"]["role"], "assistant");
|
|
assert_eq!(
|
|
fourth_json["item"]["content"][0]["type"],
|
|
Value::String("output_text".to_string())
|
|
);
|
|
assert_eq!(
|
|
fourth_json["item"]["content"][0]["text"],
|
|
Value::String("assistant context".to_string())
|
|
);
|
|
|
|
let fifth = ws
|
|
.next()
|
|
.await
|
|
.expect("fifth msg")
|
|
.expect("fifth msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let fifth_json: Value = serde_json::from_str(&fifth).expect("json");
|
|
assert_eq!(fifth_json["type"], "conversation.handoff.append");
|
|
assert_eq!(fifth_json["handoff_id"], "handoff_1");
|
|
assert_eq!(
|
|
fifth_json["output_text"],
|
|
"\"Agent Final Message\":\n\nhello from background agent"
|
|
);
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "conversation.output_audio.delta",
|
|
"delta": "AQID",
|
|
"sample_rate": 48000,
|
|
"channels": 1
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send audio");
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "conversation.input_transcript.delta",
|
|
"delta": "delegate "
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send input transcript delta");
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "conversation.input_transcript.delta",
|
|
"delta": "now"
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send input transcript delta");
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "conversation.output_transcript.delta",
|
|
"delta": "working"
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send output transcript delta");
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "conversation.handoff.requested",
|
|
"handoff_id": "handoff_1",
|
|
"item_id": "item_2",
|
|
"input_transcript": "delegate now"
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send item added");
|
|
});
|
|
|
|
let provider = Provider {
|
|
name: "test".to_string(),
|
|
base_url: format!("http://{addr}"),
|
|
query_params: Some(HashMap::new()),
|
|
headers: HeaderMap::new(),
|
|
retry: crate::provider::RetryConfig {
|
|
max_attempts: 1,
|
|
base_delay: Duration::from_millis(1),
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
};
|
|
let client = test_client(provider);
|
|
let connection = client
|
|
.connect(
|
|
RealtimeSessionConfig {
|
|
instructions: "backend prompt".to_string(),
|
|
initial_items: Vec::new(),
|
|
delegation_ack_filler: None,
|
|
model: Some("realtime-test-model".to_string()),
|
|
session_id: Some("conv_1".to_string()),
|
|
event_parser: RealtimeEventParser::V1,
|
|
session_mode: RealtimeSessionMode::Conversational,
|
|
output_modality: RealtimeOutputModality::Audio,
|
|
voice: RealtimeVoice::Breeze,
|
|
},
|
|
HeaderMap::new(),
|
|
HeaderMap::new(),
|
|
)
|
|
.await
|
|
.expect("connect");
|
|
|
|
let created = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
created,
|
|
RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_mock".to_string(),
|
|
instructions: Some("backend prompt".to_string()),
|
|
}
|
|
);
|
|
|
|
connection
|
|
.send_audio_frame(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 48000,
|
|
num_channels: 1,
|
|
samples_per_channel: Some(960),
|
|
item_id: None,
|
|
})
|
|
.await
|
|
.expect("send audio");
|
|
connection
|
|
.send_conversation_item_create(
|
|
"hello agent".to_string(),
|
|
ConversationTextRole::Developer,
|
|
)
|
|
.await
|
|
.expect("send item");
|
|
connection
|
|
.send_conversation_item_create(
|
|
"assistant context".to_string(),
|
|
ConversationTextRole::Assistant,
|
|
)
|
|
.await
|
|
.expect("send assistant item");
|
|
connection
|
|
.send_conversation_function_call_output(
|
|
"handoff_1".to_string(),
|
|
"hello from background agent".to_string(),
|
|
)
|
|
.await
|
|
.expect("send handoff");
|
|
|
|
let audio_event = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
audio_event,
|
|
RealtimeEvent::AudioOut(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 48000,
|
|
num_channels: 1,
|
|
samples_per_channel: None,
|
|
item_id: None,
|
|
})
|
|
);
|
|
|
|
let input_delta_event = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
input_delta_event,
|
|
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "delegate ".to_string(),
|
|
})
|
|
);
|
|
|
|
let input_delta_event = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
input_delta_event,
|
|
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "now".to_string(),
|
|
})
|
|
);
|
|
|
|
let output_delta_event = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
output_delta_event,
|
|
RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta {
|
|
delta: "working".to_string(),
|
|
})
|
|
);
|
|
|
|
let added_event = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
added_event,
|
|
RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
|
handoff_id: "handoff_1".to_string(),
|
|
item_id: "item_2".to_string(),
|
|
input_transcript: "delegate now".to_string(),
|
|
active_transcript: vec![
|
|
RealtimeTranscriptEntry {
|
|
role: "user".to_string(),
|
|
text: "delegate now".to_string(),
|
|
},
|
|
RealtimeTranscriptEntry {
|
|
role: "assistant".to_string(),
|
|
text: "working".to_string(),
|
|
},
|
|
],
|
|
})
|
|
);
|
|
|
|
connection.close().await.expect("close");
|
|
server.await.expect("server task");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn realtime_v2_session_update_includes_background_agent_tool_and_handoff_output_item() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
|
|
let server = tokio::spawn(async move {
|
|
let (stream, _) = listener.accept().await.expect("accept");
|
|
let mut ws = accept_async(stream).await.expect("accept ws");
|
|
|
|
let first = ws
|
|
.next()
|
|
.await
|
|
.expect("first msg")
|
|
.expect("first msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let first_json: Value = serde_json::from_str(&first).expect("json");
|
|
assert_eq!(first_json["type"], "session.update");
|
|
assert_eq!(
|
|
first_json["session"]["type"],
|
|
Value::String("realtime".to_string())
|
|
);
|
|
assert_eq!(first_json["session"]["output_modalities"], json!(["audio"]));
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["format"],
|
|
json!({
|
|
"type": "audio/pcm",
|
|
"rate": 24_000,
|
|
})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["noise_reduction"],
|
|
json!({
|
|
"type": "near_field",
|
|
})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["transcription"],
|
|
json!({
|
|
"model": "gpt-4o-mini-transcribe",
|
|
})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["turn_detection"],
|
|
json!({
|
|
"type": "server_vad",
|
|
"interrupt_response": true,
|
|
"create_response": true,
|
|
"silence_duration_ms": 500,
|
|
})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["output"]["format"],
|
|
json!({
|
|
"type": "audio/pcm",
|
|
"rate": 24_000,
|
|
})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["output"]["voice"],
|
|
Value::String("cedar".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][0]["type"],
|
|
Value::String("function".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][0]["name"],
|
|
Value::String("background_agent".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][0]["parameters"]["required"],
|
|
json!(["prompt"])
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][1]["type"],
|
|
Value::String("function".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][1]["name"],
|
|
Value::String("remain_silent".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tools"][1]["parameters"]["properties"],
|
|
json!({})
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["tool_choice"],
|
|
Value::String("auto".to_string())
|
|
);
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_v2", "instructions": "backend prompt"}
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send session.updated");
|
|
|
|
let second = ws
|
|
.next()
|
|
.await
|
|
.expect("second msg")
|
|
.expect("second msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let second_json: Value = serde_json::from_str(&second).expect("json");
|
|
assert_eq!(second_json["type"], "conversation.item.create");
|
|
assert_eq!(second_json["item"]["role"], "developer");
|
|
assert_eq!(
|
|
second_json["item"]["type"],
|
|
Value::String("message".to_string())
|
|
);
|
|
assert_eq!(
|
|
second_json["item"]["content"][0]["type"],
|
|
Value::String("input_text".to_string())
|
|
);
|
|
assert_eq!(
|
|
second_json["item"]["content"][0]["text"],
|
|
Value::String("delegate this".to_string())
|
|
);
|
|
|
|
let third = ws
|
|
.next()
|
|
.await
|
|
.expect("third msg")
|
|
.expect("third msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let third_json: Value = serde_json::from_str(&third).expect("json");
|
|
assert_eq!(third_json["type"], "conversation.item.create");
|
|
assert_eq!(third_json["item"]["role"], "assistant");
|
|
assert_eq!(
|
|
third_json["item"]["content"][0]["type"],
|
|
Value::String("output_text".to_string())
|
|
);
|
|
assert_eq!(
|
|
third_json["item"]["content"][0]["text"],
|
|
Value::String("assistant context".to_string())
|
|
);
|
|
|
|
let fourth = ws
|
|
.next()
|
|
.await
|
|
.expect("fourth msg")
|
|
.expect("fourth msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
|
|
assert_eq!(fourth_json["type"], "conversation.item.create");
|
|
assert_eq!(
|
|
fourth_json["item"]["type"],
|
|
Value::String("function_call_output".to_string())
|
|
);
|
|
assert_eq!(
|
|
fourth_json["item"]["call_id"],
|
|
Value::String("call_1".to_string())
|
|
);
|
|
assert_eq!(
|
|
fourth_json["item"]["output"],
|
|
Value::String("delegated result".to_string())
|
|
);
|
|
});
|
|
|
|
let provider = Provider {
|
|
name: "test".to_string(),
|
|
base_url: format!("http://{addr}"),
|
|
query_params: Some(HashMap::new()),
|
|
headers: HeaderMap::new(),
|
|
retry: crate::provider::RetryConfig {
|
|
max_attempts: 1,
|
|
base_delay: Duration::from_millis(1),
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
};
|
|
let client = test_client(provider);
|
|
let connection = client
|
|
.connect(
|
|
RealtimeSessionConfig {
|
|
instructions: "backend prompt".to_string(),
|
|
initial_items: Vec::new(),
|
|
delegation_ack_filler: None,
|
|
model: Some("realtime-test-model".to_string()),
|
|
session_id: Some("conv_1".to_string()),
|
|
event_parser: RealtimeEventParser::RealtimeV2,
|
|
session_mode: RealtimeSessionMode::Conversational,
|
|
output_modality: RealtimeOutputModality::Audio,
|
|
voice: RealtimeVoice::Cedar,
|
|
},
|
|
HeaderMap::new(),
|
|
HeaderMap::new(),
|
|
)
|
|
.await
|
|
.expect("connect");
|
|
|
|
let created = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
created,
|
|
RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_v2".to_string(),
|
|
instructions: Some("backend prompt".to_string()),
|
|
}
|
|
);
|
|
|
|
connection
|
|
.send_conversation_item_create(
|
|
"delegate this".to_string(),
|
|
ConversationTextRole::Developer,
|
|
)
|
|
.await
|
|
.expect("send text item");
|
|
connection
|
|
.send_conversation_item_create(
|
|
"assistant context".to_string(),
|
|
ConversationTextRole::Assistant,
|
|
)
|
|
.await
|
|
.expect("send assistant item");
|
|
connection
|
|
.send_conversation_function_call_output(
|
|
"call_1".to_string(),
|
|
"delegated result".to_string(),
|
|
)
|
|
.await
|
|
.expect("send handoff output");
|
|
|
|
connection.close().await.expect("close");
|
|
server.await.expect("server task");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn transcription_mode_session_update_omits_output_audio_and_instructions() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
|
|
let server = tokio::spawn(async move {
|
|
let (stream, _) = listener.accept().await.expect("accept");
|
|
let mut ws = accept_async(stream).await.expect("accept ws");
|
|
|
|
let first = ws
|
|
.next()
|
|
.await
|
|
.expect("first msg")
|
|
.expect("first msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let first_json: Value = serde_json::from_str(&first).expect("json");
|
|
assert_eq!(first_json["type"], "session.update");
|
|
assert_eq!(
|
|
first_json["session"]["type"],
|
|
Value::String("transcription".to_string())
|
|
);
|
|
assert!(first_json["session"].get("instructions").is_none());
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["input"]["transcription"],
|
|
json!({
|
|
"model": "gpt-4o-mini-transcribe",
|
|
})
|
|
);
|
|
assert!(first_json["session"]["audio"].get("output").is_none());
|
|
assert!(first_json["session"].get("tools").is_none());
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_transcription"}
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send session.updated");
|
|
|
|
let second = ws
|
|
.next()
|
|
.await
|
|
.expect("second msg")
|
|
.expect("second msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let second_json: Value = serde_json::from_str(&second).expect("json");
|
|
assert_eq!(second_json["type"], "input_audio_buffer.append");
|
|
});
|
|
|
|
let provider = Provider {
|
|
name: "test".to_string(),
|
|
base_url: format!("http://{addr}"),
|
|
query_params: Some(HashMap::new()),
|
|
headers: HeaderMap::new(),
|
|
retry: crate::provider::RetryConfig {
|
|
max_attempts: 1,
|
|
base_delay: Duration::from_millis(1),
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
};
|
|
let client = test_client(provider);
|
|
let connection = client
|
|
.connect(
|
|
RealtimeSessionConfig {
|
|
instructions: "backend prompt".to_string(),
|
|
initial_items: Vec::new(),
|
|
delegation_ack_filler: None,
|
|
model: Some("realtime-test-model".to_string()),
|
|
session_id: Some("conv_1".to_string()),
|
|
event_parser: RealtimeEventParser::RealtimeV2,
|
|
session_mode: RealtimeSessionMode::Transcription,
|
|
output_modality: RealtimeOutputModality::Audio,
|
|
voice: RealtimeVoice::Marin,
|
|
},
|
|
HeaderMap::new(),
|
|
HeaderMap::new(),
|
|
)
|
|
.await
|
|
.expect("connect");
|
|
|
|
let created = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
created,
|
|
RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_transcription".to_string(),
|
|
instructions: None,
|
|
}
|
|
);
|
|
|
|
connection
|
|
.send_audio_frame(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 24_000,
|
|
num_channels: 1,
|
|
samples_per_channel: Some(480),
|
|
item_id: None,
|
|
})
|
|
.await
|
|
.expect("send audio");
|
|
|
|
connection.close().await.expect("close");
|
|
server.await.expect("server task");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn v1_transcription_mode_is_treated_as_conversational() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
|
|
let server = tokio::spawn(async move {
|
|
let (stream, _) = listener.accept().await.expect("accept");
|
|
let mut ws = accept_async(stream).await.expect("accept ws");
|
|
|
|
let first = ws
|
|
.next()
|
|
.await
|
|
.expect("first msg")
|
|
.expect("first msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let first_json: Value = serde_json::from_str(&first).expect("json");
|
|
assert_eq!(first_json["type"], "session.update");
|
|
assert_eq!(
|
|
first_json["session"]["type"],
|
|
Value::String("quicksilver".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["instructions"],
|
|
Value::String("backend prompt".to_string())
|
|
);
|
|
assert_eq!(
|
|
first_json["session"]["audio"]["output"]["voice"],
|
|
Value::String("cove".to_string())
|
|
);
|
|
assert!(first_json["session"].get("tools").is_none());
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_v1_mode"}
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send session.updated");
|
|
});
|
|
|
|
let provider = Provider {
|
|
name: "test".to_string(),
|
|
base_url: format!("http://{addr}"),
|
|
query_params: Some(HashMap::new()),
|
|
headers: HeaderMap::new(),
|
|
retry: crate::provider::RetryConfig {
|
|
max_attempts: 1,
|
|
base_delay: Duration::from_millis(1),
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
};
|
|
let client = test_client(provider);
|
|
let connection = client
|
|
.connect(
|
|
RealtimeSessionConfig {
|
|
instructions: "backend prompt".to_string(),
|
|
initial_items: Vec::new(),
|
|
delegation_ack_filler: None,
|
|
model: Some("realtime-test-model".to_string()),
|
|
session_id: Some("conv_1".to_string()),
|
|
event_parser: RealtimeEventParser::V1,
|
|
session_mode: RealtimeSessionMode::Transcription,
|
|
output_modality: RealtimeOutputModality::Audio,
|
|
voice: RealtimeVoice::Cove,
|
|
},
|
|
HeaderMap::new(),
|
|
HeaderMap::new(),
|
|
)
|
|
.await
|
|
.expect("connect");
|
|
|
|
let created = connection
|
|
.next_event()
|
|
.await
|
|
.expect("next event")
|
|
.expect("event");
|
|
assert_eq!(
|
|
created,
|
|
RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_v1_mode".to_string(),
|
|
instructions: None,
|
|
}
|
|
);
|
|
|
|
connection.close().await.expect("close");
|
|
server.await.expect("server task");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn send_does_not_block_while_next_event_waits_for_inbound_data() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
|
|
let server = tokio::spawn(async move {
|
|
let (stream, _) = listener.accept().await.expect("accept");
|
|
let mut ws = accept_async(stream).await.expect("accept ws");
|
|
|
|
let first = ws
|
|
.next()
|
|
.await
|
|
.expect("first msg")
|
|
.expect("first msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let first_json: Value = serde_json::from_str(&first).expect("json");
|
|
assert_eq!(first_json["type"], "session.update");
|
|
|
|
let second = ws
|
|
.next()
|
|
.await
|
|
.expect("second msg")
|
|
.expect("second msg ok")
|
|
.into_text()
|
|
.expect("text");
|
|
let second_json: Value = serde_json::from_str(&second).expect("json");
|
|
assert_eq!(second_json["type"], "input_audio_buffer.append");
|
|
|
|
ws.send(Message::Text(
|
|
json!({
|
|
"type": "session.updated",
|
|
"session": {"id": "sess_after_send", "instructions": "backend prompt"}
|
|
})
|
|
.to_string()
|
|
.into(),
|
|
))
|
|
.await
|
|
.expect("send session.updated");
|
|
});
|
|
|
|
let provider = Provider {
|
|
name: "test".to_string(),
|
|
base_url: format!("http://{addr}"),
|
|
query_params: Some(HashMap::new()),
|
|
headers: HeaderMap::new(),
|
|
retry: crate::provider::RetryConfig {
|
|
max_attempts: 1,
|
|
base_delay: Duration::from_millis(1),
|
|
retry_429: false,
|
|
retry_5xx: false,
|
|
retry_transport: false,
|
|
},
|
|
stream_idle_timeout: Duration::from_secs(5),
|
|
};
|
|
let client = test_client(provider);
|
|
let connection = client
|
|
.connect(
|
|
RealtimeSessionConfig {
|
|
instructions: "backend prompt".to_string(),
|
|
initial_items: Vec::new(),
|
|
delegation_ack_filler: None,
|
|
model: Some("realtime-test-model".to_string()),
|
|
session_id: Some("conv_1".to_string()),
|
|
event_parser: RealtimeEventParser::V1,
|
|
session_mode: RealtimeSessionMode::Conversational,
|
|
output_modality: RealtimeOutputModality::Audio,
|
|
voice: RealtimeVoice::Cove,
|
|
},
|
|
HeaderMap::new(),
|
|
HeaderMap::new(),
|
|
)
|
|
.await
|
|
.expect("connect");
|
|
|
|
let (send_result, next_result) = tokio::join!(
|
|
async {
|
|
tokio::time::timeout(
|
|
Duration::from_millis(200),
|
|
connection.send_audio_frame(RealtimeAudioFrame {
|
|
data: "AQID".to_string(),
|
|
sample_rate: 48000,
|
|
num_channels: 1,
|
|
samples_per_channel: Some(960),
|
|
item_id: None,
|
|
}),
|
|
)
|
|
.await
|
|
},
|
|
connection.next_event()
|
|
);
|
|
|
|
send_result
|
|
.expect("send should not block on next_event")
|
|
.expect("send audio");
|
|
let next_event = next_result.expect("next event").expect("event");
|
|
assert_eq!(
|
|
next_event,
|
|
RealtimeEvent::SessionUpdated {
|
|
realtime_session_id: "sess_after_send".to_string(),
|
|
instructions: Some("backend prompt".to_string()),
|
|
}
|
|
);
|
|
|
|
connection.close().await.expect("close");
|
|
server.await.expect("server task");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "methods_auth_tests.rs"]
|
|
mod auth_tests;
|