mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
## Why A pending realtime connection handshake can hold up subsequent attachment requests. Callers need to cancel obsolete attempts or limit how long they may take to establish a connection. ## What changed - Add `RealtimeAttachmentOptions` with an optional cancellation receiver and `start_before` deadline to `Op::RealtimeConversationAttach`. - Interrupt pending connection work and finish cleanup before acknowledging cancellation. Reject cancelled or expired requests before replacing an existing conversation, and recheck cancellation and the deadline before publishing a connection. - Preserve a successfully published connection when cancellation arrives later. Dropping a cancellation sender whose value is `false` lets the attempt continue. - Prepare the daemon startup test's socket directory with `codex_uds::prepare_private_socket_directory`. ## Testing Add integration coverage for cancelling a held handshake, skipping cancelled or expired queued requests, preserving the next usable call, and releasing a pending handshake when its deadline expires without a cancellation receiver. GitOrigin-RevId: 7144c1237bcbe3b859f10b654d919ae9ff20cbbb
3133 lines
110 KiB
Rust
3133 lines
110 KiB
Rust
use crate::client::ModelClient;
|
||
use crate::client::X_CODEX_TURN_METADATA_HEADER;
|
||
use crate::context::ContextualUserFragment;
|
||
use crate::context::RealtimeDelegation;
|
||
use crate::context::RealtimeDelegationSource;
|
||
use crate::realtime_context::build_realtime_startup_context;
|
||
use crate::realtime_context::truncate_realtime_text_to_token_budget;
|
||
use crate::realtime_prompt::prepare_realtime_backend_prompt;
|
||
use crate::responses_metadata::THREAD_SOURCE_KEY;
|
||
use crate::session::session::Session;
|
||
use anyhow::Context;
|
||
use async_channel::Receiver;
|
||
use async_channel::RecvError;
|
||
use async_channel::Sender;
|
||
use async_channel::TrySendError;
|
||
use base64::Engine;
|
||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||
use codex_api::ApiError;
|
||
use codex_api::Provider as ApiProvider;
|
||
use codex_api::RealtimeAudioFrame;
|
||
use codex_api::RealtimeContextAppendChannel;
|
||
use codex_api::RealtimeEvent;
|
||
use codex_api::RealtimeEventParser;
|
||
use codex_api::RealtimeSessionConfig;
|
||
use codex_api::RealtimeSessionMode;
|
||
use codex_api::RealtimeWebsocketClient;
|
||
use codex_api::RealtimeWebsocketConnection;
|
||
use codex_api::RealtimeWebsocketEvents;
|
||
use codex_api::RealtimeWebsocketWriter;
|
||
use codex_api::build_session_headers;
|
||
use codex_api::map_api_error;
|
||
use codex_async_utils::OrCancelExt;
|
||
use codex_config::config_toml::RealtimeWsMode;
|
||
use codex_config::config_toml::RealtimeWsVersion;
|
||
use codex_features::Feature;
|
||
use codex_http_client::HttpClientFactory;
|
||
use codex_login::CodexAuth;
|
||
use codex_login::auth::is_fedramp_api_key;
|
||
use codex_login::default_client::add_originator_header;
|
||
use codex_login::default_client::default_headers;
|
||
use codex_login::read_openai_api_key_from_env;
|
||
use codex_model_provider::resolve_api_provider;
|
||
use codex_model_provider_info::ModelProviderInfo;
|
||
use codex_protocol::auth::AuthMode;
|
||
use codex_protocol::error::CodexErr;
|
||
use codex_protocol::error::Result as CodexResult;
|
||
use codex_protocol::models::MessagePhase;
|
||
use codex_protocol::protocol::CodexErrorInfo;
|
||
use codex_protocol::protocol::CodexResponseHandoffMode;
|
||
use codex_protocol::protocol::ConversationAudioParams;
|
||
use codex_protocol::protocol::ConversationSpeechParams;
|
||
use codex_protocol::protocol::ConversationStartParams;
|
||
use codex_protocol::protocol::ConversationStartTransport;
|
||
use codex_protocol::protocol::ConversationTextParams;
|
||
use codex_protocol::protocol::ConversationTextRole;
|
||
use codex_protocol::protocol::ErrorEvent;
|
||
use codex_protocol::protocol::Event;
|
||
use codex_protocol::protocol::EventMsg;
|
||
use codex_protocol::protocol::RealtimeAttachmentOptions;
|
||
use codex_protocol::protocol::RealtimeConversationClosedEvent;
|
||
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
|
||
use codex_protocol::protocol::RealtimeConversationSdpEvent;
|
||
use codex_protocol::protocol::RealtimeConversationStartedEvent;
|
||
use codex_protocol::protocol::RealtimeHandoffRequested;
|
||
use codex_protocol::protocol::RealtimeOutputModality;
|
||
use codex_protocol::protocol::RealtimeTranscriptEntry;
|
||
use codex_protocol::protocol::RealtimeVoice;
|
||
use codex_protocol::protocol::RealtimeVoicesList;
|
||
use codex_utils_output_truncation::approx_bytes_for_tokens;
|
||
use codex_utils_string::approx_token_count;
|
||
use codex_utils_string::take_bytes_at_char_boundary;
|
||
use codex_utils_string::to_ascii_json_string;
|
||
use http::HeaderMap;
|
||
use http::HeaderValue;
|
||
use http::header::AUTHORIZATION;
|
||
use serde_json::json;
|
||
use std::collections::BTreeMap;
|
||
use std::collections::HashMap;
|
||
use std::sync::Arc;
|
||
use std::sync::atomic::AtomicBool;
|
||
use std::sync::atomic::Ordering;
|
||
use std::time::Duration;
|
||
use std::time::Instant;
|
||
use std::time::SystemTime;
|
||
use tokio::sync::Mutex;
|
||
use tokio::sync::Semaphore;
|
||
use tokio::sync::watch;
|
||
use tokio::task::JoinHandle;
|
||
use tokio_util::sync::CancellationToken;
|
||
use tracing::debug;
|
||
use tracing::error;
|
||
use tracing::info;
|
||
use tracing::warn;
|
||
|
||
mod bem;
|
||
mod caller_sideband;
|
||
mod existing_call;
|
||
mod sideband;
|
||
mod transcript_tail;
|
||
|
||
use self::bem::ChannelParser as BemChannelParser;
|
||
use self::bem::message_phase as bem_message_phase;
|
||
use self::sideband::spawn_webrtc_sideband_input_task;
|
||
|
||
const AUDIO_IN_QUEUE_CAPACITY: usize = 256;
|
||
const TEXT_IN_QUEUE_CAPACITY: usize = 64;
|
||
const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64;
|
||
const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256;
|
||
const REALTIME_THREAD_SOURCE_MAX_BYTES: usize = 256;
|
||
const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_300;
|
||
const REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET: usize = 1_000;
|
||
const REALTIME_REASONING_STATUS_MAX_BYTES: usize = 256;
|
||
const REALTIME_INITIAL_ITEMS_MAX_COUNT: usize = 128;
|
||
const REALTIME_INITIAL_ITEMS_MAX_TOKENS: usize = 8_192;
|
||
const REALTIME_MODE_INSTRUCTIONS_MAX_TOKENS: usize = 8_192;
|
||
const HANDOFF_STREAM_FLUSH_INTERVAL: Duration = Duration::from_millis(200);
|
||
const HANDOFF_STREAM_TRUNCATION_MARKER: &str = "\n…output truncated…\n";
|
||
const AGENT_FINAL_MESSAGE_PREFIX: &str = "\"Agent Final Message\":\n\n";
|
||
const STANDALONE_HANDOFF_ID: &str = "codex";
|
||
const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5";
|
||
const DEFAULT_FRAMELESS_REALTIME_MODEL: &str = "gpt-live-1-codex";
|
||
pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] ";
|
||
pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] ";
|
||
const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str =
|
||
"Background agent finished. Use the preceding [BACKEND] messages as the result.";
|
||
const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str =
|
||
"This was sent to steer the previous background agent task.";
|
||
const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str =
|
||
"Conversation already has an active response in progress:";
|
||
const REALTIME_SESSION_ENDED_HANDOFF_INSTRUCTION: &str = "The user just ended their realtime session. Here is the remaining handoff/transcript tail. You probably do not have to do anything; acknowledge the handoff unless the transcript itself asks for something.";
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
enum RealtimeConversationEnd {
|
||
Requested,
|
||
TransportClosed,
|
||
Error,
|
||
}
|
||
|
||
enum RealtimeFanoutTaskStop {
|
||
Await,
|
||
Detach,
|
||
}
|
||
|
||
/// Serializes turn admission with the safety cutoff for one voice session.
|
||
struct RealtimeHandoffAdmission {
|
||
gate: Semaphore,
|
||
retired: AtomicBool,
|
||
}
|
||
|
||
impl RealtimeHandoffAdmission {
|
||
fn new() -> Self {
|
||
Self {
|
||
gate: Semaphore::new(1),
|
||
retired: AtomicBool::new(false),
|
||
}
|
||
}
|
||
|
||
async fn route(&self, session: &Arc<Session>, text: String) -> Result<(), &'static str> {
|
||
let Ok(_permit) = self.gate.acquire().await else {
|
||
return Ok(());
|
||
};
|
||
if !self.retired.load(Ordering::Acquire) {
|
||
// Bound the layout of the handoff future while retaining the admission permit.
|
||
Box::pin(session.route_realtime_text_input(text)).await?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
async fn record_transcript(&self, session: &Session, text: String) -> std::io::Result<()> {
|
||
let Ok(_permit) = self.gate.acquire().await else {
|
||
return Ok(());
|
||
};
|
||
if !self.retired.load(Ordering::Acquire) {
|
||
transcript_tail::record(session, text).await?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
async fn retire(&self) {
|
||
let Ok(_permit) = self.gate.acquire().await else {
|
||
return;
|
||
};
|
||
self.retired.store(true, Ordering::Release);
|
||
}
|
||
}
|
||
|
||
pub(crate) struct RealtimeConversationManager {
|
||
state: Mutex<RealtimeConversationManagerState>,
|
||
}
|
||
|
||
struct RealtimeConversationManagerState {
|
||
conversation: Option<ConversationState>,
|
||
mode_instructions: Option<RealtimeModeInstructions>,
|
||
// Keep context active until the terminal transcript is recorded.
|
||
context_active: bool,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub(crate) struct RealtimeModeInstructions {
|
||
pub(crate) start: Option<String>,
|
||
pub(crate) end: Option<String>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub(crate) struct RealtimeConversationSnapshot {
|
||
pub(crate) active: bool,
|
||
pub(crate) mode_instructions: Option<RealtimeModeInstructions>,
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
enum RealtimeSessionKind {
|
||
V1,
|
||
V2,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
struct RealtimeHandoffState {
|
||
output_tx: Sender<RealtimeOutbound>,
|
||
last_output: Arc<Mutex<Option<RealtimeHandoffOutput>>>,
|
||
stream: Arc<Mutex<RealtimeHandoffStreamState>>,
|
||
client_managed_handoffs: bool,
|
||
codex_responses_as_items: bool,
|
||
codex_response_item_prefix: Option<String>,
|
||
codex_response_handoff_mode: CodexResponseHandoffMode,
|
||
backend_reasoning_status: bool,
|
||
codex_response_handoff_channel_prefixes: Arc<BTreeMap<String, Vec<String>>>,
|
||
session_kind: RealtimeSessionKind,
|
||
event_parser: RealtimeEventParser,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
struct RealtimeHandoffOutput {
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
}
|
||
|
||
#[derive(Debug, Default)]
|
||
struct RealtimeHandoffStreamState {
|
||
active_handoff: Option<String>,
|
||
items: HashMap<String, RealtimeStreamedItem>,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
struct RealtimeStreamedItem {
|
||
handoff_id: String,
|
||
phase: Option<MessagePhase>,
|
||
bem_channel_parser: Option<BemChannelParser>,
|
||
prefix_final_message: bool,
|
||
sent_bytes: usize,
|
||
buffered_text: String,
|
||
tail_text: String,
|
||
truncated: bool,
|
||
last_flush_at: Instant,
|
||
flush_scheduled: bool,
|
||
}
|
||
|
||
impl RealtimeStreamedItem {
|
||
fn next_flush_delay(&self) -> Duration {
|
||
HANDOFF_STREAM_FLUSH_INTERVAL.saturating_sub(self.last_flush_at.elapsed())
|
||
}
|
||
|
||
fn output_prefix(&self) -> &'static str {
|
||
if self.prefix_final_message
|
||
&& self.sent_bytes == 0
|
||
&& matches!(self.phase, Some(MessagePhase::FinalAnswer) | None)
|
||
{
|
||
AGENT_FINAL_MESSAGE_PREFIX
|
||
} else {
|
||
""
|
||
}
|
||
}
|
||
|
||
fn stream_head_byte_limit(&self) -> usize {
|
||
let output_byte_limit = approx_bytes_for_tokens(REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET);
|
||
output_byte_limit.saturating_sub(HANDOFF_STREAM_TRUNCATION_MARKER.len()) / 2
|
||
}
|
||
|
||
fn tail_byte_limit(&self) -> usize {
|
||
approx_bytes_for_tokens(REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET)
|
||
.saturating_sub(self.stream_head_byte_limit())
|
||
.saturating_sub(HANDOFF_STREAM_TRUNCATION_MARKER.len())
|
||
}
|
||
|
||
fn streamable_text_bytes(&self) -> usize {
|
||
self.stream_head_byte_limit()
|
||
.saturating_sub(self.sent_bytes)
|
||
.saturating_sub(self.output_prefix().len())
|
||
}
|
||
|
||
fn push_text(&mut self, text: &str) {
|
||
if text.is_empty() {
|
||
return;
|
||
}
|
||
|
||
let text = if let Some(parser) = self.bem_channel_parser.as_mut() {
|
||
let Some(text) = parser.push(text) else {
|
||
return;
|
||
};
|
||
self.phase = parser.phase();
|
||
text
|
||
} else {
|
||
text.to_string()
|
||
};
|
||
self.push_output_text(&text);
|
||
}
|
||
|
||
fn finish_input(&mut self) {
|
||
let Some(parser) = self.bem_channel_parser.as_mut() else {
|
||
return;
|
||
};
|
||
self.phase = parser.phase();
|
||
let text = parser.finish();
|
||
if self.phase.is_none() && !text.is_empty() {
|
||
warn!("BEM output ended before a recognized channel header was received");
|
||
self.phase = Some(MessagePhase::FinalAnswer);
|
||
}
|
||
self.push_output_text(&text);
|
||
}
|
||
|
||
fn push_output_text(&mut self, text: &str) {
|
||
if text.is_empty() {
|
||
return;
|
||
}
|
||
if self.truncated {
|
||
self.tail_text.push_str(text);
|
||
self.tail_text =
|
||
take_last_bytes_at_char_boundary(&self.tail_text, self.tail_byte_limit())
|
||
.to_string();
|
||
return;
|
||
}
|
||
|
||
self.buffered_text.push_str(text);
|
||
let output_byte_limit = approx_bytes_for_tokens(REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET);
|
||
let remaining_text_bytes = output_byte_limit
|
||
.saturating_sub(self.sent_bytes)
|
||
.saturating_sub(self.output_prefix().len());
|
||
if self.buffered_text.len() <= remaining_text_bytes {
|
||
return;
|
||
}
|
||
|
||
let head_bytes =
|
||
take_bytes_at_char_boundary(&self.buffered_text, self.streamable_text_bytes()).len();
|
||
self.tail_text =
|
||
take_last_bytes_at_char_boundary(&self.buffered_text, self.tail_byte_limit())
|
||
.to_string();
|
||
self.buffered_text.truncate(head_bytes);
|
||
self.truncated = true;
|
||
}
|
||
|
||
fn drain_stream_chunk(&mut self) -> Option<String> {
|
||
let prefix = self.output_prefix();
|
||
let available_text_bytes = self.streamable_text_bytes();
|
||
if self.buffered_text.is_empty() || available_text_bytes == 0 {
|
||
return None;
|
||
}
|
||
|
||
let requested_bytes = available_text_bytes.min(self.buffered_text.len());
|
||
let split_at = take_bytes_at_char_boundary(&self.buffered_text, requested_bytes).len();
|
||
if split_at == 0 {
|
||
return None;
|
||
}
|
||
let text = self.buffered_text.drain(..split_at).collect::<String>();
|
||
let text = format!("{prefix}{text}");
|
||
self.sent_bytes += text.len();
|
||
Some(text)
|
||
}
|
||
|
||
fn drain_final_chunk(&mut self) -> Option<String> {
|
||
let prefix = self.output_prefix();
|
||
if !self.truncated {
|
||
if self.buffered_text.is_empty() {
|
||
return None;
|
||
}
|
||
let text = self.buffered_text.drain(..).collect::<String>();
|
||
let text = format!("{prefix}{text}");
|
||
self.sent_bytes += text.len();
|
||
return Some(text);
|
||
}
|
||
|
||
let head = self.buffered_text.drain(..).collect::<String>();
|
||
let tail = self.tail_text.drain(..).collect::<String>();
|
||
let text = format!("{prefix}{head}{HANDOFF_STREAM_TRUNCATION_MARKER}{tail}");
|
||
self.sent_bytes += text.len();
|
||
Some(text)
|
||
}
|
||
}
|
||
|
||
fn take_last_bytes_at_char_boundary(text: &str, max_bytes: usize) -> &str {
|
||
let mut start = text.len().saturating_sub(max_bytes);
|
||
while start < text.len() && !text.is_char_boundary(start) {
|
||
start += 1;
|
||
}
|
||
&text[start..]
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
enum RealtimeOutbound {
|
||
QuietReasoningStatus {
|
||
handoff_id: String,
|
||
text: String,
|
||
},
|
||
StandaloneHandoff {
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
},
|
||
StandaloneSpeech {
|
||
text: String,
|
||
},
|
||
HandoffUpdate {
|
||
handoff_id: String,
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
},
|
||
HandoffAppend {
|
||
handoff_id: String,
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
},
|
||
CompletedHandoff {
|
||
handoff_id: String,
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
},
|
||
ConversationItem {
|
||
text: String,
|
||
phase: Option<MessagePhase>,
|
||
},
|
||
HandoffCompleteAck {
|
||
handoff_id: String,
|
||
},
|
||
}
|
||
|
||
#[derive(Debug, PartialEq, Eq)]
|
||
struct OutputAudioState {
|
||
item_id: String,
|
||
audio_end_ms: u32,
|
||
}
|
||
|
||
#[derive(Default)]
|
||
struct RealtimeResponseCreateQueue {
|
||
active_default_response: bool,
|
||
pending_create: bool,
|
||
}
|
||
|
||
impl RealtimeResponseCreateQueue {
|
||
async fn request_create(
|
||
&mut self,
|
||
writer: &RealtimeWebsocketWriter,
|
||
reason: &str,
|
||
) -> anyhow::Result<()> {
|
||
if self.active_default_response {
|
||
self.pending_create = true;
|
||
return Ok(());
|
||
}
|
||
self.send_create_now(writer, reason).await
|
||
}
|
||
|
||
fn mark_started(&mut self) {
|
||
self.active_default_response = true;
|
||
}
|
||
|
||
async fn mark_finished(
|
||
&mut self,
|
||
writer: &RealtimeWebsocketWriter,
|
||
reason: &str,
|
||
) -> anyhow::Result<()> {
|
||
self.active_default_response = false;
|
||
if !self.pending_create {
|
||
return Ok(());
|
||
}
|
||
self.pending_create = false;
|
||
self.send_create_now(writer, reason).await
|
||
}
|
||
|
||
async fn send_create_now(
|
||
&mut self,
|
||
writer: &RealtimeWebsocketWriter,
|
||
reason: &str,
|
||
) -> anyhow::Result<()> {
|
||
if let Err(err) = writer.send_response_create().await {
|
||
if matches!(&err, ApiError::Stream(message) if message.starts_with(REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX))
|
||
{
|
||
warn!("realtime response.create raced an active response; deferring");
|
||
self.active_default_response = true;
|
||
self.pending_create = true;
|
||
return Ok(());
|
||
}
|
||
warn!("failed to send {reason} response.create: {err}");
|
||
return Err(err.into());
|
||
}
|
||
self.active_default_response = true;
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
// Any completed transcript travels with its event so the fanout persists it before later handoffs.
|
||
type RealtimeOutput = (RealtimeEvent, Option<String>);
|
||
|
||
struct RealtimeInputTask {
|
||
writer: RealtimeWebsocketWriter,
|
||
events: RealtimeWebsocketEvents,
|
||
text_rx: Receiver<ConversationTextParams>,
|
||
handoff_output_rx: Receiver<RealtimeOutbound>,
|
||
audio_rx: Receiver<RealtimeAudioFrame>,
|
||
events_tx: Sender<RealtimeOutput>,
|
||
handoff_state: RealtimeHandoffState,
|
||
session_kind: RealtimeSessionKind,
|
||
event_parser: RealtimeEventParser,
|
||
stop_token: CancellationToken,
|
||
}
|
||
|
||
struct RealtimeTranscriptTailFlush {
|
||
enabled: bool,
|
||
event_parser: RealtimeEventParser,
|
||
tx: Sender<String>,
|
||
}
|
||
|
||
struct RealtimeInputChannels {
|
||
text_rx: Receiver<ConversationTextParams>,
|
||
handoff_output_rx: Receiver<RealtimeOutbound>,
|
||
audio_rx: Receiver<RealtimeAudioFrame>,
|
||
}
|
||
|
||
impl RealtimeHandoffState {
|
||
fn streams_handoff_append(&self) -> bool {
|
||
self.event_parser == RealtimeEventParser::FramelessBidi
|
||
&& !self.client_managed_handoffs
|
||
&& !self.codex_responses_as_items
|
||
}
|
||
|
||
fn routes_handoff_by_bem(&self) -> bool {
|
||
self.event_parser == RealtimeEventParser::FramelessBidi
|
||
&& self.codex_response_handoff_mode == CodexResponseHandoffMode::BemTags
|
||
}
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
struct ConversationState {
|
||
session_id: Option<String>,
|
||
sub_id: String,
|
||
audio_tx: Sender<RealtimeAudioFrame>,
|
||
text_tx: Sender<ConversationTextParams>,
|
||
session_kind: RealtimeSessionKind,
|
||
handoff: RealtimeHandoffState,
|
||
input_task: JoinHandle<()>,
|
||
fanout_task: Option<JoinHandle<()>>,
|
||
realtime_active: Arc<AtomicBool>,
|
||
// A misalignment failure retires only this session's queued handoffs.
|
||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||
stop_token: CancellationToken,
|
||
}
|
||
|
||
#[derive(Clone, Default)]
|
||
struct RealtimeAttachmentControl {
|
||
token: CancellationToken,
|
||
options: RealtimeAttachmentOptions,
|
||
}
|
||
|
||
impl RealtimeAttachmentControl {
|
||
fn is_cancelled(&self) -> bool {
|
||
self.token.is_cancelled()
|
||
|| self
|
||
.options
|
||
.cancel
|
||
.as_ref()
|
||
.is_some_and(|cancellation| *cancellation.borrow())
|
||
|| self
|
||
.options
|
||
.start_before
|
||
.is_some_and(|deadline| SystemTime::now() >= deadline)
|
||
}
|
||
}
|
||
|
||
struct RealtimeStart {
|
||
attachment_control: RealtimeAttachmentControl,
|
||
sideband: Option<codex_protocol::protocol::RealtimeSidebandTransport>,
|
||
api_provider: ApiProvider,
|
||
http_client_factory: HttpClientFactory,
|
||
realtime_sideband_base_url: Option<String>,
|
||
extra_headers: Option<HeaderMap>,
|
||
websocket_auth: Option<codex_api::SharedAuthProvider>,
|
||
client_managed_handoffs: bool,
|
||
flush_transcript_tail_on_session_end: bool,
|
||
codex_responses_as_items: bool,
|
||
codex_response_item_prefix: Option<String>,
|
||
codex_response_handoff_mode: CodexResponseHandoffMode,
|
||
backend_reasoning_status: bool,
|
||
codex_response_handoff_channel_prefixes: Option<BTreeMap<String, Vec<String>>>,
|
||
realtime_call_api_provider: Option<ApiProvider>,
|
||
session_config: RealtimeSessionConfig,
|
||
model_client: ModelClient,
|
||
sdp: Option<String>,
|
||
existing_call_id: Option<String>,
|
||
auth_context: codex_model_provider::ProviderAuthContext,
|
||
}
|
||
|
||
struct RealtimeStartOutput {
|
||
realtime_active: Arc<AtomicBool>,
|
||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||
events_rx: Receiver<RealtimeOutput>,
|
||
transcript_tail_rx: Receiver<String>,
|
||
sdp: Option<String>,
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
impl RealtimeConversationManager {
|
||
pub(crate) fn new() -> Self {
|
||
Self {
|
||
state: Mutex::new(RealtimeConversationManagerState {
|
||
conversation: None,
|
||
mode_instructions: None,
|
||
context_active: false,
|
||
}),
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn snapshot(&self) -> RealtimeConversationSnapshot {
|
||
let state = self.state.lock().await;
|
||
RealtimeConversationSnapshot {
|
||
active: state.context_active,
|
||
mode_instructions: state.mode_instructions.clone(),
|
||
}
|
||
}
|
||
|
||
#[tracing::instrument(
|
||
name = "realtime_conversation.running_state",
|
||
level = "trace",
|
||
skip_all
|
||
)]
|
||
pub(crate) async fn running_state(&self) -> Option<()> {
|
||
let state = self.state.lock().await;
|
||
state
|
||
.conversation
|
||
.as_ref()
|
||
.and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(()))
|
||
}
|
||
|
||
pub(crate) async fn is_running_v2(&self) -> bool {
|
||
let state = self.state.lock().await;
|
||
matches!(
|
||
state.conversation.as_ref(),
|
||
Some(state)
|
||
if state.realtime_active.load(Ordering::Relaxed)
|
||
&& state.session_kind == RealtimeSessionKind::V2
|
||
)
|
||
}
|
||
|
||
/// Prevent already parsed handoffs from starting turns after a safety failure.
|
||
/// Keep the gate on the old session so a later voice session starts fresh.
|
||
pub(crate) async fn retire_handoffs_for_misalignment(&self) {
|
||
let route_handoffs = self
|
||
.state
|
||
.lock()
|
||
.await
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| Arc::clone(&state.route_handoffs));
|
||
if let Some(route_handoffs) = route_handoffs {
|
||
route_handoffs.retire().await;
|
||
}
|
||
}
|
||
|
||
async fn start(
|
||
&self,
|
||
start: RealtimeStart,
|
||
mode_instructions: RealtimeModeInstructions,
|
||
sess: &Arc<Session>,
|
||
sub_id: &str,
|
||
) -> CodexResult<RealtimeStartOutput> {
|
||
let previous_state = {
|
||
let mut guard = self.state.lock().await;
|
||
if start.attachment_control.is_cancelled() {
|
||
return Err(CodexErr::TurnAborted);
|
||
}
|
||
guard.conversation.take()
|
||
};
|
||
if let Some(state) = previous_state {
|
||
let previous_sub_id = state.sub_id.clone();
|
||
stop_conversation_state(state, RealtimeFanoutTaskStop::Await).await;
|
||
send_realtime_conversation_closed(
|
||
sess,
|
||
previous_sub_id,
|
||
RealtimeConversationEnd::Requested,
|
||
)
|
||
.await;
|
||
}
|
||
|
||
self.start_inner(start, mode_instructions, sub_id).await
|
||
}
|
||
|
||
async fn start_inner(
|
||
&self,
|
||
start: RealtimeStart,
|
||
mode_instructions: RealtimeModeInstructions,
|
||
sub_id: &str,
|
||
) -> CodexResult<RealtimeStartOutput> {
|
||
let session_id = start.session_config.session_id.clone();
|
||
let RealtimeStart {
|
||
attachment_control,
|
||
sideband,
|
||
auth_context,
|
||
api_provider,
|
||
http_client_factory,
|
||
realtime_sideband_base_url,
|
||
extra_headers,
|
||
websocket_auth,
|
||
client_managed_handoffs,
|
||
flush_transcript_tail_on_session_end,
|
||
codex_responses_as_items,
|
||
codex_response_item_prefix,
|
||
codex_response_handoff_mode,
|
||
backend_reasoning_status,
|
||
codex_response_handoff_channel_prefixes,
|
||
realtime_call_api_provider,
|
||
session_config,
|
||
model_client,
|
||
sdp,
|
||
existing_call_id,
|
||
} = start;
|
||
if attachment_control.is_cancelled() {
|
||
return Err(CodexErr::TurnAborted);
|
||
}
|
||
let event_parser = session_config.event_parser;
|
||
let session_kind = match event_parser {
|
||
RealtimeEventParser::V1 | RealtimeEventParser::FramelessBidi => RealtimeSessionKind::V1,
|
||
RealtimeEventParser::RealtimeV2 => RealtimeSessionKind::V2,
|
||
};
|
||
|
||
let (audio_tx, audio_rx) =
|
||
async_channel::bounded::<RealtimeAudioFrame>(AUDIO_IN_QUEUE_CAPACITY);
|
||
let (text_tx, text_rx) =
|
||
async_channel::bounded::<ConversationTextParams>(TEXT_IN_QUEUE_CAPACITY);
|
||
let (handoff_output_tx, handoff_output_rx) =
|
||
async_channel::bounded::<RealtimeOutbound>(HANDOFF_OUT_QUEUE_CAPACITY);
|
||
let (events_tx, events_rx) =
|
||
async_channel::bounded::<RealtimeOutput>(OUTPUT_EVENTS_QUEUE_CAPACITY);
|
||
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
|
||
|
||
let realtime_active = Arc::new(AtomicBool::new(true));
|
||
let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
|
||
let stop_token = CancellationToken::new();
|
||
let handoff = RealtimeHandoffState {
|
||
output_tx: handoff_output_tx,
|
||
last_output: Arc::new(Mutex::new(None)),
|
||
stream: Arc::new(Mutex::new(RealtimeHandoffStreamState::default())),
|
||
client_managed_handoffs,
|
||
codex_responses_as_items,
|
||
codex_response_item_prefix,
|
||
codex_response_handoff_mode,
|
||
backend_reasoning_status,
|
||
codex_response_handoff_channel_prefixes: Arc::new(
|
||
codex_response_handoff_channel_prefixes.unwrap_or_default(),
|
||
),
|
||
session_kind,
|
||
event_parser,
|
||
};
|
||
let input_channels = RealtimeInputChannels {
|
||
text_rx,
|
||
handoff_output_rx,
|
||
audio_rx,
|
||
};
|
||
|
||
let sideband_base_url =
|
||
realtime_sideband_base_url.unwrap_or_else(|| "https://api.openai.com/v1".to_string());
|
||
let client = RealtimeWebsocketClient::new(api_provider, http_client_factory.clone())
|
||
.with_webrtc_sideband_base_url(sideband_base_url.clone());
|
||
let transcript_tail_flush = RealtimeTranscriptTailFlush {
|
||
enabled: flush_transcript_tail_on_session_end,
|
||
event_parser,
|
||
tx: transcript_tail_tx,
|
||
};
|
||
// Use the sideband transport passed from outside.
|
||
let (task, sdp) = if let Some(transport) = sideband {
|
||
// Existing naming uses websocket, actual link is defined by the transport.
|
||
let connection =
|
||
codex_api::RealtimeWebsocketConnection::from_sideband(transport, event_parser);
|
||
|
||
let task = caller_sideband::spawn_input_task(
|
||
RealtimeInputTask {
|
||
writer: connection.writer(),
|
||
events: connection.events(),
|
||
text_rx: input_channels.text_rx,
|
||
handoff_output_rx: input_channels.handoff_output_rx,
|
||
audio_rx: input_channels.audio_rx,
|
||
events_tx,
|
||
handoff_state: handoff.clone(),
|
||
session_kind,
|
||
event_parser,
|
||
stop_token: stop_token.clone(),
|
||
},
|
||
transcript_tail_flush,
|
||
);
|
||
(task, None)
|
||
} else if let Some(sdp) = sdp {
|
||
let call = model_client
|
||
.create_realtime_call_with_headers(
|
||
sdp,
|
||
session_config.clone(),
|
||
extra_headers.unwrap_or_default(),
|
||
realtime_call_api_provider,
|
||
&sideband_base_url,
|
||
&auth_context,
|
||
http_client_factory,
|
||
)
|
||
.or_cancel(&attachment_control.token)
|
||
.await??;
|
||
let task = spawn_webrtc_sideband_input_task(RealtimeWebrtcSidebandInputTask {
|
||
client: client
|
||
.with_http_client_factory(call.sideband_http_client_factory)
|
||
.with_auth(call.sideband_auth),
|
||
session_config,
|
||
call_id: call.call_id,
|
||
sideband_headers: call.sideband_headers,
|
||
session_initialization: RealtimeSidebandSessionInitialization::CoreCreated,
|
||
input_channels,
|
||
events_tx,
|
||
handoff_state: handoff.clone(),
|
||
session_kind,
|
||
event_parser,
|
||
realtime_active: Arc::clone(&realtime_active),
|
||
transcript_tail_flush,
|
||
stop_token: stop_token.clone(),
|
||
});
|
||
(task, Some(call.sdp))
|
||
} else if let Some(call_id) = existing_call_id {
|
||
let task = existing_call::attach(existing_call::ExistingCallAttachment {
|
||
attachment_cancellation: attachment_control.token.clone(),
|
||
client,
|
||
model_client,
|
||
auth_context,
|
||
http_client_factory,
|
||
sideband_base_url,
|
||
session_config,
|
||
call_id,
|
||
extra_headers: extra_headers.unwrap_or_default(),
|
||
input_channels,
|
||
events_tx,
|
||
handoff_state: handoff.clone(),
|
||
session_kind,
|
||
event_parser,
|
||
realtime_active: Arc::clone(&realtime_active),
|
||
transcript_tail_flush,
|
||
stop_token: stop_token.clone(),
|
||
})
|
||
.await?;
|
||
(task, None)
|
||
} else {
|
||
let extra_headers = extra_headers.unwrap_or_default();
|
||
let client = match websocket_auth {
|
||
Some(auth) => client.with_auth(auth),
|
||
None => client,
|
||
};
|
||
let connection = client
|
||
.connect(session_config, extra_headers, default_headers())
|
||
.or_cancel(&attachment_control.token)
|
||
.await?
|
||
.map_err(map_api_error)?;
|
||
let task = spawn_realtime_input_task(
|
||
RealtimeInputTask {
|
||
writer: connection.writer(),
|
||
events: connection.events(),
|
||
text_rx: input_channels.text_rx,
|
||
handoff_output_rx: input_channels.handoff_output_rx,
|
||
audio_rx: input_channels.audio_rx,
|
||
events_tx,
|
||
handoff_state: handoff.clone(),
|
||
session_kind,
|
||
event_parser,
|
||
stop_token: stop_token.clone(),
|
||
},
|
||
transcript_tail_flush,
|
||
);
|
||
(task, None)
|
||
};
|
||
|
||
let conversation = ConversationState {
|
||
session_id,
|
||
sub_id: sub_id.to_owned(),
|
||
audio_tx,
|
||
text_tx,
|
||
session_kind,
|
||
handoff,
|
||
input_task: task,
|
||
fanout_task: None,
|
||
realtime_active: Arc::clone(&realtime_active),
|
||
route_handoffs: Arc::clone(&route_handoffs),
|
||
stop_token,
|
||
};
|
||
let uncommitted = {
|
||
let mut state = self.state.lock().await;
|
||
// Read the original sources at commit, even if their wakeup has not
|
||
// been polled. Hold the watch guard through synchronous publication.
|
||
let cancellation = attachment_control
|
||
.options
|
||
.cancel
|
||
.as_ref()
|
||
.map(watch::Receiver::borrow);
|
||
if cancellation.as_ref().is_some_and(|cancelled| **cancelled)
|
||
|| attachment_control.token.is_cancelled()
|
||
|| attachment_control
|
||
.options
|
||
.start_before
|
||
.is_some_and(|deadline| SystemTime::now() >= deadline)
|
||
{
|
||
Some(conversation)
|
||
} else {
|
||
// Publication commits success; later cancellation must preserve
|
||
// residency until the caller projects the Started event.
|
||
state.conversation = Some(conversation);
|
||
state.mode_instructions = Some(mode_instructions);
|
||
state.context_active = true;
|
||
None
|
||
}
|
||
};
|
||
if let Some(conversation) = uncommitted {
|
||
drop(events_rx);
|
||
drop(transcript_tail_rx);
|
||
stop_conversation_state(conversation, RealtimeFanoutTaskStop::Await).await;
|
||
return Err(CodexErr::TurnAborted);
|
||
}
|
||
Ok(RealtimeStartOutput {
|
||
realtime_active,
|
||
route_handoffs,
|
||
events_rx,
|
||
transcript_tail_rx,
|
||
sdp,
|
||
})
|
||
}
|
||
|
||
pub(crate) async fn register_fanout_task(
|
||
&self,
|
||
realtime_active: &Arc<AtomicBool>,
|
||
fanout_task: JoinHandle<()>,
|
||
) {
|
||
let mut fanout_task = Some(fanout_task);
|
||
{
|
||
let mut guard = self.state.lock().await;
|
||
if let Some(state) = guard.conversation.as_mut()
|
||
// Only attach the task to the conversation currently stored in the manager.
|
||
&& Arc::ptr_eq(&state.realtime_active, realtime_active)
|
||
{
|
||
state.fanout_task = fanout_task.take();
|
||
}
|
||
}
|
||
|
||
if let Some(fanout_task) = fanout_task {
|
||
fanout_task.abort();
|
||
let _ = fanout_task.await;
|
||
}
|
||
}
|
||
|
||
#[expect(
|
||
clippy::await_holding_invalid_type,
|
||
reason = "Publish the matching close before a replacement can start its history segment."
|
||
)]
|
||
async fn finish_if_active(
|
||
&self,
|
||
sess: &Arc<Session>,
|
||
realtime_active: &Arc<AtomicBool>,
|
||
sub_id: String,
|
||
end: RealtimeConversationEnd,
|
||
) {
|
||
// Keep selection and its close event together. An old fanout must not
|
||
// close the history segment that belongs to a replacement.
|
||
let mut guard = self.state.lock().await;
|
||
if !guard
|
||
.conversation
|
||
.as_ref()
|
||
.is_some_and(|state| Arc::ptr_eq(&state.realtime_active, realtime_active))
|
||
{
|
||
return;
|
||
}
|
||
let Some(state) = guard.conversation.take() else {
|
||
return;
|
||
};
|
||
stop_conversation_state(state, RealtimeFanoutTaskStop::Detach).await;
|
||
send_realtime_conversation_closed(sess, sub_id, end).await;
|
||
}
|
||
|
||
pub(crate) async fn audio_in(&self, frame: RealtimeAudioFrame) -> CodexResult<()> {
|
||
let sender = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.audio_tx.clone())
|
||
};
|
||
|
||
let Some(sender) = sender else {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
));
|
||
};
|
||
|
||
match sender.try_send(frame) {
|
||
Ok(()) => Ok(()),
|
||
Err(TrySendError::Full(_)) => {
|
||
warn!("dropping input audio frame due to full queue");
|
||
Ok(())
|
||
}
|
||
Err(TrySendError::Closed(_)) => Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
)),
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn text_in(&self, mut params: ConversationTextParams) -> CodexResult<()> {
|
||
let sender = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| (state.text_tx.clone(), state.session_kind))
|
||
};
|
||
|
||
let Some((sender, session_kind)) = sender else {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
));
|
||
};
|
||
|
||
if params.role == ConversationTextRole::User {
|
||
params.text =
|
||
prefix_realtime_text(params.text, REALTIME_USER_TEXT_PREFIX, session_kind);
|
||
}
|
||
sender
|
||
.send(params)
|
||
.await
|
||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?;
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn send_reasoning_status(&self, text: &str) -> CodexResult<()> {
|
||
let handoff = self
|
||
.state
|
||
.lock()
|
||
.await
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.handoff.clone());
|
||
let Some(handoff) = handoff.filter(|handoff| {
|
||
handoff.backend_reasoning_status
|
||
&& !handoff.client_managed_handoffs
|
||
&& handoff.event_parser == RealtimeEventParser::FramelessBidi
|
||
}) else {
|
||
return Ok(());
|
||
};
|
||
let Some(handoff_id) = handoff.stream.lock().await.active_handoff.clone() else {
|
||
return Ok(());
|
||
};
|
||
let text = take_bytes_at_char_boundary(text, REALTIME_REASONING_STATUS_MAX_BYTES);
|
||
handoff
|
||
.output_tx
|
||
.send(RealtimeOutbound::QuietReasoningStatus {
|
||
handoff_id,
|
||
text: format!("[STATUS] {text}"),
|
||
})
|
||
.await
|
||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))
|
||
}
|
||
|
||
pub(crate) async fn handoff_out(
|
||
&self,
|
||
output_text: String,
|
||
phase: Option<MessagePhase>,
|
||
) -> CodexResult<()> {
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
let Some(state) = guard.conversation.as_ref() else {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
));
|
||
};
|
||
state.handoff.clone()
|
||
};
|
||
|
||
if handoff.client_managed_handoffs {
|
||
return Ok(());
|
||
}
|
||
let phase = if handoff.routes_handoff_by_bem()
|
||
&& !matches!(phase, Some(MessagePhase::PartialAnswer))
|
||
{
|
||
match bem_message_phase(
|
||
&output_text,
|
||
&handoff.codex_response_handoff_channel_prefixes,
|
||
) {
|
||
Some(phase) => Some(phase),
|
||
None => {
|
||
warn!("BEM output did not contain a recognized channel header");
|
||
Some(MessagePhase::FinalAnswer)
|
||
}
|
||
}
|
||
} else {
|
||
phase
|
||
};
|
||
let is_nonterminal = matches!(
|
||
phase,
|
||
Some(MessagePhase::Commentary | MessagePhase::PartialAnswer)
|
||
);
|
||
let active_handoff = handoff.stream.lock().await.active_handoff.clone();
|
||
let output = match active_handoff {
|
||
Some(handoff_id) => {
|
||
let output_text = realtime_backend_output(output_text, handoff.session_kind);
|
||
*handoff.last_output.lock().await = Some(RealtimeHandoffOutput {
|
||
text: output_text.clone(),
|
||
phase: phase.clone(),
|
||
});
|
||
if handoff.codex_responses_as_items {
|
||
RealtimeOutbound::ConversationItem {
|
||
text: realtime_backend_item(
|
||
output_text,
|
||
handoff.codex_response_item_prefix.as_deref(),
|
||
),
|
||
phase,
|
||
}
|
||
} else if handoff.event_parser == RealtimeEventParser::V1 && is_nonterminal {
|
||
RealtimeOutbound::HandoffAppend {
|
||
handoff_id,
|
||
text: output_text,
|
||
phase,
|
||
}
|
||
} else {
|
||
RealtimeOutbound::HandoffUpdate {
|
||
handoff_id,
|
||
text: output_text,
|
||
phase,
|
||
}
|
||
}
|
||
}
|
||
None if output_text.trim().is_empty() => return Ok(()),
|
||
None => {
|
||
let output_text = realtime_backend_output(output_text, handoff.session_kind);
|
||
if handoff.codex_responses_as_items {
|
||
RealtimeOutbound::ConversationItem {
|
||
text: realtime_backend_item(
|
||
output_text,
|
||
handoff.codex_response_item_prefix.as_deref(),
|
||
),
|
||
phase,
|
||
}
|
||
} else {
|
||
RealtimeOutbound::StandaloneHandoff {
|
||
text: if handoff.event_parser == RealtimeEventParser::V1 && !is_nonterminal
|
||
{
|
||
format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}")
|
||
} else {
|
||
output_text
|
||
},
|
||
phase,
|
||
}
|
||
}
|
||
}
|
||
};
|
||
handoff
|
||
.output_tx
|
||
.send(output)
|
||
.await
|
||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?;
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn register_handoff_stream_item(
|
||
&self,
|
||
item_id: String,
|
||
phase: Option<MessagePhase>,
|
||
initial_text: String,
|
||
) {
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.handoff.clone())
|
||
};
|
||
let Some(handoff) = handoff else {
|
||
return;
|
||
};
|
||
if !handoff.streams_handoff_append() {
|
||
return;
|
||
}
|
||
let flush_delay = {
|
||
let mut stream = handoff.stream.lock().await;
|
||
let Some(handoff_id) = stream.active_handoff.clone() else {
|
||
return;
|
||
};
|
||
// An explicit stable fragment is answer text, even when it starts
|
||
// with characters that legacy BEM treats as a private channel tag.
|
||
let parse_bem = handoff.routes_handoff_by_bem()
|
||
&& !matches!(phase, Some(MessagePhase::PartialAnswer));
|
||
let mut streamed_item = RealtimeStreamedItem {
|
||
handoff_id,
|
||
phase: if parse_bem { None } else { phase },
|
||
bem_channel_parser: parse_bem.then(|| {
|
||
BemChannelParser::new(Arc::clone(
|
||
&handoff.codex_response_handoff_channel_prefixes,
|
||
))
|
||
}),
|
||
prefix_final_message: handoff.event_parser == RealtimeEventParser::V1,
|
||
sent_bytes: 0,
|
||
buffered_text: String::new(),
|
||
tail_text: String::new(),
|
||
truncated: false,
|
||
last_flush_at: Instant::now(),
|
||
flush_scheduled: false,
|
||
};
|
||
streamed_item.push_text(&initial_text);
|
||
let flush_delay = if streamed_item.buffered_text.is_empty() {
|
||
None
|
||
} else {
|
||
streamed_item.flush_scheduled = true;
|
||
Some(streamed_item.next_flush_delay())
|
||
};
|
||
stream.items.insert(item_id.clone(), streamed_item);
|
||
flush_delay
|
||
};
|
||
if let Some(flush_delay) = flush_delay {
|
||
schedule_streamed_handoff_flush(&handoff, item_id, flush_delay);
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn stream_handoff_delta(
|
||
&self,
|
||
item_id: &str,
|
||
delta: String,
|
||
) -> CodexResult<()> {
|
||
if delta.is_empty() {
|
||
return Ok(());
|
||
}
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
let Some(state) = guard.conversation.as_ref() else {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
));
|
||
};
|
||
state.handoff.clone()
|
||
};
|
||
if !handoff.streams_handoff_append() {
|
||
return Ok(());
|
||
}
|
||
let flush_delay = {
|
||
let mut stream = handoff.stream.lock().await;
|
||
let Some(streamed_item) = stream.items.get_mut(item_id) else {
|
||
return Ok(());
|
||
};
|
||
streamed_item.push_text(&delta);
|
||
if streamed_item.flush_scheduled || streamed_item.streamable_text_bytes() == 0 {
|
||
None
|
||
} else {
|
||
streamed_item.flush_scheduled = true;
|
||
Some(streamed_item.next_flush_delay())
|
||
}
|
||
};
|
||
if let Some(flush_delay) = flush_delay {
|
||
schedule_streamed_handoff_flush(&handoff, item_id.to_string(), flush_delay);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn finish_handoff_stream_item(&self, item_id: &str) -> bool {
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.handoff.clone())
|
||
};
|
||
let Some(handoff) = handoff else {
|
||
return false;
|
||
};
|
||
if !handoff.streams_handoff_append() {
|
||
return false;
|
||
}
|
||
let Some(mut streamed_item) = handoff.stream.lock().await.items.remove(item_id) else {
|
||
return false;
|
||
};
|
||
streamed_item.finish_input();
|
||
let chunk = streamed_item.drain_final_chunk();
|
||
let sent_output = streamed_item.sent_bytes > 0;
|
||
if let Some(text) = chunk {
|
||
let _ = handoff
|
||
.output_tx
|
||
.send(RealtimeOutbound::HandoffAppend {
|
||
handoff_id: streamed_item.handoff_id,
|
||
text,
|
||
phase: streamed_item.phase,
|
||
})
|
||
.await;
|
||
}
|
||
sent_output
|
||
}
|
||
|
||
pub(crate) async fn append_speech(&self, text: String) -> CodexResult<()> {
|
||
if text.trim().is_empty() {
|
||
return Ok(());
|
||
}
|
||
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
let Some(state) = guard.conversation.as_ref() else {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"conversation is not running".to_string(),
|
||
));
|
||
};
|
||
state.handoff.clone()
|
||
};
|
||
|
||
handoff
|
||
.output_tx
|
||
.send(RealtimeOutbound::StandaloneSpeech {
|
||
text: realtime_backend_output(text, handoff.session_kind),
|
||
})
|
||
.await
|
||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?;
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn handoff_complete(&self) -> CodexResult<()> {
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.handoff.clone())
|
||
};
|
||
let Some(handoff) = handoff else {
|
||
return Ok(());
|
||
};
|
||
if handoff.client_managed_handoffs {
|
||
return Ok(());
|
||
}
|
||
match handoff.session_kind {
|
||
RealtimeSessionKind::V1 => return Ok(()),
|
||
RealtimeSessionKind::V2 => {}
|
||
}
|
||
|
||
let Some(handoff_id) = handoff.stream.lock().await.active_handoff.clone() else {
|
||
return Ok(());
|
||
};
|
||
let Some(last_output) = handoff.last_output.lock().await.clone() else {
|
||
return Ok(());
|
||
};
|
||
|
||
let output = if handoff.codex_responses_as_items {
|
||
RealtimeOutbound::HandoffCompleteAck { handoff_id }
|
||
} else {
|
||
RealtimeOutbound::CompletedHandoff {
|
||
handoff_id,
|
||
text: last_output.text,
|
||
phase: last_output.phase,
|
||
}
|
||
};
|
||
|
||
handoff
|
||
.output_tx
|
||
.send(output)
|
||
.await
|
||
.map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))
|
||
}
|
||
|
||
pub(crate) async fn clear_active_handoff(&self) {
|
||
let handoff = {
|
||
let guard = self.state.lock().await;
|
||
guard
|
||
.conversation
|
||
.as_ref()
|
||
.map(|state| state.handoff.clone())
|
||
};
|
||
if let Some(handoff) = handoff {
|
||
{
|
||
let mut stream = handoff.stream.lock().await;
|
||
stream.active_handoff = None;
|
||
stream.items.clear();
|
||
}
|
||
*handoff.last_output.lock().await = None;
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn shutdown(&self) -> CodexResult<()> {
|
||
let state = {
|
||
let mut guard = self.state.lock().await;
|
||
guard.conversation.take()
|
||
};
|
||
|
||
if let Some(state) = state {
|
||
stop_conversation_state(state, RealtimeFanoutTaskStop::Await).await;
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
async fn stop_conversation_state(
|
||
mut state: ConversationState,
|
||
fanout_task_stop: RealtimeFanoutTaskStop,
|
||
) {
|
||
state.realtime_active.store(false, Ordering::Relaxed);
|
||
state.stop_token.cancel();
|
||
let _ = state.input_task.await;
|
||
|
||
if let Some(fanout_task) = state.fanout_task.take() {
|
||
match fanout_task_stop {
|
||
RealtimeFanoutTaskStop::Await => {
|
||
let _ = fanout_task.await;
|
||
}
|
||
RealtimeFanoutTaskStop::Detach => {}
|
||
}
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn handle_start(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationStartParams,
|
||
) -> CodexResult<()> {
|
||
let prepared_start = match prepare_realtime_start(sess, params).await {
|
||
Ok(prepared_start) => prepared_start,
|
||
Err(err) => {
|
||
error!("failed to prepare realtime conversation: {err}");
|
||
let message = err.to_string();
|
||
sess.send_event_raw(Event {
|
||
id: sub_id,
|
||
msg: EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||
payload: RealtimeEvent::Error(message),
|
||
}),
|
||
})
|
||
.await;
|
||
return Ok(());
|
||
}
|
||
};
|
||
|
||
if let Err(err) = handle_start_inner(sess, &sub_id, prepared_start).await {
|
||
error!("failed to start realtime conversation: {err}");
|
||
let message = err.to_string();
|
||
sess.send_event_raw(Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||
payload: RealtimeEvent::Error(message),
|
||
}),
|
||
})
|
||
.await;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
struct PreparedRealtimeConversationStart {
|
||
attachment_control: RealtimeAttachmentControl,
|
||
sideband: Option<codex_protocol::protocol::RealtimeSidebandTransport>,
|
||
api_provider: ApiProvider,
|
||
http_client_factory: HttpClientFactory,
|
||
realtime_sideband_base_url: Option<String>,
|
||
extra_headers: Option<HeaderMap>,
|
||
websocket_auth: Option<codex_api::SharedAuthProvider>,
|
||
client_managed_handoffs: bool,
|
||
flush_transcript_tail_on_session_end: bool,
|
||
codex_responses_as_items: bool,
|
||
codex_response_item_prefix: Option<String>,
|
||
codex_response_handoff_mode: CodexResponseHandoffMode,
|
||
backend_reasoning_status: bool,
|
||
codex_response_handoff_channel_prefixes: Option<BTreeMap<String, Vec<String>>>,
|
||
realtime_start_instructions: Option<String>,
|
||
realtime_end_instructions: Option<String>,
|
||
realtime_call_api_provider: Option<ApiProvider>,
|
||
requested_realtime_session_id: Option<String>,
|
||
version: RealtimeWsVersion,
|
||
session_config: RealtimeSessionConfig,
|
||
transport: ConversationStartTransport,
|
||
auth_context: codex_model_provider::ProviderAuthContext,
|
||
}
|
||
|
||
#[derive(Clone, Copy)]
|
||
pub(crate) enum ConfiguredRealtimeVoice {
|
||
Use,
|
||
Ignore,
|
||
}
|
||
|
||
async fn prepare_realtime_start(
|
||
sess: &Arc<Session>,
|
||
params: ConversationStartParams,
|
||
) -> CodexResult<PreparedRealtimeConversationStart> {
|
||
sess.services
|
||
.model_client
|
||
.validate_required_cyber_access_program(/*programs*/ None)?;
|
||
let provider = sess.provider().await;
|
||
let auth_manager = sess
|
||
.services
|
||
.model_client
|
||
.auth_manager()
|
||
.unwrap_or_else(|| Arc::clone(&sess.services.auth_manager));
|
||
let config = sess.get_config().await;
|
||
let auth_context = codex_model_provider::ProviderAuthContext {
|
||
credentials: Arc::clone(&auth_manager),
|
||
runtime_auth: if config.features.enabled(Feature::RuntimeAgentInstances) {
|
||
let runtime = sess.services.runtime_auth.get().ok_or_else(|| {
|
||
CodexErr::InvalidRequest("runtime authentication is not initialized".to_string())
|
||
})?;
|
||
Some((Arc::clone(runtime), config.runtime_auth_mode))
|
||
} else {
|
||
None
|
||
},
|
||
};
|
||
let (auth, mut http_client_factory) = match auth_manager.auth_with_http_client_factory().await {
|
||
Some((auth, captured_factory)) if auth.is_chatgpt_auth() || auth.is_api_key_auth() => {
|
||
let factory = config.http_client_factory();
|
||
let policy = factory
|
||
.network_policy()
|
||
.clone()
|
||
.with_account_binding(captured_factory.network_policy())
|
||
.ok_or_else(|| {
|
||
CodexErr::InvalidRequest(
|
||
"realtime credentials and transport must share the application network policy"
|
||
.into(),
|
||
)
|
||
})?;
|
||
(Some(auth), factory.with_network_policy(policy))
|
||
}
|
||
Some((auth, _)) => (Some(auth), config.http_client_factory()),
|
||
None => (None, config.http_client_factory()),
|
||
};
|
||
let transport = params
|
||
.transport
|
||
.clone()
|
||
.unwrap_or(ConversationStartTransport::Websocket);
|
||
let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?;
|
||
let realtime_sideband_base_url = match &transport {
|
||
ConversationStartTransport::ExistingCall {
|
||
sideband_base_url, ..
|
||
} => sideband_base_url
|
||
.clone()
|
||
.or_else(|| config.experimental_realtime_ws_base_url.clone()),
|
||
ConversationStartTransport::Websocket | ConversationStartTransport::Webrtc { .. } => {
|
||
config.experimental_realtime_ws_base_url.clone()
|
||
}
|
||
};
|
||
if let Some(realtime_ws_base_url) = &realtime_sideband_base_url {
|
||
api_provider.base_url = realtime_ws_base_url.clone();
|
||
}
|
||
let realtime_call_api_provider =
|
||
if let Some(realtime_call_base_url) = &config.experimental_realtime_webrtc_call_base_url {
|
||
let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?;
|
||
api_provider.base_url = realtime_call_base_url.clone();
|
||
Some(api_provider)
|
||
} else {
|
||
None
|
||
};
|
||
let version = params.version.unwrap_or(match &transport {
|
||
ConversationStartTransport::Websocket => config.realtime.version,
|
||
ConversationStartTransport::Webrtc { .. }
|
||
| ConversationStartTransport::ExistingCall { .. } => RealtimeWsVersion::V1,
|
||
});
|
||
match &transport {
|
||
ConversationStartTransport::Webrtc { .. } => {
|
||
validate_avas_webrtc_start(version, config.realtime.session_type)?;
|
||
}
|
||
ConversationStartTransport::ExistingCall { .. } => {
|
||
if version == RealtimeWsVersion::V2 {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"AVAS realtime calls require realtime v1 or v3".to_string(),
|
||
));
|
||
}
|
||
if params.include_startup_context
|
||
|| params.prompt.is_some()
|
||
|| !params.initial_items.is_empty()
|
||
|| params.model.is_some()
|
||
|| params.voice.is_some()
|
||
|| params.delegation_ack_filler.is_some()
|
||
{
|
||
return Err(CodexErr::InvalidRequest(
|
||
"existing realtime calls do not support session configuration options"
|
||
.to_string(),
|
||
));
|
||
}
|
||
}
|
||
ConversationStartTransport::Websocket => {}
|
||
}
|
||
let configured_voice = match (&transport, params.version) {
|
||
(ConversationStartTransport::ExistingCall { .. }, _)
|
||
| (ConversationStartTransport::Webrtc { .. }, None) => ConfiguredRealtimeVoice::Ignore,
|
||
(ConversationStartTransport::Webrtc { .. } | ConversationStartTransport::Websocket, _) => {
|
||
ConfiguredRealtimeVoice::Use
|
||
}
|
||
};
|
||
let mut session_config =
|
||
build_realtime_session_config(sess, ¶ms, version, configured_voice).await?;
|
||
if matches!(&transport, ConversationStartTransport::ExistingCall { .. }) {
|
||
session_config.session_id = params.realtime_session_id.clone();
|
||
}
|
||
let requested_realtime_session_id = session_config.session_id.clone();
|
||
let event_parser = session_config.event_parser;
|
||
let originator = sess.originator().await;
|
||
let mut websocket_auth = None;
|
||
let mut extra_headers = match transport {
|
||
ConversationStartTransport::Websocket => {
|
||
let realtime_api_key = realtime_api_key(auth.as_ref(), &provider)?;
|
||
if realtime_sideband_base_url.is_none() {
|
||
api_provider = resolve_api_provider(
|
||
&provider,
|
||
Some(&CodexAuth::from_api_key(&realtime_api_key)),
|
||
)?;
|
||
}
|
||
if let Some((manager, mode)) = &auth_context.runtime_auth
|
||
&& crate::runtime_auth::is_openai_realtime_url(&api_provider.base_url)?
|
||
{
|
||
codex_model_provider::validate_runtime_environment(
|
||
manager,
|
||
Some(&api_provider.base_url),
|
||
)?;
|
||
// Saved and command-provided keys keep their existing credential lifecycle.
|
||
let runtime = if provider.env_key.is_none()
|
||
&& provider.experimental_bearer_token.is_none()
|
||
&& auth.as_ref().and_then(CodexAuth::api_key).is_some()
|
||
{
|
||
manager.bind(Arc::clone(&auth_manager))
|
||
} else {
|
||
manager.bind_api_key(&realtime_api_key, &auth_manager.runtime_config())
|
||
};
|
||
let selected = codex_model_provider::select_runtime_auth_for(
|
||
&runtime,
|
||
&CodexAuth::from_api_key(&realtime_api_key),
|
||
&api_provider.headers,
|
||
)
|
||
.await?;
|
||
codex_model_provider::validate_runtime_selectors(
|
||
&api_provider.headers,
|
||
selected.runtime().actor().info(),
|
||
)?;
|
||
http_client_factory = selected
|
||
.bind_http_client_factory(http_client_factory)
|
||
.map_err(|error| codex_api::map_api_error(error.into()))?;
|
||
websocket_auth = Some(selected.into_auth_provider(*mode));
|
||
}
|
||
realtime_request_headers(
|
||
requested_realtime_session_id.as_deref(),
|
||
params.voice_session_id.as_deref(),
|
||
Some(realtime_api_key.as_str()),
|
||
event_parser,
|
||
originator.as_str(),
|
||
)?
|
||
}
|
||
ConversationStartTransport::Webrtc { .. }
|
||
| ConversationStartTransport::ExistingCall { .. } => {
|
||
realtime_request_headers(
|
||
requested_realtime_session_id.as_deref(),
|
||
params.voice_session_id.as_deref(),
|
||
/*api_key*/ None,
|
||
event_parser,
|
||
originator.as_str(),
|
||
)?
|
||
}
|
||
}
|
||
.unwrap_or_default();
|
||
extra_headers.extend(build_session_headers(
|
||
Some(sess.session_id().to_string()),
|
||
Some(sess.thread_id().to_string()),
|
||
));
|
||
// Voice calls can span zero or many backing turns; send only the saved thread source.
|
||
if let Some(thread_source) = sess.thread_config_snapshot().await.thread_source
|
||
&& thread_source.as_str().len() <= REALTIME_THREAD_SOURCE_MAX_BYTES
|
||
&& let Ok(metadata) = to_ascii_json_string(&json!({ THREAD_SOURCE_KEY: thread_source }))
|
||
&& let Ok(metadata) = HeaderValue::from_str(&metadata)
|
||
{
|
||
extra_headers.insert(X_CODEX_TURN_METADATA_HEADER, metadata);
|
||
}
|
||
Ok(PreparedRealtimeConversationStart {
|
||
attachment_control: RealtimeAttachmentControl::default(),
|
||
sideband: None,
|
||
auth_context,
|
||
api_provider,
|
||
http_client_factory,
|
||
realtime_sideband_base_url,
|
||
extra_headers: Some(extra_headers),
|
||
websocket_auth,
|
||
client_managed_handoffs: params.client_managed_handoffs,
|
||
flush_transcript_tail_on_session_end: params.flush_transcript_tail_on_session_end,
|
||
codex_responses_as_items: params.codex_responses_as_items,
|
||
codex_response_item_prefix: params.codex_response_item_prefix,
|
||
codex_response_handoff_mode: params.codex_response_handoff_mode,
|
||
backend_reasoning_status: params.backend_reasoning_status,
|
||
codex_response_handoff_channel_prefixes: params.codex_response_handoff_channel_prefixes,
|
||
realtime_start_instructions: params.realtime_start_instructions,
|
||
realtime_end_instructions: params.realtime_end_instructions,
|
||
realtime_call_api_provider,
|
||
requested_realtime_session_id,
|
||
version,
|
||
session_config,
|
||
transport,
|
||
})
|
||
}
|
||
|
||
fn validate_avas_webrtc_start(
|
||
version: RealtimeWsVersion,
|
||
session_type: RealtimeWsMode,
|
||
) -> CodexResult<()> {
|
||
if version == RealtimeWsVersion::V2 {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"AVAS realtime calls require realtime v1 or v3".to_string(),
|
||
));
|
||
}
|
||
if session_type != RealtimeWsMode::Conversational {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"AVAS realtime calls require conversational realtime".to_string(),
|
||
));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn build_realtime_session_config(
|
||
sess: &Arc<Session>,
|
||
params: &ConversationStartParams,
|
||
version: RealtimeWsVersion,
|
||
configured_voice: ConfiguredRealtimeVoice,
|
||
) -> CodexResult<RealtimeSessionConfig> {
|
||
for (name, instructions) in [
|
||
(
|
||
"realtime start instructions",
|
||
params.realtime_start_instructions.as_deref(),
|
||
),
|
||
(
|
||
"realtime end instructions",
|
||
params.realtime_end_instructions.as_deref(),
|
||
),
|
||
] {
|
||
if instructions.is_some_and(|instructions| {
|
||
approx_token_count(instructions) > REALTIME_MODE_INSTRUCTIONS_MAX_TOKENS
|
||
}) {
|
||
return Err(CodexErr::InvalidRequest(format!(
|
||
"{name} must not exceed {REALTIME_MODE_INSTRUCTIONS_MAX_TOKENS} estimated tokens"
|
||
)));
|
||
}
|
||
}
|
||
|
||
let config = sess.get_config().await;
|
||
let prompt = prepare_realtime_backend_prompt(
|
||
params.prompt.clone(),
|
||
config.experimental_realtime_ws_backend_prompt.clone(),
|
||
);
|
||
let startup_context = if params.include_startup_context {
|
||
match config.experimental_realtime_ws_startup_context.clone() {
|
||
Some(startup_context) => startup_context,
|
||
None => {
|
||
build_realtime_startup_context(sess.as_ref(), REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET)
|
||
.await
|
||
.unwrap_or_default()
|
||
}
|
||
}
|
||
} else {
|
||
String::new()
|
||
};
|
||
let prompt = match (prompt.is_empty(), startup_context.is_empty()) {
|
||
(true, true) => String::new(),
|
||
(true, false) => startup_context,
|
||
(false, true) => prompt,
|
||
(false, false) => format!("{prompt}\n\n{startup_context}"),
|
||
};
|
||
if version != RealtimeWsVersion::V3 && !params.initial_items.is_empty() {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"initial realtime items require realtime v3".to_string(),
|
||
));
|
||
}
|
||
if params.initial_items.len() > REALTIME_INITIAL_ITEMS_MAX_COUNT {
|
||
return Err(CodexErr::InvalidRequest(format!(
|
||
"initial realtime items must contain no more than {REALTIME_INITIAL_ITEMS_MAX_COUNT} items"
|
||
)));
|
||
}
|
||
let mut total_initial_item_tokens: usize = 0;
|
||
for item in ¶ms.initial_items {
|
||
let item_tokens = approx_token_count(&item.text);
|
||
if item_tokens > REALTIME_INITIAL_ITEMS_MAX_TOKENS {
|
||
return Err(CodexErr::InvalidRequest(format!(
|
||
"each initial realtime item must not exceed {REALTIME_INITIAL_ITEMS_MAX_TOKENS} estimated tokens"
|
||
)));
|
||
}
|
||
total_initial_item_tokens = total_initial_item_tokens.saturating_add(item_tokens);
|
||
}
|
||
if total_initial_item_tokens > REALTIME_INITIAL_ITEMS_MAX_TOKENS {
|
||
return Err(CodexErr::InvalidRequest(format!(
|
||
"initial realtime items must not exceed {REALTIME_INITIAL_ITEMS_MAX_TOKENS} estimated tokens in total"
|
||
)));
|
||
}
|
||
let model = Some(
|
||
params
|
||
.model
|
||
.clone()
|
||
.or_else(|| config.experimental_realtime_ws_model.clone())
|
||
.unwrap_or_else(|| match version {
|
||
RealtimeWsVersion::V1 | RealtimeWsVersion::V2 => DEFAULT_REALTIME_MODEL.to_string(),
|
||
RealtimeWsVersion::V3 => DEFAULT_FRAMELESS_REALTIME_MODEL.to_string(),
|
||
}),
|
||
);
|
||
let event_parser = match version {
|
||
RealtimeWsVersion::V1 => RealtimeEventParser::V1,
|
||
RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2,
|
||
RealtimeWsVersion::V3 => RealtimeEventParser::FramelessBidi,
|
||
};
|
||
if version != RealtimeWsVersion::V2
|
||
&& matches!(params.output_modality, RealtimeOutputModality::Text)
|
||
{
|
||
return Err(CodexErr::InvalidRequest(
|
||
"text realtime output modality requires realtime v2".to_string(),
|
||
));
|
||
}
|
||
let session_mode = match config.realtime.session_type {
|
||
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
|
||
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
|
||
};
|
||
let config_voice = match configured_voice {
|
||
ConfiguredRealtimeVoice::Use => config.realtime.voice,
|
||
ConfiguredRealtimeVoice::Ignore => None,
|
||
};
|
||
let voice = params
|
||
.voice
|
||
.or(config_voice)
|
||
.unwrap_or_else(|| default_realtime_voice(version));
|
||
validate_realtime_voice(version, voice)?;
|
||
Ok(RealtimeSessionConfig {
|
||
instructions: prompt,
|
||
initial_items: params.initial_items.clone(),
|
||
delegation_ack_filler: params.delegation_ack_filler,
|
||
model,
|
||
session_id: Some(
|
||
params
|
||
.realtime_session_id
|
||
.clone()
|
||
.unwrap_or_else(|| sess.thread_id.to_string()),
|
||
),
|
||
event_parser,
|
||
session_mode,
|
||
output_modality: params.output_modality,
|
||
voice,
|
||
})
|
||
}
|
||
|
||
fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice {
|
||
let voices = RealtimeVoicesList::builtin();
|
||
match version {
|
||
RealtimeWsVersion::V1 | RealtimeWsVersion::V3 => voices.default_v1,
|
||
RealtimeWsVersion::V2 => voices.default_v2,
|
||
}
|
||
}
|
||
|
||
fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessionKind) -> String {
|
||
if session_kind != RealtimeSessionKind::V2 || text.is_empty() || text.starts_with(prefix) {
|
||
return text;
|
||
}
|
||
format!("{prefix}{text}")
|
||
}
|
||
|
||
fn realtime_backend_output(output_text: String, session_kind: RealtimeSessionKind) -> String {
|
||
let output_text = prefix_realtime_text(output_text, REALTIME_BACKEND_TEXT_PREFIX, session_kind);
|
||
truncate_realtime_text_to_token_budget(&output_text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET)
|
||
}
|
||
|
||
fn realtime_backend_item(text: String, prefix: Option<&str>) -> String {
|
||
let text = match prefix.filter(|prefix| !prefix.is_empty()) {
|
||
Some(prefix) => format!("{prefix}\n\n{text}"),
|
||
None => text,
|
||
};
|
||
truncate_realtime_text_to_token_budget(&text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET)
|
||
}
|
||
|
||
fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> {
|
||
let voices = RealtimeVoicesList::builtin();
|
||
let allowed = match version {
|
||
RealtimeWsVersion::V1 => &voices.v1,
|
||
RealtimeWsVersion::V2 => &voices.v2,
|
||
RealtimeWsVersion::V3 => &voices.v3,
|
||
};
|
||
if allowed.contains(&voice) {
|
||
return Ok(());
|
||
}
|
||
|
||
let version = match version {
|
||
RealtimeWsVersion::V1 => "v1",
|
||
RealtimeWsVersion::V2 => "v2",
|
||
RealtimeWsVersion::V3 => "v3",
|
||
};
|
||
let allowed = allowed
|
||
.iter()
|
||
.map(|voice| voice.wire_name())
|
||
.collect::<Vec<_>>()
|
||
.join(", ");
|
||
Err(CodexErr::InvalidRequest(format!(
|
||
"realtime voice `{}` is not supported for {version}; supported voices: {allowed}",
|
||
voice.wire_name()
|
||
)))
|
||
}
|
||
|
||
async fn handle_start_inner(
|
||
sess: &Arc<Session>,
|
||
sub_id: &str,
|
||
prepared_start: PreparedRealtimeConversationStart,
|
||
) -> CodexResult<()> {
|
||
let PreparedRealtimeConversationStart {
|
||
attachment_control,
|
||
sideband,
|
||
auth_context,
|
||
api_provider,
|
||
http_client_factory,
|
||
realtime_sideband_base_url,
|
||
extra_headers,
|
||
websocket_auth,
|
||
client_managed_handoffs,
|
||
flush_transcript_tail_on_session_end,
|
||
codex_responses_as_items,
|
||
codex_response_item_prefix,
|
||
codex_response_handoff_mode,
|
||
backend_reasoning_status,
|
||
codex_response_handoff_channel_prefixes,
|
||
realtime_start_instructions,
|
||
realtime_end_instructions,
|
||
realtime_call_api_provider,
|
||
requested_realtime_session_id,
|
||
version,
|
||
session_config,
|
||
transport,
|
||
} = prepared_start;
|
||
info!("starting realtime conversation");
|
||
let (sdp, existing_call_id) = match transport {
|
||
ConversationStartTransport::Websocket => (None, None),
|
||
ConversationStartTransport::Webrtc { sdp } => (Some(sdp), None),
|
||
ConversationStartTransport::ExistingCall { call_id, .. } => (None, Some(call_id)),
|
||
};
|
||
let mode_instructions = RealtimeModeInstructions {
|
||
start: realtime_start_instructions,
|
||
end: realtime_end_instructions,
|
||
};
|
||
let event_parser = session_config.event_parser;
|
||
let start = RealtimeStart {
|
||
attachment_control,
|
||
sideband,
|
||
auth_context,
|
||
api_provider,
|
||
http_client_factory,
|
||
realtime_sideband_base_url,
|
||
extra_headers,
|
||
websocket_auth,
|
||
client_managed_handoffs,
|
||
flush_transcript_tail_on_session_end,
|
||
codex_responses_as_items,
|
||
codex_response_item_prefix,
|
||
codex_response_handoff_mode,
|
||
backend_reasoning_status,
|
||
codex_response_handoff_channel_prefixes,
|
||
realtime_call_api_provider,
|
||
session_config,
|
||
model_client: sess.services.model_client.clone(),
|
||
sdp,
|
||
existing_call_id,
|
||
};
|
||
let start_output = sess
|
||
.conversation
|
||
.start(start, mode_instructions, sess, sub_id)
|
||
.await?;
|
||
|
||
info!("realtime conversation started");
|
||
|
||
sess.send_event_raw(Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent {
|
||
realtime_session_id: requested_realtime_session_id,
|
||
version,
|
||
}),
|
||
})
|
||
.await;
|
||
|
||
let RealtimeStartOutput {
|
||
realtime_active,
|
||
route_handoffs,
|
||
events_rx,
|
||
transcript_tail_rx,
|
||
sdp,
|
||
} = start_output;
|
||
if let Some(sdp) = sdp {
|
||
sess.send_event_raw(Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::RealtimeConversationSdp(RealtimeConversationSdpEvent { sdp }),
|
||
})
|
||
.await;
|
||
}
|
||
|
||
let sess_clone = Arc::clone(sess);
|
||
let sub_id = sub_id.to_string();
|
||
let fanout_realtime_active = Arc::clone(&realtime_active);
|
||
let fanout_task = tokio::spawn(async move {
|
||
let ev = |msg| Event {
|
||
id: sub_id.clone(),
|
||
msg,
|
||
};
|
||
let mut end = RealtimeConversationEnd::TransportClosed;
|
||
let mut handoff_error = None;
|
||
// Drain already-parsed events so a queued handoff is routed before the final tail.
|
||
while let Ok((event, completed)) = events_rx.recv().await {
|
||
if let Some(delta) = completed {
|
||
let text = wrap_realtime_delegation_input(
|
||
"",
|
||
Some(&delta),
|
||
RealtimeDelegationSource::Handoff,
|
||
/*preserve_transcript*/ true,
|
||
);
|
||
if let Err(err) = route_handoffs.record_transcript(&sess_clone, text).await {
|
||
warn!("failed to save realtime transcript: {err}");
|
||
handoff_error = Some("failed to save the realtime transcript");
|
||
break;
|
||
}
|
||
}
|
||
match &event {
|
||
RealtimeEvent::AudioOut(_) => {}
|
||
_ => {
|
||
// Transcripts, handoffs, and conversation items may contain
|
||
// credentials. Keep this receipt free of event payloads.
|
||
info!("received realtime conversation event");
|
||
}
|
||
}
|
||
if let RealtimeEvent::Error(_) = &event {
|
||
end = RealtimeConversationEnd::Error;
|
||
}
|
||
let maybe_routed_text = match &event {
|
||
RealtimeEvent::HandoffRequested(handoff) => realtime_delegation_from_handoff(
|
||
handoff,
|
||
event_parser == RealtimeEventParser::FramelessBidi,
|
||
),
|
||
_ => None,
|
||
};
|
||
if let Some(text) = maybe_routed_text {
|
||
// The routed text can contain spoken prompts or workspace secrets.
|
||
debug!("[realtime-text] realtime conversation text output");
|
||
handoff_error = route_handoffs.route(&sess_clone, text).await.err();
|
||
}
|
||
sess_clone
|
||
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
|
||
RealtimeConversationRealtimeEvent {
|
||
payload: event.clone(),
|
||
},
|
||
)))
|
||
.await;
|
||
if handoff_error.is_some() {
|
||
break;
|
||
}
|
||
}
|
||
drop(events_rx);
|
||
if handoff_error.is_none()
|
||
&& let Ok(text) = transcript_tail_rx.recv().await
|
||
&& let Err(err) = route_handoffs.record_transcript(&sess_clone, text).await
|
||
{
|
||
warn!("failed to flush realtime transcript before closure: {err}");
|
||
handoff_error = Some("failed to save the realtime transcript before closure");
|
||
}
|
||
// Make realtime_end eligible only after the tail has reached history.
|
||
{
|
||
let mut state = sess_clone.conversation.state.lock().await;
|
||
if state.conversation.as_ref().is_none_or(|current| {
|
||
Arc::ptr_eq(¤t.realtime_active, &fanout_realtime_active)
|
||
}) {
|
||
state.context_active = false;
|
||
}
|
||
}
|
||
if let Some(error) = handoff_error {
|
||
end = RealtimeConversationEnd::Error;
|
||
sess_clone
|
||
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
|
||
RealtimeConversationRealtimeEvent {
|
||
payload: RealtimeEvent::Error(error.to_string()),
|
||
},
|
||
)))
|
||
.await;
|
||
}
|
||
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
|
||
match end {
|
||
RealtimeConversationEnd::TransportClosed => {
|
||
info!("realtime conversation transport closed");
|
||
}
|
||
RealtimeConversationEnd::Requested | RealtimeConversationEnd::Error => {}
|
||
}
|
||
sess_clone
|
||
.conversation
|
||
.finish_if_active(&sess_clone, &fanout_realtime_active, sub_id, end)
|
||
.await;
|
||
}
|
||
});
|
||
sess.conversation
|
||
.register_fanout_task(&realtime_active, fanout_task)
|
||
.await;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
pub(crate) async fn handle_audio(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationAudioParams,
|
||
) {
|
||
if let Err(err) = sess.conversation.audio_in(params.frame).await {
|
||
error!("failed to append realtime audio: {err}");
|
||
if sess.conversation.running_state().await.is_some() {
|
||
warn!("realtime audio input failed while the session was already ending");
|
||
} else {
|
||
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest)
|
||
.await;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn realtime_transcript_delta_from_handoff(handoff: &RealtimeHandoffRequested) -> Option<String> {
|
||
realtime_transcript_delta(&handoff.active_transcript)
|
||
}
|
||
|
||
fn realtime_transcript_delta(active_transcript: &[RealtimeTranscriptEntry]) -> Option<String> {
|
||
let active_transcript = active_transcript
|
||
.iter()
|
||
.map(|entry| format!("{role}: {text}", role = entry.role, text = entry.text))
|
||
.collect::<Vec<_>>()
|
||
.join("\n");
|
||
(!active_transcript.is_empty()).then_some(active_transcript)
|
||
}
|
||
|
||
fn realtime_text_from_handoff_request(handoff: &RealtimeHandoffRequested) -> Option<String> {
|
||
(!handoff.input_transcript.is_empty())
|
||
.then_some(handoff.input_transcript.clone())
|
||
.or_else(|| realtime_transcript_delta_from_handoff(handoff))
|
||
}
|
||
|
||
fn realtime_delegation_from_handoff(
|
||
handoff: &RealtimeHandoffRequested,
|
||
preserve_transcript: bool,
|
||
) -> Option<String> {
|
||
let input = realtime_text_from_handoff_request(handoff)?;
|
||
Some(wrap_realtime_delegation_input(
|
||
&input,
|
||
realtime_transcript_delta_from_handoff(handoff).as_deref(),
|
||
RealtimeDelegationSource::Handoff,
|
||
preserve_transcript,
|
||
))
|
||
}
|
||
|
||
fn wrap_realtime_delegation_input(
|
||
input: &str,
|
||
transcript_delta: Option<&str>,
|
||
source: RealtimeDelegationSource,
|
||
preserve_transcript: bool,
|
||
) -> String {
|
||
RealtimeDelegation::new(input, transcript_delta, source, preserve_transcript)
|
||
.render()
|
||
.into_text()
|
||
.unwrap_or_default()
|
||
}
|
||
|
||
fn realtime_api_key(auth: Option<&CodexAuth>, provider: &ModelProviderInfo) -> CodexResult<String> {
|
||
if let Some(api_key) = provider.api_key()? {
|
||
return Ok(api_key);
|
||
}
|
||
|
||
if let Some(token) = provider.experimental_bearer_token.clone() {
|
||
return Ok(token.into_inner());
|
||
}
|
||
|
||
if let Some(api_key) = auth.and_then(CodexAuth::api_key) {
|
||
return Ok(api_key.to_string());
|
||
}
|
||
|
||
// TODO(aibrahim): Remove this temporary fallback once realtime auth no longer
|
||
// requires API key auth for ChatGPT/SIWC sessions.
|
||
if provider.is_openai()
|
||
&& let Some(api_key) = read_openai_api_key_from_env()
|
||
{
|
||
return Ok(api_key);
|
||
}
|
||
|
||
Err(CodexErr::InvalidRequest(
|
||
"realtime conversation requires API key auth".to_string(),
|
||
))
|
||
}
|
||
|
||
fn realtime_request_headers(
|
||
realtime_session_id: Option<&str>,
|
||
voice_session_id: Option<&str>,
|
||
api_key: Option<&str>,
|
||
event_parser: RealtimeEventParser,
|
||
originator: &str,
|
||
) -> CodexResult<Option<HeaderMap>> {
|
||
let mut headers = HeaderMap::new();
|
||
|
||
match event_parser {
|
||
RealtimeEventParser::V1 => {
|
||
headers.insert("openai-alpha", HeaderValue::from_static("quicksilver=v1"));
|
||
}
|
||
RealtimeEventParser::FramelessBidi => {
|
||
headers.insert("openai-alpha", HeaderValue::from_static("quicksilver=v2"));
|
||
}
|
||
RealtimeEventParser::RealtimeV2 => {}
|
||
}
|
||
|
||
if let Some(realtime_session_id) = realtime_session_id
|
||
&& let Ok(realtime_session_id) = HeaderValue::from_str(realtime_session_id)
|
||
{
|
||
headers.insert("x-session-id", realtime_session_id);
|
||
}
|
||
|
||
if let Some(voice_session_id) = voice_session_id
|
||
&& let Ok(voice_session_id) = HeaderValue::from_str(voice_session_id)
|
||
{
|
||
headers.insert("voice-session-id", voice_session_id);
|
||
}
|
||
|
||
if let Some(api_key) = api_key {
|
||
let auth_value = HeaderValue::from_str(&format!("Bearer {api_key}")).map_err(|err| {
|
||
CodexErr::InvalidRequest(format!("invalid realtime api key header: {err}"))
|
||
})?;
|
||
headers.insert(AUTHORIZATION, auth_value);
|
||
if is_fedramp_api_key(api_key) {
|
||
headers.insert("X-OpenAI-Fedramp", HeaderValue::from_static("true"));
|
||
}
|
||
}
|
||
|
||
add_originator_header(&mut headers, originator);
|
||
|
||
Ok(Some(headers))
|
||
}
|
||
|
||
pub(crate) async fn handle_text(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationTextParams,
|
||
) {
|
||
debug!(text = %params.text, "[realtime-text] appending realtime conversation text input");
|
||
if let Err(err) = sess.conversation.text_in(params).await {
|
||
error!("failed to append realtime text: {err}");
|
||
if sess.conversation.running_state().await.is_some() {
|
||
warn!("realtime text input failed while the session was already ending");
|
||
} else {
|
||
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest)
|
||
.await;
|
||
}
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn handle_speech(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationSpeechParams,
|
||
) {
|
||
debug!("[realtime-text] appending realtime speech");
|
||
if let Err(err) = sess.conversation.append_speech(params.text).await {
|
||
error!("failed to append realtime speech: {err}");
|
||
if sess.conversation.running_state().await.is_some() {
|
||
warn!("realtime speech append failed while the session was already ending");
|
||
} else {
|
||
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest)
|
||
.await;
|
||
}
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn handle_close(sess: &Arc<Session>, sub_id: String) {
|
||
handle_detach(sess, /*expected*/ None, &sub_id).await;
|
||
}
|
||
|
||
/// Completes after the connection is published or the cancelled attempt is cleaned up.
|
||
pub(crate) async fn handle_attach(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationStartParams,
|
||
options: RealtimeAttachmentOptions,
|
||
) -> CodexResult<()> {
|
||
let attachment_control = RealtimeAttachmentControl {
|
||
token: CancellationToken::new(),
|
||
options,
|
||
};
|
||
if attachment_control.is_cancelled() {
|
||
return Err(CodexErr::TurnAborted);
|
||
}
|
||
let remaining = attachment_control
|
||
.options
|
||
.start_before
|
||
.map(|deadline| deadline.duration_since(SystemTime::now()))
|
||
.transpose()
|
||
.map_err(|_| CodexErr::TurnAborted)?;
|
||
if remaining == Some(Duration::ZERO) {
|
||
return Err(CodexErr::TurnAborted);
|
||
}
|
||
let attach = async {
|
||
let mut prepared = prepare_realtime_start(sess, params)
|
||
.or_cancel(&attachment_control.token)
|
||
.await??;
|
||
prepared.attachment_control = attachment_control.clone();
|
||
handle_start_inner(sess, &sub_id, prepared).await
|
||
};
|
||
tokio::pin!(attach);
|
||
let cancelled = async {
|
||
let Some(mut cancellation) = attachment_control.options.cancel.clone() else {
|
||
return false;
|
||
};
|
||
cancellation.wait_for(|cancelled| *cancelled).await.is_ok()
|
||
};
|
||
let expired = async {
|
||
match remaining {
|
||
Some(remaining) => tokio::time::sleep(remaining).await,
|
||
None => std::future::pending::<()>().await,
|
||
}
|
||
};
|
||
tokio::select! {
|
||
biased;
|
||
true = cancelled => {}
|
||
_ = expired => {}
|
||
result = &mut attach => return result,
|
||
}
|
||
attachment_control.token.cancel();
|
||
attach.await
|
||
}
|
||
|
||
/// Attaches Core to an existing call using the caller’s channels to receive
|
||
/// events and send commands.
|
||
pub(crate) async fn handle_attach_stream(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
params: ConversationStartParams,
|
||
transport: codex_protocol::protocol::RealtimeSidebandTransport,
|
||
) -> CodexResult<()> {
|
||
if !matches!(
|
||
¶ms.transport,
|
||
Some(ConversationStartTransport::ExistingCall { .. })
|
||
) {
|
||
return Err(CodexErr::InvalidRequest(
|
||
"sideband stream requires an existing call".into(),
|
||
));
|
||
}
|
||
|
||
let mut prepared = prepare_realtime_start(sess, params).await?;
|
||
prepared.sideband = Some(transport);
|
||
handle_start_inner(sess, &sub_id, prepared).await
|
||
}
|
||
|
||
pub(crate) async fn handle_detach(sess: &Arc<Session>, expected: Option<&str>, stop_sub_id: &str) {
|
||
let current = {
|
||
let mut state = sess.conversation.state.lock().await;
|
||
if expected.is_some_and(|expected| {
|
||
state
|
||
.conversation
|
||
.as_ref()
|
||
.and_then(|current| current.session_id.as_deref())
|
||
!= Some(expected)
|
||
}) {
|
||
return;
|
||
}
|
||
state.conversation.take()
|
||
};
|
||
let sub_id = if let Some(current) = current {
|
||
let sub_id = current.sub_id.clone();
|
||
stop_conversation_state(current, RealtimeFanoutTaskStop::Await).await;
|
||
sub_id
|
||
} else {
|
||
stop_sub_id.to_owned()
|
||
};
|
||
send_realtime_conversation_closed(sess, sub_id, RealtimeConversationEnd::Requested).await;
|
||
}
|
||
|
||
fn spawn_realtime_input_task(
|
||
input: RealtimeInputTask,
|
||
transcript_tail_flush: RealtimeTranscriptTailFlush,
|
||
) -> JoinHandle<()> {
|
||
tokio::spawn(async move {
|
||
let events_tx = input.events_tx.clone();
|
||
let transcript_state = input.events.transcript_state();
|
||
let exit = run_realtime_input_task(input, /*pending_outbound*/ None).await;
|
||
if let RealtimeInputTaskExit::TransportLost { err, .. } = exit {
|
||
report_realtime_transport_loss(&events_tx, err).await;
|
||
}
|
||
let transcript_tail = transcript_state.take_transcript_tail().await;
|
||
flush_realtime_transcript_tail(&transcript_tail_flush, &transcript_tail).await;
|
||
})
|
||
}
|
||
|
||
enum RealtimeInputTaskExit {
|
||
Terminal,
|
||
TransportLost {
|
||
err: ApiError,
|
||
pending_outbound: Option<Box<RealtimePendingOutbound>>,
|
||
},
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
enum RealtimePendingOutbound {
|
||
Text(ConversationTextParams),
|
||
Handoff(RealtimeOutbound),
|
||
}
|
||
|
||
struct RealtimeWebrtcSidebandInputTask {
|
||
client: RealtimeWebsocketClient,
|
||
session_config: RealtimeSessionConfig,
|
||
call_id: String,
|
||
sideband_headers: HeaderMap,
|
||
session_initialization: RealtimeSidebandSessionInitialization,
|
||
input_channels: RealtimeInputChannels,
|
||
events_tx: Sender<RealtimeOutput>,
|
||
handoff_state: RealtimeHandoffState,
|
||
session_kind: RealtimeSessionKind,
|
||
event_parser: RealtimeEventParser,
|
||
realtime_active: Arc<AtomicBool>,
|
||
transcript_tail_flush: RealtimeTranscriptTailFlush,
|
||
stop_token: CancellationToken,
|
||
}
|
||
|
||
enum RealtimeSidebandSessionInitialization {
|
||
CoreCreated,
|
||
ExistingCall {
|
||
initial_connection: Option<RealtimeWebsocketConnection>,
|
||
},
|
||
}
|
||
|
||
// The main loop for realtime inputs, including user audio/text and model commentary/results.
|
||
async fn run_realtime_input_task(
|
||
input: RealtimeInputTask,
|
||
pending_outbound: Option<Box<RealtimePendingOutbound>>,
|
||
) -> RealtimeInputTaskExit {
|
||
let RealtimeInputTask {
|
||
writer,
|
||
events,
|
||
text_rx,
|
||
handoff_output_rx,
|
||
audio_rx,
|
||
events_tx,
|
||
handoff_state,
|
||
session_kind,
|
||
event_parser,
|
||
stop_token,
|
||
} = input;
|
||
|
||
let mut output_audio_state: Option<OutputAudioState> = None;
|
||
let mut response_create_queue = RealtimeResponseCreateQueue::default();
|
||
|
||
// Send any previously unsent messages before entering the main event loop.
|
||
if let Some(pending_outbound) = pending_outbound
|
||
&& let Err(err) = send_realtime_pending_outbound(
|
||
&pending_outbound,
|
||
&writer,
|
||
&handoff_state,
|
||
event_parser,
|
||
&mut response_create_queue,
|
||
)
|
||
.await
|
||
{
|
||
return classify_realtime_input_error_with_pending(err, Some(pending_outbound));
|
||
}
|
||
|
||
loop {
|
||
let result = tokio::select! {
|
||
_ = stop_token.cancelled() => Err(RealtimeInputTaskExit::Terminal),
|
||
realtime_event = events.next_event() => {
|
||
match realtime_event {
|
||
Ok(Some(event)) => {
|
||
let completed = events.take_completed_transcript(&event, RealtimeDelegation::FIELD_BYTES).await;
|
||
handle_realtime_server_event(
|
||
(event, realtime_transcript_delta(&completed)),
|
||
&writer,
|
||
&events_tx,
|
||
&handoff_state,
|
||
session_kind,
|
||
&mut output_audio_state,
|
||
&mut response_create_queue,
|
||
)
|
||
.await
|
||
.map_err(classify_realtime_input_error)
|
||
}
|
||
Ok(None) => Err(RealtimeInputTaskExit::Terminal),
|
||
Err(err) => Err(RealtimeInputTaskExit::TransportLost {
|
||
err,
|
||
pending_outbound: None,
|
||
}),
|
||
}
|
||
}
|
||
// Text input that should be sent into realtime.
|
||
text = text_rx.recv() => {
|
||
let pending_outbound = text
|
||
.as_ref()
|
||
.ok()
|
||
.cloned()
|
||
.map(RealtimePendingOutbound::Text)
|
||
.map(Box::new);
|
||
handle_text_input(
|
||
text,
|
||
&writer,
|
||
)
|
||
.await
|
||
.map_err(|err| {
|
||
classify_realtime_input_error_with_pending(err, pending_outbound)
|
||
})
|
||
}
|
||
// Background agent progress or final output that should be sent back to realtime.
|
||
background_agent_output = handoff_output_rx.recv() => {
|
||
let pending_outbound = background_agent_output
|
||
.as_ref()
|
||
.ok()
|
||
.cloned()
|
||
.map(RealtimePendingOutbound::Handoff)
|
||
.map(Box::new);
|
||
handle_handoff_output(
|
||
background_agent_output,
|
||
&writer,
|
||
&handoff_state,
|
||
event_parser,
|
||
&mut response_create_queue,
|
||
)
|
||
.await
|
||
.map_err(|err| {
|
||
classify_realtime_input_error_with_pending(err, pending_outbound)
|
||
})
|
||
}
|
||
// Audio frames captured from the user microphone.
|
||
user_audio_frame = audio_rx.recv() => {
|
||
handle_user_audio_input(user_audio_frame, &writer)
|
||
.await
|
||
.map_err(classify_realtime_input_error)
|
||
}
|
||
};
|
||
if let Err(exit) = result {
|
||
break exit;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn classify_realtime_input_error(err: anyhow::Error) -> RealtimeInputTaskExit {
|
||
classify_realtime_input_error_with_pending(err, /*pending_outbound*/ None)
|
||
}
|
||
|
||
fn classify_realtime_input_error_with_pending(
|
||
err: anyhow::Error,
|
||
pending_outbound: Option<Box<RealtimePendingOutbound>>,
|
||
) -> RealtimeInputTaskExit {
|
||
match err.downcast::<ApiError>() {
|
||
Ok(err) => RealtimeInputTaskExit::TransportLost {
|
||
err,
|
||
pending_outbound,
|
||
},
|
||
Err(err) => {
|
||
warn!("realtime input task stopped: {err}");
|
||
RealtimeInputTaskExit::Terminal
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn send_realtime_pending_outbound(
|
||
pending_outbound: &RealtimePendingOutbound,
|
||
writer: &RealtimeWebsocketWriter,
|
||
handoff_state: &RealtimeHandoffState,
|
||
event_parser: RealtimeEventParser,
|
||
response_create_queue: &mut RealtimeResponseCreateQueue,
|
||
) -> anyhow::Result<()> {
|
||
match pending_outbound {
|
||
RealtimePendingOutbound::Text(params) => writer
|
||
.send_conversation_item_create(params.text.clone(), params.role)
|
||
.await
|
||
.map_err(anyhow::Error::from),
|
||
RealtimePendingOutbound::Handoff(output) => {
|
||
handle_handoff_output(
|
||
Ok(output.clone()),
|
||
writer,
|
||
handoff_state,
|
||
event_parser,
|
||
response_create_queue,
|
||
)
|
||
.await
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn report_realtime_transport_loss(events_tx: &Sender<RealtimeOutput>, err: ApiError) {
|
||
let mapped_error = map_api_error(err);
|
||
error!("realtime stream closed: {mapped_error}");
|
||
let _ = events_tx
|
||
.send((RealtimeEvent::Error(mapped_error.to_string()), None))
|
||
.await;
|
||
}
|
||
|
||
async fn flush_realtime_transcript_tail(
|
||
transcript_tail_flush: &RealtimeTranscriptTailFlush,
|
||
transcript_tail: &[RealtimeTranscriptEntry],
|
||
) {
|
||
if transcript_tail_flush.enabled
|
||
&& let Some(transcript_delta) = realtime_transcript_delta(transcript_tail)
|
||
{
|
||
let _ = transcript_tail_flush
|
||
.tx
|
||
.send(wrap_realtime_delegation_input(
|
||
REALTIME_SESSION_ENDED_HANDOFF_INSTRUCTION,
|
||
Some(&transcript_delta),
|
||
RealtimeDelegationSource::TranscriptTailFlush,
|
||
transcript_tail_flush.event_parser == RealtimeEventParser::FramelessBidi,
|
||
))
|
||
.await;
|
||
}
|
||
}
|
||
|
||
async fn handle_text_input(
|
||
params: Result<ConversationTextParams, RecvError>,
|
||
writer: &RealtimeWebsocketWriter,
|
||
) -> anyhow::Result<()> {
|
||
let params = params.context("text input channel closed")?;
|
||
writer
|
||
.send_conversation_item_create(params.text, params.role)
|
||
.await
|
||
.map_err(anyhow::Error::from)?;
|
||
Ok(())
|
||
}
|
||
|
||
async fn flush_streamed_handoff_item(handoff: &RealtimeHandoffState, item_id: &str) {
|
||
let (handoff_id, text, phase) = {
|
||
let mut stream = handoff.stream.lock().await;
|
||
let Some(streamed_item) = stream.items.get_mut(item_id) else {
|
||
return;
|
||
};
|
||
streamed_item.flush_scheduled = false;
|
||
let Some(text) = streamed_item.drain_stream_chunk() else {
|
||
return;
|
||
};
|
||
streamed_item.last_flush_at = Instant::now();
|
||
(
|
||
streamed_item.handoff_id.clone(),
|
||
text,
|
||
streamed_item.phase.clone(),
|
||
)
|
||
};
|
||
let _ = handoff
|
||
.output_tx
|
||
.send(RealtimeOutbound::HandoffAppend {
|
||
handoff_id,
|
||
text,
|
||
phase,
|
||
})
|
||
.await;
|
||
}
|
||
|
||
fn schedule_streamed_handoff_flush(
|
||
handoff: &RealtimeHandoffState,
|
||
item_id: String,
|
||
flush_delay: Duration,
|
||
) {
|
||
let handoff = handoff.clone();
|
||
let _flush_task = tokio::spawn(async move {
|
||
tokio::time::sleep(flush_delay).await;
|
||
flush_streamed_handoff_item(&handoff, &item_id).await;
|
||
});
|
||
}
|
||
|
||
fn v3_output_writer(
|
||
writer: &RealtimeWebsocketWriter,
|
||
phase: Option<&MessagePhase>,
|
||
handoff_mode: CodexResponseHandoffMode,
|
||
) -> RealtimeWebsocketWriter {
|
||
let channel = match handoff_mode {
|
||
CodexResponseHandoffMode::Thinking => None,
|
||
CodexResponseHandoffMode::Commentary => Some(RealtimeContextAppendChannel::Commentary),
|
||
CodexResponseHandoffMode::BemTags => match phase {
|
||
Some(MessagePhase::PartialAnswer | MessagePhase::FinalAnswer) => {
|
||
Some(RealtimeContextAppendChannel::Speakable)
|
||
}
|
||
Some(MessagePhase::Commentary) => Some(RealtimeContextAppendChannel::Commentary),
|
||
None => Some(RealtimeContextAppendChannel::Speakable),
|
||
},
|
||
};
|
||
match channel {
|
||
Some(channel) => writer.clone().with_context_append_channel(channel),
|
||
None => writer.clone(),
|
||
}
|
||
}
|
||
|
||
async fn handle_handoff_output(
|
||
handoff_output: Result<RealtimeOutbound, RecvError>,
|
||
writer: &RealtimeWebsocketWriter,
|
||
handoff_state: &RealtimeHandoffState,
|
||
event_parser: RealtimeEventParser,
|
||
response_create_queue: &mut RealtimeResponseCreateQueue,
|
||
) -> anyhow::Result<()> {
|
||
let handoff_output = handoff_output.context("handoff output channel closed")?;
|
||
let result = match event_parser {
|
||
RealtimeEventParser::V1 => match handoff_output {
|
||
RealtimeOutbound::QuietReasoningStatus { .. } => Ok(()),
|
||
RealtimeOutbound::StandaloneHandoff { text, phase: _ } => {
|
||
writer
|
||
.send_standalone_handoff(STANDALONE_HANDOFF_ID.to_string(), text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::StandaloneSpeech { text } => {
|
||
writer
|
||
.send_standalone_handoff(STANDALONE_HANDOFF_ID.to_string(), text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffUpdate {
|
||
handoff_id,
|
||
text,
|
||
phase: _,
|
||
}
|
||
| RealtimeOutbound::CompletedHandoff {
|
||
handoff_id,
|
||
text,
|
||
phase: _,
|
||
} => {
|
||
writer
|
||
.send_conversation_function_call_output(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffAppend {
|
||
handoff_id,
|
||
text,
|
||
phase: _,
|
||
} => {
|
||
writer
|
||
.send_conversation_handoff_append(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::ConversationItem { text, phase: _ } => {
|
||
writer
|
||
.send_conversation_item_create(text, ConversationTextRole::Developer)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffCompleteAck { .. } => Ok(()),
|
||
},
|
||
RealtimeEventParser::FramelessBidi => match handoff_output {
|
||
RealtimeOutbound::QuietReasoningStatus { handoff_id, text } => {
|
||
if handoff_state.stream.lock().await.active_handoff.as_deref() != Some(&handoff_id)
|
||
{
|
||
return Ok(());
|
||
}
|
||
// An omitted channel defaults to speakable on this wire protocol.
|
||
writer
|
||
.clone()
|
||
.with_context_append_channel(RealtimeContextAppendChannel::Commentary)
|
||
.send_conversation_handoff_append(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::StandaloneHandoff { text, phase } => {
|
||
v3_output_writer(
|
||
writer,
|
||
phase.as_ref(),
|
||
handoff_state.codex_response_handoff_mode,
|
||
)
|
||
.send_standalone_handoff(STANDALONE_HANDOFF_ID.to_string(), text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::StandaloneSpeech { text } => {
|
||
writer
|
||
.clone()
|
||
.with_context_append_channel(RealtimeContextAppendChannel::Speakable)
|
||
.send_standalone_handoff(STANDALONE_HANDOFF_ID.to_string(), text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffUpdate {
|
||
handoff_id,
|
||
text,
|
||
phase,
|
||
} => {
|
||
v3_output_writer(
|
||
writer,
|
||
phase.as_ref(),
|
||
handoff_state.codex_response_handoff_mode,
|
||
)
|
||
.send_conversation_function_call_output(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffAppend {
|
||
handoff_id,
|
||
text,
|
||
phase,
|
||
} => {
|
||
v3_output_writer(
|
||
writer,
|
||
phase.as_ref(),
|
||
handoff_state.codex_response_handoff_mode,
|
||
)
|
||
.send_conversation_handoff_append(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::CompletedHandoff {
|
||
handoff_id,
|
||
text,
|
||
phase,
|
||
} => {
|
||
v3_output_writer(
|
||
writer,
|
||
phase.as_ref(),
|
||
handoff_state.codex_response_handoff_mode,
|
||
)
|
||
.send_conversation_function_call_output(handoff_id, text)
|
||
.await
|
||
}
|
||
RealtimeOutbound::ConversationItem { text, phase } => {
|
||
v3_output_writer(
|
||
writer,
|
||
phase.as_ref(),
|
||
handoff_state.codex_response_handoff_mode,
|
||
)
|
||
.send_conversation_item_create(text, ConversationTextRole::Developer)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffCompleteAck { .. } => Ok(()),
|
||
},
|
||
RealtimeEventParser::RealtimeV2 => match handoff_output {
|
||
RealtimeOutbound::QuietReasoningStatus { .. } => Ok(()),
|
||
RealtimeOutbound::StandaloneHandoff { text, phase: _ } => {
|
||
if let Err(err) = writer
|
||
.send_conversation_item_create(text, ConversationTextRole::User)
|
||
.await
|
||
{
|
||
Err(err)
|
||
} else {
|
||
return response_create_queue
|
||
.request_create(writer, "standalone handoff")
|
||
.await;
|
||
}
|
||
}
|
||
RealtimeOutbound::StandaloneSpeech { text } => {
|
||
if let Err(err) = writer
|
||
.send_conversation_item_create(text, ConversationTextRole::User)
|
||
.await
|
||
{
|
||
Err(err)
|
||
} else {
|
||
return response_create_queue
|
||
.request_create(writer, "standalone handoff")
|
||
.await;
|
||
}
|
||
}
|
||
RealtimeOutbound::HandoffUpdate {
|
||
handoff_id,
|
||
text,
|
||
phase: _,
|
||
}
|
||
| RealtimeOutbound::HandoffAppend {
|
||
handoff_id,
|
||
text,
|
||
phase: _,
|
||
} => {
|
||
let active_handoff = handoff_state.stream.lock().await.active_handoff.clone();
|
||
match active_handoff {
|
||
Some(active_handoff) if active_handoff == handoff_id => {}
|
||
Some(_) | None => {
|
||
debug!("dropping stale realtime handoff progress update");
|
||
return Ok(());
|
||
}
|
||
}
|
||
writer
|
||
.send_conversation_item_create(text, ConversationTextRole::User)
|
||
.await
|
||
}
|
||
RealtimeOutbound::CompletedHandoff {
|
||
handoff_id,
|
||
text: _,
|
||
phase: _,
|
||
} => {
|
||
if let Err(err) = writer
|
||
.send_conversation_function_call_output(
|
||
handoff_id,
|
||
REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(),
|
||
)
|
||
.await
|
||
{
|
||
Err(err)
|
||
} else {
|
||
return response_create_queue
|
||
.request_create(writer, "handoff")
|
||
.await;
|
||
}
|
||
}
|
||
RealtimeOutbound::ConversationItem { text, phase: _ } => {
|
||
writer
|
||
.send_conversation_item_create(text, ConversationTextRole::Developer)
|
||
.await
|
||
}
|
||
RealtimeOutbound::HandoffCompleteAck { handoff_id } => {
|
||
writer
|
||
.send_conversation_function_call_output(handoff_id, String::new())
|
||
.await
|
||
}
|
||
},
|
||
};
|
||
if let Err(err) = result {
|
||
warn!("failed to send handoff output: {err}");
|
||
return Err(err.into());
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
async fn handle_realtime_server_event(
|
||
(event, completed): RealtimeOutput,
|
||
writer: &RealtimeWebsocketWriter,
|
||
events_tx: &Sender<RealtimeOutput>,
|
||
handoff_state: &RealtimeHandoffState,
|
||
session_kind: RealtimeSessionKind,
|
||
output_audio_state: &mut Option<OutputAudioState>,
|
||
response_create_queue: &mut RealtimeResponseCreateQueue,
|
||
) -> anyhow::Result<()> {
|
||
let should_stop = match &event {
|
||
RealtimeEvent::AudioOut(frame) => {
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => {
|
||
update_output_audio_state(output_audio_state, frame);
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::InputAudioSpeechStarted(event) => {
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => {
|
||
if let Some(output_audio_state) = output_audio_state.take()
|
||
&& event
|
||
.item_id
|
||
.as_deref()
|
||
.is_none_or(|item_id| item_id == output_audio_state.item_id)
|
||
{
|
||
writer
|
||
.send_payload(
|
||
json!({
|
||
"type": "conversation.item.truncate",
|
||
"item_id": output_audio_state.item_id,
|
||
"content_index": 0,
|
||
"audio_end_ms": output_audio_state.audio_end_ms,
|
||
})
|
||
.to_string(),
|
||
)
|
||
.await
|
||
.map_err(anyhow::Error::from)?;
|
||
}
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::ResponseCreated(_) => {
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => response_create_queue.mark_started(),
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::ResponseCancelled(_) => {
|
||
*output_audio_state = None;
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => {
|
||
response_create_queue
|
||
.mark_finished(writer, "deferred")
|
||
.await?;
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::ResponseDone(_) => {
|
||
*output_audio_state = None;
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => {
|
||
response_create_queue
|
||
.mark_finished(writer, "deferred")
|
||
.await?;
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::HandoffRequested(handoff) => {
|
||
*output_audio_state = None;
|
||
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {
|
||
let mut stream = handoff_state.stream.lock().await;
|
||
stream.items.clear();
|
||
stream.active_handoff = Some(handoff.handoff_id.clone());
|
||
}
|
||
RealtimeSessionKind::V2 => {
|
||
let active_handoff = handoff_state.stream.lock().await.active_handoff.clone();
|
||
match active_handoff {
|
||
Some(_) => {
|
||
if let Err(err) = writer
|
||
.send_conversation_function_call_output(
|
||
handoff.handoff_id.clone(),
|
||
REALTIME_V2_STEER_ACKNOWLEDGEMENT.to_string(),
|
||
)
|
||
.await
|
||
{
|
||
warn!("failed to send handoff steering acknowledgement: {err}");
|
||
return Err(err.into());
|
||
}
|
||
response_create_queue
|
||
.request_create(writer, "handoff steering")
|
||
.await?;
|
||
}
|
||
None => {
|
||
handoff_state.stream.lock().await.active_handoff =
|
||
Some(handoff.handoff_id.clone());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::NoopRequested(noop) => {
|
||
*output_audio_state = None;
|
||
|
||
match session_kind {
|
||
RealtimeSessionKind::V1 => {}
|
||
RealtimeSessionKind::V2 => {
|
||
if let Err(err) = writer
|
||
.send_conversation_function_call_output(noop.call_id.clone(), String::new())
|
||
.await
|
||
{
|
||
warn!("failed to send realtime noop function output: {err}");
|
||
return Err(err.into());
|
||
}
|
||
}
|
||
}
|
||
false
|
||
}
|
||
RealtimeEvent::Error(_) => true,
|
||
RealtimeEvent::SessionUpdated {
|
||
realtime_session_id,
|
||
..
|
||
} => {
|
||
info!(realtime_session_id = %realtime_session_id, "realtime session updated");
|
||
false
|
||
}
|
||
RealtimeEvent::InputTranscriptDelta(_)
|
||
| RealtimeEvent::InputTranscriptDone(_)
|
||
| RealtimeEvent::OutputTranscriptDelta(_)
|
||
| RealtimeEvent::OutputTranscriptDone(_)
|
||
| RealtimeEvent::ConversationItemAdded(_)
|
||
| RealtimeEvent::ConversationItemDone { .. }
|
||
| RealtimeEvent::HistoryItemStarted(_)
|
||
| RealtimeEvent::HistoryTranscriptDelta { .. }
|
||
| RealtimeEvent::HistoryItemCompleted(_) => false,
|
||
};
|
||
|
||
if events_tx.send((event, completed)).await.is_err() {
|
||
anyhow::bail!("realtime output event channel closed");
|
||
}
|
||
if should_stop {
|
||
error!("realtime stream error event received");
|
||
anyhow::bail!("realtime stream error event received");
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
async fn handle_user_audio_input(
|
||
frame: Result<RealtimeAudioFrame, RecvError>,
|
||
writer: &RealtimeWebsocketWriter,
|
||
) -> anyhow::Result<()> {
|
||
let frame = frame.context("user audio input channel closed")?;
|
||
writer
|
||
.send_audio_frame(frame)
|
||
.await
|
||
.map_err(anyhow::Error::from)?;
|
||
Ok(())
|
||
}
|
||
|
||
fn update_output_audio_state(
|
||
output_audio_state: &mut Option<OutputAudioState>,
|
||
frame: &RealtimeAudioFrame,
|
||
) {
|
||
let Some(item_id) = frame.item_id.clone() else {
|
||
return;
|
||
};
|
||
let audio_end_ms = audio_duration_ms(frame);
|
||
if audio_end_ms == 0 {
|
||
return;
|
||
}
|
||
|
||
if let Some(current) = output_audio_state.as_mut()
|
||
&& current.item_id == item_id
|
||
{
|
||
current.audio_end_ms = current.audio_end_ms.saturating_add(audio_end_ms);
|
||
return;
|
||
}
|
||
|
||
*output_audio_state = Some(OutputAudioState {
|
||
item_id,
|
||
audio_end_ms,
|
||
});
|
||
}
|
||
|
||
fn audio_duration_ms(frame: &RealtimeAudioFrame) -> u32 {
|
||
let Some(samples_per_channel) = frame
|
||
.samples_per_channel
|
||
.or(decoded_samples_per_channel(frame))
|
||
else {
|
||
return 0;
|
||
};
|
||
let sample_rate = u64::from(frame.sample_rate.max(1));
|
||
((u64::from(samples_per_channel) * 1_000) / sample_rate) as u32
|
||
}
|
||
|
||
fn decoded_samples_per_channel(frame: &RealtimeAudioFrame) -> Option<u32> {
|
||
let bytes = BASE64_STANDARD.decode(&frame.data).ok()?;
|
||
let channels = usize::from(frame.num_channels.max(1));
|
||
let samples = bytes.len().checked_div(2)?.checked_div(channels)?;
|
||
u32::try_from(samples).ok()
|
||
}
|
||
|
||
async fn send_conversation_error(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
message: String,
|
||
codex_error_info: CodexErrorInfo,
|
||
) {
|
||
sess.send_event_raw(Event {
|
||
id: sub_id,
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
misalignment: None,
|
||
message,
|
||
codex_error_info: Some(codex_error_info),
|
||
rejected_audio_file_ids: Vec::new(),
|
||
}),
|
||
})
|
||
.await;
|
||
}
|
||
|
||
async fn send_realtime_conversation_closed(
|
||
sess: &Arc<Session>,
|
||
sub_id: String,
|
||
end: RealtimeConversationEnd,
|
||
) {
|
||
let reason = match end {
|
||
RealtimeConversationEnd::Requested => Some("requested".to_string()),
|
||
RealtimeConversationEnd::TransportClosed => Some("transport_closed".to_string()),
|
||
RealtimeConversationEnd::Error => Some("error".to_string()),
|
||
};
|
||
|
||
sess.send_event_raw(Event {
|
||
id: sub_id,
|
||
msg: EventMsg::RealtimeConversationClosed(RealtimeConversationClosedEvent { reason }),
|
||
})
|
||
.await;
|
||
}
|
||
|
||
#[cfg(test)]
|
||
#[path = "realtime_conversation_tests.rs"]
|
||
mod tests;
|