mirror of
https://github.com/openai/codex.git
synced 2026-10-26 18:38:47 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
committed by
copyberry
parent
4110342321
commit
b01c3986fd
@@ -76,6 +76,7 @@ use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::debug;
|
||||
@@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop {
|
||||
Detach,
|
||||
}
|
||||
|
||||
/// Serializes turn admission with the safety cutoff for one voice session.
|
||||
struct RealtimeHandoffAdmission {
|
||||
gate: Semaphore,
|
||||
retired: AtomicBool,
|
||||
}
|
||||
|
||||
impl RealtimeHandoffAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
gate: Semaphore::new(1),
|
||||
retired: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
async fn route(&self, session: &Arc<Session>, text: String) {
|
||||
let Ok(_permit) = self.gate.acquire().await else {
|
||||
return;
|
||||
};
|
||||
if !self.retired.load(Ordering::Acquire) {
|
||||
session.route_realtime_text_input(text).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn retire(&self) {
|
||||
let Ok(_permit) = self.gate.acquire().await else {
|
||||
return;
|
||||
};
|
||||
self.retired.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RealtimeConversationManager {
|
||||
state: Mutex<Option<ConversationState>>,
|
||||
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
|
||||
@@ -467,6 +499,8 @@ struct ConversationState {
|
||||
input_task: JoinHandle<()>,
|
||||
fanout_task: Option<JoinHandle<()>>,
|
||||
realtime_active: Arc<AtomicBool>,
|
||||
// A misalignment failure retires only this session's queued handoffs.
|
||||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||||
stop_token: CancellationToken,
|
||||
}
|
||||
|
||||
@@ -489,6 +523,7 @@ struct RealtimeStart {
|
||||
|
||||
struct RealtimeStartOutput {
|
||||
realtime_active: Arc<AtomicBool>,
|
||||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||||
events_rx: Receiver<RealtimeEvent>,
|
||||
transcript_tail_rx: Receiver<String>,
|
||||
sdp: Option<String>,
|
||||
@@ -529,6 +564,20 @@ impl RealtimeConversationManager {
|
||||
)
|
||||
}
|
||||
|
||||
/// Prevent already parsed handoffs from starting turns after a safety failure.
|
||||
/// Keep the gate on the old session so a later voice session starts fresh.
|
||||
pub(crate) async fn retire_handoffs_for_misalignment(&self) {
|
||||
let route_handoffs = self
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.as_ref()
|
||||
.map(|state| Arc::clone(&state.route_handoffs));
|
||||
if let Some(route_handoffs) = route_handoffs {
|
||||
route_handoffs.retire().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn start(
|
||||
&self,
|
||||
start: RealtimeStart,
|
||||
@@ -581,6 +630,7 @@ impl RealtimeConversationManager {
|
||||
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
|
||||
|
||||
let realtime_active = Arc::new(AtomicBool::new(true));
|
||||
let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
|
||||
let stop_token = CancellationToken::new();
|
||||
let handoff = RealtimeHandoffState {
|
||||
output_tx: handoff_output_tx,
|
||||
@@ -690,10 +740,12 @@ impl RealtimeConversationManager {
|
||||
input_task: task,
|
||||
fanout_task: None,
|
||||
realtime_active: Arc::clone(&realtime_active),
|
||||
route_handoffs: Arc::clone(&route_handoffs),
|
||||
stop_token,
|
||||
});
|
||||
Ok(RealtimeStartOutput {
|
||||
realtime_active,
|
||||
route_handoffs,
|
||||
events_rx,
|
||||
transcript_tail_rx,
|
||||
sdp,
|
||||
@@ -1571,6 +1623,7 @@ async fn handle_start_inner(
|
||||
|
||||
let RealtimeStartOutput {
|
||||
realtime_active,
|
||||
route_handoffs,
|
||||
events_rx,
|
||||
transcript_tail_rx,
|
||||
sdp,
|
||||
@@ -1597,10 +1650,9 @@ async fn handle_start_inner(
|
||||
match &event {
|
||||
RealtimeEvent::AudioOut(_) => {}
|
||||
_ => {
|
||||
info!(
|
||||
event = ?event,
|
||||
"received realtime conversation event"
|
||||
);
|
||||
// Transcripts, handoffs, and conversation items may contain
|
||||
// credentials. Keep this receipt free of event payloads.
|
||||
info!("received realtime conversation event");
|
||||
}
|
||||
}
|
||||
if let RealtimeEvent::Error(_) = &event {
|
||||
@@ -1613,9 +1665,9 @@ async fn handle_start_inner(
|
||||
_ => None,
|
||||
};
|
||||
if let Some(text) = maybe_routed_text {
|
||||
debug!(text = %text, "[realtime-text] realtime conversation text output");
|
||||
let sess_for_routed_text = Arc::clone(&sess_clone);
|
||||
sess_for_routed_text.route_realtime_text_input(text).await;
|
||||
// The routed text can contain spoken prompts or workspace secrets.
|
||||
debug!("[realtime-text] realtime conversation text output");
|
||||
route_handoffs.route(&sess_clone, text).await;
|
||||
}
|
||||
sess_clone
|
||||
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
|
||||
@@ -1626,7 +1678,7 @@ async fn handle_start_inner(
|
||||
.await;
|
||||
}
|
||||
if let Ok(text) = transcript_tail_rx.recv().await {
|
||||
sess_clone.route_realtime_text_input(text).await;
|
||||
route_handoffs.route(&sess_clone, text).await;
|
||||
}
|
||||
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
|
||||
match end {
|
||||
@@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech(
|
||||
sub_id: String,
|
||||
params: ConversationSpeechParams,
|
||||
) {
|
||||
debug!(text = %params.text, "[realtime-text] appending realtime speech");
|
||||
debug!("[realtime-text] appending realtime speech");
|
||||
if let Err(err) = sess.conversation.append_speech(params.text).await {
|
||||
error!("failed to append realtime speech: {err}");
|
||||
if sess.conversation.running_state().await.is_some() {
|
||||
|
||||
@@ -636,6 +636,12 @@ pub(crate) async fn run_turn(
|
||||
}
|
||||
Err(e) => {
|
||||
info!("Turn error: {e:#}");
|
||||
if matches!(
|
||||
e.details(),
|
||||
CodexErrorDetails::MisalignmentPolicyViolation { .. }
|
||||
) {
|
||||
sess.conversation.retire_handoffs_for_misalignment().await;
|
||||
}
|
||||
let error = e.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
|
||||
Reference in New Issue
Block a user