Add live WebRTC voice conversations to the TUI (#43581)

## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson
2026-09-07 20:25:30 +00:00
committed by copyberry
parent 4110342321
commit b01c3986fd
70 changed files with 4421 additions and 69 deletions
+61 -9
View File
@@ -76,6 +76,7 @@ use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::debug;
@@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop {
Detach,
}
/// Serializes turn admission with the safety cutoff for one voice session.
struct RealtimeHandoffAdmission {
gate: Semaphore,
retired: AtomicBool,
}
impl RealtimeHandoffAdmission {
fn new() -> Self {
Self {
gate: Semaphore::new(1),
retired: AtomicBool::new(false),
}
}
async fn route(&self, session: &Arc<Session>, text: String) {
let Ok(_permit) = self.gate.acquire().await else {
return;
};
if !self.retired.load(Ordering::Acquire) {
session.route_realtime_text_input(text).await;
}
}
async fn retire(&self) {
let Ok(_permit) = self.gate.acquire().await else {
return;
};
self.retired.store(true, Ordering::Release);
}
}
pub(crate) struct RealtimeConversationManager {
state: Mutex<Option<ConversationState>>,
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
@@ -467,6 +499,8 @@ struct ConversationState {
input_task: JoinHandle<()>,
fanout_task: Option<JoinHandle<()>>,
realtime_active: Arc<AtomicBool>,
// A misalignment failure retires only this session's queued handoffs.
route_handoffs: Arc<RealtimeHandoffAdmission>,
stop_token: CancellationToken,
}
@@ -489,6 +523,7 @@ struct RealtimeStart {
struct RealtimeStartOutput {
realtime_active: Arc<AtomicBool>,
route_handoffs: Arc<RealtimeHandoffAdmission>,
events_rx: Receiver<RealtimeEvent>,
transcript_tail_rx: Receiver<String>,
sdp: Option<String>,
@@ -529,6 +564,20 @@ impl RealtimeConversationManager {
)
}
/// Prevent already parsed handoffs from starting turns after a safety failure.
/// Keep the gate on the old session so a later voice session starts fresh.
pub(crate) async fn retire_handoffs_for_misalignment(&self) {
let route_handoffs = self
.state
.lock()
.await
.as_ref()
.map(|state| Arc::clone(&state.route_handoffs));
if let Some(route_handoffs) = route_handoffs {
route_handoffs.retire().await;
}
}
async fn start(
&self,
start: RealtimeStart,
@@ -581,6 +630,7 @@ impl RealtimeConversationManager {
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
let realtime_active = Arc::new(AtomicBool::new(true));
let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
let stop_token = CancellationToken::new();
let handoff = RealtimeHandoffState {
output_tx: handoff_output_tx,
@@ -690,10 +740,12 @@ impl RealtimeConversationManager {
input_task: task,
fanout_task: None,
realtime_active: Arc::clone(&realtime_active),
route_handoffs: Arc::clone(&route_handoffs),
stop_token,
});
Ok(RealtimeStartOutput {
realtime_active,
route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1571,6 +1623,7 @@ async fn handle_start_inner(
let RealtimeStartOutput {
realtime_active,
route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1597,10 +1650,9 @@ async fn handle_start_inner(
match &event {
RealtimeEvent::AudioOut(_) => {}
_ => {
info!(
event = ?event,
"received realtime conversation event"
);
// Transcripts, handoffs, and conversation items may contain
// credentials. Keep this receipt free of event payloads.
info!("received realtime conversation event");
}
}
if let RealtimeEvent::Error(_) = &event {
@@ -1613,9 +1665,9 @@ async fn handle_start_inner(
_ => None,
};
if let Some(text) = maybe_routed_text {
debug!(text = %text, "[realtime-text] realtime conversation text output");
let sess_for_routed_text = Arc::clone(&sess_clone);
sess_for_routed_text.route_realtime_text_input(text).await;
// The routed text can contain spoken prompts or workspace secrets.
debug!("[realtime-text] realtime conversation text output");
route_handoffs.route(&sess_clone, text).await;
}
sess_clone
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
@@ -1626,7 +1678,7 @@ async fn handle_start_inner(
.await;
}
if let Ok(text) = transcript_tail_rx.recv().await {
sess_clone.route_realtime_text_input(text).await;
route_handoffs.route(&sess_clone, text).await;
}
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
match end {
@@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech(
sub_id: String,
params: ConversationSpeechParams,
) {
debug!(text = %params.text, "[realtime-text] appending realtime speech");
debug!("[realtime-text] appending realtime speech");
if let Err(err) = sess.conversation.append_speech(params.text).await {
error!("failed to append realtime speech: {err}");
if sess.conversation.running_state().await.is_some() {
+6
View File
@@ -636,6 +636,12 @@ pub(crate) async fn run_turn(
}
Err(e) => {
info!("Turn error: {e:#}");
if matches!(
e.details(),
CodexErrorDetails::MisalignmentPolicyViolation { .. }
) {
sess.conversation.retire_handoffs_for_misalignment().await;
}
let error = e.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;