Add live WebRTC voice conversations to the TUI (#43581)

## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson
2026-09-07 20:25:30 +00:00
committed by copyberry
parent 4110342321
commit b01c3986fd
70 changed files with 4421 additions and 69 deletions
+61 -9
View File
@@ -76,6 +76,7 @@ use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::debug;
@@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop {
Detach,
}
/// Serializes turn admission with the safety cutoff for one voice session.
struct RealtimeHandoffAdmission {
gate: Semaphore,
retired: AtomicBool,
}
impl RealtimeHandoffAdmission {
fn new() -> Self {
Self {
gate: Semaphore::new(1),
retired: AtomicBool::new(false),
}
}
async fn route(&self, session: &Arc<Session>, text: String) {
let Ok(_permit) = self.gate.acquire().await else {
return;
};
if !self.retired.load(Ordering::Acquire) {
session.route_realtime_text_input(text).await;
}
}
async fn retire(&self) {
let Ok(_permit) = self.gate.acquire().await else {
return;
};
self.retired.store(true, Ordering::Release);
}
}
pub(crate) struct RealtimeConversationManager {
state: Mutex<Option<ConversationState>>,
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
@@ -467,6 +499,8 @@ struct ConversationState {
input_task: JoinHandle<()>,
fanout_task: Option<JoinHandle<()>>,
realtime_active: Arc<AtomicBool>,
// A misalignment failure retires only this session's queued handoffs.
route_handoffs: Arc<RealtimeHandoffAdmission>,
stop_token: CancellationToken,
}
@@ -489,6 +523,7 @@ struct RealtimeStart {
struct RealtimeStartOutput {
realtime_active: Arc<AtomicBool>,
route_handoffs: Arc<RealtimeHandoffAdmission>,
events_rx: Receiver<RealtimeEvent>,
transcript_tail_rx: Receiver<String>,
sdp: Option<String>,
@@ -529,6 +564,20 @@ impl RealtimeConversationManager {
)
}
/// Prevent already parsed handoffs from starting turns after a safety failure.
/// Keep the gate on the old session so a later voice session starts fresh.
pub(crate) async fn retire_handoffs_for_misalignment(&self) {
let route_handoffs = self
.state
.lock()
.await
.as_ref()
.map(|state| Arc::clone(&state.route_handoffs));
if let Some(route_handoffs) = route_handoffs {
route_handoffs.retire().await;
}
}
async fn start(
&self,
start: RealtimeStart,
@@ -581,6 +630,7 @@ impl RealtimeConversationManager {
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
let realtime_active = Arc::new(AtomicBool::new(true));
let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
let stop_token = CancellationToken::new();
let handoff = RealtimeHandoffState {
output_tx: handoff_output_tx,
@@ -690,10 +740,12 @@ impl RealtimeConversationManager {
input_task: task,
fanout_task: None,
realtime_active: Arc::clone(&realtime_active),
route_handoffs: Arc::clone(&route_handoffs),
stop_token,
});
Ok(RealtimeStartOutput {
realtime_active,
route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1571,6 +1623,7 @@ async fn handle_start_inner(
let RealtimeStartOutput {
realtime_active,
route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1597,10 +1650,9 @@ async fn handle_start_inner(
match &event {
RealtimeEvent::AudioOut(_) => {}
_ => {
info!(
event = ?event,
"received realtime conversation event"
);
// Transcripts, handoffs, and conversation items may contain
// credentials. Keep this receipt free of event payloads.
info!("received realtime conversation event");
}
}
if let RealtimeEvent::Error(_) = &event {
@@ -1613,9 +1665,9 @@ async fn handle_start_inner(
_ => None,
};
if let Some(text) = maybe_routed_text {
debug!(text = %text, "[realtime-text] realtime conversation text output");
let sess_for_routed_text = Arc::clone(&sess_clone);
sess_for_routed_text.route_realtime_text_input(text).await;
// The routed text can contain spoken prompts or workspace secrets.
debug!("[realtime-text] realtime conversation text output");
route_handoffs.route(&sess_clone, text).await;
}
sess_clone
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
@@ -1626,7 +1678,7 @@ async fn handle_start_inner(
.await;
}
if let Ok(text) = transcript_tail_rx.recv().await {
sess_clone.route_realtime_text_input(text).await;
route_handoffs.route(&sess_clone, text).await;
}
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
match end {
@@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech(
sub_id: String,
params: ConversationSpeechParams,
) {
debug!(text = %params.text, "[realtime-text] appending realtime speech");
debug!("[realtime-text] appending realtime speech");
if let Err(err) = sess.conversation.append_speech(params.text).await {
error!("failed to append realtime speech: {err}");
if sess.conversation.running_state().await.is_some() {
+6
View File
@@ -636,6 +636,12 @@ pub(crate) async fn run_turn(
}
Err(e) => {
info!("Turn error: {e:#}");
if matches!(
e.details(),
CodexErrorDetails::MisalignmentPolicyViolation { .. }
) {
sess.conversation.retire_handoffs_for_misalignment().await;
}
let error = e.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
+1
View File
@@ -128,6 +128,7 @@ mod prompt_debug_tests;
mod quota_exceeded;
mod realtime_conversation;
mod realtime_initial_items;
mod realtime_misalignment;
mod realtime_sideband_endpoint;
mod reasoning_effort_override;
mod remote_env;
@@ -0,0 +1,151 @@
//! A failed turn must not admit handoffs still arriving from its voice session.
use anyhow::Context;
use anyhow::Result;
use codex_config::config_toml::RealtimeWsVersion;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::protocol::RealtimeOutputModality;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event_match;
use futures::SinkExt;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::oneshot;
use tokio_tungstenite::accept_async;
use tokio_tungstenite::tungstenite::Message;
use wiremock::ResponseTemplate;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn misalignment_retires_late_voice_handoff_before_it_starts_a_turn() -> Result<()> {
skip_if_no_network!(Ok(()));
let api_server = responses::start_mock_server().await;
let first_response = responses::mount_response_once(
&api_server,
ResponseTemplate::new(403).set_body_json(json!({
"error": {
"message": "This request violated the misalignment policy.",
"type": "invalid_request_error",
"code": "misalignment_policy_violation"
}
})),
)
.await;
let listener = TcpListener::bind("127.0.0.1:0").await?;
let realtime_url = format!("ws://{}", listener.local_addr()?);
let (late_handoff_tx, mut late_handoff_rx) = oneshot::channel();
let (finish_tx, finish_rx) = oneshot::channel();
let sideband = tokio::spawn(async move {
let (stream, _) = listener.accept().await?;
let mut websocket = accept_async(stream).await?;
websocket
.send(Message::Text(
json!({
"type": "session.updated",
"session": {"id": "session_misalignment", "instructions": "backend prompt"}
})
.to_string()
.into(),
))
.await?;
for (handoff_id, text) in [
("initial_handoff", "first request"),
("late_handoff", "must not start a turn"),
] {
if handoff_id == "late_handoff" {
(&mut late_handoff_rx).await?;
}
websocket
.send(Message::Text(
json!({
"type": "conversation.handoff.requested",
"handoff_id": handoff_id,
"item_id": handoff_id,
"input_transcript": text
})
.to_string()
.into(),
))
.await?;
}
finish_rx.await?;
websocket.close(None).await?;
Ok::<_, anyhow::Error>(())
});
let mut builder = test_codex().with_config(move |config| {
config.experimental_realtime_ws_base_url = Some(realtime_url);
config.realtime.version = RealtimeWsVersion::V1;
});
let test = builder.build_with_auto_env(&api_server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
client_managed_handoffs: false,
delegation_ack_filler: None,
flush_transcript_tail_on_session_end: false,
codex_responses_as_items: false,
codex_response_item_prefix: None,
codex_response_handoff_mode:
codex_protocol::protocol::CodexResponseHandoffMode::Thinking,
codex_response_handoff_channel_prefixes: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: true,
initial_items: Vec::new(),
realtime_start_instructions: None,
realtime_end_instructions: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
}))
.await?;
wait_for_event_match(&test.codex, |event| match event {
EventMsg::Error(error)
if error.codex_error_info == Some(CodexErrorInfo::MisalignmentPolicyViolation) =>
{
Some(())
}
_ => None,
})
.await;
assert_eq!(first_response.requests().len(), 1);
late_handoff_tx.send(()).expect("sideband still open");
wait_for_event_match(&test.codex, |event| match event {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::HandoffRequested(handoff),
}) if handoff.handoff_id == "late_handoff" => Some(()),
_ => None,
})
.await;
finish_tx.send(()).expect("sideband still open");
sideband.await??;
// The fanout forwards a handoff only after deciding whether to route it.
// Give any spawned turn a chance to reach the mock API before checking.
tokio::time::sleep(Duration::from_millis(200)).await;
let requests = api_server
.received_requests()
.await
.context("API requests")?;
assert_eq!(
requests
.iter()
.filter(|request| request.url.path() == "/v1/responses")
.count(),
1,
);
Ok(())
}