Add live WebRTC voice conversations to the TUI (#43581)

## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson
2026-09-07 20:25:30 +00:00
committed by copyberry
parent 4110342321
commit b01c3986fd
70 changed files with 4421 additions and 69 deletions
@@ -0,0 +1,91 @@
//! App-server signaling for TUI-owned realtime WebRTC sessions.
use super::AppServerSession;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams;
use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse;
use codex_app_server_protocol::ThreadRealtimeStartParams;
use codex_app_server_protocol::ThreadRealtimeStartResponse;
use codex_app_server_protocol::ThreadRealtimeStartTransport;
use codex_app_server_protocol::ThreadRealtimeStopParams;
use codex_app_server_protocol::ThreadRealtimeStopResponse;
use codex_protocol::ThreadId;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeOutputModality;
use color_eyre::eyre::Result;
use color_eyre::eyre::WrapErr;
impl AppServerSession {
pub(crate) async fn thread_realtime_start(
&mut self,
thread_id: ThreadId,
offer_sdp: String,
model: Option<String>,
) -> Result<()> {
let request_id = self.next_request_id();
let _: ThreadRealtimeStartResponse = self
.client
.request_typed(ClientRequest::ThreadRealtimeStart {
request_id,
params: ThreadRealtimeStartParams {
thread_id: thread_id.to_string(),
client_managed_handoffs: Some(true),
delegation_ack_filler: None,
flush_transcript_tail_on_session_end: None,
codex_responses_as_items: None,
codex_response_item_prefix: None,
codex_response_handoff_mode: None,
codex_response_handoff_channel_prefixes: None,
model,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: Some(false),
initial_items: None,
realtime_start_instructions: None,
realtime_end_instructions: None,
prompt: None,
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp }),
version: Some(RealtimeConversationVersion::V3),
voice: None,
},
})
.await
.wrap_err("thread/realtime/start failed in TUI")?;
Ok(())
}
pub(crate) async fn thread_realtime_stop(&mut self, thread_id: ThreadId) -> Result<()> {
let request_id = self.next_request_id();
let _: ThreadRealtimeStopResponse = self
.client
.request_typed(ClientRequest::ThreadRealtimeStop {
request_id,
params: ThreadRealtimeStopParams {
thread_id: thread_id.to_string(),
},
})
.await
.wrap_err("thread/realtime/stop failed in TUI")?;
Ok(())
}
pub(crate) async fn thread_realtime_append_speech(
&mut self,
thread_id: ThreadId,
text: String,
) -> Result<()> {
let request_id = self.next_request_id();
let _: ThreadRealtimeAppendSpeechResponse = self
.client
.request_typed(ClientRequest::ThreadRealtimeAppendSpeech {
request_id,
params: ThreadRealtimeAppendSpeechParams {
thread_id: thread_id.to_string(),
text,
},
})
.await
.wrap_err("thread/realtime/appendSpeech failed in TUI")?;
Ok(())
}
}