Add live WebRTC voice conversations to the TUI (#43581)

## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson
2026-09-07 20:25:30 +00:00
committed by copyberry
parent 4110342321
commit b01c3986fd
70 changed files with 4421 additions and 69 deletions
+91
View File
@@ -1,3 +1,4 @@
use std::fmt;
use std::path::PathBuf;
use codex_app_server_protocol::AskForApproval;
@@ -9,6 +10,7 @@ use codex_app_server_protocol::ReviewTarget;
use codex_app_server_protocol::ToolRequestUserInputResponse;
use codex_app_server_protocol::UserInput;
use codex_config::types::ApprovalsReviewer;
use codex_protocol::ThreadId;
use codex_protocol::approvals::GuardianAssessmentEvent;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::Personality;
@@ -19,13 +21,98 @@ use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
use codex_protocol::request_permissions::RequestPermissionsResponse;
use serde::Serialize;
use serde::Serializer;
use serde_json::Value;
/// Keeps ICE credentials out of command diagnostics and session recordings.
/// Convert back to a string only when sending the offer to app-server signaling.
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct RealtimeOfferSdp(String);
impl From<String> for RealtimeOfferSdp {
fn from(value: String) -> Self {
Self(value)
}
}
impl From<RealtimeOfferSdp> for String {
fn from(value: RealtimeOfferSdp) -> Self {
value.0
}
}
impl fmt::Debug for RealtimeOfferSdp {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("<redacted>")
}
}
impl Serialize for RealtimeOfferSdp {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str("<redacted>")
}
}
/// Keeps spoken answers out of command diagnostics and session recordings.
/// Convert back to a string only at the app-server signaling boundary.
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct RealtimeSpeechText(String);
impl From<String> for RealtimeSpeechText {
fn from(value: String) -> Self {
Self(value)
}
}
impl From<RealtimeSpeechText> for String {
fn from(value: RealtimeSpeechText) -> Self {
value.0
}
}
impl RealtimeSpeechText {
pub(crate) fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Debug for RealtimeSpeechText {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("<redacted>")
}
}
impl Serialize for RealtimeSpeechText {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str("<redacted>")
}
}
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Clone, PartialEq, Serialize)]
pub(crate) enum AppCommand {
Interrupt,
CleanBackgroundTerminals,
RealtimeConversationStart {
thread_id: ThreadId,
offer_sdp: RealtimeOfferSdp,
},
RealtimeConversationStop {
thread_id: ThreadId,
},
RealtimeConversationSpeech {
thread_id: ThreadId,
attempt_id: u64,
input_generation: u64,
delivery_id: u64,
text: RealtimeSpeechText,
},
RunUserShellCommand {
command: String,
},
@@ -252,3 +339,7 @@ impl From<&AppCommand> for AppCommand {
value.clone()
}
}
#[cfg(test)]
#[path = "app_command_tests.rs"]
mod tests;