mirror of
https://github.com/openai/codex.git
synced 2026-10-26 18:38:47 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
committed by
copyberry
parent
4110342321
commit
b01c3986fd
@@ -1,3 +1,4 @@
|
||||
use std::fmt;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use codex_app_server_protocol::AskForApproval;
|
||||
@@ -9,6 +10,7 @@ use codex_app_server_protocol::ReviewTarget;
|
||||
use codex_app_server_protocol::ToolRequestUserInputResponse;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_config::types::ApprovalsReviewer;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::approvals::GuardianAssessmentEvent;
|
||||
use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::Personality;
|
||||
@@ -19,13 +21,98 @@ use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
|
||||
use codex_protocol::request_permissions::RequestPermissionsResponse;
|
||||
use serde::Serialize;
|
||||
use serde::Serializer;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Keeps ICE credentials out of command diagnostics and session recordings.
|
||||
/// Convert back to a string only when sending the offer to app-server signaling.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RealtimeOfferSdp(String);
|
||||
|
||||
impl From<String> for RealtimeOfferSdp {
|
||||
fn from(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RealtimeOfferSdp> for String {
|
||||
fn from(value: RealtimeOfferSdp) -> Self {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RealtimeOfferSdp {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("<redacted>")
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for RealtimeOfferSdp {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str("<redacted>")
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps spoken answers out of command diagnostics and session recordings.
|
||||
/// Convert back to a string only at the app-server signaling boundary.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RealtimeSpeechText(String);
|
||||
|
||||
impl From<String> for RealtimeSpeechText {
|
||||
fn from(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RealtimeSpeechText> for String {
|
||||
fn from(value: RealtimeSpeechText) -> Self {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl RealtimeSpeechText {
|
||||
pub(crate) fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RealtimeSpeechText {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("<redacted>")
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for RealtimeSpeechText {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str("<redacted>")
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub(crate) enum AppCommand {
|
||||
Interrupt,
|
||||
CleanBackgroundTerminals,
|
||||
RealtimeConversationStart {
|
||||
thread_id: ThreadId,
|
||||
offer_sdp: RealtimeOfferSdp,
|
||||
},
|
||||
RealtimeConversationStop {
|
||||
thread_id: ThreadId,
|
||||
},
|
||||
RealtimeConversationSpeech {
|
||||
thread_id: ThreadId,
|
||||
attempt_id: u64,
|
||||
input_generation: u64,
|
||||
delivery_id: u64,
|
||||
text: RealtimeSpeechText,
|
||||
},
|
||||
RunUserShellCommand {
|
||||
command: String,
|
||||
},
|
||||
@@ -252,3 +339,7 @@ impl From<&AppCommand> for AppCommand {
|
||||
value.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "app_command_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
Reference in New Issue
Block a user