mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
committed by
copyberry
parent
4110342321
commit
b01c3986fd
@@ -0,0 +1,91 @@
|
||||
//! App-server signaling for TUI-owned realtime WebRTC sessions.
|
||||
|
||||
use super::AppServerSession;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams;
|
||||
use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse;
|
||||
use codex_app_server_protocol::ThreadRealtimeStartParams;
|
||||
use codex_app_server_protocol::ThreadRealtimeStartResponse;
|
||||
use codex_app_server_protocol::ThreadRealtimeStartTransport;
|
||||
use codex_app_server_protocol::ThreadRealtimeStopParams;
|
||||
use codex_app_server_protocol::ThreadRealtimeStopResponse;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::RealtimeConversationVersion;
|
||||
use codex_protocol::protocol::RealtimeOutputModality;
|
||||
use color_eyre::eyre::Result;
|
||||
use color_eyre::eyre::WrapErr;
|
||||
|
||||
impl AppServerSession {
|
||||
pub(crate) async fn thread_realtime_start(
|
||||
&mut self,
|
||||
thread_id: ThreadId,
|
||||
offer_sdp: String,
|
||||
model: Option<String>,
|
||||
) -> Result<()> {
|
||||
let request_id = self.next_request_id();
|
||||
let _: ThreadRealtimeStartResponse = self
|
||||
.client
|
||||
.request_typed(ClientRequest::ThreadRealtimeStart {
|
||||
request_id,
|
||||
params: ThreadRealtimeStartParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
client_managed_handoffs: Some(true),
|
||||
delegation_ack_filler: None,
|
||||
flush_transcript_tail_on_session_end: None,
|
||||
codex_responses_as_items: None,
|
||||
codex_response_item_prefix: None,
|
||||
codex_response_handoff_mode: None,
|
||||
codex_response_handoff_channel_prefixes: None,
|
||||
model,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
include_startup_context: Some(false),
|
||||
initial_items: None,
|
||||
realtime_start_instructions: None,
|
||||
realtime_end_instructions: None,
|
||||
prompt: None,
|
||||
realtime_session_id: None,
|
||||
transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp }),
|
||||
version: Some(RealtimeConversationVersion::V3),
|
||||
voice: None,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.wrap_err("thread/realtime/start failed in TUI")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn thread_realtime_stop(&mut self, thread_id: ThreadId) -> Result<()> {
|
||||
let request_id = self.next_request_id();
|
||||
let _: ThreadRealtimeStopResponse = self
|
||||
.client
|
||||
.request_typed(ClientRequest::ThreadRealtimeStop {
|
||||
request_id,
|
||||
params: ThreadRealtimeStopParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
},
|
||||
})
|
||||
.await
|
||||
.wrap_err("thread/realtime/stop failed in TUI")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn thread_realtime_append_speech(
|
||||
&mut self,
|
||||
thread_id: ThreadId,
|
||||
text: String,
|
||||
) -> Result<()> {
|
||||
let request_id = self.next_request_id();
|
||||
let _: ThreadRealtimeAppendSpeechResponse = self
|
||||
.client
|
||||
.request_typed(ClientRequest::ThreadRealtimeAppendSpeech {
|
||||
request_id,
|
||||
params: ThreadRealtimeAppendSpeechParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
text,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.wrap_err("thread/realtime/appendSpeech failed in TUI")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user