mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
committed by
copyberry
parent
4110342321
commit
b01c3986fd
@@ -76,6 +76,7 @@ use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::debug;
|
||||
@@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop {
|
||||
Detach,
|
||||
}
|
||||
|
||||
/// Serializes turn admission with the safety cutoff for one voice session.
|
||||
struct RealtimeHandoffAdmission {
|
||||
gate: Semaphore,
|
||||
retired: AtomicBool,
|
||||
}
|
||||
|
||||
impl RealtimeHandoffAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
gate: Semaphore::new(1),
|
||||
retired: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
async fn route(&self, session: &Arc<Session>, text: String) {
|
||||
let Ok(_permit) = self.gate.acquire().await else {
|
||||
return;
|
||||
};
|
||||
if !self.retired.load(Ordering::Acquire) {
|
||||
session.route_realtime_text_input(text).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn retire(&self) {
|
||||
let Ok(_permit) = self.gate.acquire().await else {
|
||||
return;
|
||||
};
|
||||
self.retired.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RealtimeConversationManager {
|
||||
state: Mutex<Option<ConversationState>>,
|
||||
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
|
||||
@@ -467,6 +499,8 @@ struct ConversationState {
|
||||
input_task: JoinHandle<()>,
|
||||
fanout_task: Option<JoinHandle<()>>,
|
||||
realtime_active: Arc<AtomicBool>,
|
||||
// A misalignment failure retires only this session's queued handoffs.
|
||||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||||
stop_token: CancellationToken,
|
||||
}
|
||||
|
||||
@@ -489,6 +523,7 @@ struct RealtimeStart {
|
||||
|
||||
struct RealtimeStartOutput {
|
||||
realtime_active: Arc<AtomicBool>,
|
||||
route_handoffs: Arc<RealtimeHandoffAdmission>,
|
||||
events_rx: Receiver<RealtimeEvent>,
|
||||
transcript_tail_rx: Receiver<String>,
|
||||
sdp: Option<String>,
|
||||
@@ -529,6 +564,20 @@ impl RealtimeConversationManager {
|
||||
)
|
||||
}
|
||||
|
||||
/// Prevent already parsed handoffs from starting turns after a safety failure.
|
||||
/// Keep the gate on the old session so a later voice session starts fresh.
|
||||
pub(crate) async fn retire_handoffs_for_misalignment(&self) {
|
||||
let route_handoffs = self
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.as_ref()
|
||||
.map(|state| Arc::clone(&state.route_handoffs));
|
||||
if let Some(route_handoffs) = route_handoffs {
|
||||
route_handoffs.retire().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn start(
|
||||
&self,
|
||||
start: RealtimeStart,
|
||||
@@ -581,6 +630,7 @@ impl RealtimeConversationManager {
|
||||
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
|
||||
|
||||
let realtime_active = Arc::new(AtomicBool::new(true));
|
||||
let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
|
||||
let stop_token = CancellationToken::new();
|
||||
let handoff = RealtimeHandoffState {
|
||||
output_tx: handoff_output_tx,
|
||||
@@ -690,10 +740,12 @@ impl RealtimeConversationManager {
|
||||
input_task: task,
|
||||
fanout_task: None,
|
||||
realtime_active: Arc::clone(&realtime_active),
|
||||
route_handoffs: Arc::clone(&route_handoffs),
|
||||
stop_token,
|
||||
});
|
||||
Ok(RealtimeStartOutput {
|
||||
realtime_active,
|
||||
route_handoffs,
|
||||
events_rx,
|
||||
transcript_tail_rx,
|
||||
sdp,
|
||||
@@ -1571,6 +1623,7 @@ async fn handle_start_inner(
|
||||
|
||||
let RealtimeStartOutput {
|
||||
realtime_active,
|
||||
route_handoffs,
|
||||
events_rx,
|
||||
transcript_tail_rx,
|
||||
sdp,
|
||||
@@ -1597,10 +1650,9 @@ async fn handle_start_inner(
|
||||
match &event {
|
||||
RealtimeEvent::AudioOut(_) => {}
|
||||
_ => {
|
||||
info!(
|
||||
event = ?event,
|
||||
"received realtime conversation event"
|
||||
);
|
||||
// Transcripts, handoffs, and conversation items may contain
|
||||
// credentials. Keep this receipt free of event payloads.
|
||||
info!("received realtime conversation event");
|
||||
}
|
||||
}
|
||||
if let RealtimeEvent::Error(_) = &event {
|
||||
@@ -1613,9 +1665,9 @@ async fn handle_start_inner(
|
||||
_ => None,
|
||||
};
|
||||
if let Some(text) = maybe_routed_text {
|
||||
debug!(text = %text, "[realtime-text] realtime conversation text output");
|
||||
let sess_for_routed_text = Arc::clone(&sess_clone);
|
||||
sess_for_routed_text.route_realtime_text_input(text).await;
|
||||
// The routed text can contain spoken prompts or workspace secrets.
|
||||
debug!("[realtime-text] realtime conversation text output");
|
||||
route_handoffs.route(&sess_clone, text).await;
|
||||
}
|
||||
sess_clone
|
||||
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
|
||||
@@ -1626,7 +1678,7 @@ async fn handle_start_inner(
|
||||
.await;
|
||||
}
|
||||
if let Ok(text) = transcript_tail_rx.recv().await {
|
||||
sess_clone.route_realtime_text_input(text).await;
|
||||
route_handoffs.route(&sess_clone, text).await;
|
||||
}
|
||||
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
|
||||
match end {
|
||||
@@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech(
|
||||
sub_id: String,
|
||||
params: ConversationSpeechParams,
|
||||
) {
|
||||
debug!(text = %params.text, "[realtime-text] appending realtime speech");
|
||||
debug!("[realtime-text] appending realtime speech");
|
||||
if let Err(err) = sess.conversation.append_speech(params.text).await {
|
||||
error!("failed to append realtime speech: {err}");
|
||||
if sess.conversation.running_state().await.is_some() {
|
||||
|
||||
@@ -636,6 +636,12 @@ pub(crate) async fn run_turn(
|
||||
}
|
||||
Err(e) => {
|
||||
info!("Turn error: {e:#}");
|
||||
if matches!(
|
||||
e.details(),
|
||||
CodexErrorDetails::MisalignmentPolicyViolation { .. }
|
||||
) {
|
||||
sess.conversation.retire_handoffs_for_misalignment().await;
|
||||
}
|
||||
let error = e.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
|
||||
@@ -128,6 +128,7 @@ mod prompt_debug_tests;
|
||||
mod quota_exceeded;
|
||||
mod realtime_conversation;
|
||||
mod realtime_initial_items;
|
||||
mod realtime_misalignment;
|
||||
mod realtime_sideband_endpoint;
|
||||
mod reasoning_effort_override;
|
||||
mod remote_env;
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! A failed turn must not admit handoffs still arriving from its voice session.
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use codex_config::config_toml::RealtimeWsVersion;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::ConversationStartParams;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
|
||||
use codex_protocol::protocol::RealtimeEvent;
|
||||
use codex_protocol::protocol::RealtimeOutputModality;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event_match;
|
||||
use futures::SinkExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio_tungstenite::accept_async;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn misalignment_retires_late_voice_handoff_before_it_starts_a_turn() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let api_server = responses::start_mock_server().await;
|
||||
let first_response = responses::mount_response_once(
|
||||
&api_server,
|
||||
ResponseTemplate::new(403).set_body_json(json!({
|
||||
"error": {
|
||||
"message": "This request violated the misalignment policy.",
|
||||
"type": "invalid_request_error",
|
||||
"code": "misalignment_policy_violation"
|
||||
}
|
||||
})),
|
||||
)
|
||||
.await;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let realtime_url = format!("ws://{}", listener.local_addr()?);
|
||||
let (late_handoff_tx, mut late_handoff_rx) = oneshot::channel();
|
||||
let (finish_tx, finish_rx) = oneshot::channel();
|
||||
let sideband = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await?;
|
||||
let mut websocket = accept_async(stream).await?;
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "session.updated",
|
||||
"session": {"id": "session_misalignment", "instructions": "backend prompt"}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await?;
|
||||
for (handoff_id, text) in [
|
||||
("initial_handoff", "first request"),
|
||||
("late_handoff", "must not start a turn"),
|
||||
] {
|
||||
if handoff_id == "late_handoff" {
|
||||
(&mut late_handoff_rx).await?;
|
||||
}
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": handoff_id,
|
||||
"item_id": handoff_id,
|
||||
"input_transcript": text
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
finish_rx.await?;
|
||||
websocket.close(None).await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
});
|
||||
|
||||
let mut builder = test_codex().with_config(move |config| {
|
||||
config.experimental_realtime_ws_base_url = Some(realtime_url);
|
||||
config.realtime.version = RealtimeWsVersion::V1;
|
||||
});
|
||||
let test = builder.build_with_auto_env(&api_server).await?;
|
||||
test.codex
|
||||
.submit(Op::RealtimeConversationStart(ConversationStartParams {
|
||||
client_managed_handoffs: false,
|
||||
delegation_ack_filler: None,
|
||||
flush_transcript_tail_on_session_end: false,
|
||||
codex_responses_as_items: false,
|
||||
codex_response_item_prefix: None,
|
||||
codex_response_handoff_mode:
|
||||
codex_protocol::protocol::CodexResponseHandoffMode::Thinking,
|
||||
codex_response_handoff_channel_prefixes: None,
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
include_startup_context: true,
|
||||
initial_items: Vec::new(),
|
||||
realtime_start_instructions: None,
|
||||
realtime_end_instructions: None,
|
||||
prompt: Some(Some("backend prompt".to_string())),
|
||||
realtime_session_id: None,
|
||||
transport: None,
|
||||
version: None,
|
||||
voice: None,
|
||||
}))
|
||||
.await?;
|
||||
|
||||
wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::Error(error)
|
||||
if error.codex_error_info == Some(CodexErrorInfo::MisalignmentPolicyViolation) =>
|
||||
{
|
||||
Some(())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(first_response.requests().len(), 1);
|
||||
late_handoff_tx.send(()).expect("sideband still open");
|
||||
wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||||
payload: RealtimeEvent::HandoffRequested(handoff),
|
||||
}) if handoff.handoff_id == "late_handoff" => Some(()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
finish_tx.send(()).expect("sideband still open");
|
||||
sideband.await??;
|
||||
|
||||
// The fanout forwards a handoff only after deciding whether to route it.
|
||||
// Give any spawned turn a chance to reach the mock API before checking.
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let requests = api_server
|
||||
.received_requests()
|
||||
.await
|
||||
.context("API requests")?;
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/v1/responses")
|
||||
.count(),
|
||||
1,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user